[x265] [PATCH] Fix possible NULL pointer dereferencing in cudata init

[Ashok Kumar Mishra]

On Fri, Dec 22, 2017 at 3:07 PM, <santhoshini at multicorewareinc.com> wrote:

> # HG changeset patch
> # User Santhoshini Sekar <santhoshini at multicorewareinc.com>
> # Date 1513927117 -19800
> #      Fri Dec 22 12:48:37 2017 +0530
> # Branch stable
> # Node ID f41671b53ae084a8207e24a91d3a7cf2a96cd1ce
> # Parent  f7498acb38746d9b799d1b45343a3db56cdba306
> Fix possible NULL pointer dereferencing in cudata init
>
> diff --git a/source/common/framedata.cpp b/source/common/framedata.cpp
> --- a/source/common/framedata.cpp
> +++ b/source/common/framedata.cpp
> @@ -41,9 +41,12 @@
>      if (param.rc.bStatWrite)
>          m_spsrps = const_cast<RPS*>(sps.spsrps);
>
> -    m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param);
> -    for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++)
> -        m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr);
> +    bool isallocated = m_cuMemPool.create(0, param.internalCsp,
> sps.numCUsInFrame, param);
> +    if (isallocated)
> +        for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++)
> +            m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr);
> +    else
> +        return false;
>
>      CHECKED_MALLOC_ZERO(m_cuStat, RCStatCU, sps.numCUsInFrame);
>      CHECKED_MALLOC(m_rowStat, RCStatRow, sps.numCuInHeight);
> diff --git a/source/encoder/analysis.cpp b/source/encoder/analysis.cpp
> --- a/source/encoder/analysis.cpp
> +++ b/source/encoder/analysis.cpp
> @@ -101,15 +101,17 @@
>      {
>          ModeDepth &md = m_modeDepth[depth];
>
> -        md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param);
> +        ok &= md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param);
>          ok &= md.fencYuv.create(cuSize, csp);
> -
> -        for (int j = 0; j < MAX_PRED_TYPES; j++)
> +        if (ok)
>          {
> -            md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j);
> -            ok &= md.pred[j].predYuv.create(cuSize, csp);
> -            ok &= md.pred[j].reconYuv.create(cuSize, csp);
> -            md.pred[j].fencYuv = &md.fencYuv;
> +            for (int j = 0; j < MAX_PRED_TYPES; j++)
> +            {
> +                md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param,
> j);
> +                ok &= md.pred[j].predYuv.create(cuSize, csp);
> +                ok &= md.pred[j].reconYuv.create(cuSize, csp);
> +                md.pred[j].fencYuv = &md.fencYuv;
> +            }
>          }
>      }
>      if (m_param->sourceHeight >= 1080)
> diff --git a/source/encoder/api.cpp b/source/encoder/api.cpp
> --- a/source/encoder/api.cpp
> +++ b/source/encoder/api.cpp
> @@ -286,7 +286,7 @@
>      else if (pi_nal)
>          *pi_nal = 0;
>
> -    if (numEncoded && encoder->m_param->csvLogLevel)
> +    if (numEncoded > 0 && encoder->m_param->csvLogLevel)
>          x265_csvlog_frame(encoder->m_param, pic_out);
>
>      if (numEncoded < 0)
> diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
> --- a/source/encoder/encoder.cpp
> +++ b/source/encoder/encoder.cpp
> @@ -1342,7 +1342,9 @@
>              }
>              else
>              {
> -                frameEnc->allocEncodeData(m_reconfigure ? m_latestParam
> : m_param, m_sps);
> +                bool isallocated = frameEnc->allocEncodeData(m_reconfigure
> ? m_latestParam : m_param, m_sps);
> +                if (!isallocated)
> +                    return -1;
>                  Slice* slice = frameEnc->m_encData->m_slice;
>                  slice->m_sps = &m_sps;
>                  slice->m_pps = &m_pps;
>
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel
>
>
Pushed.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20171222/c9768f13/attachment.html>