[x265] [PATCH] Fix possible NULL pointer dereferencing in cudata init
Ashok Kumar Mishra
ashok at multicorewareinc.com
Fri Dec 22 10:47:47 CET 2017
On Fri, Dec 22, 2017 at 3:07 PM, <santhoshini at multicorewareinc.com> wrote:
> # HG changeset patch
> # User Santhoshini Sekar <santhoshini at multicorewareinc.com>
> # Date 1513927117 -19800
> # Fri Dec 22 12:48:37 2017 +0530
> # Branch stable
> # Node ID f41671b53ae084a8207e24a91d3a7cf2a96cd1ce
> # Parent f7498acb38746d9b799d1b45343a3db56cdba306
> Fix possible NULL pointer dereferencing in cudata init
>
> diff --git a/source/common/framedata.cpp b/source/common/framedata.cpp
> --- a/source/common/framedata.cpp
> +++ b/source/common/framedata.cpp
> @@ -41,9 +41,12 @@
> if (param.rc.bStatWrite)
> m_spsrps = const_cast<RPS*>(sps.spsrps);
>
> - m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param);
> - for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++)
> - m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr);
> + bool isallocated = m_cuMemPool.create(0, param.internalCsp,
> sps.numCUsInFrame, param);
> + if (isallocated)
> + for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++)
> + m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr);
> + else
> + return false;
>
> CHECKED_MALLOC_ZERO(m_cuStat, RCStatCU, sps.numCUsInFrame);
> CHECKED_MALLOC(m_rowStat, RCStatRow, sps.numCuInHeight);
> diff --git a/source/encoder/analysis.cpp b/source/encoder/analysis.cpp
> --- a/source/encoder/analysis.cpp
> +++ b/source/encoder/analysis.cpp
> @@ -101,15 +101,17 @@
> {
> ModeDepth &md = m_modeDepth[depth];
>
> - md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param);
> + ok &= md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param);
> ok &= md.fencYuv.create(cuSize, csp);
> -
> - for (int j = 0; j < MAX_PRED_TYPES; j++)
> + if (ok)
> {
> - md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j);
> - ok &= md.pred[j].predYuv.create(cuSize, csp);
> - ok &= md.pred[j].reconYuv.create(cuSize, csp);
> - md.pred[j].fencYuv = &md.fencYuv;
> + for (int j = 0; j < MAX_PRED_TYPES; j++)
> + {
> + md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param,
> j);
> + ok &= md.pred[j].predYuv.create(cuSize, csp);
> + ok &= md.pred[j].reconYuv.create(cuSize, csp);
> + md.pred[j].fencYuv = &md.fencYuv;
> + }
> }
> }
> if (m_param->sourceHeight >= 1080)
> diff --git a/source/encoder/api.cpp b/source/encoder/api.cpp
> --- a/source/encoder/api.cpp
> +++ b/source/encoder/api.cpp
> @@ -286,7 +286,7 @@
> else if (pi_nal)
> *pi_nal = 0;
>
> - if (numEncoded && encoder->m_param->csvLogLevel)
> + if (numEncoded > 0 && encoder->m_param->csvLogLevel)
> x265_csvlog_frame(encoder->m_param, pic_out);
>
> if (numEncoded < 0)
> diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
> --- a/source/encoder/encoder.cpp
> +++ b/source/encoder/encoder.cpp
> @@ -1342,7 +1342,9 @@
> }
> else
> {
> - frameEnc->allocEncodeData(m_reconfigure ? m_latestParam
> : m_param, m_sps);
> + bool isallocated = frameEnc->allocEncodeData(m_reconfigure
> ? m_latestParam : m_param, m_sps);
> + if (!isallocated)
> + return -1;
> Slice* slice = frameEnc->m_encData->m_slice;
> slice->m_sps = &m_sps;
> slice->m_pps = &m_pps;
>
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel
>
>
Pushed.
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20171222/c9768f13/attachment.html>
More information about the x265-devel
mailing list