[x265] [PATCH] Fix possible NULL pointer dereferencing in cudata init
santhoshini at multicorewareinc.com
santhoshini at multicorewareinc.com
Fri Dec 22 10:37:11 CET 2017
# HG changeset patch
# User Santhoshini Sekar <santhoshini at multicorewareinc.com>
# Date 1513927117 -19800
# Fri Dec 22 12:48:37 2017 +0530
# Branch stable
# Node ID f41671b53ae084a8207e24a91d3a7cf2a96cd1ce
# Parent f7498acb38746d9b799d1b45343a3db56cdba306
Fix possible NULL pointer dereferencing in cudata init
diff --git a/source/common/framedata.cpp b/source/common/framedata.cpp
--- a/source/common/framedata.cpp
+++ b/source/common/framedata.cpp
@@ -41,9 +41,12 @@
if (param.rc.bStatWrite)
m_spsrps = const_cast<RPS*>(sps.spsrps);
- m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param);
- for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++)
- m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr);
+ bool isallocated = m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param);
+ if (isallocated)
+ for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++)
+ m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr);
+ else
+ return false;
CHECKED_MALLOC_ZERO(m_cuStat, RCStatCU, sps.numCUsInFrame);
CHECKED_MALLOC(m_rowStat, RCStatRow, sps.numCuInHeight);
diff --git a/source/encoder/analysis.cpp b/source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp
+++ b/source/encoder/analysis.cpp
@@ -101,15 +101,17 @@
{
ModeDepth &md = m_modeDepth[depth];
- md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param);
+ ok &= md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param);
ok &= md.fencYuv.create(cuSize, csp);
-
- for (int j = 0; j < MAX_PRED_TYPES; j++)
+ if (ok)
{
- md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j);
- ok &= md.pred[j].predYuv.create(cuSize, csp);
- ok &= md.pred[j].reconYuv.create(cuSize, csp);
- md.pred[j].fencYuv = &md.fencYuv;
+ for (int j = 0; j < MAX_PRED_TYPES; j++)
+ {
+ md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j);
+ ok &= md.pred[j].predYuv.create(cuSize, csp);
+ ok &= md.pred[j].reconYuv.create(cuSize, csp);
+ md.pred[j].fencYuv = &md.fencYuv;
+ }
}
}
if (m_param->sourceHeight >= 1080)
diff --git a/source/encoder/api.cpp b/source/encoder/api.cpp
--- a/source/encoder/api.cpp
+++ b/source/encoder/api.cpp
@@ -286,7 +286,7 @@
else if (pi_nal)
*pi_nal = 0;
- if (numEncoded && encoder->m_param->csvLogLevel)
+ if (numEncoded > 0 && encoder->m_param->csvLogLevel)
x265_csvlog_frame(encoder->m_param, pic_out);
if (numEncoded < 0)
diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
--- a/source/encoder/encoder.cpp
+++ b/source/encoder/encoder.cpp
@@ -1342,7 +1342,9 @@
}
else
{
- frameEnc->allocEncodeData(m_reconfigure ? m_latestParam : m_param, m_sps);
+ bool isallocated = frameEnc->allocEncodeData(m_reconfigure ? m_latestParam : m_param, m_sps);
+ if (!isallocated)
+ return -1;
Slice* slice = frameEnc->m_encData->m_slice;
slice->m_sps = &m_sps;
slice->m_pps = &m_pps;
-------------- next part --------------
A non-text attachment was scrubbed...
Name: x265.patch
Type: text/x-patch
Size: 3317 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20171222/bc8e14ce/attachment.bin>
More information about the x265-devel
mailing list