[x265] [PATCH] Fix possible NULL pointer dereferencing in cudata init

[santhoshini at multicorewareinc.com]

# HG changeset patch
# User Santhoshini Sekar <santhoshini at multicorewareinc.com>
# Date 1513927117 -19800
#      Fri Dec 22 12:48:37 2017 +0530
# Branch stable
# Node ID f41671b53ae084a8207e24a91d3a7cf2a96cd1ce
# Parent  f7498acb38746d9b799d1b45343a3db56cdba306
Fix possible NULL pointer dereferencing in cudata init

diff --git a/source/common/framedata.cpp b/source/common/framedata.cpp
--- a/source/common/framedata.cpp
+++ b/source/common/framedata.cpp
@@ -41,9 +41,12 @@
     if (param.rc.bStatWrite)
         m_spsrps = const_cast<RPS*>(sps.spsrps);
 
-    m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param);
-    for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++)
-        m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr);
+    bool isallocated = m_cuMemPool.create(0, param.internalCsp, sps.numCUsInFrame, param);
+    if (isallocated)
+        for (uint32_t ctuAddr = 0; ctuAddr < sps.numCUsInFrame; ctuAddr++)
+            m_picCTU[ctuAddr].initialize(m_cuMemPool, 0, param, ctuAddr);
+    else
+        return false;
 
     CHECKED_MALLOC_ZERO(m_cuStat, RCStatCU, sps.numCUsInFrame);
     CHECKED_MALLOC(m_rowStat, RCStatRow, sps.numCuInHeight);
diff --git a/source/encoder/analysis.cpp b/source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp
+++ b/source/encoder/analysis.cpp
@@ -101,15 +101,17 @@
     {
         ModeDepth &md = m_modeDepth[depth];
 
-        md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param);
+        ok &= md.cuMemPool.create(depth, csp, MAX_PRED_TYPES, *m_param);
         ok &= md.fencYuv.create(cuSize, csp);
-
-        for (int j = 0; j < MAX_PRED_TYPES; j++)
+        if (ok)
         {
-            md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j);
-            ok &= md.pred[j].predYuv.create(cuSize, csp);
-            ok &= md.pred[j].reconYuv.create(cuSize, csp);
-            md.pred[j].fencYuv = &md.fencYuv;
+            for (int j = 0; j < MAX_PRED_TYPES; j++)
+            {
+                md.pred[j].cu.initialize(md.cuMemPool, depth, *m_param, j);
+                ok &= md.pred[j].predYuv.create(cuSize, csp);
+                ok &= md.pred[j].reconYuv.create(cuSize, csp);
+                md.pred[j].fencYuv = &md.fencYuv;
+            }
         }
     }
     if (m_param->sourceHeight >= 1080)
diff --git a/source/encoder/api.cpp b/source/encoder/api.cpp
--- a/source/encoder/api.cpp
+++ b/source/encoder/api.cpp
@@ -286,7 +286,7 @@
     else if (pi_nal)
         *pi_nal = 0;
 
-    if (numEncoded && encoder->m_param->csvLogLevel)
+    if (numEncoded > 0 && encoder->m_param->csvLogLevel)
         x265_csvlog_frame(encoder->m_param, pic_out);
 
     if (numEncoded < 0)
diff --git a/source/encoder/encoder.cpp b/source/encoder/encoder.cpp
--- a/source/encoder/encoder.cpp
+++ b/source/encoder/encoder.cpp
@@ -1342,7 +1342,9 @@
             }
             else
             {
-                frameEnc->allocEncodeData(m_reconfigure ? m_latestParam : m_param, m_sps);
+                bool isallocated = frameEnc->allocEncodeData(m_reconfigure ? m_latestParam : m_param, m_sps);
+                if (!isallocated)
+                    return -1;
                 Slice* slice = frameEnc->m_encData->m_slice;
                 slice->m_sps = &m_sps;
                 slice->m_pps = &m_pps;
-------------- next part --------------
A non-text attachment was scrubbed...
Name: x265.patch
Type: text/x-patch
Size: 3317 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20171222/bc8e14ce/attachment.bin>