[x265] [PATCH] Fix: segmentation fault for hist-scenecut option

[srikanth.kurapati at multicorewareinc.com]

# HG changeset patch
# User Srikanth Kurapati
# Date 1584430274 -19800
#      Tue Mar 17 13:01:14 2020 +0530
# Node ID b7a02a40bc5d97308d41b925f9c37df414ab6ecb
# Parent  b7b0de75112b8022789590640662f2fab956cfbe
Fix: segmentation fault for hist-scenecut option

fixes plane size calculation for chroma planes using source resolution and not
padded resolution.

diff -r b7b0de75112b -r b7a02a40bc5d source/encoder/encoder.cpp
--- a/source/encoder/encoder.cpp	Fri Feb 21 17:02:18 2020 +0530
+++ b/source/encoder/encoder.cpp	Tue Mar 17 13:01:14 2020 +0530
@@ -218,10 +218,7 @@
 
     if (m_param->bHistBasedSceneCut)
     {
-        for (int i = 0; i < x265_cli_csps[m_param->internalCsp].planes; i++)
-        {
-            m_planeSizes[i] = (m_param->sourceWidth >> x265_cli_csps[p->internalCsp].width[i]) * (m_param->sourceHeight >> x265_cli_csps[m_param->internalCsp].height[i]);
-        }
+        m_planeSizes[0] = (m_param->sourceWidth >> x265_cli_csps[p->internalCsp].width[0]) * (m_param->sourceHeight >> x265_cli_csps[m_param->internalCsp].height[0]);
         uint32_t pixelbytes = m_param->internalBitDepth > 8 ? 2 : 1;
         m_edgePic = X265_MALLOC(pixel, m_planeSizes[0] * pixelbytes);
         m_edgeHistThreshold = m_param->edgeTransitionThreshold;
@@ -1605,6 +1602,14 @@
         if (m_param->bHistBasedSceneCut && pic_in)
         {
             x265_picture *pic = (x265_picture *) pic_in;
+
+            if (pic->poc == 0)
+            {
+                /* for entire encode compute the chroma plane sizes only once */
+                for (int i = 1; i < x265_cli_csps[m_param->internalCsp].planes; i++)
+                    m_planeSizes[i] = (pic->width >> x265_cli_csps[m_param->internalCsp].width[i]) * (pic->height >> x265_cli_csps[m_param->internalCsp].height[i]);
+            }
+
             if (computeHistograms(pic))
             {
                 double maxUVSad = 0.0, edgeSad = 0.0;
-------------- next part --------------
A non-text attachment was scrubbed...
Name: x265-default.patch
Type: text/x-patch
Size: 1921 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20200317/84cc8bba/attachment.bin>