[x265] [PATCH 1 of 2] api: fix memory allocation of distortionData

[Ma0]

# HG changeset patch
# User Ma0 <mateuszb at poczta.onet.pl>
# Date 1536497279 -7200
#      Sun Sep 09 14:47:59 2018 +0200
# Node ID deb35c64bdc56e4c9405d34bff516dc5bdaae5ae
# Parent  88ee12651e3031dc1fc2f3f6a8bbac5f67839579
api: fix memory allocation of distortionData

sse_t type depends of x265 default bit-depth
if 8bit x265 try to allocate distortionData for 10bit encoding
(possible in multilib build) we need double memory size for
sse_t type tables (uint32_t -> uint64_t)

diff -r 88ee12651e30 -r deb35c64bdc5 source/encoder/api.cpp
--- a/source/encoder/api.cpp	Thu Aug 16 18:27:01 2018 +0530
+++ b/source/encoder/api.cpp	Sun Sep 09 14:47:59 2018 +0200
@@ -412,12 +412,20 @@
     int numDir = 2; //irrespective of P or B slices set direction as 2
     uint32_t numPlanes = param->internalCsp == X265_CSP_I400 ? 1 : 3;
 
+#if X265_DEPTH < 10 && (LINKED_10BIT || LINKED_12BIT)
+    uint32_t numCUs_sse_t = param->internalBitDepth > 8 ? analysis->numCUsInFrame << 1 : analysis->numCUsInFrame;
+#elif X265_DEPTH >= 10 && LINKED_8BIT
+    uint32_t numCUs_sse_t = param->internalBitDepth > 8 ? analysis->numCUsInFrame : (analysis->numCUsInFrame + 1U) >> 1;
+#else
+    uint32_t numCUs_sse_t = analysis->numCUsInFrame;
+#endif
+
     //Allocate memory for distortionData pointer
     CHECKED_MALLOC_ZERO(distortionData, x265_analysis_distortion_data, 1);
-    CHECKED_MALLOC_ZERO(distortionData->distortion, sse_t, analysis->numPartitions * analysis->numCUsInFrame);
+    CHECKED_MALLOC_ZERO(distortionData->distortion, sse_t, analysis->numPartitions * numCUs_sse_t);
     if (param->rc.bStatRead)
     {
-        CHECKED_MALLOC_ZERO(distortionData->ctuDistortion, sse_t, analysis->numCUsInFrame);
+        CHECKED_MALLOC_ZERO(distortionData->ctuDistortion, sse_t, numCUs_sse_t);
         CHECKED_MALLOC_ZERO(distortionData->scaledDistortion, double, analysis->numCUsInFrame);
         CHECKED_MALLOC_ZERO(distortionData->offset, double, analysis->numCUsInFrame);
         CHECKED_MALLOC_ZERO(distortionData->threshold, double, analysis->numCUsInFrame);