[x265] [PATCH] ssim-rd: fix for high bit depth

[ashok at multicorewareinc.com]

# HG changeset patch
# User Ashok Kumar Mishra <ashok at multicorewareinc.com>
# Date 1483524608 -19800
#      Wed Jan 04 15:40:08 2017 +0530
# Node ID 255a921f8f894b4cd23dc00a5fef97b9289c9fad
# Parent  fff226628dc2c743f56d8fcd301db17c767bba13
ssim-rd: fix for high bit depth

diff -r fff226628dc2 -r 255a921f8f89 source/common/quant.cpp
--- a/source/common/quant.cpp	Tue Jan 03 16:31:00 2017 +0530
+++ b/source/common/quant.cpp	Wed Jan 04 15:40:08 2017 +0530
@@ -483,6 +483,7 @@
 {
     static const int ssim_c1 = (int)(.01 * .01 * PIXEL_MAX * PIXEL_MAX * 64 + .5); // 416
     static const int ssim_c2 = (int)(.03 * .03 * PIXEL_MAX * PIXEL_MAX * 64 * 63 + .5); // 235963
+    int shift = (X265_DEPTH - 8);
 
     int trSize = 1 << log2TrSize;
     uint64_t ssDc = 0, ssBlock = 0, ssAc = 0;
@@ -521,7 +522,7 @@
     {
         for (int block_xx = 0; block_xx < trSize; block_xx += 4)
         {
-            uint32_t temp = fenc[block_yy * fStride + block_xx];
+            uint32_t temp = fenc[block_yy * fStride + block_xx] >> shift;
             dc_k += temp * temp;
         }
     }
@@ -539,7 +540,7 @@
     {
         for (int block_xx = 0; block_xx < trSize; block_xx += 1)
         {
-            uint32_t temp = fenc[block_yy * fStride + block_xx];
+            uint32_t temp = fenc[block_yy * fStride + block_xx] >> shift;
             ac_k += temp * temp;
         }
     }
diff -r fff226628dc2 -r 255a921f8f89 source/encoder/analysis.cpp
--- a/source/encoder/analysis.cpp	Tue Jan 03 16:31:00 2017 +0530
+++ b/source/encoder/analysis.cpp	Wed Jan 04 15:40:08 2017 +0530
@@ -2919,6 +2919,7 @@
 {
     static const int ssim_c1 = (int)(.01 * .01 * PIXEL_MAX * PIXEL_MAX * 64 + .5); // 416
     static const int ssim_c2 = (int)(.03 * .03 * PIXEL_MAX * PIXEL_MAX * 64 * 63 + .5); // 235963
+    int shift = (X265_DEPTH - 8);
 
     double s = 1 + 0.005 * qp;
 
@@ -2931,7 +2932,7 @@
     {
         for (uint32_t block_xx = 0; block_xx < blockSize; block_xx += 4)
         {
-            uint32_t temp = src[block_yy * blockSize + block_xx];
+            uint32_t temp = src[block_yy * blockSize + block_xx] >> shift;
             z_o += temp * temp; // 2 * (Z(0)) pow(2)
         }
     }
@@ -2944,7 +2945,7 @@
     {
         for (uint32_t block_xx = 0; block_xx < blockSize; block_xx += 1)
         {
-            uint32_t temp = src[block_yy * blockSize + block_xx];
+            uint32_t temp = src[block_yy * blockSize + block_xx] >> shift;
             z_k += temp * temp;
         }
     }