[x265] [PATCH] ssd: fix overflow in ssd_s for 10 and 12 bit

[Divya Manivannan]

# HG changeset patch
# User Divya Manivannan <divya at multicorewareinc.com>
# Date 1443610501 -19800
#      Wed Sep 30 16:25:01 2015 +0530
# Node ID d7552147b1440db4dd2676c5ffeb4c905f7b70ee
# Parent  28bd3020855e0e2329738b82309d4113d091dd3f
ssd: fix overflow in ssd_s for 10 and 12 bit

diff -r 28bd3020855e -r d7552147b144 source/common/pixel.cpp
--- a/source/common/pixel.cpp	Wed Sep 30 16:24:59 2015 +0530
+++ b/source/common/pixel.cpp	Wed Sep 30 16:25:01 2015 +0530
@@ -403,9 +403,9 @@
 }
 
 template<int size>
-int pixel_ssd_s_c(const int16_t* a, intptr_t dstride)
+sse_ret_t pixel_ssd_s_c(const int16_t* a, intptr_t dstride)
 {
-    int sum = 0;
+    sse_ret_t sum = 0;
     for (int y = 0; y < size; y++)
     {
         for (int x = 0; x < size; x++)
diff -r 28bd3020855e -r d7552147b144 source/common/primitives.h
--- a/source/common/primitives.h	Wed Sep 30 16:24:59 2015 +0530
+++ b/source/common/primitives.h	Wed Sep 30 16:25:01 2015 +0530
@@ -114,7 +114,7 @@
 typedef int  (*pixelcmp_ss_t)(const int16_t* fenc, intptr_t fencstride, const int16_t* fref, intptr_t frefstride);
 typedef sse_ret_t (*pixel_sse_t)(const pixel* fenc, intptr_t fencstride, const pixel* fref, intptr_t frefstride); // fenc is aligned
 typedef sse_ret_t (*pixel_sse_ss_t)(const int16_t* fenc, intptr_t fencstride, const int16_t* fref, intptr_t frefstride);
-typedef int  (*pixel_ssd_s_t)(const int16_t* fenc, intptr_t fencstride);
+typedef sse_ret_t (*pixel_ssd_s_t)(const int16_t* fenc, intptr_t fencstride);
 typedef void (*pixelcmp_x4_t)(const pixel* fenc, const pixel* fref0, const pixel* fref1, const pixel* fref2, const pixel* fref3, intptr_t frefstride, int32_t* res);
 typedef void (*pixelcmp_x3_t)(const pixel* fenc, const pixel* fref0, const pixel* fref1, const pixel* fref2, intptr_t frefstride, int32_t* res);
 typedef void (*blockfill_s_t)(int16_t* dst, intptr_t dstride, int16_t val);
diff -r 28bd3020855e -r d7552147b144 source/common/x86/asm-primitives.cpp
--- a/source/common/x86/asm-primitives.cpp	Wed Sep 30 16:24:59 2015 +0530
+++ b/source/common/x86/asm-primitives.cpp	Wed Sep 30 16:25:01 2015 +0530
@@ -956,7 +956,10 @@
         ALL_LUMA_TU_S(cpy1Dto2D_shl, cpy1Dto2D_shl_, sse2);
         ALL_LUMA_TU_S(cpy2Dto1D_shr, cpy2Dto1D_shr_, sse2);
         ALL_LUMA_TU_S(cpy2Dto1D_shl, cpy2Dto1D_shl_, sse2);
-        ALL_LUMA_TU_S(ssd_s, pixel_ssd_s_, sse2);
+
+        // ssd_s need to be fixed for 10 and 12 bit
+        //ALL_LUMA_TU_S(ssd_s, pixel_ssd_s_, sse2);
+
         ALL_LUMA_TU_S(calcresidual, getResidual, sse2);
         ALL_LUMA_TU_S(transpose, transpose, sse2);
 
@@ -1533,8 +1536,9 @@
         p.chroma[X265_CSP_I422].pu[CHROMA_422_16x8].satd = PFX(pixel_satd_16x8_avx2);
         p.chroma[X265_CSP_I422].pu[CHROMA_422_32x16].satd = PFX(pixel_satd_32x16_avx2);
 
-        p.cu[BLOCK_16x16].ssd_s = PFX(pixel_ssd_s_16_avx2);
-        p.cu[BLOCK_32x32].ssd_s = PFX(pixel_ssd_s_32_avx2);
+        // ssd_s need to be fixed for 10 and 12 bit
+        //p.cu[BLOCK_16x16].ssd_s = PFX(pixel_ssd_s_16_avx2);
+        //p.cu[BLOCK_32x32].ssd_s = PFX(pixel_ssd_s_32_avx2);
 
         // sse_ss primitive need to be fixed for 10 and 12 bit
         //p.cu[BLOCK_16x16].sse_ss = PFX(pixel_ssd_ss_16x16_avx2);
diff -r 28bd3020855e -r d7552147b144 source/common/x86/pixel.h
--- a/source/common/x86/pixel.h	Wed Sep 30 16:24:59 2015 +0530
+++ b/source/common/x86/pixel.h	Wed Sep 30 16:25:01 2015 +0530
@@ -50,8 +50,8 @@
     FUNCDEF_CHROMA_PU(int, pixel_sad, cpu, const pixel*, intptr_t, const pixel*, intptr_t); \
     FUNCDEF_CHROMA_PU(sse_ret_t, pixel_ssd_ss, cpu, const int16_t*, intptr_t, const int16_t*, intptr_t); \
     FUNCDEF_CHROMA_PU(void, addAvg, cpu, const int16_t*, const int16_t*, pixel*, intptr_t, intptr_t, intptr_t); \
-    FUNCDEF_CHROMA_PU(int, pixel_ssd_s, cpu, const int16_t*, intptr_t); \
-    FUNCDEF_TU_S(int, pixel_ssd_s, cpu, const int16_t*, intptr_t); \
+    FUNCDEF_CHROMA_PU(sse_ret_t, pixel_ssd_s, cpu, const int16_t*, intptr_t); \
+    FUNCDEF_TU_S(sse_ret_t, pixel_ssd_s, cpu, const int16_t*, intptr_t); \
     FUNCDEF_TU(uint64_t, pixel_var, cpu, const pixel*, intptr_t); \
     FUNCDEF_TU(int, psyCost_pp, cpu, const pixel* source, intptr_t sstride, const pixel* recon, intptr_t rstride); \
     FUNCDEF_TU(int, psyCost_ss, cpu, const int16_t* source, intptr_t sstride, const int16_t* recon, intptr_t rstride)
diff -r 28bd3020855e -r d7552147b144 source/test/pixelharness.cpp
--- a/source/test/pixelharness.cpp	Wed Sep 30 16:24:59 2015 +0530
+++ b/source/test/pixelharness.cpp	Wed Sep 30 16:25:01 2015 +0530
@@ -228,8 +228,8 @@
     {
         // NOTE: stride must be multiple of 16, because minimum block is 4x4
         int stride = (STRIDE + (rand() % STRIDE)) & ~15;
-        int cres = ref(sbuf1 + j, stride);
-        int vres = (int)checked(opt, sbuf1 + j, (intptr_t)stride);
+        sse_ret_t cres = ref(sbuf1 + j, stride);
+        sse_ret_t vres = (sse_ret_t)checked(opt, sbuf1 + j, (intptr_t)stride);
 
         if (cres != vres)
             return false;