[x264-devel] Fix pixel_var2 with 4:2:2 encoding

[Jason Garrett-Glaser]

x264 | branch: master | Jason Garrett-Glaser <jason at x264.com> | Wed Oct 12 09:16:32 2011 -0700| [d2594831dd858d6ed8efcfd4160ea5ac7f1357c7] | committer: Jason Garrett-Glaser

Fix pixel_var2 with 4:2:2 encoding
Might have caused artifacts or suboptimal chroma compression.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=d2594831dd858d6ed8efcfd4160ea5ac7f1357c7
---

 common/pixel.c |    8 ++++----
 1 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/common/pixel.c b/common/pixel.c
index 138fc23..19d6d2f 100644
--- a/common/pixel.c
+++ b/common/pixel.c
@@ -193,7 +193,7 @@ PIXEL_VAR_C( x264_pixel_var_8x8,    8,  8 )
 /****************************************************************************
  * pixel_var2_wxh
  ****************************************************************************/
-#define PIXEL_VAR2_C( name, w, h ) \
+#define PIXEL_VAR2_C( name, w, h, shift ) \
 static int name( pixel *pix1, int i_stride1, pixel *pix2, int i_stride2, int *ssd ) \
 { \
     uint32_t var = 0, sum = 0, sqr = 0; \
@@ -209,13 +209,13 @@ static int name( pixel *pix1, int i_stride1, pixel *pix2, int i_stride2, int *ss
         pix2 += i_stride2; \
     } \
     sum = abs(sum); \
-    var = sqr - ((uint64_t)sum * sum >> 6); \
+    var = sqr - ((uint64_t)sum * sum >> shift); \
     *ssd = sqr; \
     return var; \
 }
 
-PIXEL_VAR2_C( x264_pixel_var2_8x16, 8, 16 )
-PIXEL_VAR2_C( x264_pixel_var2_8x8,  8,  8 )
+PIXEL_VAR2_C( x264_pixel_var2_8x16, 8, 16, 7 )
+PIXEL_VAR2_C( x264_pixel_var2_8x8,  8,  8, 6 )
 
 #if BIT_DEPTH > 8
     typedef uint32_t sum_t;