[x264-devel] commit: Fix integer overflow in chroma SSD check (Jason Garrett-Glaser )

[git version control]

x264 | branch: master | Jason Garrett-Glaser <darkshikari at gmail.com> | Mon Feb 22 11:21:51 2010 -0800| [28e6eb67ffaa002469f60c40e2b5d58b2a758f9c] | committer: Jason Garrett-Glaser 

Fix integer overflow in chroma SSD check
Could cause bad skips at very high quantizers on extreme inputs.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=28e6eb67ffaa002469f60c40e2b5d58b2a758f9c
---

 encoder/rdo.c |    4 ++--
 1 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/encoder/rdo.c b/encoder/rdo.c
index 3ed4a47..e15f47d 100644
--- a/encoder/rdo.c
+++ b/encoder/rdo.c
@@ -131,7 +131,7 @@ static inline int ssd_plane( x264_t *h, int size, int p, int x, int y )
 static inline int ssd_mb( x264_t *h )
 {
     int chromassd = ssd_plane(h, PIXEL_8x8, 1, 0, 0) + ssd_plane(h, PIXEL_8x8, 2, 0, 0);
-    chromassd = (chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8;
+    chromassd = ((uint64_t)chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8;
     return ssd_plane(h, PIXEL_16x16, 0, 0, 0) + chromassd;
 }
 
@@ -223,7 +223,7 @@ uint64_t x264_rd_cost_part( x264_t *h, int i_lambda2, int i4, int i_pixel )
 
     chromassd = ssd_plane( h, i_pixel+3, 1, (i8&1)*4, (i8>>1)*4 )
               + ssd_plane( h, i_pixel+3, 2, (i8&1)*4, (i8>>1)*4 );
-    chromassd = (chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8;
+    chromassd = ((uint64_t)chromassd * h->mb.i_chroma_lambda2_offset + 128) >> 8;
     i_ssd = ssd_plane( h, i_pixel,   0, (i8&1)*8, (i8>>1)*8 ) + chromassd;
 
     if( h->param.b_cabac )