[x264-devel] Fix overflow in 8-bit x86 vsad asm function

[Anton Mitrofanov]

x264 | branch: master | Anton Mitrofanov <BugMaster at narod.ru> | Fri Dec  9 01:54:22 2011 +0400| [489a9b2d04c4828877930d2a9104ce93dde8cb85] | committer: Jason Garrett-Glaser

Fix overflow in 8-bit x86 vsad asm function

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=489a9b2d04c4828877930d2a9104ce93dde8cb85
---

 common/x86/sad-a.asm |   12 +++++++-----
 1 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/common/x86/sad-a.asm b/common/x86/sad-a.asm
index 0fad5b6..70e067e 100644
--- a/common/x86/sad-a.asm
+++ b/common/x86/sad-a.asm
@@ -287,10 +287,11 @@ cglobal pixel_vsad_mmx2, 3,3
     psadbw    m3, m5
     psadbw    m4, m6
     psadbw    m5, m7
-    paddw     m0, m2
-    paddw     m0, m3
-    paddw     m0, m4
-    paddw     m0, m5
+    ;max sum: 31*16*255(pixel_max)=126480
+    paddd     m0, m2
+    paddd     m0, m3
+    paddd     m0, m4
+    paddd     m0, m5
     mova      m2, m6
     mova      m3, m7
     sub      r2d, 2
@@ -321,7 +322,8 @@ cglobal pixel_vsad_sse2, 3,3
     jg .loop
 .end:
     movhlps   m1, m0
-    paddw     m0, m1
+    ;max sum: 31*16*255(pixel_max)=126480
+    paddd     m0, m1
     movd     eax, m0
     RET