[x264-devel] commit: Fix bug in hadamard_ac SSE assembly (Loren Merritt )

[git version control]

x264 | branch: master | Loren Merritt <pengvado at akuvian.org> | Thu Oct 30 00:47:09 2008 -0700| [dbc5ef040b1f8a83e7491dc8a2fc8943b1e20c07] | committer: Jason Garrett-Glaser 

Fix bug in hadamard_ac SSE assembly
Some extreme inputs could cause overflows.

> http://git.videolan.org/gitweb.cgi/x264.git/?a=commit;h=dbc5ef040b1f8a83e7491dc8a2fc8943b1e20c07
---

 common/x86/pixel-a.asm |    3 ++-
 1 files changed, 2 insertions(+), 1 deletions(-)

diff --git a/common/x86/pixel-a.asm b/common/x86/pixel-a.asm
index 5c64658..f1594be 100644
--- a/common/x86/pixel-a.asm
+++ b/common/x86/pixel-a.asm
@@ -1463,12 +1463,13 @@ cglobal x264_pixel_hadamard_ac_%1x%2_%3, 2,3
     paddusw m1, [rsp+0x60]
     paddusw m0, [rsp+0x70]
     paddusw m1, [rsp+0x80]
+    psrlw m0, 1
 %endif
     HADDW m0, m2
     HADDW m1, m3
     movd edx, m0
     movd eax, m1
-    shr  edx, 2
+    shr  edx, 2 - (%1*%2 >> 8)
     shr  eax, 1
 %ifdef ARCH_X86_64
     shl  rdx, 32