[x264-devel] x86_64, sse2 deblock & valgrind uninitialized variable problem

[Jindrich Makovicka]

Hi,

because Valgrind invalidates the data below the stack upon any CALL,
it resets the temporary data in x264_deblock_h_luma_sse2 to an
uninitialized state.

I am not sure if the following patch is acceptable, as it adds
otherwise useless instructions, but in the case you refuse it, at
least add some comments that the code is really ok :)

--- deblock-a.asm.orig  2007-02-15 11:28:47.000000000 +0000
+++ deblock-a.asm       2007-05-18 10:53:27.838820785 +0000
@@ -326,7 +326,9 @@
     lea    r11, [r10+r10*2]
     lea    rax, [rdi-4]
     lea    r9,  [rdi-4+r11]
-    %define pix_tmp rsp-104 ; 16x6 for the buffer + 8 for
x264_deblock_v_luma_sse2's return address
+    %define pix_tmp rsp
+
+    sub rsp, 96             ; 16x6 for the tmp buffer

     ; transpose 6x16 -> tmp space
     TRANSPOSE6x8_MEM  PASS8ROWS(rax, r9, r10, r11), pix_tmp
@@ -360,6 +362,8 @@
     movq   mm3, [pix_tmp+0x40]
     TRANSPOSE8x4_STORE  PASS8ROWS(rax, r9, r10, r11)

+    add rsp, 96
+
     ret


-- 
Jindrich Makovicka

-- 
This is the x264-devel mailing-list
To unsubscribe, go to: http://developers.videolan.org/lists.html