[x264-devel] Stack alignment issue with libx264 compiled with mingw linked with a program compiled with MSVC

[Benjamin Pracht]

I seem to have a stack alignment issue when using x264 in the vlc
browser plugin (both compiled with mingw32 4.2.1), in firefox and IE.

I get a segfault in predict_8x8_ddl_sse2 with the following disassembly:

Program received signal SIGSEGV, Segmentation fault.
[Switching to thread 3736.0x3ec]
0x6fe2ce38 in predict_8x8_ddl_sse2 () from /cygdrive/c/Program Files/Veodia/vlc/plugins/libx264_plugin.dll

#0  0x6fe2ce38 in predict_8x8_ddl_sse2 () from /cygdrive/c/Program Files/Veodia/vlc/plugins/libx264_plugin.dll
#1  0x6fdd5152 in x264_slicetype_mb_cost (h=0xe0ad6c0, a=0x105adab8, frames=0x105ada70, p0=0, p1=0, b=0,
    dist_scale_factor=128) at encoder/slicetype.c:219
Backtrace stopped: frame did not save the PC
(gdb) disassemble
Dump of assembler code for function predict_8x8_ddl_sse2:
0x6fe2ce30 <predict_8x8_ddl_sse2+0>:    mov    0x4(%esp),%eax
0x6fe2ce34 <predict_8x8_ddl_sse2+4>:    mov    0x8(%esp),%ecx
0x6fe2ce38 <predict_8x8_ddl_sse2+8>:    movaps 0x10(%ecx),%xmm3
0x6fe2ce3c <predict_8x8_ddl_sse2+12>:   movups 0x11(%ecx),%xmm2
0x6fe2ce40 <predict_8x8_ddl_sse2+16>:   movaps %xmm3,%xmm1
0x6fe2ce43 <predict_8x8_ddl_sse2+19>:   pslldq $0x1,%xmm1
0x6fe2ce48 <predict_8x8_ddl_sse2+24>:   movaps %xmm1,%xmm4
0x6fe2ce4b <predict_8x8_ddl_sse2+27>:   pavgb  %xmm2,%xmm1
0x6fe2ce4f <predict_8x8_ddl_sse2+31>:   pxor   %xmm4,%xmm2
0x6fe2ce53 <predict_8x8_ddl_sse2+35>:   movaps %xmm3,%xmm0
0x6fe2ce56 <predict_8x8_ddl_sse2+38>:   pand   0x6fe47080,%xmm2
0x6fe2ce5e <predict_8x8_ddl_sse2+46>:   psubusb %xmm2,%xmm1
0x6fe2ce62 <predict_8x8_ddl_sse2+50>:   pavgb  %xmm1,%xmm0
0x6fe2ce66 <predict_8x8_ddl_sse2+54>:   psrldq $0x1,%xmm0
0x6fe2ce6b <predict_8x8_ddl_sse2+59>:   movq   %xmm0,(%eax)
0x6fe2ce6f <predict_8x8_ddl_sse2+63>:   psrldq $0x1,%xmm0
0x6fe2ce74 <predict_8x8_ddl_sse2+68>:   movq   %xmm0,0x20(%eax)
0x6fe2ce79 <predict_8x8_ddl_sse2+73>:   psrldq $0x1,%xmm0
0x6fe2ce7e <predict_8x8_ddl_sse2+78>:   movq   %xmm0,0x40(%eax)
0x6fe2ce83 <predict_8x8_ddl_sse2+83>:   psrldq $0x1,%xmm0
0x6fe2ce88 <predict_8x8_ddl_sse2+88>:   movq   %xmm0,0x60(%eax)
0x6fe2ce8d <predict_8x8_ddl_sse2+93>:   psrldq $0x1,%xmm0
0x6fe2ce92 <predict_8x8_ddl_sse2+98>:   movq   %xmm0,0x80(%eax)
0x6fe2ce9a <predict_8x8_ddl_sse2+106>:  psrldq $0x1,%xmm0
0x6fe2ce9f <predict_8x8_ddl_sse2+111>:  movq   %xmm0,0xa0(%eax)
0x6fe2cea7 <predict_8x8_ddl_sse2+119>:  psrldq $0x1,%xmm0
0x6fe2ceac <predict_8x8_ddl_sse2+124>:  movq   %xmm0,0xc0(%eax)
0x6fe2ceb4 <predict_8x8_ddl_sse2+132>:  psrldq $0x1,%xmm0
0x6fe2ceb9 <predict_8x8_ddl_sse2+137>:  movq   %xmm0,0xe0(%eax)
0x6fe2cec1 <predict_8x8_ddl_sse2+145>:  ret
0x6fe2cec2 <predict_8x8_ddl_sse2+146>:  jmp    0x6fe2ced0
<predict_8x8_ddr_sse2>
0x6fe2cec4 <predict_8x8_ddl_sse2+148>:  nop
0x6fe2cec5 <predict_8x8_ddl_sse2+149>:  nop
0x6fe2cec6 <predict_8x8_ddl_sse2+150>:  nop
0x6fe2cec7 <predict_8x8_ddl_sse2+151>:  nop
0x6fe2cec8 <predict_8x8_ddl_sse2+152>:  nop
0x6fe2cec9 <predict_8x8_ddl_sse2+153>:  nop
0x6fe2ceca <predict_8x8_ddl_sse2+154>:  nop
0x6fe2cecb <predict_8x8_ddl_sse2+155>:  nop
0x6fe2cecc <predict_8x8_ddl_sse2+156>:  nop
0x6fe2cecd <predict_8x8_ddl_sse2+157>:  nop
0x6fe2cece <predict_8x8_ddl_sse2+158>:  nop
0x6fe2cecf <predict_8x8_ddl_sse2+159>:  nop
End of assembler dump.

(gdb) p $ecx
$2 = 274388936
(gdb)


Basically, this looks to me like 

http://lists.mplayerhq.hu/pipermail/ffmpeg-devel/2006-October/017915.html

Is there any workaround?

Thanks.

-- 
BigBen