[x264-devel] segfault in predict_8x8_ddl_sse2 in shared lib

[Dave Brueck]

Hello,

I get a crash in predict_8x8_ddl_sse2 when I load x264 as a shared  
library, but encoding the same file using the x264 command line binary  
does not crash. It may just be that I am doing something incorrect  
when loading libx264.so (i.e. I'm not 100% sure this is x264's  
problem), so I'm mostly looking for any tips you all could give me on  
where to look next.

My program is a small Python program that loads libx264 using ctypes  
(a foreign function interface library for Python) to perform the  
encoding. I'm stumped because my program works fine if I disable SSE2  
optimizations.

The code crashes every time in the same spot. I'm using a daily  
snapshot (x264-snapshot-20080911-2245.tar.bz2) that I built like this:

./configure --prefix=/usr --enable-debug --enable-pic --enable-shared

I'm running gcc 4.2.3 and yasm 0.7.1.2093 on Ubuntu 8.04 on a dual  
quad-core Xeon machine.

Details below. Thanks for any pointers you can give me!
-Dave


Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0xad406b90 (LWP 19977)]
0xb0c96128 in predict_8x8_ddl_sse2 () from /usr/lib/libx264.so.61
Current language:  auto; currently asm
(gdb) bt
#0  0xb0c96128 in predict_8x8_ddl_sse2 () from /usr/lib/libx264.so.61
#1  0xb0c48635 in x264_slicetype_mb_cost (h=0x100308f0, a=0xad400ee4,  
frames=<value optimized out>, p0=0, p1=0, b=0, dist_scale_factor=128)
     at encoder/slicetype.c:218
#2  0xb0c48a77 in x264_slicetype_frame_cost (h=0x100308f0,  
a=0xad400ee4, frames=0xad40304c, p0=0, p1=0, b=0, b_intra_penalty=0)
     at encoder/slicetype.c:302
#3  0xb0c48d4e in x264_rc_analyse_slice (h=0x100308f0) at encoder/ 
slicetype.c:540
#4  0xb0c6f607 in rate_estimate_qscale (h=0x100308f0) at encoder/ 
ratecontrol.c:1468
#5  0xb0c6fe0c in x264_ratecontrol_start (h=0x100308f0, i_force_qp=0)  
at encoder/ratecontrol.c:803
#6  0xb0c8264b in x264_encoder_encode (h=0x100308f0,  
pp_nal=0xad403438, pi_nal=0xad403434, pic_in=0xad40349c,  
pic_out=0xad4033f0)
     at encoder/encoder.c:1470
[snip]

(gdb) disassemble
Dump of assembler code for function predict_8x8_ddl_sse2:
0xb0c96120 <predict_8x8_ddl_sse2+0>:	mov    0x4(%esp),%eax
0xb0c96124 <predict_8x8_ddl_sse2+4>:	mov    0x8(%esp),%ecx
0xb0c96128 <predict_8x8_ddl_sse2+8>:	movaps 0x10(%ecx),%xmm3
0xb0c9612c <predict_8x8_ddl_sse2+12>:	movups 0x11(%ecx),%xmm2
0xb0c96130 <predict_8x8_ddl_sse2+16>:	movaps %xmm3,%xmm1
0xb0c96133 <predict_8x8_ddl_sse2+19>:	pslldq $0x1,%xmm1
0xb0c96138 <predict_8x8_ddl_sse2+24>:	movaps %xmm1,%xmm4
0xb0c9613b <predict_8x8_ddl_sse2+27>:	pavgb  %xmm2,%xmm1
0xb0c9613f <predict_8x8_ddl_sse2+31>:	pxor   %xmm4,%xmm2
0xb0c96143 <predict_8x8_ddl_sse2+35>:	movaps %xmm3,%xmm0
0xb0c96146 <predict_8x8_ddl_sse2+38>:	pand   0xb0ca6160,%xmm2
0xb0c9614e <predict_8x8_ddl_sse2+46>:	psubusb %xmm2,%xmm1
0xb0c96152 <predict_8x8_ddl_sse2+50>:	pavgb  %xmm1,%xmm0
0xb0c96156 <predict_8x8_ddl_sse2+54>:	psrldq $0x1,%xmm0
0xb0c9615b <predict_8x8_ddl_sse2+59>:	movq   %xmm0,(%eax)
0xb0c9615f <predict_8x8_ddl_sse2+63>:	psrldq $0x1,%xmm0
0xb0c96164 <predict_8x8_ddl_sse2+68>:	movq   %xmm0,0x20(%eax)
0xb0c96169 <predict_8x8_ddl_sse2+73>:	psrldq $0x1,%xmm0
0xb0c9616e <predict_8x8_ddl_sse2+78>:	movq   %xmm0,0x40(%eax)
0xb0c96173 <predict_8x8_ddl_sse2+83>:	psrldq $0x1,%xmm0
0xb0c96178 <predict_8x8_ddl_sse2+88>:	movq   %xmm0,0x60(%eax)
0xb0c9617d <predict_8x8_ddl_sse2+93>:	psrldq $0x1,%xmm0
0xb0c96182 <predict_8x8_ddl_sse2+98>:	movq   %xmm0,0x80(%eax)
0xb0c9618a <predict_8x8_ddl_sse2+106>:	psrldq $0x1,%xmm0
0xb0c9618f <predict_8x8_ddl_sse2+111>:	movq   %xmm0,0xa0(%eax)
0xb0c96197 <predict_8x8_ddl_sse2+119>:	psrldq $0x1,%xmm0
0xb0c9619c <predict_8x8_ddl_sse2+124>:	movq   %xmm0,0xc0(%eax)
0xb0c961a4 <predict_8x8_ddl_sse2+132>:	psrldq $0x1,%xmm0
0xb0c961a9 <predict_8x8_ddl_sse2+137>:	movq   %xmm0,0xe0(%eax)
0xb0c961b1 <predict_8x8_ddl_sse2+145>:	ret
0xb0c961b2 <predict_8x8_ddl_sse2+146>:	jmp    0xb0c961c0  
<predict_8x8_ddr_sse2>
0xb0c961b4 <predict_8x8_ddl_sse2+148>:	nop
0xb0c961b5 <predict_8x8_ddl_sse2+149>:	nop
0xb0c961b6 <predict_8x8_ddl_sse2+150>:	nop
0xb0c961b7 <predict_8x8_ddl_sse2+151>:	nop
0xb0c961b8 <predict_8x8_ddl_sse2+152>:	nop
0xb0c961b9 <predict_8x8_ddl_sse2+153>:	nop
0xb0c961ba <predict_8x8_ddl_sse2+154>:	nop
0xb0c961bb <predict_8x8_ddl_sse2+155>:	nop
0xb0c961bc <predict_8x8_ddl_sse2+156>:	nop
0xb0c961bd <predict_8x8_ddl_sse2+157>:	nop
0xb0c961be <predict_8x8_ddl_sse2+158>:	nop
0xb0c961bf <predict_8x8_ddl_sse2+159>:	nop
End of assembler dump.

(gdb) info registers
eax            0xad400d2c	-1388311252
ecx            0xad400e24	-1388311004
edx            0x100308f0	268634352
ebx            0xb0ca7bb4	-1328907340
esp            0xad400bb0	0xad400bb0
ebp            0xad400e6c	0xad400e6c
esi            0xad400d2c	-1388311252
edi            0x0	0
eip            0xb0c96128	0xb0c96128 <predict_8x8_ddl_sse2+8>
eflags         0x10292	[ AF SF IF RF ]
cs             0x73	115
ss             0x7b	123
ds             0x7b	123
es             0x7b	123
fs             0x0	0
gs             0x33	51