[x264-devel] [Bug Report] Segmentation Fault in 2nd Pass

Rodolfo J. Quesada Zumbado rquesada at roqz.net
Fri Jun 6 18:33:42 CEST 2008


Sure, here it is:

0xb7ee27b0 in x264_rd_cost_i8x8_chroma () from /usr/lib/libx264.so.59
(gdb) disass $pc-100 $pc+100
Dump of assembler code from 0xb7ee274c to 0xb7ee2814:
0xb7ee274c <x264_rd_cost_i8x8_chroma+540>:	xor    %eax,(%eax)
0xb7ee274e <x264_rd_cost_i8x8_chroma+542>:	add    %cl,-0x76ebdbac(%ebx)
0xb7ee2754 <x264_rd_cost_i8x8_chroma+548>:	stc
0xb7ee2755 <x264_rd_cost_i8x8_chroma+549>:	mov    %esi,%eax
0xb7ee2757 <x264_rd_cost_i8x8_chroma+551>:	add    $0x1,%edi
0xb7ee275a <x264_rd_cost_i8x8_chroma+554>:	mov    %ebx,(%esp)
0xb7ee275d <x264_rd_cost_i8x8_chroma+557>:	add    $0x20,%ebx
0xb7ee2760 <x264_rd_cost_i8x8_chroma+560>:	movl   $0xf,0x4(%esp)
0xb7ee2768 <x264_rd_cost_i8x8_chroma+568>:	call   0xb7ee2040 
<block_residual_write_cavlc>
0xb7ee276d <x264_rd_cost_i8x8_chroma+573>:	cmp    $0x18,%edi
0xb7ee2770 <x264_rd_cost_i8x8_chroma+576>:	jne    0xb7ee2690 
<x264_rd_cost_i8x8_chroma+352>
0xb7ee2776 <x264_rd_cost_i8x8_chroma+582>:	jmp    0xb7ee25ec 
<x264_rd_cost_i8x8_chroma+188>
0xb7ee277b <x264_rd_cost_i8x8_chroma+587>:	nop
0xb7ee277c <x264_rd_cost_i8x8_chroma+588>:	lea    0x0(%esi),%esi
0xb7ee2780 <x264_rd_cost_i8x8_chroma+592>:	lea    0x1050(%esi),%eax
0xb7ee2786 <x264_rd_cost_i8x8_chroma+598>:	mov    %eax,0x4(%esp)
0xb7ee278a <x264_rd_cost_i8x8_chroma+602>:	lea    0x40(%esp),%eax
0xb7ee278e <x264_rd_cost_i8x8_chroma+606>:	movl   $0x1d0,0x8(%esp)
0xb7ee2796 <x264_rd_cost_i8x8_chroma+614>:	mov    %eax,(%esp)
---Type <return> to continue, or q <return> to quit---
0xb7ee2799 <x264_rd_cost_i8x8_chroma+617>:	call   *0x42f4(%esi)
0xb7ee279f <x264_rd_cost_i8x8_chroma+623>:	mov    0x1dfc(%esi),%ebx
0xb7ee27a5 <x264_rd_cost_i8x8_chroma+629>:	xor    %ecx,%ecx
0xb7ee27a7 <x264_rd_cost_i8x8_chroma+631>:	mov    0x1fe4(%esi),%eax
0xb7ee27ad <x264_rd_cost_i8x8_chroma+637>:	test   $0x1,%bl
0xb7ee27b0 <x264_rd_cost_i8x8_chroma+640>:	movzbl -0x480c1173(%eax),%edi
0xb7ee27b7 <x264_rd_cost_i8x8_chroma+647>:	jne    0xb7ee2a58 
<x264_rd_cost_i8x8_chroma+1320>
0xb7ee27bd <x264_rd_cost_i8x8_chroma+653>:	and    $0x2,%ebx
0xb7ee27c0 <x264_rd_cost_i8x8_chroma+656>:	jne    0xb7ee2a40 
<x264_rd_cost_i8x8_chroma+1296>
0xb7ee27c6 <x264_rd_cost_i8x8_chroma+662>:	xor    %edx,%edx
0xb7ee27c8 <x264_rd_cost_i8x8_chroma+664>:	test   %edi,%edi
0xb7ee27ca <x264_rd_cost_i8x8_chroma+666>:	setg   %dl
0xb7ee27cd <x264_rd_cost_i8x8_chroma+669>:	add    $0x40,%ecx
0xb7ee27d0 <x264_rd_cost_i8x8_chroma+672>:	movzbl 0x44(%esp,%ecx,1),%eax
0xb7ee27d5 <x264_rd_cost_i8x8_chroma+677>:	lea    (%edx,%eax,2),%eax
0xb7ee27d8 <x264_rd_cost_i8x8_chroma+680>:	movzbl -0x480c36c0(%eax),%edx
0xb7ee27df <x264_rd_cost_i8x8_chroma+687>:	mov    %dl,0x44(%esp,%ecx,1)
0xb7ee27e3 <x264_rd_cost_i8x8_chroma+691>:	movzwl 
-0x480c3580(%eax,%eax,1),%ecx
0xb7ee27eb <x264_rd_cost_i8x8_chroma+699>:	add    0x40(%esp),%ecx
0xb7ee27ef <x264_rd_cost_i8x8_chroma+703>:	test   %edi,%edi
---Type <return> to continue, or q <return> to quit---
0xb7ee27f1 <x264_rd_cost_i8x8_chroma+705>:	mov    %ecx,0x40(%esp)
0xb7ee27f5 <x264_rd_cost_i8x8_chroma+709>:	je     0xb7ee2856 
<x264_rd_cost_i8x8_chroma+806>
0xb7ee27f7 <x264_rd_cost_i8x8_chroma+711>:	movzbl 0x87(%esp),%edx
0xb7ee27ff <x264_rd_cost_i8x8_chroma+719>:	xor    %eax,%eax
0xb7ee2801 <x264_rd_cost_i8x8_chroma+721>:	cmp    $0x1,%edi
0xb7ee2804 <x264_rd_cost_i8x8_chroma+724>:	setg   %al
0xb7ee2807 <x264_rd_cost_i8x8_chroma+727>:	lea    (%eax,%edx,2),%edx
0xb7ee280a <x264_rd_cost_i8x8_chroma+730>:	movzwl 
-0x480c3580(%edx,%edx,1),%eax
0xb7ee2812 <x264_rd_cost_i8x8_chroma+738>:	movzbl -0x480c36c0(%edx),%ebx
End of assembler dump.
(gdb)

Here is 'info all-registers' too:

(gdb) info all-registers
eax            0xbd048ad3	-1123775789
ecx            0x0	0
edx            0x0	0
ebx            0x1	1
esp            0xbff886e0	0xbff886e0
ebp            0x2db	0x2db
esi            0x8bc2240	146547264
edi            0x0	0
eip            0xb7ee27b0	0xb7ee27b0 <x264_rd_cost_i8x8_chroma+640>
eflags         0x10202	[ IF RF ]
cs             0x73	115
ss             0x7b	123
ds             0x7b	123
es             0x7b	123
fs             0x0	0
gs             0x33	51
st0            -nan(0xc4e380c00000000)	(raw 0xffff0c4e380c00000000)
st1            -nan(0x403c342c48584e38)	(raw 0xffff403c342c48584e38)
st2            -nan(0x313c3f43310a1c21)	(raw 0xffff313c3f43310a1c21)
st3            -nan(0x672731534943413f)	(raw 0xffff672731534943413f)
st4            -inf	(raw 0xffff0000000000000000)
st5            -nan(0x8585858585858585)	(raw 0xffff8585858585858585)
st6            -nan(0x8585858585858585)	(raw 0xffff8585858585858585)
---Type <return> to continue, or q <return> to quit---
st7            -inf	(raw 0xffff0000000000000000)
fctrl          0x37f	895
fstat          0x20	32
ftag           0xaaaa	43690
fiseg          0x0	0
fioff          0x0	0
foseg          0x0	0
fooff          0x0	0
fop            0x0	0
xmm0           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
   v16_int8 = {0xb0, 0x2, 0x0, 0x0, 0x58, 0x1, 0x0, 0x0, 0x58, 0x1, 0x0, 
0x0,
     0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x2b0, 0x0, 0x158, 0x0, 0x158, 
0x0, 0x0,
     0x0}, v4_int32 = {0x2b0, 0x158, 0x158, 0x0}, v2_int64 = 
{0x158000002b0,
     0x158}, uint128 = 0x000000000000015800000158000002b0}
xmm1           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
   v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 
0x0,
     0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 
0x0},
   uint128 = 0x00000000000000000000000000000000}
xmm2           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
   v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 
0x0,
     0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 
0x0},
   uint128 = 0x00000000000000000000000000000000}
xmm3           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
---Type <return> to continue, or q <return> to quit---
   v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 
0x0,
     0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 
0x0},
   uint128 = 0x00000000000000000000000000000000}
xmm4           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
   v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 
0x0,
     0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 
0x0},
   uint128 = 0x00000000000000000000000000000000}
xmm5           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
   v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 
0x0,
     0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 
0x0},
   uint128 = 0x00000000000000000000000000000000}
xmm6           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
   v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 
0x0,
     0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 
0x0},
   uint128 = 0x00000000000000000000000000000000}
xmm7           {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
   v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 
0x0,
     0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 
0x0},
   uint128 = 0x00000000000000000000000000000000}
mxcsr          0x1fa0	[ PE IM DM ZM OM UM PM ]
mm0            {uint64 = 0xc4e380c00000000, v2_int32 = {0x0, 0xc4e380c},
   v4_int16 = {0x0, 0x0, 0x380c, 0xc4e}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 
0xc,
     0x38, 0x4e, 0xc}}
---Type <return> to continue, or q <return> to quit---
mm1            {uint64 = 0x403c342c48584e38, v2_int32 = {0x48584e38,
     0x403c342c}, v4_int16 = {0x4e38, 0x4858, 0x342c, 0x403c}, v8_int8 = 
{0x38,
     0x4e, 0x58, 0x48, 0x2c, 0x34, 0x3c, 0x40}}
mm2            {uint64 = 0x313c3f43310a1c21, v2_int32 = {0x310a1c21,
     0x313c3f43}, v4_int16 = {0x1c21, 0x310a, 0x3f43, 0x313c}, v8_int8 = 
{0x21,
     0x1c, 0xa, 0x31, 0x43, 0x3f, 0x3c, 0x31}}
mm3            {uint64 = 0x672731534943413f, v2_int32 = {0x4943413f,
     0x67273153}, v4_int16 = {0x413f, 0x4943, 0x3153, 0x6727}, v8_int8 = 
{0x3f,
     0x41, 0x43, 0x49, 0x53, 0x31, 0x27, 0x67}}
mm4            {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0,
     0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}
mm5            {uint64 = 0x8585858585858585, v2_int32 = {0x85858585,
     0x85858585}, v4_int16 = {0x8585, 0x8585, 0x8585, 0x8585}, v8_int8 = 
{0x85,
     0x85, 0x85, 0x85, 0x85, 0x85, 0x85, 0x85}}
mm6            {uint64 = 0x8585858585858585, v2_int32 = {0x85858585,
     0x85858585}, v4_int16 = {0x8585, 0x8585, 0x8585, 0x8585}, v8_int8 = 
{0x85,
     0x85, 0x85, 0x85, 0x85, 0x85, 0x85, 0x85}}
mm7            {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0,
     0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}
(gdb)



Jason Garrett-Glaser wrote:
>> 0xb7fd57b0 in x264_rd_cost_i8x8_chroma () from /usr/lib/libx264.so.59
>> (gdb) bt
>> #0  0xb7fd57b0 in x264_rd_cost_i8x8_chroma () from /usr/lib/libx264.so.59
>> #1  0x00000000 in ?? ()
> 
> Can you give a disass, e.g. "disass $pc-50 $pc+50 or something like
> that?  The function is relatively small so the asm might be useful.
> 
> Dark Shikari
> _______________________________________________
> x264-devel mailing list
> x264-devel at videolan.org
> http://mailman.videolan.org/listinfo/x264-devel

-- 
Rodolfo J. Quesada Zumbado
rquesada at roqz.net


More information about the x264-devel mailing list