[x264-devel] [Bug Report] Segmentation Fault in 2nd Pass
Rodolfo J. Quesada Zumbado
rquesada at roqz.net
Fri Jun 6 18:33:42 CEST 2008
Sure, here it is:
0xb7ee27b0 in x264_rd_cost_i8x8_chroma () from /usr/lib/libx264.so.59
(gdb) disass $pc-100 $pc+100
Dump of assembler code from 0xb7ee274c to 0xb7ee2814:
0xb7ee274c <x264_rd_cost_i8x8_chroma+540>: xor %eax,(%eax)
0xb7ee274e <x264_rd_cost_i8x8_chroma+542>: add %cl,-0x76ebdbac(%ebx)
0xb7ee2754 <x264_rd_cost_i8x8_chroma+548>: stc
0xb7ee2755 <x264_rd_cost_i8x8_chroma+549>: mov %esi,%eax
0xb7ee2757 <x264_rd_cost_i8x8_chroma+551>: add $0x1,%edi
0xb7ee275a <x264_rd_cost_i8x8_chroma+554>: mov %ebx,(%esp)
0xb7ee275d <x264_rd_cost_i8x8_chroma+557>: add $0x20,%ebx
0xb7ee2760 <x264_rd_cost_i8x8_chroma+560>: movl $0xf,0x4(%esp)
0xb7ee2768 <x264_rd_cost_i8x8_chroma+568>: call 0xb7ee2040
<block_residual_write_cavlc>
0xb7ee276d <x264_rd_cost_i8x8_chroma+573>: cmp $0x18,%edi
0xb7ee2770 <x264_rd_cost_i8x8_chroma+576>: jne 0xb7ee2690
<x264_rd_cost_i8x8_chroma+352>
0xb7ee2776 <x264_rd_cost_i8x8_chroma+582>: jmp 0xb7ee25ec
<x264_rd_cost_i8x8_chroma+188>
0xb7ee277b <x264_rd_cost_i8x8_chroma+587>: nop
0xb7ee277c <x264_rd_cost_i8x8_chroma+588>: lea 0x0(%esi),%esi
0xb7ee2780 <x264_rd_cost_i8x8_chroma+592>: lea 0x1050(%esi),%eax
0xb7ee2786 <x264_rd_cost_i8x8_chroma+598>: mov %eax,0x4(%esp)
0xb7ee278a <x264_rd_cost_i8x8_chroma+602>: lea 0x40(%esp),%eax
0xb7ee278e <x264_rd_cost_i8x8_chroma+606>: movl $0x1d0,0x8(%esp)
0xb7ee2796 <x264_rd_cost_i8x8_chroma+614>: mov %eax,(%esp)
---Type <return> to continue, or q <return> to quit---
0xb7ee2799 <x264_rd_cost_i8x8_chroma+617>: call *0x42f4(%esi)
0xb7ee279f <x264_rd_cost_i8x8_chroma+623>: mov 0x1dfc(%esi),%ebx
0xb7ee27a5 <x264_rd_cost_i8x8_chroma+629>: xor %ecx,%ecx
0xb7ee27a7 <x264_rd_cost_i8x8_chroma+631>: mov 0x1fe4(%esi),%eax
0xb7ee27ad <x264_rd_cost_i8x8_chroma+637>: test $0x1,%bl
0xb7ee27b0 <x264_rd_cost_i8x8_chroma+640>: movzbl -0x480c1173(%eax),%edi
0xb7ee27b7 <x264_rd_cost_i8x8_chroma+647>: jne 0xb7ee2a58
<x264_rd_cost_i8x8_chroma+1320>
0xb7ee27bd <x264_rd_cost_i8x8_chroma+653>: and $0x2,%ebx
0xb7ee27c0 <x264_rd_cost_i8x8_chroma+656>: jne 0xb7ee2a40
<x264_rd_cost_i8x8_chroma+1296>
0xb7ee27c6 <x264_rd_cost_i8x8_chroma+662>: xor %edx,%edx
0xb7ee27c8 <x264_rd_cost_i8x8_chroma+664>: test %edi,%edi
0xb7ee27ca <x264_rd_cost_i8x8_chroma+666>: setg %dl
0xb7ee27cd <x264_rd_cost_i8x8_chroma+669>: add $0x40,%ecx
0xb7ee27d0 <x264_rd_cost_i8x8_chroma+672>: movzbl 0x44(%esp,%ecx,1),%eax
0xb7ee27d5 <x264_rd_cost_i8x8_chroma+677>: lea (%edx,%eax,2),%eax
0xb7ee27d8 <x264_rd_cost_i8x8_chroma+680>: movzbl -0x480c36c0(%eax),%edx
0xb7ee27df <x264_rd_cost_i8x8_chroma+687>: mov %dl,0x44(%esp,%ecx,1)
0xb7ee27e3 <x264_rd_cost_i8x8_chroma+691>: movzwl
-0x480c3580(%eax,%eax,1),%ecx
0xb7ee27eb <x264_rd_cost_i8x8_chroma+699>: add 0x40(%esp),%ecx
0xb7ee27ef <x264_rd_cost_i8x8_chroma+703>: test %edi,%edi
---Type <return> to continue, or q <return> to quit---
0xb7ee27f1 <x264_rd_cost_i8x8_chroma+705>: mov %ecx,0x40(%esp)
0xb7ee27f5 <x264_rd_cost_i8x8_chroma+709>: je 0xb7ee2856
<x264_rd_cost_i8x8_chroma+806>
0xb7ee27f7 <x264_rd_cost_i8x8_chroma+711>: movzbl 0x87(%esp),%edx
0xb7ee27ff <x264_rd_cost_i8x8_chroma+719>: xor %eax,%eax
0xb7ee2801 <x264_rd_cost_i8x8_chroma+721>: cmp $0x1,%edi
0xb7ee2804 <x264_rd_cost_i8x8_chroma+724>: setg %al
0xb7ee2807 <x264_rd_cost_i8x8_chroma+727>: lea (%eax,%edx,2),%edx
0xb7ee280a <x264_rd_cost_i8x8_chroma+730>: movzwl
-0x480c3580(%edx,%edx,1),%eax
0xb7ee2812 <x264_rd_cost_i8x8_chroma+738>: movzbl -0x480c36c0(%edx),%ebx
End of assembler dump.
(gdb)
Here is 'info all-registers' too:
(gdb) info all-registers
eax 0xbd048ad3 -1123775789
ecx 0x0 0
edx 0x0 0
ebx 0x1 1
esp 0xbff886e0 0xbff886e0
ebp 0x2db 0x2db
esi 0x8bc2240 146547264
edi 0x0 0
eip 0xb7ee27b0 0xb7ee27b0 <x264_rd_cost_i8x8_chroma+640>
eflags 0x10202 [ IF RF ]
cs 0x73 115
ss 0x7b 123
ds 0x7b 123
es 0x7b 123
fs 0x0 0
gs 0x33 51
st0 -nan(0xc4e380c00000000) (raw 0xffff0c4e380c00000000)
st1 -nan(0x403c342c48584e38) (raw 0xffff403c342c48584e38)
st2 -nan(0x313c3f43310a1c21) (raw 0xffff313c3f43310a1c21)
st3 -nan(0x672731534943413f) (raw 0xffff672731534943413f)
st4 -inf (raw 0xffff0000000000000000)
st5 -nan(0x8585858585858585) (raw 0xffff8585858585858585)
st6 -nan(0x8585858585858585) (raw 0xffff8585858585858585)
---Type <return> to continue, or q <return> to quit---
st7 -inf (raw 0xffff0000000000000000)
fctrl 0x37f 895
fstat 0x20 32
ftag 0xaaaa 43690
fiseg 0x0 0
fioff 0x0 0
foseg 0x0 0
fooff 0x0 0
fop 0x0 0
xmm0 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0xb0, 0x2, 0x0, 0x0, 0x58, 0x1, 0x0, 0x0, 0x58, 0x1, 0x0,
0x0,
0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x2b0, 0x0, 0x158, 0x0, 0x158,
0x0, 0x0,
0x0}, v4_int32 = {0x2b0, 0x158, 0x158, 0x0}, v2_int64 =
{0x158000002b0,
0x158}, uint128 = 0x000000000000015800000158000002b0}
xmm1 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0,
0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0,
0x0},
uint128 = 0x00000000000000000000000000000000}
xmm2 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0,
0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0,
0x0},
uint128 = 0x00000000000000000000000000000000}
xmm3 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
---Type <return> to continue, or q <return> to quit---
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0,
0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0,
0x0},
uint128 = 0x00000000000000000000000000000000}
xmm4 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0,
0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0,
0x0},
uint128 = 0x00000000000000000000000000000000}
xmm5 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0,
0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0,
0x0},
uint128 = 0x00000000000000000000000000000000}
xmm6 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0,
0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0,
0x0},
uint128 = 0x00000000000000000000000000000000}
xmm7 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0,
0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0,
0x0},
uint128 = 0x00000000000000000000000000000000}
mxcsr 0x1fa0 [ PE IM DM ZM OM UM PM ]
mm0 {uint64 = 0xc4e380c00000000, v2_int32 = {0x0, 0xc4e380c},
v4_int16 = {0x0, 0x0, 0x380c, 0xc4e}, v8_int8 = {0x0, 0x0, 0x0, 0x0,
0xc,
0x38, 0x4e, 0xc}}
---Type <return> to continue, or q <return> to quit---
mm1 {uint64 = 0x403c342c48584e38, v2_int32 = {0x48584e38,
0x403c342c}, v4_int16 = {0x4e38, 0x4858, 0x342c, 0x403c}, v8_int8 =
{0x38,
0x4e, 0x58, 0x48, 0x2c, 0x34, 0x3c, 0x40}}
mm2 {uint64 = 0x313c3f43310a1c21, v2_int32 = {0x310a1c21,
0x313c3f43}, v4_int16 = {0x1c21, 0x310a, 0x3f43, 0x313c}, v8_int8 =
{0x21,
0x1c, 0xa, 0x31, 0x43, 0x3f, 0x3c, 0x31}}
mm3 {uint64 = 0x672731534943413f, v2_int32 = {0x4943413f,
0x67273153}, v4_int16 = {0x413f, 0x4943, 0x3153, 0x6727}, v8_int8 =
{0x3f,
0x41, 0x43, 0x49, 0x53, 0x31, 0x27, 0x67}}
mm4 {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0,
0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}
mm5 {uint64 = 0x8585858585858585, v2_int32 = {0x85858585,
0x85858585}, v4_int16 = {0x8585, 0x8585, 0x8585, 0x8585}, v8_int8 =
{0x85,
0x85, 0x85, 0x85, 0x85, 0x85, 0x85, 0x85}}
mm6 {uint64 = 0x8585858585858585, v2_int32 = {0x85858585,
0x85858585}, v4_int16 = {0x8585, 0x8585, 0x8585, 0x8585}, v8_int8 =
{0x85,
0x85, 0x85, 0x85, 0x85, 0x85, 0x85, 0x85}}
mm7 {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0,
0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}}
(gdb)
Jason Garrett-Glaser wrote:
>> 0xb7fd57b0 in x264_rd_cost_i8x8_chroma () from /usr/lib/libx264.so.59
>> (gdb) bt
>> #0 0xb7fd57b0 in x264_rd_cost_i8x8_chroma () from /usr/lib/libx264.so.59
>> #1 0x00000000 in ?? ()
>
> Can you give a disass, e.g. "disass $pc-50 $pc+50 or something like
> that? The function is relatively small so the asm might be useful.
>
> Dark Shikari
> _______________________________________________
> x264-devel mailing list
> x264-devel at videolan.org
> http://mailman.videolan.org/listinfo/x264-devel
--
Rodolfo J. Quesada Zumbado
rquesada at roqz.net
More information about the x264-devel
mailing list