[x264-devel] Segmentation fault in x264_predict_8x8_filter_ssse3()
Natko Kalisnik
nkalisni at inet.hr
Thu Dec 22 15:16:45 CET 2011
I'm trying to encode 1920x1080i video in x264, but I have problems.
You can download a file with that same frames here (raw format, 252 frames
in it):
ftp://ftp.ldv.ei.tum.de/videolab/public/SVT_Test_Set/1080i/1080i25_stockholm_ter.yuv
The x264 version is: x264-snapshot-20111206-2245
The ffmpeg version is 0.9
Builded with gcc version 4.6.1 on Windows XP (MinGW)
For each frame I wrote to console "Frame: frame_number" just before calling
the avcodec_encode_video().
You will see a crash in x264_predict_8x8_filter_ssse3(). I experimented with
the C implementation too.
It also crashes.
If you need more info on this subject, I can provide it...
Here follows the gdb output:
C:\Documents and Settings\foo\workspace\ffmpeg_debugging\Debug>gdb
ffmpeg_debugging.exe
GNU gdb (GDB) 7.3.1
Copyright (C) 2011 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later
<http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law. Type "show copying"
and "show warranty" for details.
This GDB was configured as "mingw32".
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>...
Reading symbols from c:\documents and
settings\foo\workspace\ffmpeg_debugging\debug\ffmpeg_debugging.exe...done.
(gdb) r
Starting program: c:\documents and
settings\foo\workspace\ffmpeg_debugging\debug\ffmpeg_debugging.exe
[New Thread 5968.0x1304]
[libx264 @ 003e2bc0] using cpu capabilities: MMX2 SSE2Fast SSSE3 Cache64
[New Thread 5968.0x9e0]
[New Thread 5968.0x1544]
[New Thread 5968.0x17ec]
[New Thread 5968.0x148c]
[libx264 @ 003e2bc0] profile High, level 4.0
[libx264 @ 003e2bc0] 264 - core 120 - H.264/MPEG-4 AVC codec - Copyleft
2003-2011 - http://www.videolan.org/x264.html - options: c
abac=1 ref=4 deblock=1:0:0 analyse=0x3:0x113 me=umh subme=8 psy=1
psy_rd=1.00:0.00 mixed_ref=1 me_range=16 chroma_me=0 trellis=1 8
x8dct=1 cqm=0 deadzone=21,11 fast_pskip=1 chroma_qp_offset=-2 threads=3
sliced_threads=0 nr=0 decimate=1 interlaced=tff bluray_com
pat=0 constrained_intra=0 bframes=3 b_pyramid=0 b_adapt=1 b_bias=0 direct=3
weightb=1 open_gop=0 weightp=0 keyint=250 keyint_min=2
5 scenecut=40 intra_refresh=0 rc_lookahead=40 rc=crf mbtree=1 crf=22.0
qcomp=0.60 qpmin=10 qpmax=51 qpstep=4 ip_ratio=1.40 aq=1:1.
00
Frame: 0
Frame: 1
Frame: 2
Frame: 3
Frame: 4
Frame: 5
Frame: 6
Frame: 7
Frame: 8
Frame: 9
Frame: 10
Frame: 11
Frame: 12
Frame: 13
Frame: 14
Frame: 15
Frame: 16
Frame: 17
Frame: 18
Frame: 19
Frame: 20
Frame: 21
Frame: 22
Frame: 23
Frame: 24
Frame: 25
Frame: 26
Frame: 27
Frame: 28
Frame: 29
Frame: 30
Frame: 31
Frame: 32
Frame: 33
Frame: 34
Frame: 35
Frame: 36
Frame: 37
Frame: 38
Frame: 39
Frame: 40
Frame: 41
Frame: 42
Frame: 43
Frame: 44
Frame: 45
Frame: 46
Frame: 47
Program received signal SIGSEGV, Segmentation fault.
[Switching to Thread 5968.0x1544]
0x009262ef in x264_predict_8x8_filter_ssse3 ()
(gdb) bt
#0 0x009262ef in x264_predict_8x8_filter_ssse3 ()
#1 0x008f7890 in x264_intra_rd_refine (h=<optimized out>, a=0x1fac7f0) at
encoder/analyse.c:1260
#2 0x008fd429 in x264_macroblock_analyse (h=0x16305f0) at
encoder/analyse.c:3018
#3 0x008a9739 in x264_slice_write (h=0x16305f0) at encoder/encoder.c:2168
#4 0x008b68e2 in x264_stack_align ()
#5 0x008a4503 in x264_slices_write (h=0x16305f0) at encoder/encoder.c:2469
#6 0x008c7d92 in x264_threadpool_thread (pool=0x136d8e0) at
common/threadpool.c:69
#7 0x00913ae9 in x264_win32thread_worker (arg=0x3edfa0) at
common/win32thread.c:65
#8 0x77c3a3b0 in msvcrt!_endthreadex () from C:\WINDOWS\system32\msvcrt.dll
#9 0x7c80b729 in KERNEL32!GetModuleFileNameA () from
C:\WINDOWS\system32\kernel32.dll
#10 0x00000000 in ?? ()
(gdb) disass $pc-64,$pc+64
Dump of assembler code from 0x9262af to 0x92632f:
0x009262af <x264_predict_8x8_filter_ssse3+63>: and %cl,(%edi)
0x009262b1 <x264_predict_8x8_filter_ssse3+65>: push $0x586f0f10
0x009262b6 <x264_predict_8x8_filter_ssse3+70>: pusha
0x009262b7 <x264_predict_8x8_filter_ssse3+71>: punpckhbw
0x40(%eax),%mm3
0x009262bb <x264_predict_8x8_filter_ssse3+75>: punpckhwd %mm2,%mm3
0x009262be <x264_predict_8x8_filter_ssse3+78>: punpckhdq %mm1,%mm3
0x009262c1 <x264_predict_8x8_filter_ssse3+81>: movq
0x80(%eax),%mm0
0x009262c8 <x264_predict_8x8_filter_ssse3+88>:
movq -0x78(%eax),%mm1
0x009262cc <x264_predict_8x8_filter_ssse3+92>: movq %mm3,%mm4
0x009262cf <x264_predict_8x8_filter_ssse3+95>: palignr
$0x7,%mm0,%mm4
0x009262d4 <x264_predict_8x8_filter_ssse3+100>: palignr
$0x1,%mm3,%mm1
0x009262d9 <x264_predict_8x8_filter_ssse3+105>: movq %mm1,%mm5
0x009262dc <x264_predict_8x8_filter_ssse3+108>: pavgb %mm4,%mm1
0x009262df <x264_predict_8x8_filter_ssse3+111>: pxor %mm5,%mm4
0x009262e2 <x264_predict_8x8_filter_ssse3+114>: pand 0xa71910,%mm4
0x009262e9 <x264_predict_8x8_filter_ssse3+121>: psubusb %mm4,%mm1
0x009262ec <x264_predict_8x8_filter_ssse3+124>: pavgb %mm1,%mm3
=> 0x009262ef <x264_predict_8x8_filter_ssse3+127>: movq
%mm3,0x8(%esi)
0x009262f3 <x264_predict_8x8_filter_ssse3+131>: movzbl
0x87(%eax),%ecx
0x009262fa <x264_predict_8x8_filter_ssse3+138>: movzbl
0x67(%eax),%edi
0x009262fe <x264_predict_8x8_filter_ssse3+142>: lea
0x2(%ecx,%ecx,2),%ecx
0x00926302 <x264_predict_8x8_filter_ssse3+146>: add %edi,%ecx
0x00926304 <x264_predict_8x8_filter_ssse3+148>: shr $0x2,%ecx
0x00926307 <x264_predict_8x8_filter_ssse3+151>: mov %cl,0x7(%esi)
0x0092630a <x264_predict_8x8_filter_ssse3+154>: mov %cl,0x6(%esi)
0x0092630d <x264_predict_8x8_filter_ssse3+157>: test $0x2,%bl
0x00926310 <x264_predict_8x8_filter_ssse3+160>: je 0x926375
<x264_predict_8x8_filter_ssse3+261>
0x00926312 <x264_predict_8x8_filter_ssse3+162>:
movdqu -0x78(%eax),%xmm3
0x00926317 <x264_predict_8x8_filter_ssse3+167>:
movhps -0x80(%eax),%xmm0
0x0092631b <x264_predict_8x8_filter_ssse3+171>: test $0x8,%dl
0x0092631e <x264_predict_8x8_filter_ssse3+174>: je 0x926379
<x264_predict_8x8_filter_ssse3+265>
0x00926320 <x264_predict_8x8_filter_ssse3+176>: and $0x4,%edx
0x00926323 <x264_predict_8x8_filter_ssse3+179>: pshufb
0xa713a0(,%edx,4),%xmm3
0x0092632d <x264_predict_8x8_filter_ssse3+189>: movdqa %xmm3,%xmm1
End of assembler dump.
(gdb) info all-registers
eax 0x16354c0 23286976
ecx 0x0 0
edx 0x1 1
ebx 0xf 15
esp 0x1fac010 0x1fac010
ebp 0x16305f0 0x16305f0
esi 0x0 0
edi 0x1 1
eip 0x9262ef 0x9262ef <x264_predict_8x8_filter_ssse3+127>
eflags 0x10246 [ PF ZF IF RF ]
cs 0x1b 27
ss 0x23 35
ds 0x23 35
es 0x23 35
fs 0x3b 59
gs 0x0 0
st0 -nan(0x1010101010101010) (raw 0xffff1010101010101010)
st1 -nan(0xe10101010101010) (raw 0xffff0e10101010101010)
st2 -nan(0x1010101010101010) (raw 0xffff1010101010101010)
st3 -nan(0xf10101010101010) (raw 0xffff0f10101010101010)
st4 -nan(0x100000000000000) (raw 0xffff0100000000000000)
st5 -nan(0xd10101010101010) (raw 0xffff0d10101010101010)
st6 -inf (raw 0xffff0000000000000000)
st7 -inf (raw 0xffff0000000000000000)
fctrl 0xffff027f -64897
fstat 0xffff0000 -65536
ftag 0xffffaaaa -21846
fiseg 0x0 0
fioff 0x0 0
foseg 0xffff0000 -65536
fooff 0x0 0
fop 0x493 1171
xmm0 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double =
{0x8000000000000000, 0x8000000000000000}, v16_int8 = {
0xff <repeats 16 times>}, v8_int16 = {0xffff, 0xffff, 0xffff, 0xffff,
0xffff, 0xffff, 0xffff, 0xffff}, v4_int32 = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff}, v2_int64 =
{0xffffffffffffffff, 0xffffffffffffffff},
uint128 = 0xffffffffffffffffffffffffffffffff}
xmm1 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0},
v2_int64 = {0x0, 0x0},
uint128 = 0x00000000000000000000000000000000}
xmm2 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0},
v2_int64 = {0x0, 0x0},
uint128 = 0x00000000000000000000000000000000}
xmm3 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x39, 0x0, 0x51, 0x5, 0x39, 0x0, 0x51,
0x5, 0xab, 0x53, 0x8a, 0x5, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x39,
0x551, 0x39, 0x551, 0x53ab, 0x58a, 0x0, 0x0},
v4_int32 = {0x5510039, 0x5510039, 0x58a53ab, 0x0}, v2_int64 =
{0x551003905510039, 0x58a53ab},
uint128 = 0x00000000058a53ab0551003905510039}
xmm4 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int16 = {0x1, 0x0, 0x0, 0x0,
0x2, 0x0, 0x0, 0x0}, v4_int32 = {0x1, 0x0, 0x2,
0x0}, v2_int64 = {0x1, 0x2}, uint128 =
0x00000000000000020000000000000001}
xmm5 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10,
0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0}, v8_int16 = {0x10,
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}, v4_int32 = {
0x100010, 0x100010, 0x100010, 0x100010}, v2_int64 = {0x10001000100010,
0x10001000100010},
uint128 = 0x00100010001000100010001000100010}
xmm6 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10,
0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0, 0x10, 0x0}, v8_int16 = {0x10,
0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}, v4_int32 = {
0x100010, 0x100010, 0x100010, 0x100010}, v2_int64 = {0x10001000100010,
0x10001000100010},
uint128 = 0x00100010001000100010001000100010}
xmm7 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0xff, 0x0, 0xff, 0x0, 0xff, 0x0, 0xff,
0x0, 0xff, 0x0, 0xff, 0x0, 0xff, 0x0, 0xff, 0x0}, v8_int16 = {0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}, v4_int32 = {
0xff00ff, 0xff00ff, 0xff00ff, 0xff00ff}, v2_int64 = {0xff00ff00ff00ff,
0xff00ff00ff00ff},
uint128 = 0x00ff00ff00ff00ff00ff00ff00ff00ff}
mxcsr 0x1fa0 [ PE IM DM ZM OM UM PM ]
mm0 {uint64 = 0x1010101010101010, v2_int32 = {0x10101010,
0x10101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0x1010},
v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}}
mm1 {uint64 = 0xe10101010101010, v2_int32 = {0x10101010,
0xe101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xe10},
v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xe}}
mm2 {uint64 = 0x1010101010101010, v2_int32 = {0x10101010,
0x10101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0x1010},
v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10}}
mm3 {uint64 = 0xf10101010101010, v2_int32 = {0x10101010,
0xf101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xf10},
v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xf}}
mm4 {uint64 = 0x100000000000000, v2_int32 = {0x0, 0x1000000},
v4_int16 = {0x0, 0x0, 0x0, 0x100}, v8_int8 = {0x0,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}}
mm5 {uint64 = 0xd10101010101010, v2_int32 = {0x10101010,
0xd101010}, v4_int16 = {0x1010, 0x1010, 0x1010, 0xd10},
v8_int8 = {0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0x10, 0xd}}
mm6 {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0,
0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0}}
mm7 {uint64 = 0x0, v2_int32 = {0x0, 0x0}, v4_int16 = {0x0, 0x0,
0x0, 0x0}, v8_int8 = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0}}
(gdb)
If I can provide more info, please let me know.
Regards,
Natko
More information about the x264-devel
mailing list