[x264-devel] x264 segmentation fault when 4:4:4 10 bits cavlc encode.
Masaharu Tamura
tamura at pegasys-inc.com
Wed Aug 15 05:55:45 CEST 2018
Hi,
I have a problem when I try to encode random pixel video.
Using x264 options is:
--input-res 704x480 --fps 30/1 --profile high444 --input-csp i444
--input-depth 10 --input-range tv --output-csp i444 --output-depth 10
--no-cabac --crf 1 --keyint 1 --subme 11 --trellis 2 -o output.264
output.yuv
I attached back trace, disassemble and register information.
Here is the public downloadable link for the test file.
https://drive.google.com/file/d/1RZaR_ecrpu6D4TD0PXJB3iTdMoz-lKtT/view?usp=sharing
Sincerely,
Masaharu Tamura.
-------------- next part --------------
(gdb) bt
#0 0x0000000140046c7a in x264_10_coeff_level_run16_lzcnt ()
#1 0x00000001400e053c in cavlc_block_residual_internal (h=h at entry=0x4cd00c0,
ctx_block_cat=ctx_block_cat at entry=2, l=l at entry=0x4cd6800, nC=1)
at encoder/cavlc.c:131
#2 0x00000001400ed4eb in cavlc_partition_luma_residual (p=<optimized out>,
i8=1, h=0x4cd00c0) at encoder/cavlc.c:301
#3 partition_i8x8_size_cavlc (i_mode=<optimized out>, i8=<optimized out>,
h=<optimized out>) at encoder/cavlc.c:691
#4 rd_cost_i8x8 (edge=0x3cc6a20, i_mode=0, i8=1, i_lambda2=9289, h=0x4cd00c0)
at encoder/rdo.c:302
#5 intra_rd_refine (h=h at entry=0x4cd00c0, a=a at entry=0x3cc8140)
at encoder/analyse.c:1164
#6 0x00000001400f27ae in x264_10_macroblock_analyse (h=h at entry=0x4cd00c0)
at encoder/analyse.c:2939
#7 0x0000000140031bd1 in slice_write (h=h at entry=0x4cd00c0)
at encoder/encoder.c:2750
#8 0x0000000140031cfd in slices_write (h=0x4cd00c0) at encoder/encoder.c:3090
#9 0x000000014003936a in threadpool_thread_internal (pool=0x1e5580)
at common/threadpool.c:69
#10 threadpool_thread (pool=0x1e5580) at common/threadpool.c:77
#11 0x00000001400136c3 in win32thread_worker (arg=<optimized out>)
at common/win32thread.c:57
#12 0x00007ffe6804aa96 in msvcrt!_beginthreadex ()
from C:\WINDOWS\System32\msvcrt.dll
#13 0x00007ffe6804ab6c in msvcrt!_endthreadex ()
from C:\WINDOWS\System32\msvcrt.dll
#14 0x00007ffe68153034 in KERNEL32!BaseThreadInitThunk ()
from C:\WINDOWS\System32\kernel32.dll
#15 0x00007ffe68301431 in ntdll!RtlUserThreadStart ()
from C:\WINDOWS\SYSTEM32\ntdll.dll
#16 0x0000000000000000 in ?? ()
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
(gdb) disass $pc-32,$pc+32
Dump of assembler code from 0x140046c5a to 0x140046c9a:
0x0000000140046c5a <x264_10_coeff_level_run16_lzcnt+58>: jrcxz 0x140046c6c <x264_10_coeff_level_run16_lzcnt+76>
0x0000000140046c5c <x264_10_coeff_level_run16_lzcnt+60>: mov $0xf,%r10d
0x0000000140046c62 <x264_10_coeff_level_run16_lzcnt+66>: lzcnt %r11d,%ecx
0x0000000140046c67 <x264_10_coeff_level_run16_lzcnt+71>: xor %eax,%eax
0x0000000140046c69 <x264_10_coeff_level_run16_lzcnt+73>: add %r11d,%r11d
0x0000000140046c6c <x264_10_coeff_level_run16_lzcnt+76>: sub %ecx,%r10d
0x0000000140046c6f <x264_10_coeff_level_run16_lzcnt+79>: shl %cl,%r11d
0x0000000140046c72 <x264_10_coeff_level_run16_lzcnt+82>: mov %r10d,(%rdx)
0x0000000140046c75 <x264_10_coeff_level_run16_lzcnt+85>: lzcnt %r11d,%ecx
=> 0x0000000140046c7a <x264_10_coeff_level_run16_lzcnt+90>: mov (%r9,%r10,4),%r8d
0x0000000140046c7e <x264_10_coeff_level_run16_lzcnt+94>: inc %ecx
0x0000000140046c80 <x264_10_coeff_level_run16_lzcnt+96>: shl %cl,%r11d
0x0000000140046c83 <x264_10_coeff_level_run16_lzcnt+99>: mov %r8d,0x10(%rdx,%rax,4)
0x0000000140046c88 <x264_10_coeff_level_run16_lzcnt+104>: inc %eax
0x0000000140046c8a <x264_10_coeff_level_run16_lzcnt+106>: sub %ecx,%r10d
0x0000000140046c8d <x264_10_coeff_level_run16_lzcnt+109>: jge 0x140046c75 <x264_10_coeff_level_run16_lzcnt+85>
0x0000000140046c8f <x264_10_coeff_level_run16_lzcnt+111>: repz retq
0x0000000140046c91 <x264_10_coeff_level_run16_lzcnt+113>: add %al,(%rax)
0x0000000140046c93 <x264_10_coeff_level_run16_lzcnt+115>: add %al,(%rax)
0x0000000140046c95 <x264_10_coeff_level_run16_lzcnt+117>: add %al,(%rax)
0x0000000140046c97 <x264_10_coeff_level_run16_lzcnt+119>: add %al,(%rax)
0x0000000140046c99 <x264_10_coeff_level_run16_lzcnt+121>: add %al,(%rax)
End of assembler dump.
(gdb) info all-registers
rax 0x0 0
rbx 0x4cd00c0 80543936
rcx 0x20 32
rdx 0x3cc6740 63727424
rsi 0x2 2
rdi 0x1401ae034 5370470452
rbp 0x1 0x1
rsp 0x3cc6708 0x3cc6708
r8 0x4cd6800 80570368
r9 0x4cd6800 80570368
r10 0xffffffef 4294967279
r11 0x0 0
r12 0x1401adbf0 5370469360
r13 0x4cd00c0 80543936
r14 0x1401a1ee0 5370420960
r15 0x5 5
rip 0x140046c7a 0x140046c7a <x264_10_coeff_level_run16_lzcnt+90>
eflags 0x10203 [ CF IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x2b 43
es 0x2b 43
fs 0x53 83
gs 0x2b 43
st0 <invalid float value> (raw 0xffff0000000000000000)
st1 <invalid float value> (raw 0xffff0000000000000000)
st2 <invalid float value> (raw 0xffff0000000000000000)
st3 <invalid float value> (raw 0xffff0000000000000000)
st4 <invalid float value> (raw 0xffff0000000000000000)
st5 <invalid float value> (raw 0xffff0000000000000000)
st6 <invalid float value> (raw 0xffff0000000000000000)
st7 <invalid float value> (raw 0xffff0000000000000000)
fctrl 0x27f 639
fstat 0xff0000 16711680
ftag 0xff 255
fiseg 0x0 0
fioff 0x0 0
foseg 0x0 0
fooff 0x0 0
fop 0x0 0
xmm0 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {
0x8000000000000000, 0x8000000000000000}, v16_int8 = {
0xff <repeats 16 times>}, v8_int16 = {0xffff, 0xffff, 0xffff, 0xffff,
0xffff, 0xffff, 0xffff, 0xffff}, v4_int32 = {0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff}, v2_int64 = {0xffffffffffffffff,
0xffffffffffffffff}, uint128 = 0xffffffffffffffffffffffffffffffff}
xmm1 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm2 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm3 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm4 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm5 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm6 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm7 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm8 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm9 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm10 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm11 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm12 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm13 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm14 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
xmm15 {v4_float = {0x0, 0x0, 0x0, 0x0}, v2_double = {0x0, 0x0},
v16_int8 = {0x0 <repeats 16 times>}, v8_int16 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int32 = {0x0, 0x0, 0x0, 0x0}, v2_int64 = {0x0, 0x0},
uint128 = 0x0}
mxcsr 0x1fa0 [ PE IM DM ZM OM UM PM ]
(gdb)
More information about the x264-devel
mailing list