[FFmpeg-trac] #6838(avcodec:open): avcodec:ff_prores_idct_put_10_sse2 segfault on decoding a mov
FFmpeg
trac at avcodec.org
Thu Nov 16 01:56:55 EET 2017
#6838: avcodec:ff_prores_idct_put_10_sse2 segfault on decoding a mov
-------------------------------------+-------------------------------------
Reporter: j13r | Owner:
Type: defect | Status: open
Priority: important | Component: avcodec
Version: git-master | Resolution:
Keywords: prores | Blocked By:
crash SIGSEGV regression | Reproduced by developer: 1
Blocking: |
Analyzed by developer: 0 |
-------------------------------------+-------------------------------------
Changes (by cehoyos):
* keywords: => prores crash SIGSEGV regression
* priority: normal => important
* version: unspecified => git-master
* status: new => open
* reproduced: 0 => 1
Comment:
Regression since bebaf4ea, reproducible with different compilers.
{{{
(gdb) r -i fate-suite/prores/Sequence_1-Apple_ProRes_422.mov
Starting program: ffmpeg_g -i fate-
suite/prores/Sequence_1-Apple_ProRes_422.mov
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib64/libthread_db.so.1".
ffmpeg version N-89061-g6d00905 Copyright (c) 2000-2017 the FFmpeg
developers
built with gcc 6.3.0 (GCC)
configuration: --enable-small --toolchain=hardened --disable-avx
libavutil 56. 0.100 / 56. 0.100
libavcodec 58. 3.102 / 58. 3.102
libavformat 58. 2.100 / 58. 2.100
libavdevice 58. 0.100 / 58. 0.100
libavfilter 7. 0.101 / 7. 0.101
libswscale 5. 0.101 / 5. 0.101
libswresample 3. 0.101 / 3. 0.101
Program received signal SIGSEGV, Segmentation fault.
0x0000555555de7c37 in ff_prores_idct_put_10_sse2 ()
(gdb) bt
#0 0x0000555555de7c37 in ff_prores_idct_put_10_sse2 ()
#1 0x0000555555a4ef32 in decode_slice_luma
(avctx=avctx at entry=0x555556dd8d60, dst=0x7ffff4d60f10,
dst_stride=dst_stride at entry=7680, buf=buf at entry=0x555556de80a6
"\006\240\217\377\377\377\300\202?\377\202?\377\060\002",
buf_size=<optimized out>, qmat=0x7fffffffcfb8,
slice=<optimized out>) at libavcodec/proresdec2.c:389
#2 0x0000555555a5023b in decode_slice_thread (avctx=0x555556dd8d60,
arg=<optimized out>, jobnr=<optimized out>, threadnr=<optimized out>) at
libavcodec/proresdec2.c:581
#3 0x0000555555ac9bad in avcodec_default_execute2 (c=0x555556dd8d60,
func=0x555555a4fff6 <decode_slice_thread>, arg=0x0, ret=0x0,
count=<optimized out>) at libavcodec/utils.c:536
#4 0x0000555555a4fc19 in decode_picture (avctx=0x555556dd8d60) at
libavcodec/proresdec2.c:625
#5 decode_frame (avctx=0x555556dd8d60, data=<optimized out>,
got_frame=0x7fffffffd200, avpkt=0x555556ddd420) at
libavcodec/proresdec2.c:677
#6 0x00005555558f5679 in decode_simple_internal (frame=0x555556ddd1a0,
avctx=0x555556dd8d60) at libavcodec/decode.c:397
#7 decode_simple_receive_frame (frame=<optimized out>, avctx=<optimized
out>) at libavcodec/decode.c:593
#8 decode_receive_frame_internal (avctx=avctx at entry=0x555556dd8d60,
frame=0x555556ddd1a0) at libavcodec/decode.c:611
#9 0x00005555558f7789 in avcodec_send_packet
(avctx=avctx at entry=0x555556dd8d60, avpkt=avpkt at entry=0x7fffffffd2d0) at
libavcodec/decode.c:673
#10 0x000055555588f0aa in try_decode_frame (s=s at entry=0x555556dd71a0,
st=st at entry=0x555556dd85d0, avpkt=avpkt at entry=0x7fffffffd418,
options=<optimized out>) at libavformat/utils.c:3006
#11 0x000055555589543b in avformat_find_stream_info (ic=0x555556dd71a0,
options=0x555556dd7cd0) at libavformat/utils.c:3831
#12 0x000055555569f30d in open_input_file (o=o at entry=0x7fffffffd6f8,
filename=0x7fffffffe1cf "fate-
suite/prores/Sequence_1-Apple_ProRes_422.mov") at
fftools/ffmpeg_opt.c:1078
#13 0x000055555569d150 in open_files (l=0x555556dd7028,
inout=inout at entry=0x55555614745d "input",
open_file=open_file at entry=0x55555569eda3 <open_input_file>) at
fftools/ffmpeg_opt.c:3281
#14 0x00005555556a3471 in ffmpeg_parse_options (argc=argc at entry=3,
argv=argv at entry=0x7fffffffdd38) at fftools/ffmpeg_opt.c:3321
#15 0x000055555569ae6c in main (argc=3, argv=0x7fffffffdd38) at
fftools/ffmpeg.c:4775
(gdb) disass $pc-23,$pc+32
Dump of assembler code from 0x555555de7c20 to 0x555555de7c57:
0x0000555555de7c20 <ff_prores_idct_put_10_sse2+0>: movdqa
(%rdx),%xmm10
0x0000555555de7c25 <ff_prores_idct_put_10_sse2+5>: movdqa
0x20(%rdx),%xmm8
0x0000555555de7c2b <ff_prores_idct_put_10_sse2+11>: movdqa
0x40(%rdx),%xmm13
0x0000555555de7c31 <ff_prores_idct_put_10_sse2+17>: movdqa
0x60(%rdx),%xmm12
=> 0x0000555555de7c37 <ff_prores_idct_put_10_sse2+23>: pmullw
(%rcx),%xmm10
0x0000555555de7c3c <ff_prores_idct_put_10_sse2+28>: pmullw
0x20(%rcx),%xmm8
0x0000555555de7c42 <ff_prores_idct_put_10_sse2+34>: pmullw
0x40(%rcx),%xmm13
0x0000555555de7c48 <ff_prores_idct_put_10_sse2+40>: pmullw
0x60(%rcx),%xmm12
0x0000555555de7c4e <ff_prores_idct_put_10_sse2+46>: paddw
0x352f69(%rip),%xmm10 # 0x55555613abc0 <ff_pw_1>
End of assembler dump.
(gdb) info all-register
rax 0x555556ddd4e0 93825017959648
rbx 0x1e00 7680
rcx 0x7fffffffcfb8 140737488342968
rdx 0x7fffffffbee0 140737488338656
rsi 0x1e00 7680
rdi 0x7ffff4d60f10 140737301057296
rbp 0x7fffffffcf30 0x7fffffffcf30
rsp 0x7fffffffbe58 0x7fffffffbe58
r8 0x0 0
r9 0x1f 31
r10 0x0 0
r11 0x0 0
r12 0x7fffffffbee0 140737488338656
r13 0xf000 61440
r14 0xf010 61456
r15 0x1 1
rip 0x555555de7c37 0x555555de7c37
<ff_prores_idct_put_10_sse2+23>
eflags 0x10202 [ IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
st0 0 (raw 0x00000000000000000000)
st1 0 (raw 0x00000000000000000000)
st2 0 (raw 0x00000000000000000000)
st3 0 (raw 0x00000000000000000000)
st4 0 (raw 0x00000000000000000000)
st5 0 (raw 0x00000000000000000000)
st6 0 (raw 0x00000000000000000000)
st7 0 (raw 0x00000000000000000000)
fctrl 0x37f 895
fstat 0x0 0
ftag 0xffff 65535
fiseg 0x0 0
fioff 0x0 0
foseg 0x0 0
fooff 0x0 0
fop 0x0 0
mxcsr 0x1fa0 [ PE IM DM ZM OM UM PM ]
ymm0 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {
0x00000000000000000000000000000000,
0x00000000000000000000000000000000}}
ymm1 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0xff, 0xff, 0xff, 0xff, 0x0
<repeats 28 times>}, v16_int16 = {0xffff, 0xffff, 0x0 <repeats 14 times>},
v8_int32 = {0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_int64 = {0xffffffff, 0x0, 0x0, 0x0}, v2_int128 =
{0x000000000000000000000000ffffffff, 0x00000000000000000000000000000000}}
ymm2 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x6e, 0x75, 0x6c, 0x6c, 0x0,
0x7f, 0x0, 0x0, 0x88, 0x6, 0x70, 0xf5, 0xff, 0x7f, 0x0 <repeats 18
times>}, v16_int16 = {0x756e, 0x6c6c, 0x7f00, 0x0, 0x688, 0xf570,
0x7fff, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 =
{0x6c6c756e, 0x7f00, 0xf5700688, 0x7fff, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0x7f006c6c756e, 0x7ffff5700688, 0x0, 0x0}, v2_int128 =
{0x00007ffff570068800007f006c6c756e, 0x00000000000000000000000000000000}}
ymm3 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {
0x00000000000000000000000000000000,
0x00000000000000000000000000000000}}
ymm4 {v8_float = {0x3, 0x351597c0, 0x0, 0x454c0000, 0x0, 0x0,
0x0, 0x0}, v4_double = {0x8000000000000000, 0x8000000000000000, 0x0, 0x0},
v32_int8 = {0x0, 0x0, 0x40, 0x40, 0x5f, 0x56, 0x54, 0x4e, 0x52, 0x3d,
0x38, 0x0, 0x4c, 0x45, 0x53, 0x53, 0x0 <repeats 16 times>},
v16_int16 = {0x0, 0x4040, 0x565f, 0x4e54, 0x3d52, 0x38, 0x454c, 0x5353,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v8_int32 = {0x40400000,
0x4e54565f, 0x383d52, 0x5353454c, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0x4e54565f40400000, 0x5353454c00383d52, 0x0, 0x0}, v2_int128 = {
0x5353454c00383d524e54565f40400000,
0x00000000000000000000000000000000}}
ymm5 {v8_float = {0x0, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x4c, 0x82, 0x9c,
0xe3, 0x92, 0x99, 0xd9, 0xbf, 0x0 <repeats 24 times>}, v16_int16 =
{0x824c, 0xe39c, 0x9992, 0xbfd9, 0x0 <repeats 12 times>}, v8_int32 = {
0xe39c824c, 0xbfd99992, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0xbfd99992e39c824c, 0x0, 0x0, 0x0}, v2_int128 =
{0x0000000000000000bfd99992e39c824c, 0x00000000000000000000000000000000}}
ymm6 {v8_float = {0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x2540be400, 0x0, 0x0, 0x0}, v32_int8 = {0x0, 0x0, 0x0, 0x20,
0x5f, 0xa0, 0x2, 0x42, 0x0 <repeats 24 times>}, v16_int16 = {0x0, 0x2000,
0xa05f, 0x4202, 0x0 <repeats 12 times>}, v8_int32 = {
0x20000000, 0x4202a05f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0x4202a05f20000000, 0x0, 0x0, 0x0}, v2_int128 =
{0x00000000000000004202a05f20000000, 0x00000000000000000000000000000000}}
ymm7 {v8_float = {0x0, 0xffffffff, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0xf9, 0xc8, 0xde,
0xfc, 0xd1, 0x21, 0x89, 0xbf, 0x0 <repeats 24 times>}, v16_int16 =
{0xc8f9, 0xfcde, 0x21d1, 0xbf89, 0x0 <repeats 12 times>}, v8_int32 = {
0xfcdec8f9, 0xbf8921d1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0xbf8921d1fcdec8f9, 0x0, 0x0, 0x0}, v2_int128 =
{0x0000000000000000bf8921d1fcdec8f9, 0x00000000000000000000000000000000}}
ymm8 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {
0x00000000000000000000000000000000,
0x00000000000000000000000000000000}}
ymm9 {v8_float = {0xba340000, 0xffffffff, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x8d, 0x2e, 0x44,
0x54, 0xfb, 0x21, 0x89, 0xbf, 0x0 <repeats 24 times>}, v16_int16 =
{0x2e8d, 0x5444, 0x21fb, 0xbf89, 0x0 <repeats 12 times>},
v8_int32 = {0x54442e8d, 0xbf8921fb, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_int64 = {0xbf8921fb54442e8d, 0x0, 0x0, 0x0}, v2_int128 =
{0x0000000000000000bf8921fb54442e8d, 0x00000000000000000000000000000000}}
ymm10 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x40, 0x3, 0x0 <repeats 30
times>}, v16_int16 = {0x340, 0x0 <repeats 15 times>}, v8_int32 = {0x340,
0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 = {0x340, 0x0,
0x0, 0x0}, v2_int128 = {0x00000000000000000000000000000340,
0x00000000000000000000000000000000}}
ymm11 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0xd9, 0x7e, 0x9a, 0x7b,
0xe2, 0x1d, 0xc7, 0x3e, 0x0 <repeats 24 times>}, v16_int16 = {0x7ed9,
0x7b9a, 0x1de2, 0x3ec7, 0x0 <repeats 12 times>}, v8_int32 = {
0x7b9a7ed9, 0x3ec71de2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0x3ec71de27b9a7ed9, 0x0, 0x0, 0x0}, v2_int128 =
{0x00000000000000003ec71de27b9a7ed9, 0x00000000000000000000000000000000}}
ymm12 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {
0x00000000000000000000000000000000,
0x00000000000000000000000000000000}}
ymm13 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x0 <repeats 32 times>},
v16_int16 = {0x0 <repeats 16 times>}, v8_int32 = {0x0, 0x0, 0x0, 0x0, 0x0,
0x0, 0x0, 0x0}, v4_int64 = {0x0, 0x0, 0x0, 0x0}, v2_int128 = {
0x00000000000000000000000000000000,
0x00000000000000000000000000000000}}
ymm14 {v8_float = {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0},
v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x46, 0xb0, 0x42, 0xa4,
0x99, 0xe4, 0xd3, 0x3e, 0x0 <repeats 24 times>}, v16_int16 = {0xb046,
0xa442, 0xe499, 0x3ed3, 0x0 <repeats 12 times>}, v8_int32 = {
0xa442b046, 0x3ed3e499, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0x3ed3e499a442b046, 0x0, 0x0, 0x0}, v2_int128 =
{0x00000000000000003ed3e499a442b046, 0x00000000000000000000000000000000}}
ymm15 {v8_float = {0x92854080, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0,
0x0}, v4_double = {0x0, 0x0, 0x0, 0x0}, v32_int8 = {0x7f, 0xf5, 0xda,
0xce, 0xf0, 0x39, 0xc1, 0x3f, 0x0 <repeats 24 times>}, v16_int16 =
{0xf57f, 0xceda, 0x39f0, 0x3fc1, 0x0 <repeats 12 times>}, v8_int32 = {
0xcedaf57f, 0x3fc139f0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, v4_int64 =
{0x3fc139f0cedaf57f, 0x0, 0x0, 0x0}, v2_int128 =
{0x00000000000000003fc139f0cedaf57f, 0x00000000000000000000000000000000}}
}}}
--
Ticket URL: <https://trac.ffmpeg.org/ticket/6838#comment:2>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker
More information about the FFmpeg-trac
mailing list