[FFmpeg-trac] #3143(avcodec:open): H.261 encoding crashes with trellis

FFmpeg trac at avcodec.org
Sun Nov 17 04:32:01 CET 2013


#3143: H.261 encoding crashes with trellis
-------------------------------------+-------------------------------------
             Reporter:  maikmerten   |                    Owner:
                 Type:  defect       |                   Status:  open
             Priority:  important    |                Component:  avcodec
              Version:  git-master   |               Resolution:
             Keywords:  h261 crash   |               Blocked By:
  SIGSEGV                            |  Reproduced by developer:  1
             Blocking:               |
Analyzed by developer:  0            |
-------------------------------------+-------------------------------------
Changes (by cehoyos):

 * keywords:  crash SIGSEGV => h261 crash SIGSEGV
 * priority:  normal => important
 * status:  new => open
 * reproduced:  0 => 1


Comment:

 {{{
 (gdb) r -i tests/lena.pnm -s 176x144 -trellis 2 out.h261
 Starting program: ffmpeg_g -i tests/lena.pnm -s 176x144 -trellis 2
 out.h261
 [Thread debugging using libthread_db enabled]
 Using host libthread_db library "/lib64/libthread_db.so.1".
 ffmpeg version N-58200-g92cbd77 Copyright (c) 2000-2013 the FFmpeg
 developers
   built on Nov 17 2013 04:21:45 with gcc 4.7 (SUSE Linux)
   configuration: --enable-gpl
   libavutil      52. 53.100 / 52. 53.100
   libavcodec     55. 43.100 / 55. 43.100
   libavformat    55. 21.100 / 55. 21.100
   libavdevice    55.  5.100 / 55.  5.100
   libavfilter     3. 91.100 /  3. 91.100
   libswscale      2.  5.101 /  2.  5.101
   libswresample   0. 17.104 /  0. 17.104
   libpostproc    52.  3.100 / 52.  3.100
 Input #0, image2, from 'tests/lena.pnm':
   Duration: 00:00:00.04, start: 0.000000, bitrate: N/A
     Stream #0:0: Video: ppm, rgb24, 256x256, 25 tbr, 25 tbn, 25 tbc
 [New Thread 0x7ffff59eb700 (LWP 23636)]
 [New Thread 0x7ffff51ea700 (LWP 23637)]
 [New Thread 0x7ffff49e9700 (LWP 23638)]
 [New Thread 0x7ffff41e8700 (LWP 23639)]
 [New Thread 0x7ffff39e7700 (LWP 23640)]
 [New Thread 0x7ffff31e6700 (LWP 23641)]
 [New Thread 0x7ffff29e5700 (LWP 23642)]
 [New Thread 0x7ffff21e4700 (LWP 23643)]
 [New Thread 0x7ffff19e3700 (LWP 23644)]
 Output #0, h261, to 'out.h261':
   Metadata:
     encoder         : Lavf55.21.100
     Stream #0:0: Video: h261, yuv420p, 176x144, q=2-31, 200 kb/s, 90k tbn,
 25 tbc
 Stream mapping:
   Stream #0:0 -> #0:0 (ppm -> h261)
 Press [q] to stop, [?] for help

 Program received signal SIGSEGV, Segmentation fault.
 0x000000000093d2f9 in dct_quantize_trellis_c (s=0x17cb5e0,
 block=0x17ef2a0, n=0, qscale=3,
     overflow=0x7fff0000001a) at libavcodec/mpegvideo_enc.c:3619
 3619                        int score= distortion +
 length[UNI_AC_ENC_INDEX(run, level)]*lambda;
 (gdb) bt
 #0  0x000000000093d2f9 in dct_quantize_trellis_c (s=0x17cb5e0,
 block=0x17ef2a0, n=0, qscale=3,
     overflow=0x7fff0000001a) at libavcodec/mpegvideo_enc.c:3619
 #1  0x00000000009488c1 in encode_mb_internal (mb_block_count=6,
 mb_block_width=8,
     mb_block_height=8, motion_y=0, motion_x=0, s=0x17cb5e0) at
 libavcodec/mpegvideo_enc.c:2060
 #2  encode_mb (motion_y=0, motion_x=0, s=0x17cb5e0) at
 libavcodec/mpegvideo_enc.c:2168
 #3  encode_thread (c=<optimized out>, arg=<optimized out>) at
 libavcodec/mpegvideo_enc.c:3042
 #4  0x0000000000a2fda7 in avcodec_default_execute (c=0x17cafa0,
 func=0x945310 <encode_thread>,
     arg=<optimized out>, ret=<optimized out>, count=1, size=8) at
 libavcodec/utils.c:1016
 #5  0x000000000094fc21 in encode_picture (picture_number=0, s=0x17cb5e0)
     at libavcodec/mpegvideo_enc.c:3435
 #6  ff_MPV_encode_picture (avctx=0x17cafa0, pkt=0x7fffffffda60,
 pic_arg=<optimized out>,
     got_packet=0x7fffffffd91c) at libavcodec/mpegvideo_enc.c:1494
 #7  0x0000000000a30ff7 in avcodec_encode_video2
 (avctx=avctx at entry=0x17cafa0,
     avpkt=avpkt at entry=0x7fffffffda60, frame=frame at entry=0x182d3a0,
     got_packet_ptr=got_packet_ptr at entry=0x7fffffffd91c) at
 libavcodec/utils.c:1863
 #8  0x0000000000471552 in do_video_out (in_picture=0x182d3a0,
 ost=0x17cb400, s=0x17caa00)
     at ffmpeg.c:965
 #9  reap_filters () at ffmpeg.c:1110
 #10 0x0000000000461548 in transcode_step () at ffmpeg.c:3235
 #11 transcode () at ffmpeg.c:3278
 #12 main (argc=<optimized out>, argv=<optimized out>) at ffmpeg.c:3456
 (gdb) disass $pc-32,$pc+32
 Dump of assembler code from 0x93d2d9 to 0x93d319:
    0x000000000093d2d9 <dct_quantize_trellis_c+1177>:    jmp    0x93d2eb
 <dct_quantize_trellis_c+1195>
    0x000000000093d2db <dct_quantize_trellis_c+1179>:    nopl
 0x0(%rax,%rax,1)
    0x000000000093d2e0 <dct_quantize_trellis_c+1184>:    movslq %edx,%rax
    0x000000000093d2e3 <dct_quantize_trellis_c+1187>:    movslq
 0x4e0(%rsp,%rax,4),%rsi
    0x000000000093d2eb <dct_quantize_trellis_c+1195>:    mov    %r9d,%edi
    0x000000000093d2ee <dct_quantize_trellis_c+1198>:    sub    %esi,%edi
    0x000000000093d2f0 <dct_quantize_trellis_c+1200>:    mov    %edi,%eax
    0x000000000093d2f2 <dct_quantize_trellis_c+1202>:    shl    $0x7,%eax
    0x000000000093d2f5 <dct_quantize_trellis_c+1205>:    add    %ebx,%eax
    0x000000000093d2f7 <dct_quantize_trellis_c+1207>:    cltq
 => 0x000000000093d2f9 <dct_quantize_trellis_c+1209>:    movzbl
 0x0(%r13,%rax,1),%eax
    0x000000000093d2ff <dct_quantize_trellis_c+1215>:    imul   %r15d,%eax
    0x000000000093d303 <dct_quantize_trellis_c+1219>:    add    %r10d,%eax
    0x000000000093d306 <dct_quantize_trellis_c+1222>:    add
 0x3d0(%rsp,%rsi,4),%eax
    0x000000000093d30d <dct_quantize_trellis_c+1229>:    cmp    %ecx,%eax
    0x000000000093d30f <dct_quantize_trellis_c+1231>:    jge    0x93d323
 <dct_quantize_trellis_c+1251>
    0x000000000093d311 <dct_quantize_trellis_c+1233>:    mov
 %edi,0x1b0(%rsp,%r8,4)
 End of assembler dump.
 (gdb) info register
 rax            0x41     65
 rbx            0x41     65
 rcx            0x78000000       2013265920
 rdx            0x0      0
 rsi            0x1      1
 rdi            0x0      0
 rbp            0x0      0x0
 rsp            0x7fffffff0bc0   0x7fffffff0bc0
 r8             0x2      2
 r9             0x1      1
 r10            0xfffffe20       4294966816
 r11            0x1      1
 r12            0x0      0
 r13            0x0      0
 r14            0x0      0
 r15            0x24f    591
 rip            0x93d2f9 0x93d2f9 <dct_quantize_trellis_c+1209>
 eflags         0x10206  [ PF IF RF ]
 cs             0x33     51
 ss             0x2b     43
 ds             0x0      0
 es             0x0      0
 fs             0x0      0
 gs             0x0      0
 }}}

-- 
Ticket URL: <https://ffmpeg.org/trac/ffmpeg/ticket/3143#comment:5>
FFmpeg <http://ffmpeg.org>
FFmpeg issue tracker


More information about the FFmpeg-trac mailing list