[FFmpeg-trac] #2207(avcodec:open): encoding bgra ljpeg crash

[FFmpeg]

#2207: encoding bgra ljpeg crash
-------------------------------------+-------------------------------------
             Reporter:  richardpl    |                    Owner:
                 Type:  defect       |                   Status:  open
             Priority:  important    |                Component:  avcodec
              Version:  git-master   |               Resolution:
             Keywords:  regression   |               Blocked By:
  crash SIGSEGV ljpeg                |  Reproduced by developer:  1
             Blocking:               |
Analyzed by developer:  0            |
-------------------------------------+-------------------------------------
Changes (by cehoyos):

 * status:  new => open


Comment:

 {{{
 (gdb) r -i tests/lena.pnm out.ljpg
 Starting program: ffmpeg_g -i tests/lena.pnm out.ljpg
 [Thread debugging using libthread_db enabled]
 Using host libthread_db library "/lib64/libthread_db.so.1".
 ffmpeg version N-49414-g11c99c7 Copyright (c) 2000-2013 the FFmpeg
 developers
   built on Jan 29 2013 14:43:08 with gcc 4.7 (SUSE Linux)
   configuration: --enable-gpl --disable-indev=jack
   libavutil      52. 17.100 / 52. 17.100
   libavcodec     54. 91.100 / 54. 91.100
   libavformat    54. 61.104 / 54. 61.104
   libavdevice    54.  3.102 / 54.  3.102
   libavfilter     3. 34.101 /  3. 34.101
   libswscale      2.  2.100 /  2.  2.100
   libswresample   0. 17.102 /  0. 17.102
   libpostproc    52.  2.100 / 52.  2.100
 Input #0, image2, from 'tests/lena.pnm':
   Duration: 00:00:00.04, start: 0.000000, bitrate: N/A
     Stream #0:0: Video: ppm, rgb24, 256x256, 25 tbr, 25 tbn, 25 tbc
 Output #0, image2, to 'out.ljpg':
   Metadata:
     encoder         : Lavf54.61.104
     Stream #0:0: Video: ljpeg, bgr24, 256x256, q=2-31, 200 kb/s, 90k tbn,
 25 tbc
 Stream mapping:
   Stream #0:0 -> #0:0 (ppm -> ljpeg)
 Press [q] to stop, [?] for help

 Program received signal SIGSEGV, Segmentation fault.
 encode_picture_lossless (avctx=0x15fb600, pkt=0x7fffffffd920,
 pict=<optimized out>,
     got_packet=0x7fffffffd814) at libavcodec/ljpegenc.c:82
 82                  buffer[0][i]= 1 << (9 - 1);
 (gdb) bt
 #0  encode_picture_lossless (avctx=0x15fb600, pkt=0x7fffffffd920,
 pict=<optimized out>,
     got_packet=0x7fffffffd814) at libavcodec/ljpegenc.c:82
 #1  0x0000000000999e08 in avcodec_encode_video2
 (avctx=avctx at entry=0x15fb600,
     avpkt=avpkt at entry=0x7fffffffd920, frame=frame at entry=0x7fffffffd980,
     got_packet_ptr=got_packet_ptr at entry=0x7fffffffd814) at
 libavcodec/utils.c:1439
 #2  0x000000000045c51d in do_video_out (in_picture=0x1648c20,
 ost=0x1600d20, s=0x15faca0)
     at ffmpeg.c:948
 #3  reap_filters () at ffmpeg.c:1101
 #4  0x000000000044eaa5 in transcode_step () at ffmpeg.c:3093
 #5  transcode () at ffmpeg.c:3136
 #6  main (argc=<optimized out>, argv=<optimized out>) at ffmpeg.c:3311
 (gdb) disass $pc-32,$pc+32
 Dump of assembler code from 0x84b9b4 to 0x84b9f4:
    0x000000000084b9b4 <encode_picture_lossless+1396>:   decl
 0x98249c(%rbx)
    0x000000000084b9ba <encode_picture_lossless+1402>:   add    %al,(%rax)
    0x000000000084b9bc <encode_picture_lossless+1404>:   mov
 0x1838(%r13),%r8
    0x000000000084b9c3 <encode_picture_lossless+1411>:   movslq
 0x12d8(%r13),%rdx
    0x000000000084b9ca <encode_picture_lossless+1418>:   test   %ebx,%ebx
    0x000000000084b9cc <encode_picture_lossless+1420>:   mov
 %r8,0x88(%rsp)
 => 0x000000000084b9d4 <encode_picture_lossless+1428>:   movw
 $0x100,(%r8)
    0x000000000084b9da <encode_picture_lossless+1434>:   movw
 $0x100,0x2(%r8)
    0x000000000084b9e1 <encode_picture_lossless+1441>:   movw
 $0x100,0x4(%r8)
    0x000000000084b9e8 <encode_picture_lossless+1448>:   jle    0x84b8ec
 <encode_picture_lossless+1196>
    0x000000000084b9ee <encode_picture_lossless+1454>:   mov
 0x70(%rsp),%r8d
    0x000000000084b9f3 <encode_picture_lossless+1459>:   mov
 %rdx,0xa8(%rsp)
 End of assembler dump.
 (gdb) info register
 rax            0x7ffff7e77040   140737352527936
 rbx            0x100    256
 rcx            0x10f0   4336
 rdx            0x300    768
 rsi            0x8      8
 rdi            0x7ffff7e7725b   140737352528475
 rbp            0x7fffffffd980   0x7fffffffd980
 rsp            0x7fffffffd670   0x7fffffffd670
 r8             0x0      0
 r9             0x11f    287
 r10            0xfa     250
 r11            0x7ffff7e77237   140737352528439
 r12            0x0      0
 r13            0x15fba60        23050848
 r14            0x7fffffffd920   140737488345376
 r15            0x15fccf8        23055608
 rip            0x84b9d4 0x84b9d4 <encode_picture_lossless+1428>
 eflags         0x10206  [ PF IF RF ]
 cs             0x33     51
 ss             0x2b     43
 ds             0x0      0
 es             0x0      0
 fs             0x0      0
 gs             0x0      0
 }}}

-- 
Ticket URL: <https://ffmpeg.org/trac/ffmpeg/ticket/2207#comment:3>
FFmpeg <http://ffmpeg.org>
FFmpeg issue tracker