[FFmpeg-trac] #8278(undetermined:new): outside the range of representable values of type 'int' and division by zero at libavfilter/vf_bm3d.c:663

[FFmpeg]

#8278: outside the range of representable values of type 'int' and division by
zero at libavfilter/vf_bm3d.c:663
-------------------------------------+-------------------------------------
             Reporter:  Suhwan       |                     Type:  defect
               Status:  new          |                 Priority:  normal
            Component:               |                  Version:  git-
  undetermined                       |  master
             Keywords:  ubsan        |               Blocked By:
             Blocking:               |  Reproduced by developer:  0
Analyzed by developer:  0            |
-------------------------------------+-------------------------------------
 Summary of the bug:
 There is an outside the range of representable values of type 'int' and a
 division by zero at libavfilter/vf_bm3d.c:663

 I compiled ffmpeg with "--toolchain=clang-usan" to check the undefined-
 behaviours and attached log file.
 How to reproduce:
 {{{
 % ffmpeg_g -y -i $PoC -filter_complex bm3d -target dv -map 0 tmp.smjpeg

 ffmpeg version N-95385-ge1b89c76f6 Copyright (c) 2000-2019 the FFmpeg
 developers
 built with clang version 6.0.0-1ubuntu2 (tags/RELEASE_600/final)
 configuration: --cc=clang --cxx=clang++ --ld=clang --enable-debug
 --toolchain=clang-usan
 }}}

 Here's UBSAN log

 {{{
 libavfilter/vf_bm3d.c:663:45: runtime error: division by zero

 Thread 1 "ffmpeg_g" hit Breakpoint 1, 0x0000000000428860 in
 __ubsan::ScopedReport::~ScopedReport() ()
 (gdb) bt
 #0  0x0000000000428860 in __ubsan::ScopedReport::~ScopedReport() ()
 #1  0x0000000000429afe in handleDivremOverflowImpl(__ubsan::OverflowData*,
 unsigned long, unsigned long, __ubsan::ReportOptions) ()
 #2  0x000000000042ca81 in __ubsan_handle_divrem_overflow ()
 #3  0x000000000075afc4 in do_output (s=0x915ff80, dst=0x93067c0 "",
 dst_linesize=32, plane=<optimized out>, nb_jobs=0) at
 libavfilter/vf_bm3d.c:663
 #4  0x000000000074a51e in filter_frame (ctx=<optimized out>,
 out=0x7fffffffc138, in=0x9305340, ref=0x9305340) at
 libavfilter/vf_bm3d.c:768
 #5  0x00000000007474c2 in activate (ctx=0x915e080) at
 libavfilter/vf_bm3d.c:861
 #6  0x00000000005ce29c in ff_filter_activate (filter=<optimized out>) at
 libavfilter/avfilter.c:1442
 #7  0x00000000005f426f in push_frame (graph=0x9158b40) at
 libavfilter/buffersrc.c:187
 #8  av_buffersrc_add_frame_internal (ctx=<optimized out>, frame=0x9158140,
 flags=<optimized out>) at libavfilter/buffersrc.c:261
 #9  0x00000000005f2e2e in av_buffersrc_add_frame_flags (ctx=0x910a900,
 frame=0x9158140, flags=4) at libavfilter/buffersrc.c:170
 #10 0x00000000004cad10 in ifilter_send_frame (ifilter=<optimized out>,
 frame=<optimized out>) at fftools/ffmpeg.c:2186
 #11 send_frame_to_filters (ist=0x910f380, decoded_frame=0x9158140) at
 fftools/ffmpeg.c:2260
 #12 0x00000000004a076c in decode_video (ist=<optimized out>,
 pkt=<optimized out>, got_output=<optimized out>, duration_pts=<optimized
 out>,
     eof=<optimized out>, decode_failed=<optimized out>) at
 fftools/ffmpeg.c:2459
 #13 process_input_packet (ist=<optimized out>, pkt=<optimized out>,
 no_eof=<optimized out>) at fftools/ffmpeg.c:2613
 #14 0x00000000004bf0a0 in process_input (file_index=<optimized out>) at
 fftools/ffmpeg.c:4508
 #15 0x000000000048d59b in transcode_step () at fftools/ffmpeg.c:4628
 #16 transcode () at fftools/ffmpeg.c:4682
 #17 0x0000000000487d54 in main (argc=37, argv=<optimized out>) at
 fftools/ffmpeg.c:4884
 (gdb) c
 Continuing.
 libavfilter/vf_bm3d.c:663:37: runtime error: -nan is outside the range of
 representable values of type 'int'

 Thread 1 "ffmpeg_g" hit Breakpoint 1, 0x0000000000428860 in
 __ubsan::ScopedReport::~ScopedReport() ()
 (gdb) bt
 #0  0x0000000000428860 in __ubsan::ScopedReport::~ScopedReport() ()
 #1  0x000000000042a496 in handleFloatCastOverflow(void*, unsigned long,
 __ubsan::ReportOptions) ()
 #2  0x000000000042ccc1 in __ubsan_handle_float_cast_overflow ()
 #3  0x000000000075b01f in do_output (s=0x915ff80, dst=0x93067c0 "",
 dst_linesize=32, plane=<optimized out>, nb_jobs=0) at
 libavfilter/vf_bm3d.c:663
 #4  0x000000000074a51e in filter_frame (ctx=<optimized out>,
 out=0x7fffffffc138, in=0x9305340, ref=0x9305340) at
 libavfilter/vf_bm3d.c:768
 #5  0x00000000007474c2 in activate (ctx=0x915e080) at
 libavfilter/vf_bm3d.c:861
 #6  0x00000000005ce29c in ff_filter_activate (filter=<optimized out>) at
 libavfilter/avfilter.c:1442
 #7  0x00000000005f426f in push_frame (graph=0x9158b40) at
 libavfilter/buffersrc.c:187
 #8  av_buffersrc_add_frame_internal (ctx=<optimized out>, frame=0x9158140,
 flags=<optimized out>) at libavfilter/buffersrc.c:261
 #9  0x00000000005f2e2e in av_buffersrc_add_frame_flags (ctx=0x910a900,
 frame=0x9158140, flags=4) at libavfilter/buffersrc.c:170
 #10 0x00000000004cad10 in ifilter_send_frame (ifilter=<optimized out>,
 frame=<optimized out>) at fftools/ffmpeg.c:2186
 #11 send_frame_to_filters (ist=0x910f380, decoded_frame=0x9158140) at
 fftools/ffmpeg.c:2260
 #12 0x00000000004a076c in decode_video (ist=<optimized out>,
 pkt=<optimized out>, got_output=<optimized out>, duration_pts=<optimized
 out>,
     eof=<optimized out>, decode_failed=<optimized out>) at
 fftools/ffmpeg.c:2459
 #13 process_input_packet (ist=<optimized out>, pkt=<optimized out>,
 no_eof=<optimized out>) at fftools/ffmpeg.c:2613
 #14 0x00000000004bf0a0 in process_input (file_index=<optimized out>) at
 fftools/ffmpeg.c:4508
 #15 0x000000000048d59b in transcode_step () at fftools/ffmpeg.c:4628
 #16 transcode () at fftools/ffmpeg.c:4682
 #17 0x0000000000487d54 in main (argc=37, argv=<optimized out>) at
 fftools/ffmpeg.c:4884
 }}}
 Please confirm.
 Thanks

--
Ticket URL: <https://trac.ffmpeg.org/ticket/8278>
FFmpeg <https://ffmpeg.org>
FFmpeg issue tracker