[FFmpeg-cvslog] avcodec/mpegpicture: Check size of edge_emu_buffer

[Michael Niedermayer]

ffmpeg | branch: release/4.1 | Michael Niedermayer <michael at niedermayer.cc> | Sun Mar 17 15:18:20 2019 +0100| [65f94b732a27cd7479c3fcff337bc247442016d8] | committer: Michael Niedermayer

avcodec/mpegpicture: Check size of edge_emu_buffer

Fixes: OOM
Fixes: 13710/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG4_fuzzer-5633152942342144

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 635067b75fce06928431ce9b9fcaee0c9b6b7280)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=65f94b732a27cd7479c3fcff337bc247442016d8
---

 libavcodec/mpegpicture.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/libavcodec/mpegpicture.c b/libavcodec/mpegpicture.c
index c0e06900fe..ecbd77d50e 100644
--- a/libavcodec/mpegpicture.c
+++ b/libavcodec/mpegpicture.c
@@ -23,6 +23,7 @@
 #include "libavutil/avassert.h"
 #include "libavutil/common.h"
 #include "libavutil/pixdesc.h"
+#include "libavutil/imgutils.h"
 
 #include "avcodec.h"
 #include "motion_est.h"
@@ -57,6 +58,7 @@ do {\
 int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
                             ScratchpadContext *sc, int linesize)
 {
+#   define EMU_EDGE_HEIGHT (4 * 70)
     int alloc_size = FFALIGN(FFABS(linesize) + 64, 32);
 
     if (avctx->hwaccel)
@@ -67,13 +69,16 @@ int ff_mpeg_framesize_alloc(AVCodecContext *avctx, MotionEstContext *me,
         return AVERROR_PATCHWELCOME;
     }
 
+    if (av_image_check_size2(alloc_size, EMU_EDGE_HEIGHT, avctx->max_pixels, AV_PIX_FMT_NONE, 0, avctx) < 0)
+        return AVERROR(ENOMEM);
+
     // edge emu needs blocksize + filter length - 1
     // (= 17x17 for  halfpel / 21x21 for H.264)
     // VC-1 computes luma and chroma simultaneously and needs 19X19 + 9x9
     // at uvlinesize. It supports only YUV420 so 24x24 is enough
     // linesize * interlaced * MBsize
     // we also use this buffer for encoding in encode_mb_internal() needig an additional 32 lines
-    FF_ALLOCZ_ARRAY_OR_GOTO(avctx, sc->edge_emu_buffer, alloc_size, 4 * 70,
+    FF_ALLOCZ_ARRAY_OR_GOTO(avctx, sc->edge_emu_buffer, alloc_size, EMU_EDGE_HEIGHT,
                       fail);
 
     FF_ALLOCZ_ARRAY_OR_GOTO(avctx, me->scratchpad, alloc_size, 4 * 16 * 2,