[FFmpeg-devel] [PATCH 2/2] avformat/matroskadec: avoid integer overflows in SAR computation

[Michael Niedermayer]

This ignores >64bit
Alternatively we could support that if it occurs in reality

Fixes: negation of -9223372036854775808
Fixes: integer overflows
Fixes: 46072/clusterfuzz-testcase-minimized-ffmpeg_dem_MATROSKA_fuzzer-5029840966778880

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libavformat/matroskadec.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/libavformat/matroskadec.c b/libavformat/matroskadec.c
index d97fc33d44..069fba6cf7 100644
--- a/libavformat/matroskadec.c
+++ b/libavformat/matroskadec.c
@@ -2886,11 +2886,14 @@ static int matroska_parse_tracks(AVFormatContext *s)
                 mkv_stereo_mode_display_mul(track->video.stereo_mode, &display_width_mul, &display_height_mul);
 
             if (track->video.display_unit < MATROSKA_VIDEO_DISPLAYUNIT_UNKNOWN) {
-                av_reduce(&st->sample_aspect_ratio.num,
-                          &st->sample_aspect_ratio.den,
-                          st->codecpar->height * track->video.display_width  * display_width_mul,
-                          st->codecpar->width  * track->video.display_height * display_height_mul,
-                          INT_MAX);
+                if (track->video.display_width && track->video.display_height &&
+                    st->codecpar->height * (int64_t)display_width_mul  < INT64_MAX / track->video.display_width &&
+                    st->codecpar->width  * (int64_t)display_height_mul < INT64_MAX / track->video.display_height)
+                    av_reduce(&st->sample_aspect_ratio.num,
+                              &st->sample_aspect_ratio.den,
+                              st->codecpar->height * track->video.display_width  * display_width_mul,
+                              st->codecpar->width  * track->video.display_height * display_height_mul,
+                              INT_MAX);
             }
             if (st->codecpar->codec_id != AV_CODEC_ID_HEVC)
                 sti->need_parsing = AVSTREAM_PARSE_HEADERS;
-- 
2.17.1