[FFmpeg-devel] [PATCH 5/7] avcodec/jfdctint_template: Fewer integer anomalies

[Michael Niedermayer]

Fixes: signed integer overflow: 105788 * -20995 cannot be represented in type 'int'
Fixes: signed integer overflow: 923211729 + 2073948236 cannot be represented in type 'int'
Fixes: signed integer overflow: 1281179284 + 2073948236 cannot be represented in type 'int'
Fixes: 68975/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_PRORES_fuzzer-6266769177116672
Fixes: 68997/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_PRORES_KS_fuzzer-6284237161431040

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libavcodec/jfdctint_template.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/libavcodec/jfdctint_template.c b/libavcodec/jfdctint_template.c
index ca17300c324..aa2680132ee 100644
--- a/libavcodec/jfdctint_template.c
+++ b/libavcodec/jfdctint_template.c
@@ -69,7 +69,7 @@
 #define GLOBAL(x) x
 #define RIGHT_SHIFT(x, n) ((x) >> (n))
 #define MULTIPLY16C16(var,const) ((var)*(const))
-#define DESCALE(x,n)  RIGHT_SHIFT((x) + (1 << ((n) - 1)), n)
+#define DESCALE(x,n)  RIGHT_SHIFT((int)(x) + (1 << ((n) - 1)), n)
 
 
 /*
@@ -175,7 +175,7 @@
 #if BITS_IN_JSAMPLE == 8 && CONST_BITS<=13 && PASS1_BITS<=2
 #define MULTIPLY(var,const)  MULTIPLY16C16(var,const)
 #else
-#define MULTIPLY(var,const)  ((var) * (const))
+#define MULTIPLY(var,const)  (int)((var) * (unsigned)(const))
 #endif
 
 
@@ -261,7 +261,7 @@ FUNC(ff_jpeg_fdct_islow)(int16_t *data)
 {
   int tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
   int tmp10, tmp11, tmp12, tmp13;
-  int z1, z2, z3, z4, z5;
+  unsigned z1, z2, z3, z4, z5;
   int16_t *dataptr;
   int ctr;
 
-- 
2.45.2