[FFmpeg-cvslog] avcodec/simple_idct_template: Fix several integer overflows
Michael Niedermayer
git at videolan.org
Sat Aug 4 17:18:10 EEST 2018
ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Sat Jul 28 12:18:05 2018 +0200| [83dab8183e9f8f6bf8ef492ee372faa43196f8a8] | committer: Michael Niedermayer
avcodec/simple_idct_template: Fix several integer overflows
Fixes: simple_idct_template.c:184:30: runtime error: signed integer overflow: -1065517056 - 1392182838 cannot be represented in type 'int'
Fixes: simple_idct_template.c:269:21: runtime error: signed integer overflow: 16384 * 259254 cannot be represented in type 'int'
Fixes: simple_idct_template.c:164:17: runtime error: signed integer overflow: 21407 * 210162 cannot be represented in type 'int'
Fixes: simple_idct_template.c:167:17: runtime error: signed integer overflow: 21407 * 210162 cannot be represented in type 'int'
Fixes: simple_idct_template.c:169:19: runtime error: signed integer overflow: 22725 * 259190 cannot be represented in type 'int'
Fixes: simple_idct_template.c:171:19: runtime error: signed integer overflow: 19265 * 259190 cannot be represented in type 'int'
Fixes: simple_idct_template.c:173:19: runtime error: signed integer overflow: 12873 * 259190 cannot be represented in type 'int'
Fixes: simple_idct_template.c:183:28: runtime error: signed integer overflow: 1860878336 + 585177665 cannot be represented in type 'int'
Fixes: simple_idct_template.c:159:17: runtime error: signed integer overflow: 16384 * 189520 cannot be represented in type 'int'
Fixes: simple_idct_template.c:170:22: runtime error: signed integer overflow: 19265 * 130147 cannot be represented in type 'int'
Fixes: simple_idct_template.c:174:23: runtime error: signed integer overflow: -22725 * 130147 cannot be represented in type 'int'
Fixes: simple_idct_template.c:183:20: runtime error: signed integer overflow: 16384 * -175206 cannot be represented in type 'int'
Fixes: simple_idct_template.c:184:22: runtime error: signed integer overflow: -16384 * -175206 cannot be represented in type 'int'
Fixes: simple_idct_template.c:185:22: runtime error: signed integer overflow: -16384 * -175206 cannot be represented in type 'int'
Fixes: simple_idct_template.c:186:20: runtime error: signed integer overflow: 16384 * -175206 cannot be represented in type 'int'
Fixes: simple_idct_template.c:195:26: runtime error: signed integer overflow: 19265 * 150747 cannot be represented in type 'int'
Fixes: simple_idct_template.c:198:27: runtime error: signed integer overflow: -22725 * 150747 cannot be represented in type 'int'
Fixes: simple_idct_template.c:184:37: runtime error: signed integer overflow: 21407 * -171941 cannot be represented in type 'int'
Fixes: simple_idct_template.c:185:37: runtime error: signed integer overflow: 21407 * -171941 cannot be represented in type 'int'
Fixes: simple_idct_template.c:192:27: runtime error: signed integer overflow: -12873 * 206341 cannot be represented in type 'int'
Fixes: simple_idct_template.c:185:30: runtime error: signed integer overflow: 1196441600 + 1703756981 cannot be represented in type 'int'
Fixes: simple_idct_template.c:176:23: runtime error: signed integer overflow: -12873 * 168461 cannot be represented in type 'int'
Fixes: simple_idct_template.c:191:27: runtime error: signed integer overflow: -22725 * -140062 cannot be represented in type 'int'
Fixes: simple_idct_template.c:197:26: runtime error: signed integer overflow: 19265 * -140062 cannot be represented in type 'int'
Fixes: simple_idct_template.c:183:34: runtime error: signed integer overflow: 8867 * -243046 cannot be represented in type 'int'
Fixes: simple_idct_template.c:186:34: runtime error: signed integer overflow: 8867 * -243046 cannot be represented in type 'int'
Fixes: simple_idct_template.c:186:28: runtime error: signed integer overflow: -816234496 - 2139878414 cannot be represented in type 'int'
Fixes: simple_idct_template.c:188:26: runtime error: signed integer overflow: 12873 * -239872 cannot be represented in type 'int'
Fixes: simple_idct_template.c:165:16: runtime error: signed integer overflow: 8867 * -260084 cannot be represented in type 'int'
Fixes: simple_idct_template.c:166:16: runtime error: signed integer overflow: 8867 * -260084 cannot be represented in type 'int'
Fixes: 9135/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG4_fuzzer-6324422955761664
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=83dab8183e9f8f6bf8ef492ee372faa43196f8a8
---
libavcodec/simple_idct_template.c | 48 +++++++++++++++++++--------------------
1 file changed, 24 insertions(+), 24 deletions(-)
diff --git a/libavcodec/simple_idct_template.c b/libavcodec/simple_idct_template.c
index 904263fc71..35c31321c6 100644
--- a/libavcodec/simple_idct_template.c
+++ b/libavcodec/simple_idct_template.c
@@ -101,8 +101,8 @@
#define DC_SHIFT -1
# endif
-#define MUL(a, b) ((a) * (b))
-#define MAC(a, b, c) ((a) += (b) * (c))
+#define MUL(a, b) ((int)((SUINT)(a) * (b)))
+#define MAC(a, b, c) ((a) += (SUINT)(b) * (c))
#else
@@ -156,15 +156,15 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
#endif
#endif
- a0 = (W4 * row[0]) + (1 << (ROW_SHIFT + extra_shift - 1));
+ a0 = ((SUINT)W4 * row[0]) + (1 << (ROW_SHIFT + extra_shift - 1));
a1 = a0;
a2 = a0;
a3 = a0;
- a0 += W2 * row[2];
- a1 += W6 * row[2];
- a2 -= W6 * row[2];
- a3 -= W2 * row[2];
+ a0 += (SUINT)W2 * row[2];
+ a1 += (SUINT)W6 * row[2];
+ a2 -= (SUINT)W6 * row[2];
+ a3 -= (SUINT)W2 * row[2];
b0 = MUL(W1, row[1]);
MAC(b0, W3, row[3]);
@@ -180,10 +180,10 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
#else
if (AV_RN64A(row + 4)) {
#endif
- a0 += W4*row[4] + W6*row[6];
- a1 += - W4*row[4] - W2*row[6];
- a2 += - W4*row[4] + W2*row[6];
- a3 += W4*row[4] - W6*row[6];
+ a0 += (SUINT) W4*row[4] + (SUINT)W6*row[6];
+ a1 += (SUINT)- W4*row[4] - (SUINT)W2*row[6];
+ a2 += (SUINT)- W4*row[4] + (SUINT)W2*row[6];
+ a3 += (SUINT) W4*row[4] - (SUINT)W6*row[6];
MAC(b0, W5, row[5]);
MAC(b0, W7, row[7]);
@@ -209,15 +209,15 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
}
#define IDCT_COLS do { \
- a0 = W4 * (col[8*0] + ((1<<(COL_SHIFT-1))/W4)); \
+ a0 = (SUINT)W4 * (col[8*0] + ((1<<(COL_SHIFT-1))/W4)); \
a1 = a0; \
a2 = a0; \
a3 = a0; \
\
- a0 += W2*col[8*2]; \
- a1 += W6*col[8*2]; \
- a2 += -W6*col[8*2]; \
- a3 += -W2*col[8*2]; \
+ a0 += (SUINT) W2*col[8*2]; \
+ a1 += (SUINT) W6*col[8*2]; \
+ a2 += (SUINT)-W6*col[8*2]; \
+ a3 += (SUINT)-W2*col[8*2]; \
\
b0 = MUL(W1, col[8*1]); \
b1 = MUL(W3, col[8*1]); \
@@ -230,10 +230,10 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
MAC(b3, -W5, col[8*3]); \
\
if (col[8*4]) { \
- a0 += W4*col[8*4]; \
- a1 += -W4*col[8*4]; \
- a2 += -W4*col[8*4]; \
- a3 += W4*col[8*4]; \
+ a0 += (SUINT) W4*col[8*4]; \
+ a1 += (SUINT)-W4*col[8*4]; \
+ a2 += (SUINT)-W4*col[8*4]; \
+ a3 += (SUINT) W4*col[8*4]; \
} \
\
if (col[8*5]) { \
@@ -244,10 +244,10 @@ static inline void FUNC6(idctRowCondDC)(idctin *row, int extra_shift)
} \
\
if (col[8*6]) { \
- a0 += W6*col[8*6]; \
- a1 += -W2*col[8*6]; \
- a2 += W2*col[8*6]; \
- a3 += -W6*col[8*6]; \
+ a0 += (SUINT) W6*col[8*6]; \
+ a1 += (SUINT)-W2*col[8*6]; \
+ a2 += (SUINT) W2*col[8*6]; \
+ a3 += (SUINT)-W6*col[8*6]; \
} \
\
if (col[8*7]) { \
More information about the ffmpeg-cvslog
mailing list