[FFmpeg-cvslog] avcodec/mdct_*: Fix integer overflow in addition in RESCALE()
Michael Niedermayer
git at videolan.org
Mon Nov 13 20:58:44 EET 2017
ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Sun Nov 5 21:20:06 2017 +0100| [770c934fa1635f4fadf5db4fc5cc5ad15d82455a] | committer: Michael Niedermayer
avcodec/mdct_*: Fix integer overflow in addition in RESCALE()
Fixes: runtime error: signed integer overflow: 1219998458 - -1469874012 cannot be represented in type 'int'
Fixes: 3443/clusterfuzz-testcase-minimized-5369987105554432
Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=770c934fa1635f4fadf5db4fc5cc5ad15d82455a
---
libavcodec/mdct_fixed.c | 8 ++++----
libavcodec/mdct_template.c | 14 +++++++-------
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/libavcodec/mdct_fixed.c b/libavcodec/mdct_fixed.c
index a32cb00ca0..aabf0c88f8 100644
--- a/libavcodec/mdct_fixed.c
+++ b/libavcodec/mdct_fixed.c
@@ -39,13 +39,13 @@ void ff_mdct_calcw_c(FFTContext *s, FFTDouble *out, const FFTSample *input)
/* pre rotation */
for(i=0;i<n8;i++) {
- re = RSCALE(-input[2*i+n3] - input[n3-1-2*i]);
- im = RSCALE(-input[n4+2*i] + input[n4-1-2*i]);
+ re = RSCALE(-input[2*i+n3], - input[n3-1-2*i]);
+ im = RSCALE(-input[n4+2*i], + input[n4-1-2*i]);
j = revtab[i];
CMUL(x[j].re, x[j].im, re, im, -tcos[i], tsin[i]);
- re = RSCALE( input[2*i] - input[n2-1-2*i]);
- im = RSCALE(-input[n2+2*i] - input[ n-1-2*i]);
+ re = RSCALE( input[2*i] , - input[n2-1-2*i]);
+ im = RSCALE(-input[n2+2*i], - input[ n-1-2*i]);
j = revtab[n8 + i];
CMUL(x[j].re, x[j].im, re, im, -tcos[n8 + i], tsin[n8 + i]);
}
diff --git a/libavcodec/mdct_template.c b/libavcodec/mdct_template.c
index 04396b46d6..e0ad9f1e53 100644
--- a/libavcodec/mdct_template.c
+++ b/libavcodec/mdct_template.c
@@ -33,12 +33,12 @@
*/
#if FFT_FLOAT
-# define RSCALE(x) (x)
+# define RSCALE(x, y) ((x) + (y))
#else
#if FFT_FIXED_32
-# define RSCALE(x) (((x) + 32) >> 6)
+# define RSCALE(x, y) ((int)((x) + (unsigned)(y) + 32) >> 6)
#else /* FFT_FIXED_32 */
-# define RSCALE(x) ((x) >> 1)
+# define RSCALE(x, y) ((int)((x) + (unsigned)(y)) >> 1)
#endif /* FFT_FIXED_32 */
#endif
@@ -181,13 +181,13 @@ void ff_mdct_calc_c(FFTContext *s, FFTSample *out, const FFTSample *input)
/* pre rotation */
for(i=0;i<n8;i++) {
- re = RSCALE(-input[2*i+n3] - input[n3-1-2*i]);
- im = RSCALE(-input[n4+2*i] + input[n4-1-2*i]);
+ re = RSCALE(-input[2*i+n3], - input[n3-1-2*i]);
+ im = RSCALE(-input[n4+2*i], + input[n4-1-2*i]);
j = revtab[i];
CMUL(x[j].re, x[j].im, re, im, -tcos[i], tsin[i]);
- re = RSCALE( input[2*i] - input[n2-1-2*i]);
- im = RSCALE(-input[n2+2*i] - input[ n-1-2*i]);
+ re = RSCALE( input[2*i] , - input[n2-1-2*i]);
+ im = RSCALE(-input[n2+2*i], - input[ n-1-2*i]);
j = revtab[n8 + i];
CMUL(x[j].re, x[j].im, re, im, -tcos[n8 + i], tsin[n8 + i]);
}
More information about the ffmpeg-cvslog
mailing list