[FFmpeg-devel] [PATCH 4/4] avcodec/bonk: Check for undefined overflow in predictor_calc_error()

[Michael Niedermayer]

Fixes: signed integer overflow: 22 * -2107998208 cannot be represented in type 'int'
Fixes: 51363/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_BONK_fuzzer-5660734784143360

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libavcodec/bonk.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/libavcodec/bonk.c b/libavcodec/bonk.c
index 1695229dbd..40963aa7c6 100644
--- a/libavcodec/bonk.c
+++ b/libavcodec/bonk.c
@@ -278,10 +278,13 @@ static int predictor_calc_error(int *k, int *state, int order, int error)
         *state_ptr = &(state[order-2]);
 
     for (i = order-2; i >= 0; i--, k_ptr--, state_ptr--) {
-        int k_value = *k_ptr, state_value = *state_ptr;
+        int64_t k_value = *k_ptr, state_value = *state_ptr;
 
         x -= shift_down(k_value * state_value, LATTICE_SHIFT);
-        state_ptr[1] = state_value + shift_down(k_value * x, LATTICE_SHIFT);
+        k_value *= x;
+        if ((int32_t)k_value != k_value)
+            return AVERROR_INVALIDDATA;
+        state_ptr[1] = state_value + shift_down(k_value, LATTICE_SHIFT);
     }
 
     // don't drift too far, to avoid overflows
@@ -366,6 +369,8 @@ static int bonk_decode(AVCodecContext *avctx, AVFrame *frame,
             int64_t t64;
             for (int j = 0; j < s->down_sampling - 1; j++) {
                 sample[0] = predictor_calc_error(s->k, state, s->n_taps, 0);
+                if (sample[0] == AVERROR_INVALIDDATA)
+                    return sample[0];
                 sample++;
             }
 
@@ -374,6 +379,8 @@ static int bonk_decode(AVCodecContext *avctx, AVFrame *frame,
                 return AVERROR_INVALIDDATA;
 
             sample[0] = predictor_calc_error(s->k, state, s->n_taps, t64);
+            if (sample[0] == AVERROR_INVALIDDATA)
+                return sample[0];
             sample++;
         }
 
-- 
2.17.1