[FFmpeg-devel] [PATCH 2/5] swscale/output: used unsigned for bit accumulation

[Michael Niedermayer]

Fixes: Integer overflow
Fixes: 368725672/clusterfuzz-testcase-minimized-ffmpeg_SWS_fuzzer-5009093023563776

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
---
 libswscale/output.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/libswscale/output.c b/libswscale/output.c
index 31921a3ccec..ae9a50637a9 100644
--- a/libswscale/output.c
+++ b/libswscale/output.c
@@ -664,7 +664,7 @@ yuv2mono_2_c_template(SwsContext *c, const int16_t *buf[2],
 
     if (c->dither == SWS_DITHER_ED) {
         int err = 0;
-        int acc = 0;
+        unsigned acc = 0;
         for (i = 0; i < dstW; i +=2) {
             int Y;
 
@@ -686,7 +686,8 @@ yuv2mono_2_c_template(SwsContext *c, const int16_t *buf[2],
         c->dither_error[0][i] = err;
     } else {
         for (i = 0; i < dstW; i += 8) {
-            int Y, acc = 0;
+            int Y;
+            unsigned acc = 0;
 
             Y = (buf0[i + 0] * yalpha1 + buf1[i + 0] * yalpha) >> 19;
             accumulate_bit(acc, Y + d128[0]);
@@ -721,7 +722,7 @@ yuv2mono_1_c_template(SwsContext *c, const int16_t *buf0,
 
     if (c->dither == SWS_DITHER_ED) {
         int err = 0;
-        int acc = 0;
+        unsigned acc = 0;
         for (i = 0; i < dstW; i +=2) {
             int Y;
 
@@ -743,7 +744,7 @@ yuv2mono_1_c_template(SwsContext *c, const int16_t *buf0,
         c->dither_error[0][i] = err;
     } else {
         for (i = 0; i < dstW; i += 8) {
-            int acc = 0;
+            unsigned acc = 0;
             accumulate_bit(acc, ((buf0[i + 0] + 64) >> 7) + d128[0]);
             accumulate_bit(acc, ((buf0[i + 1] + 64) >> 7) + d128[1]);
             accumulate_bit(acc, ((buf0[i + 2] + 64) >> 7) + d128[2]);
-- 
2.47.0