[FFmpeg-cvslog] avutil/tx: Use unsigned in ff_tx_fft_sr_combine() to avoid undefined behavior

[Michael Niedermayer]

ffmpeg | branch: release/5.1 | Michael Niedermayer <michael at niedermayer.cc> | Mon Nov 21 00:20:14 2022 +0100| [c1780eeccf4f2985b5e6e024230d5235cedd1d8f] | committer: Michael Niedermayer

avutil/tx: Use unsigned in ff_tx_fft_sr_combine() to avoid undefined behavior

Fixes: signed integer overflow: -1284837070 - 982101618 cannot be represented in type 'int'
Fixes: 53105/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_AC3_FIXED_fuzzer-4848015827664896

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 7792825ad6b84f54f5a7fd7f90a907291363c419)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=c1780eeccf4f2985b5e6e024230d5235cedd1d8f
---

 libavutil/tx_priv.h     | 3 +++
 libavutil/tx_template.c | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/libavutil/tx_priv.h b/libavutil/tx_priv.h
index c9eda44e61..3dd748bd2c 100644
--- a/libavutil/tx_priv.h
+++ b/libavutil/tx_priv.h
@@ -34,6 +34,7 @@
 #define MULT(x, m) ((x) * (m))
 #define SCALE_TYPE float
 typedef float TXSample;
+typedef float TXUSample;
 typedef AVComplexFloat TXComplex;
 #elif defined(TX_DOUBLE)
 #define TX_TAB(x) x ## _double
@@ -45,6 +46,7 @@ typedef AVComplexFloat TXComplex;
 #define MULT(x, m) ((x) * (m))
 #define SCALE_TYPE double
 typedef double TXSample;
+typedef double TXUSample;
 typedef AVComplexDouble TXComplex;
 #elif defined(TX_INT32)
 #define TX_TAB(x) x ## _int32
@@ -56,6 +58,7 @@ typedef AVComplexDouble TXComplex;
 #define MULT(x, m) (((((int64_t)(x)) * (int64_t)(m)) + 0x40000000) >> 31)
 #define SCALE_TYPE float
 typedef int32_t TXSample;
+typedef uint32_t TXUSample;
 typedef AVComplexInt32 TXComplex;
 #else
 typedef void TXComplex;
diff --git a/libavutil/tx_template.c b/libavutil/tx_template.c
index 1e4354580b..6057644a63 100644
--- a/libavutil/tx_template.c
+++ b/libavutil/tx_template.c
@@ -502,7 +502,7 @@ static inline void TX_NAME(ff_tx_fft_sr_combine)(TXComplex *z,
     int o2 = 4*len;
     int o3 = 6*len;
     const TXSample *wim = cos + o1 - 7;
-    TXSample t1, t2, t3, t4, t5, t6, r0, i0, r1, i1;
+    TXUSample t1, t2, t3, t4, t5, t6, r0, i0, r1, i1;
 
     for (int i = 0; i < len; i += 4) {
         TRANSFORM(z[0], z[o1 + 0], z[o2 + 0], z[o3 + 0], cos[0], wim[7]);