[FFmpeg-devel] [PATCH 1/3] avcodec/aacdec_template: Fix undefined integer overflow in apply_tns()

[wm4]

On Sun,  2 Jul 2017 04:28:54 +0200
Michael Niedermayer <michael at niedermayer.cc> wrote:

> Fixes: runtime error: signed integer overflow: -2147483648 - 1202286525 cannot be represented in type 'int'
> Fixes: 2071/clusterfuzz-testcase-minimized-6036414271586304
> 
> Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
> Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
> ---
>  libavcodec/aac_defines.h     | 2 ++
>  libavcodec/aacdec_template.c | 5 +++--
>  2 files changed, 5 insertions(+), 2 deletions(-)
> 
> diff --git a/libavcodec/aac_defines.h b/libavcodec/aac_defines.h
> index 3c79a8a4a1..ee4c73a87d 100644
> --- a/libavcodec/aac_defines.h
> +++ b/libavcodec/aac_defines.h
> @@ -35,6 +35,7 @@
>  #define AAC_RENAME(x)       x ## _fixed
>  #define AAC_RENAME_32(x)    x ## _fixed_32
>  typedef int                 INTFLOAT;
> +typedef unsigned            SUINTFLOAT;
>  typedef int64_t             INT64FLOAT;
>  typedef int16_t             SHORTFLOAT;
>  typedef SoftFloat           AAC_FLOAT;
> @@ -83,6 +84,7 @@ typedef int                 AAC_SIGNE;
>  #define AAC_RENAME(x)       x
>  #define AAC_RENAME_32(x)    x
>  typedef float               INTFLOAT;
> +typedef float               SUINTFLOAT;

Not more of this damn shit.

>  typedef float               INT64FLOAT;
>  typedef float               SHORTFLOAT;
>  typedef float               AAC_FLOAT;
> diff --git a/libavcodec/aacdec_template.c b/libavcodec/aacdec_template.c
> index 4b98142536..add333e862 100644
> --- a/libavcodec/aacdec_template.c
> +++ b/libavcodec/aacdec_template.c
> @@ -2389,7 +2389,7 @@ static int decode_extension_payload(AACContext *ac, GetBitContext *gb, int cnt,
>   * @param   decode  1 if tool is used normally, 0 if tool is used in LTP.
>   * @param   coef    spectral coefficients
>   */
> -static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns,
> +static void apply_tns(INTFLOAT coef_param[1024], TemporalNoiseShaping *tns,
>                        IndividualChannelStream *ics, int decode)
>  {
>      const int mmm = FFMIN(ics->tns_max_bands, ics->max_sfb);
> @@ -2397,6 +2397,7 @@ static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns,
>      int bottom, top, order, start, end, size, inc;
>      INTFLOAT lpc[TNS_MAX_ORDER];
>      INTFLOAT tmp[TNS_MAX_ORDER+1];
> +    SUINTFLOAT *coef = coef_param;
>  
>      for (w = 0; w < ics->num_windows; w++) {
>          bottom = ics->num_swb;
> @@ -2426,7 +2427,7 @@ static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns,
>                  // ar filter
>                  for (m = 0; m < size; m++, start += inc)
>                      for (i = 1; i <= FFMIN(m, order); i++)
> -                        coef[start] -= AAC_MUL26(coef[start - i * inc], lpc[i - 1]);
> +                        coef[start] -= AAC_MUL26((INTFLOAT)coef[start - i * inc], lpc[i - 1]);
>              } else {
>                  // ma filter
>                  for (m = 0; m < size; m++, start += inc) {