[FFmpeg-cvslog] avcodec/aacenc: Tighter input checks

[Michael Niedermayer]

ffmpeg | branch: release/2.8 | Michael Niedermayer <michael at niedermayer.cc> | Tue Aug 23 11:00:29 2016 +0200| [562f2ba4ed07baed407ccd314c8358446caa5e1e] | committer: Michael Niedermayer

avcodec/aacenc: Tighter input checks

Fixes occurance of NaN/Inf leading to assertion failures and out of array access
Fixes: d1c38a09acc34845c6be3a127a5aacaf/signal_sigsegv_3982225_6121_d18bd5451d4245ee09408f04badd1b83.wmv

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>
(cherry picked from commit 77bf96b04710b98a52aaddb93bfd32da0d506191)
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=562f2ba4ed07baed407ccd314c8358446caa5e1e
---

 libavcodec/aacenc.c | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/libavcodec/aacenc.c b/libavcodec/aacenc.c
index 7b643f5..86d4085 100644
--- a/libavcodec/aacenc.c
+++ b/libavcodec/aacenc.c
@@ -493,6 +493,7 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
     int ms_mode = 0, is_mode = 0, tns_mode = 0, pred_mode = 0;
     int chan_el_counter[4];
     FFPsyWindowInfo windows[AAC_MAX_CHANNELS];
+    int k;
 
     if (s->last_frame == 2)
         return 0;
@@ -573,16 +574,11 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
 
             apply_window_and_mdct(s, &cpe->ch[ch], overlap);
 
-            if (isnan(cpe->ch[ch].coeffs[    0]) || isinf(cpe->ch[ch].coeffs[    0]) ||
-                isnan(cpe->ch[ch].coeffs[  128]) || isinf(cpe->ch[ch].coeffs[  128]) ||
-                isnan(cpe->ch[ch].coeffs[2*128]) || isinf(cpe->ch[ch].coeffs[2*128]) ||
-                isnan(cpe->ch[ch].coeffs[3*128]) || isinf(cpe->ch[ch].coeffs[3*128]) ||
-                isnan(cpe->ch[ch].coeffs[4*128]) || isinf(cpe->ch[ch].coeffs[4*128]) ||
-                isnan(cpe->ch[ch].coeffs[5*128]) || isinf(cpe->ch[ch].coeffs[5*128]) ||
-                isnan(cpe->ch[ch].coeffs[6*128]) || isinf(cpe->ch[ch].coeffs[6*128]) ||
-                isnan(cpe->ch[ch].coeffs[7*128]) || isinf(cpe->ch[ch].coeffs[7*128])) {
-                av_log(avctx, AV_LOG_ERROR, "Input contains NaN/+-Inf\n");
-                return AVERROR(EINVAL);
+            for (k = 0; k < 1024; k++) {
+                if (!(fabs(cpe->ch[ch].coeffs[k]) < 1E16)) { // Ensure headroom for energy calculation
+                    av_log(avctx, AV_LOG_ERROR, "Input contains (near) NaN/+-Inf\n");
+                    return AVERROR(EINVAL);
+                }
             }
             avoid_clipping(s, &cpe->ch[ch]);
         }