[FFmpeg-cvslog] avcodec/wmaenc: Check input for finiteness

[Michael Niedermayer]

ffmpeg | branch: master | Michael Niedermayer <michael at niedermayer.cc> | Sat Jan 16 21:40:49 2016 +0100| [aa6aa2ef091818c2669c48051286ce361401f31b] | committer: Michael Niedermayer

avcodec/wmaenc: Check input for finiteness

Fixes out of array read
Fixes: 1b79b985cdf860ffa228c00ee5497051/signal_sigsegv_1f99d24_315_f0f1fdb451264b1138e9c24eb7d49181.wv

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=aa6aa2ef091818c2669c48051286ce361401f31b
---

 libavcodec/wmaenc.c |   13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/libavcodec/wmaenc.c b/libavcodec/wmaenc.c
index 54e83b6..abe8e4b 100644
--- a/libavcodec/wmaenc.c
+++ b/libavcodec/wmaenc.c
@@ -98,7 +98,7 @@ static av_cold int encode_init(AVCodecContext *avctx)
     return 0;
 }
 
-static void apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame)
+static int apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame)
 {
     WMACodecContext *s = avctx->priv_data;
     float **audio      = (float **) frame->extended_data;
@@ -117,7 +117,13 @@ static void apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame)
                                     win, len);
         s->fdsp->vector_fmul(s->frame_out[ch], s->frame_out[ch], win, len);
         mdct->mdct_calc(mdct, s->coefs[ch], s->output);
+        if (!isfinite(s->coefs[ch][0])) {
+            av_log(avctx, AV_LOG_ERROR, "Input contains NaN/+-Inf\n");
+            return AVERROR(EINVAL);
+        }
     }
+
+    return 0;
 }
 
 // FIXME use for decoding too
@@ -364,7 +370,10 @@ static int encode_superframe(AVCodecContext *avctx, AVPacket *avpkt,
     s->block_len_bits = s->frame_len_bits; // required by non variable block len
     s->block_len      = 1 << s->block_len_bits;
 
-    apply_window_and_mdct(avctx, frame);
+    ret = apply_window_and_mdct(avctx, frame);
+
+    if (ret < 0)
+        return ret;
 
     if (s->ms_stereo) {
         float a, b;