[FFmpeg-cvslog] avcodec/shorten: properly handle bitshift > 31

[Paul B Mahol]

ffmpeg | branch: master | Paul B Mahol <onemda at gmail.com> | Mon Apr 11 13:06:10 2016 +0200| [b62ed56e25c8894a39352c82c7dadd2ebb9191f9] | committer: Paul B Mahol

avcodec/shorten: properly handle bitshift > 31

Signed-off-by: Paul B Mahol <onemda at gmail.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=b62ed56e25c8894a39352c82c7dadd2ebb9191f9
---

 libavcodec/shorten.c |   10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/libavcodec/shorten.c b/libavcodec/shorten.c
index b3b6d2a..061a74b 100644
--- a/libavcodec/shorten.c
+++ b/libavcodec/shorten.c
@@ -164,9 +164,13 @@ static void fix_bitshift(ShortenContext *s, int32_t *buffer)
 {
     int i;
 
-    if (s->bitshift != 0)
+    if (s->bitshift == 32) {
+        for (i = 0; i < s->blocksize; i++)
+            buffer[i] = 0;
+    } else if (s->bitshift != 0) {
         for (i = 0; i < s->blocksize; i++)
             buffer[i] <<= s->bitshift;
+    }
 }
 
 static int init_offset(ShortenContext *s)
@@ -602,7 +606,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
                 break;
             case FN_BITSHIFT: {
                 unsigned bitshift = get_ur_golomb_shorten(&s->gb, BITSHIFTSIZE);
-                if (bitshift > 31) {
+                if (bitshift > 32) {
                     av_log(avctx, AV_LOG_ERROR, "bitshift %d is invalid\n",
                            bitshift);
                     return AVERROR_INVALIDDATA;
@@ -680,7 +684,7 @@ static int shorten_decode_frame(AVCodecContext *avctx, void *data,
                 if (s->version < 2)
                     s->offset[channel][s->nmean - 1] = sum / s->blocksize;
                 else
-                    s->offset[channel][s->nmean - 1] = (sum / s->blocksize) << s->bitshift;
+                    s->offset[channel][s->nmean - 1] = s->bitshift == 32 ? 0 : (sum / s->blocksize) << s->bitshift;
             }
 
             /* copy wrap samples for use with next block */