[FFmpeg-cvslog] avcodec/xpmdec: do not allow number of colors to be higher than allocated

[Paul B Mahol]

ffmpeg | branch: master | Paul B Mahol <onemda at gmail.com> | Sun Mar 12 22:51:00 2017 +0100| [2b790b1c9e3bf8f4fbc56fc3a071f2015b58de21] | committer: Paul B Mahol

avcodec/xpmdec: do not allow number of colors to be higher than allocated

Signed-off-by: Paul B Mahol <onemda at gmail.com>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=2b790b1c9e3bf8f4fbc56fc3a071f2015b58de21
---

 libavcodec/xpmdec.c | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/libavcodec/xpmdec.c b/libavcodec/xpmdec.c
index 25ef992..592f81a 100644
--- a/libavcodec/xpmdec.c
+++ b/libavcodec/xpmdec.c
@@ -328,29 +328,22 @@ static int xpm_decode_frame(AVCodecContext *avctx, void *data,
     if ((ret = ff_get_buffer(avctx, p, 0)) < 0)
         return ret;
 
-    if (ncolors <= 0) {
-        av_log(avctx, AV_LOG_ERROR, "invalid number of colors: %d\n", ncolors);
+    if (cpp <= 0 || cpp >= 5) {
+        av_log(avctx, AV_LOG_ERROR, "unsupported/invalid number of chars per pixel: %d\n", cpp);
         return AVERROR_INVALIDDATA;
     }
 
-    if (cpp <= 0) {
-        av_log(avctx, AV_LOG_ERROR, "invalid number of chars per pixel: %d\n", cpp);
+    size = 1;
+    for (i = 0; i < cpp; i++)
+        size *= 94;
+
+    if (ncolors <= 0 || ncolors > size) {
+        av_log(avctx, AV_LOG_ERROR, "invalid number of colors: %d\n", ncolors);
         return AVERROR_INVALIDDATA;
     }
 
-    size = 1;
-    j = 1;
-    for (i = 0; i < cpp; i++) {
-        size += j * 94;
-        j *= 95;
-    }
     size *= 4;
 
-    if (size < 0) {
-        av_log(avctx, AV_LOG_ERROR, "unsupported number of chars per pixel: %d\n", cpp);
-        return AVERROR(ENOMEM);
-    }
-
     av_fast_padded_malloc(&x->pixels, &x->pixels_size, size);
     if (!x->pixels)
         return AVERROR(ENOMEM);