[FFmpeg-devel] [PATCH]Avoid overflows when reading pgm with maxval != 255 and 65535

[Carl Eugen Hoyos]

Hi!

libjpeg v6b can write 12bit pgm files that show white noise with ImageMagick 
and gimp, FFmpeg shows recognizable but very broken images.
The reason is that the bits above maxval (4095) are not initialized.
Attached patch is only tested for the specific 12bit gray16 case.

Please review, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/pnm.h b/libavcodec/pnm.h
index 92edf8d..c80ff01 100644
--- a/libavcodec/pnm.h
+++ b/libavcodec/pnm.h
@@ -30,6 +30,7 @@ typedef struct PNMContext {
     uint8_t *bytestream_end;
     AVFrame picture;
     int maxval;                 ///< maximum value of a pixel
+    int maskval;
     int type;
 } PNMContext;
 
diff --git a/libavcodec/pnmdec.c b/libavcodec/pnmdec.c
index d0c7295..676761e 100644
--- a/libavcodec/pnmdec.c
+++ b/libavcodec/pnmdec.c
@@ -75,8 +75,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
         n = avctx->width;
         components=1;
         sample_len=8;
-        if (s->maxval < 255)
+        if (s->maxval < 255) {
             upgrade = 1;
+            s->maskval = (2 << av_log2(s->maxval)) - 1;
+        }
         goto do_read;
     case AV_PIX_FMT_GRAY8A:
         n = avctx->width * 2;
@@ -88,8 +90,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
         n = avctx->width * 2;
         components=1;
         sample_len=16;
-        if (s->maxval < 65535)
+        if (s->maxval < 65535) {
             upgrade = 2;
+            s->maskval = (2 << av_log2(s->maxval)) - 1;
+        }
         goto do_read;
     case AV_PIX_FMT_MONOWHITE:
     case AV_PIX_FMT_MONOBLACK:
@@ -136,11 +140,11 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
             else if (upgrade == 1) {
                 unsigned int j, f = (255 * 128 + s->maxval / 2) / s->maxval;
                 for (j = 0; j < n; j++)
-                    ptr[j] = (s->bytestream[j] * f + 64) >> 7;
+                    ptr[j] = ((s->bytestream[j] & s->maskval) * f + 64) >> 7;
             } else if (upgrade == 2) {
                 unsigned int j, v, f = (65535 * 32768 + s->maxval / 2) / s->maxval;
                 for (j = 0; j < n / 2; j++) {
-                    v = av_be2ne16(((uint16_t *)s->bytestream)[j]);
+                    v = av_be2ne16(((uint16_t *)s->bytestream)[j]) & s->maskval;
                     ((uint16_t *)ptr)[j] = (v * f + 16384) >> 15;
                 }
             }