[FFmpeg-cvslog] mjpegdec: consider chroma subsampling in size check

[Andreas Cadhalpun]

ffmpeg | branch: release/2.5 | Andreas Cadhalpun <Andreas.Cadhalpun at googlemail.com> | Wed Dec  2 21:52:23 2015 +0100| [d52b5f85f2837b0de9bdefe2a650d8d1b0e02ec1] | committer: Michael Niedermayer

mjpegdec: consider chroma subsampling in size check

If the chroma components are subsampled, smaller buffers are allocated
for them. In that case the maximal block_offset for the chroma
components is not as large as for the luma component.

This fixes out of bounds writes causing segmentation faults or memory
corruption.

Reviewed-by: Michael Niedermayer <michael at niedermayer.cc>
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun at googlemail.com>
(cherry picked from commit 5adb5d9d894aa495e7bf9557b4c78350cbfc9d32)

Signed-off-by: Michael Niedermayer <michael at niedermayer.cc>

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=d52b5f85f2837b0de9bdefe2a650d8d1b0e02ec1
---

 libavcodec/mjpegdec.c |   11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/libavcodec/mjpegdec.c b/libavcodec/mjpegdec.c
index a1141e7..d2624f0 100644
--- a/libavcodec/mjpegdec.c
+++ b/libavcodec/mjpegdec.c
@@ -1222,7 +1222,7 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah,
                              int mb_bitmask_size,
                              const AVFrame *reference)
 {
-    int i, mb_x, mb_y;
+    int i, mb_x, mb_y, chroma_h_shift, chroma_v_shift, chroma_width, chroma_height;
     uint8_t *data[MAX_COMPONENTS];
     const uint8_t *reference_data[MAX_COMPONENTS];
     int linesize[MAX_COMPONENTS];
@@ -1239,6 +1239,11 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah,
 
     s->restart_count = 0;
 
+    av_pix_fmt_get_chroma_sub_sample(s->avctx->pix_fmt, &chroma_h_shift,
+                                     &chroma_v_shift);
+    chroma_width  = FF_CEIL_RSHIFT(s->width,  chroma_h_shift);
+    chroma_height = FF_CEIL_RSHIFT(s->height, chroma_v_shift);
+
     for (i = 0; i < nb_components; i++) {
         int c   = s->comp_index[i];
         data[c] = s->picture_ptr->data[c];
@@ -1275,8 +1280,8 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah,
 
                     if (s->interlaced && s->bottom_field)
                         block_offset += linesize[c] >> 1;
-                    if (   8*(h * mb_x + x) < s->width
-                        && 8*(v * mb_y + y) < s->height) {
+                    if (   8*(h * mb_x + x) < ((c == 1) || (c == 2) ? chroma_width  : s->width)
+                        && 8*(v * mb_y + y) < ((c == 1) || (c == 2) ? chroma_height : s->height)) {
                         ptr = data[c] + block_offset;
                     } else
                         ptr = NULL;