[FFmpeg-devel] [PATCH] libopenjpegenc: fix out-of-bounds reads when filling the edges

[Andreas Cadhalpun]

If x is 0, 'x - 1' is in the previous line, or worse outside the buffer
for the first line.

If y is 0, 'x - image->comps[compno].w' is outside the buffer.

Finally, image->comps[compno].w is unsigned (at least in openjpeg2), so
the calculation could silently wrap around without the explicit cast to
int.

Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun at googlemail.com>
---
 libavcodec/libopenjpegenc.c | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/libavcodec/libopenjpegenc.c b/libavcodec/libopenjpegenc.c
index 857ee1a..83c965d 100644
--- a/libavcodec/libopenjpegenc.c
+++ b/libavcodec/libopenjpegenc.c
@@ -415,13 +415,13 @@ static int libopenjpeg_copy_packed8(AVCodecContext *avctx, const AVFrame *frame,
                 frame_index += numcomps;
             }
             for (; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - 1];
+                image_line[x] = x > 0 ? image_line[x - 1] : 0;
             }
         }
         for (; y < image->comps[compno].h; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
             for (x = 0; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - image->comps[compno].w];
+                image_line[x] = y > 0 ? image_line[x - (int)image->comps[compno].w] : 0;
             }
         }
     }
@@ -455,13 +455,13 @@ static int libopenjpeg_copy_packed12(AVCodecContext *avctx, const AVFrame *frame
                 frame_index += numcomps;
             }
             for (; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - 1];
+                image_line[x] = x > 0 ? image_line[x - 1] : 0;
             }
         }
         for (; y < image->comps[compno].h; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
             for (x = 0; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - image->comps[compno].w];
+                image_line[x] = y > 0 ? image_line[x - (int)image->comps[compno].w] : 0;
             }
         }
     }
@@ -495,13 +495,13 @@ static int libopenjpeg_copy_packed16(AVCodecContext *avctx, const AVFrame *frame
                 frame_index += numcomps;
             }
             for (; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - 1];
+                image_line[x] = x > 0 ? image_line[x - 1] : 0;
             }
         }
         for (; y < image->comps[compno].h; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
             for (x = 0; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - image->comps[compno].w];
+                image_line[x] = y > 0 ? image_line[x - (int)image->comps[compno].w] : 0;
             }
         }
     }
@@ -536,13 +536,13 @@ static int libopenjpeg_copy_unpacked8(AVCodecContext *avctx, const AVFrame *fram
             for (x = 0; x < width; ++x)
                 image_line[x] = frame->data[compno][frame_index++];
             for (; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - 1];
+                image_line[x] = x > 0 ? image_line[x - 1] : 0;
             }
         }
         for (; y < image->comps[compno].h; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
             for (x = 0; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - image->comps[compno].w];
+                image_line[x] = y > 0 ? image_line[x - (int)image->comps[compno].w] : 0;
             }
         }
     }
@@ -579,13 +579,13 @@ static int libopenjpeg_copy_unpacked16(AVCodecContext *avctx, const AVFrame *fra
             for (x = 0; x < width; ++x)
                 image_line[x] = frame_ptr[frame_index++];
             for (; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - 1];
+                image_line[x] = x > 0 ? image_line[x - 1] : 0;
             }
         }
         for (; y < image->comps[compno].h; ++y) {
             image_line = image->comps[compno].data + y * image->comps[compno].w;
             for (x = 0; x < image->comps[compno].w; ++x) {
-                image_line[x] = image_line[x - image->comps[compno].w];
+                image_line[x] = y > 0 ? image_line[x - (int)image->comps[compno].w] : 0;
             }
         }
     }
-- 
2.9.3