[FFmpeg-cvslog] decode: fix the code reducing cropping to preserve alignment

[Anton Khirnov]

ffmpeg | branch: master | Anton Khirnov <anton at khirnov.net> | Fri May 19 11:47:21 2017 +0200| [02327d1237f19619272e7b04b715e807c16af6a5] | committer: James Almer

decode: fix the code reducing cropping to preserve alignment

Currently it does not work at all.

Libav Bug-Id: 1058
(cherry picked from commit 8652a2c24836ce5546b398f12b7fed45000050e1)

> http://git.videolan.org/gitweb.cgi/ffmpeg.git/?a=commit;h=02327d1237f19619272e7b04b715e807c16af6a5
---

 libavcodec/decode.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/libavcodec/decode.c b/libavcodec/decode.c
index fabdab3694..e423e68eab 100644
--- a/libavcodec/decode.c
+++ b/libavcodec/decode.c
@@ -763,6 +763,7 @@ static int apply_cropping(AVCodecContext *avctx, AVFrame *frame)
 
     /* adjust the offsets to avoid breaking alignment */
     if (!(avctx->flags & AV_CODEC_FLAG_UNALIGNED)) {
+        int log2_crop_align = frame->crop_left ? av_ctz(frame->crop_left) : INT_MAX;
         int min_log2_align = INT_MAX;
 
         for (i = 0; frame->data[i]; i++) {
@@ -770,8 +771,13 @@ static int apply_cropping(AVCodecContext *avctx, AVFrame *frame)
             min_log2_align = FFMIN(log2_align, min_log2_align);
         }
 
+        /* we assume, and it should always be true, that the data alignment is
+         * related to the cropping alignment by a constant power-of-2 factor */
+        if (log2_crop_align < min_log2_align)
+            return AVERROR_BUG;
+
         if (min_log2_align < 5) {
-            frame->crop_left &= ~((1 << min_log2_align) - 1);
+            frame->crop_left &= ~((1 << (5 + log2_crop_align - min_log2_align)) - 1);
             calc_cropping_offsets(offsets, frame, desc);
         }
     }