[FFmpeg-devel] [PATCH 1/2] avcodec/mpegvideo_enc: Reject input incompatible with chroma subsampling

[Andreas Rheinhardt]

Fixes ticket #10952.

Discovered by: Zeng Yunxiang
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt at outlook.com>
---
I am pretty sure that a lot of other encoders don't handle this well
either. Maybe we should handle this more generically in
ff_encode_preinit?

 libavcodec/mpegvideo_enc.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c
index d1b1917824..a65ecc6839 100644
--- a/libavcodec/mpegvideo_enc.c
+++ b/libavcodec/mpegvideo_enc.c
@@ -314,6 +314,7 @@ av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
     AVCPBProperties *cpb_props;
     int i, ret;
     int mb_array_size, mv_table_size;
+    int chroma_h_subsampling = 1, chroma_v_subsampling = 1;
 
     mpv_encode_defaults(s);
 
@@ -325,14 +326,25 @@ av_cold int ff_mpv_encode_init(AVCodecContext *avctx)
     case AV_PIX_FMT_YUVJ422P:
     case AV_PIX_FMT_YUV422P:
         s->chroma_format = CHROMA_422;
+        chroma_h_subsampling = 2;
         break;
     case AV_PIX_FMT_YUVJ420P:
     case AV_PIX_FMT_YUV420P:
     default:
         s->chroma_format = CHROMA_420;
+        chroma_h_subsampling = 2;
+        chroma_v_subsampling = 2;
         break;
     }
 
+    if (avctx->width &  (chroma_h_subsampling - 1) ||
+        avctx->height & (chroma_v_subsampling - 1)) {
+        av_log(avctx, AV_LOG_ERROR,
+               "Dimensions %dx%d incompatible with chroma subsampling.\n",
+               avctx->width, avctx->height);
+        return AVERROR(EINVAL);
+    }
+
     avctx->bits_per_raw_sample = av_clip(avctx->bits_per_raw_sample, 0, 8);
 
     s->bit_rate = avctx->bit_rate;
-- 
2.40.1