[FFmpeg-devel] [PATCH 11/11] avcodec/encode: enforce AVCodec capabilities at encode time

[Niklas Haas]

From: Niklas Haas <git at haasn.dev>

In general, the logic is always the same: if the codec supports only a
single format, enforce it if possible. Otherwise, throw an error when
an incompatible format is passed.

To preserve backwards compatibility and make this check less pedantic
than it needs to be, always consider 'AVCOL_*_UNSPECIFIED' a valid
value. That way, we will only throw an error if the user explicitly set
a color value that we explicitly know is unsupported.
---
 libavcodec/encode.c | 59 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)

diff --git a/libavcodec/encode.c b/libavcodec/encode.c
index 31565638fa..32eb7dd9eb 100644
--- a/libavcodec/encode.c
+++ b/libavcodec/encode.c
@@ -574,10 +574,23 @@ int attribute_align_arg avcodec_receive_packet(AVCodecContext *avctx, AVPacket *
     return 0;
 }
 
+static int fmt_is_regular_yuv(enum AVPixelFormat fmt)
+{
+    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt);
+    if (!desc || desc->nb_components < 3)
+        return 0;
+    if (desc->flags & (AV_PIX_FMT_FLAG_RGB | AV_PIX_FMT_FLAG_PAL |
+                       AV_PIX_FMT_FLAG_XYZ | AV_PIX_FMT_FLAG_FLOAT))
+        return 0;
+    return 1;
+}
+
+
 static int encode_preinit_video(AVCodecContext *avctx)
 {
     const AVCodec *c = avctx->codec;
     const AVPixFmtDescriptor *pixdesc = av_pix_fmt_desc_get(avctx->pix_fmt);
+    enum AVPixelFormat swfmt = avctx->pix_fmt;
     int i;
 
     if (!av_get_pix_fmt_name(avctx->pix_fmt)) {
@@ -605,6 +618,52 @@ static int encode_preinit_video(AVCodecContext *avctx)
         }
     }
 
+    if (pixdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
+        const AVHWFramesContext *hwfc = (void *) avctx->hw_frames_ctx->data;
+        swfmt = hwfc->sw_format;
+    }
+
+    if (fmt_is_regular_yuv(swfmt)) {
+        if (avctx->colorspace != AVCOL_SPC_UNSPECIFIED && c->color_spaces) {
+            for (i = 0; c->color_spaces[i] != AVCOL_SPC_UNSPECIFIED; i++)
+                if (avctx->colorspace == c->color_spaces[i])
+                    break;
+            if (c->color_spaces[i] == AVCOL_SPC_UNSPECIFIED) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "Specified color space %s is not supported by the %s encoder.\n",
+                       av_color_space_name(avctx->colorspace), c->name);
+                av_log(avctx, AV_LOG_ERROR, "Supported color spaces:\n");
+                for (int p = 0; c->color_spaces[p] != AVCOL_SPC_UNSPECIFIED; p++) {
+                    av_log(avctx, AV_LOG_ERROR, "  %s\n",
+                           av_color_space_name(c->color_spaces[p]));
+                }
+                return AVERROR(EINVAL);
+            }
+        }
+
+        if (avctx->color_range != AVCOL_RANGE_UNSPECIFIED && c->color_ranges) {
+            for (i = 0; c->color_ranges[i] != AVCOL_RANGE_UNSPECIFIED; i++) {
+                if (avctx->color_range == c->color_ranges[i])
+                    break;
+            }
+            if (c->color_ranges[i] == AVCOL_RANGE_UNSPECIFIED) {
+                if (i == 1 && !avctx->color_range) {
+                    avctx->color_range = c->color_ranges[0];
+                } else if (avctx->color_range) {
+                    av_log(avctx, AV_LOG_ERROR,
+                           "Specified color range %s is not supported by the %s encoder.\n",
+                           av_color_range_name(avctx->color_range), c->name);
+                    av_log(avctx, AV_LOG_ERROR, "Supported color ranges:\n");
+                    for (int p = 0; c->color_ranges[p] != AVCOL_RANGE_UNSPECIFIED; p++) {
+                        av_log(avctx, AV_LOG_ERROR, "  %s\n",
+                               av_color_range_name(c->color_ranges[p]));
+                    }
+                    return AVERROR(EINVAL);
+                }
+            }
+        }
+    }
+
     if (    avctx->bits_per_raw_sample < 0
         || (avctx->bits_per_raw_sample > 8 && pixdesc->comp[0].depth <= 8)) {
         av_log(avctx, AV_LOG_WARNING, "Specified bit depth %d not possible with the specified pixel formats depth %d\n",
-- 
2.43.0