[FFmpeg-devel] [PATCH] movenc: Fix VPCC bitdepth for hardware pixel formats

[Nikolas Bowe]

If a hardware encoder is used, the pixel format may be a hardware pixel format.
This leads to invalid VPCC atom being written, due to depth of hardware pixel formats being 0.
To work around this, fallback on bits_per_raw_sample.

Signed-off-by: Nikolas Bowe <nbowe at google.com>
---
 libavcodec/vaapi_encode.c |  2 +-
 libavformat/vpcc.c        | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c
index b4e9fadaee..cfd9413f0f 100644
--- a/libavcodec/vaapi_encode.c
+++ b/libavcodec/vaapi_encode.c
@@ -1116,7 +1116,7 @@ static av_cold int vaapi_encode_profile_entrypoint(AVCodecContext *avctx)
                ctx->input_frames->sw_format);
         return AVERROR(EINVAL);
     }
-    depth = desc->comp[0].depth;
+    avctx->bits_per_raw_sample = depth = desc->comp[0].depth;
     for (i = 1; i < desc->nb_components; i++) {
         if (desc->comp[i].depth != depth) {
             av_log(avctx, AV_LOG_ERROR, "Invalid input pixfmt (%s).\n",
diff --git a/libavformat/vpcc.c b/libavformat/vpcc.c
index e0b7f288a6..f667ca9c00 100644
--- a/libavformat/vpcc.c
+++ b/libavformat/vpcc.c
@@ -51,15 +51,20 @@ static int get_vpx_chroma_subsampling(AVFormatContext *s,
     return -1;
 }
 
-static int get_bit_depth(AVFormatContext *s, enum AVPixelFormat pixel_format)
+static int get_bit_depth(AVFormatContext *s, AVCodecParameters *par)
 {
+    enum AVPixelFormat pixel_format = par->format;
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pixel_format);
     if (desc == NULL) {
         av_log(s, AV_LOG_ERROR, "Unsupported pixel format (%d)\n",
                pixel_format);
         return -1;
     }
-    return desc->comp[0].depth;
+    if (desc->comp[0].depth) {
+        return desc->comp[0].depth;
+    }
+    // Fallback on bits_per_raw_sample if pix_fmt is a hw format.
+    return par->bits_per_raw_sample;
 }
 
 static int get_vpx_video_full_range_flag(enum AVColorRange color_range)
@@ -119,13 +124,13 @@ int ff_isom_get_vpcc_features(AVFormatContext *s, AVCodecParameters *par,
     int profile = par->profile;
     int level = par->level == FF_LEVEL_UNKNOWN ?
         get_vp9_level(par, frame_rate) : par->level;
-    int bit_depth = get_bit_depth(s, par->format);
+    int bit_depth = get_bit_depth(s, par);
     int vpx_chroma_subsampling =
         get_vpx_chroma_subsampling(s, par->format, par->chroma_location);
     int vpx_video_full_range_flag =
         get_vpx_video_full_range_flag(par->color_range);
 
-    if (bit_depth < 0 || vpx_chroma_subsampling < 0)
+    if (bit_depth <= 0 || vpx_chroma_subsampling < 0)
         return AVERROR_INVALIDDATA;
 
     if (profile == FF_PROFILE_UNKNOWN) {
-- 
2.20.1.611.gfbb209baf1-goog