[FFmpeg-devel] [PATCH] avcodec/videotoolboxenc: Add AYUV as a candidate pix_fmt for HEVC alpha

[Zhao Zhili]

From: Zhao Zhili <zhilizhao at tencent.com>

---
 libavcodec/videotoolboxenc.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c
index 55a440c7b4..3a69e44e6d 100644
--- a/libavcodec/videotoolboxenc.c
+++ b/libavcodec/videotoolboxenc.c
@@ -1335,8 +1335,10 @@ static int vtenc_create_encoder(AVCodecContext   *avctx,
         }
     }
 
-    if (vtctx->codec_id == AV_CODEC_ID_HEVC) {
-        if (avctx->pix_fmt == AV_PIX_FMT_BGRA && vtctx->alpha_quality > 0.0) {
+    if (vtctx->codec_id == AV_CODEC_ID_HEVC && vtctx->alpha_quality > 0.0) {
+        const AVPixFmtDescriptor *descriptor = av_pix_fmt_desc_get(avctx->pix_fmt);
+
+        if (descriptor->flags & AV_PIX_FMT_FLAG_ALPHA) {
             CFNumberRef alpha_quality_num = CFNumberCreate(kCFAllocatorDefault,
                                                            kCFNumberDoubleType,
                                                            &vtctx->alpha_quality);
@@ -2854,6 +2856,7 @@ static const enum AVPixelFormat hevc_pix_fmts[] = {
     AV_PIX_FMT_YUV420P,
     AV_PIX_FMT_BGRA,
     AV_PIX_FMT_P010LE,
+    AV_PIX_FMT_AYUV,
     AV_PIX_FMT_NONE
 };
 
-- 
2.46.0