[FFmpeg-devel] [PATCH] avfilter/qsvvpp: Work around a bug in MSDK where VPP processing hangs under certain conditions

[Soft Works]

These conditions are:
- Dimensions are aligned to 16/32 byte (e.g. 1920x800)
- No scaling is done
- Color format conversion (e.g. 10bit to 8bit)

Example command:
ffmpeg -c:v hevc_qsv -hwaccel qsv -i hevc_10bit_1920_800.mkv -filter_complex "scale_qsv=format=nv12" -c:v h264_qsv out.mkv

Fix:
- Increase the frame height to the next alignment value
---
 libavfilter/qsvvpp.c       | 7 ++++++-
 libavfilter/vf_scale_qsv.c | 9 ++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/libavfilter/qsvvpp.c b/libavfilter/qsvvpp.c
index 1bbb7a7e68..98d2353d1c 100644
--- a/libavfilter/qsvvpp.c
+++ b/libavfilter/qsvvpp.c
@@ -420,6 +420,7 @@ static int init_vpp_session(AVFilterContext *avctx, QSVVPPContext *s)
     mfxHandleType handle_type;
     mfxVersion ver;
     mfxIMPL impl;
+    int height_align_adjust = 0;
     int ret, i;
 
     if (inlink->hw_frames_ctx) {
@@ -463,9 +464,13 @@ static int init_vpp_session(AVFilterContext *avctx, QSVVPPContext *s)
         out_frames_ctx   = (AVHWFramesContext *)out_frames_ref->data;
         out_frames_hwctx = out_frames_ctx->hwctx;
 
+        /* work around a bug in MSDK where VPP processing hangs under certain conditions */
+        if (inlink->h == outlink->h)
+            height_align_adjust = 1;
+
         out_frames_ctx->format            = AV_PIX_FMT_QSV;
         out_frames_ctx->width             = FFALIGN(outlink->w, 32);
-        out_frames_ctx->height            = FFALIGN(outlink->h, 32);
+        out_frames_ctx->height            = FFALIGN(outlink->h + height_align_adjust, 32);
         out_frames_ctx->sw_format         = s->out_sw_format;
         out_frames_ctx->initial_pool_size = 64;
         if (avctx->extra_hw_frames > 0)
diff --git a/libavfilter/vf_scale_qsv.c b/libavfilter/vf_scale_qsv.c
index 5259104a4f..303d2101a9 100644
--- a/libavfilter/vf_scale_qsv.c
+++ b/libavfilter/vf_scale_qsv.c
@@ -181,8 +181,10 @@ static int init_out_pool(AVFilterContext *ctx,
     AVQSVFramesContext *out_frames_hwctx;
     enum AVPixelFormat in_format;
     enum AVPixelFormat out_format;
+    int height_align_adjust = 0;
     int i, ret;
 
+
     /* check that we have a hw context */
     if (!ctx->inputs[0]->hw_frames_ctx) {
         av_log(ctx, AV_LOG_ERROR, "No hw context provided on input\n");
@@ -191,6 +193,7 @@ static int init_out_pool(AVFilterContext *ctx,
     in_frames_ctx   = (AVHWFramesContext*)ctx->inputs[0]->hw_frames_ctx->data;
     in_frames_hwctx = in_frames_ctx->hwctx;
 
+    in_format     = in_frames_ctx->sw_format;
     in_format     = in_frames_ctx->sw_format;
     out_format    = (s->format == AV_PIX_FMT_NONE) ? in_format : s->format;
 
@@ -200,9 +203,13 @@ static int init_out_pool(AVFilterContext *ctx,
     out_frames_ctx   = (AVHWFramesContext*)outlink->hw_frames_ctx->data;
     out_frames_hwctx = out_frames_ctx->hwctx;
 
+    /* work around a bug in MSDK where VPP processing hangs under certain conditions */
+    if (in_frames_ctx->height == out_height)
+        height_align_adjust = 1;
+
     out_frames_ctx->format            = AV_PIX_FMT_QSV;
     out_frames_ctx->width             = FFALIGN(out_width,  16);
-    out_frames_ctx->height            = FFALIGN(out_height, 16);
+    out_frames_ctx->height            = FFALIGN(out_height + height_align_adjust, 16);
     out_frames_ctx->sw_format         = out_format;
     out_frames_ctx->initial_pool_size = 4;
 
-- 
2.26.2.windows.1