[FFmpeg-devel] [PATCH v5 2/3] avutil/hwcontext_d3d11va: pass the format value from outside for staging texture

[Tong Wu]

In d3d11va_create_staging_texture(), during the hwmap process, the
ctx->internal->priv is not initialized, resulting in the
texDesc.Format not initialized. Now pass the format value from
d3d11va_transfer_data() to fix it.

$ ffmpeg.exe -y -hwaccel qsv -init_hw_device d3d11va=d3d11 \
-init_hw_device qsv=qsv at d3d11 -c:v h264_qsv \
-i input.h264 -vf "hwmap=derive_device=d3d11va,format=d3d11,hwdownload,format=nv12" \
-f null -

Signed-off-by: Tong Wu <tong1.wu at intel.com>
---
 libavutil/hwcontext_d3d11va.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/libavutil/hwcontext_d3d11va.c b/libavutil/hwcontext_d3d11va.c
index 2b4e24bb56..bb4e9d6965 100644
--- a/libavutil/hwcontext_d3d11va.c
+++ b/libavutil/hwcontext_d3d11va.c
@@ -346,7 +346,7 @@ static int d3d11va_transfer_get_formats(AVHWFramesContext *ctx,
     return 0;
 }
 
-static int d3d11va_create_staging_texture(AVHWFramesContext *ctx)
+static int d3d11va_create_staging_texture(AVHWFramesContext *ctx, DXGI_FORMAT format)
 {
     AVD3D11VADeviceContext *device_hwctx = ctx->device_ctx->hwctx;
     D3D11VAFramesContext              *s = ctx->internal->priv;
@@ -355,7 +355,7 @@ static int d3d11va_create_staging_texture(AVHWFramesContext *ctx)
         .Width          = ctx->width,
         .Height         = ctx->height,
         .MipLevels      = 1,
-        .Format         = s->format,
+        .Format         = format,
         .SampleDesc     = { .Count = 1 },
         .ArraySize      = 1,
         .Usage          = D3D11_USAGE_STAGING,
@@ -411,7 +411,8 @@ static int d3d11va_transfer_data(AVHWFramesContext *ctx, AVFrame *dst,
     device_hwctx->lock(device_hwctx->lock_ctx);
 
     if (!s->staging_texture) {
-        int res = d3d11va_create_staging_texture(ctx);
+        ID3D11Texture2D_GetDesc((ID3D11Texture2D *)texture, &desc);
+        int res = d3d11va_create_staging_texture(ctx, desc.Format);
         if (res < 0)
             return res;
     }
-- 
2.35.1