[FFmpeg-devel] [PATCH 1/2 V2] libavcodec/qsvdec: Fix the QSV decoder can't work when using system memory

Huang, Zhengxu zhengxu.maxwell at gmail.com
Tue Jan 17 08:53:31 EET 2017


在 2017/1/12 5:00, Mark Thompson 写道:

> On 09/01/17 02:05, Huang, Zhengxu wrote:
>>  From 37629f14294125c7396e5e12970d75e895b1caba Mon Sep 17 00:00:00 2001
>> From: Zhengxu <zhengxu.maxwell at gmail.com>
>> Date: Mon, 19 Dec 2016 01:27:06 -0500
>> Subject: [PATCH 1/2] libavcodec/qsvdec: Fix the QSV decoder can't work when
>>   using system memory
>>
>> Description: ./ffmpeg -c:v h264_qsv -i in -c:v h264 output.h264 does not
>> work because the qsv decode will failed.
> Yes it does!  (Or at least, it works for me?)  In the absence of a provided device handle the dispatcher finds a device to use.
Our test environment is the MSS 16.5 PV1 and the decode indeed can't 
work in the system memory.
>
> Now I can see that it is likely to run into exactly the same device-selection issues as were being fixed in other patch, but I don't see how this is fixing it.  You are not passing anything to av_hwdevice_ctx_create(), so it will be going through exactly the same set of possible device nodes as ffmpeg did and will therefore fail to find the device in exactly the same way.
> So, to properly fix the problem I think you are having (but correct me if this is wrong), you really want the -qsv_device option added to ffmpeg to apply to this case in libavcodec as well?  That is, we get the qsv_device in ffmpeg into libavcodec somehow and then use it with the code below.  (Patch itself implementing that looks mostly fine, but I would prefer to get the methodology resolved first.  Similarly for the matching patch to the encoder.)
yes, you are right!  We do need to add the qsv_device for the 
encode/decode to choose a device handle.  The user can assign the device 
using the
below command:

/./ffmpeg -qsv_dev device -c:v h264_qsv -i in -c:v h264 output.h264/

/ ./ffmpeg -c:v h264 -i in  -qsv_dev device-c:v h264_qsv  output.h264/

Using the qsv_dev inorder to avoid conflicting with the global option 
"qsv_device".

Attached the updated patches.

Thanks.
>
> - Mark
>
>
>> Root cuase: when using the system rather than the hwaccel,ff_qsv_init_internal_session
>> does not set the vaDisplay handle for the session. The qsv decode will failed.
>>
>> Solution: when creating the internal session , call the HwContext API to get session and
>> release all resource when close the decoder.
>>
>> Signed-off-by: ChaoX A Liu <chaox.a.liu at gmail.com>
>> Signed-off-by: Huang, Zhengxu <zhengxu.maxwell at gmail.com>
>> Signed-off-by: Andrew, Zhang <huazh407 at gmail.com>
>> ---
>>   libavcodec/qsv.c          |  6 ++---
>>   libavcodec/qsv_internal.h |  5 ++++
>>   libavcodec/qsvdec.c       | 59 ++++++++++++++++++++++++++++++++++++++++++++---
>>   libavcodec/qsvdec.h       |  1 +
>>   4 files changed, 65 insertions(+), 6 deletions(-)
>>
>> diff --git a/libavcodec/qsv.c b/libavcodec/qsv.c
>> index aac6ce6..a932fc3 100644
>> --- a/libavcodec/qsv.c
>> +++ b/libavcodec/qsv.c
>> @@ -120,7 +120,7 @@ int ff_qsv_map_pixfmt(enum AVPixelFormat format, uint32_t *fourcc)
>>       }
>>   }
>>   
>> -static int qsv_load_plugins(mfxSession session, const char *load_plugins,
>> +int ff_qsv_load_plugins(mfxSession session, const char *load_plugins,
>>                               void *logctx)
>>   {
>>       if (!load_plugins || !*load_plugins)
>> @@ -185,7 +185,7 @@ int ff_qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
>>           return ff_qsv_error(ret);
>>       }
>>   
>> -    ret = qsv_load_plugins(*session, load_plugins, avctx);
>> +    ret = ff_qsv_load_plugins(*session, load_plugins, avctx);
>>       if (ret < 0) {
>>           av_log(avctx, AV_LOG_ERROR, "Error loading plugins\n");
>>           return ret;
>> @@ -329,7 +329,7 @@ int ff_qsv_init_session_hwcontext(AVCodecContext *avctx, mfxSession *psession,
>>           }
>>       }
>>   
>> -    ret = qsv_load_plugins(session, load_plugins, avctx);
>> +    ret = ff_qsv_load_plugins(session, load_plugins, avctx);
>>       if (ret < 0) {
>>           av_log(avctx, AV_LOG_ERROR, "Error loading plugins\n");
>>           return ret;
>> diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
>> index 5d2a216..13f23ef 100644
>> --- a/libavcodec/qsv_internal.h
>> +++ b/libavcodec/qsv_internal.h
>> @@ -50,6 +50,10 @@ typedef struct QSVFrame {
>>       struct QSVFrame *next;
>>   } QSVFrame;
>>   
>> +typedef struct QSVDeviceContext {
>> +    AVBufferRef *hw_device_ctx;
>> +} QSVDeviceContext;
>> +
>>   typedef struct QSVFramesContext {
>>       AVBufferRef *hw_frames_ctx;
>>       mfxFrameInfo info;
>> @@ -73,5 +77,6 @@ int ff_qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
>>   int ff_qsv_init_session_hwcontext(AVCodecContext *avctx, mfxSession *session,
>>                                     QSVFramesContext *qsv_frames_ctx,
>>                                     const char *load_plugins, int opaque);
>> +int ff_qsv_load_plugins(mfxSession session, const char *load_plugins, void *logctx);
>>   
>>   #endif /* AVCODEC_QSV_INTERNAL_H */
>> diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
>> index 258042d..ccd5e26 100644
>> --- a/libavcodec/qsvdec.c
>> +++ b/libavcodec/qsvdec.c
>> @@ -41,6 +41,58 @@
>>   #include "qsv_internal.h"
>>   #include "qsvdec.h"
>>   
>> +static int qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
>> +                                 QSVContext *q, const char *load_plugins)
>> +{
>> +    mfxIMPL impl   = MFX_IMPL_AUTO_ANY;
>> +    mfxVersion ver = { { QSV_VERSION_MINOR, QSV_VERSION_MAJOR } };
>> +
>> +    const char *desc;
>> +    int ret;
>> +    AVHWDeviceContext *device_hw;
>> +    AVQSVDeviceContext *hwctx;
>> +    AVBufferRef *hw_device_ctx;
>> +
>> +    ret = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_QSV, NULL, NULL, 0);
>> +    if (ret < 0) {
>> +        av_log(NULL, AV_LOG_ERROR, "Failed to create a QSV device\n");
>> +        return ret;
>> +    }
>> +    device_hw = (AVHWDeviceContext*)hw_device_ctx->data;
>> +    hwctx = device_hw->hwctx;
>> +
>> +    *session = hwctx->session;
>> +
>> +    q->device_ctx.hw_device_ctx = hw_device_ctx;
>> +
>> +    ret = ff_qsv_load_plugins(*session, load_plugins, avctx);
>> +    if (ret < 0) {
>> +        av_log(avctx, AV_LOG_ERROR, "Error loading plugins\n");
>> +        return ret;
>> +    }
>> +    MFXQueryIMPL(*session, &impl);
>> +
>> +    switch (MFX_IMPL_BASETYPE(impl)) {
>> +    case MFX_IMPL_SOFTWARE:
>> +        desc = "software";
>> +        break;
>> +    case MFX_IMPL_HARDWARE:
>> +    case MFX_IMPL_HARDWARE2:
>> +    case MFX_IMPL_HARDWARE3:
>> +    case MFX_IMPL_HARDWARE4:
>> +        desc = "hardware accelerated";
>> +        break;
>> +    default:
>> +        desc = "unknown";
>> +    }
>> +
>> +    av_log(avctx, AV_LOG_VERBOSE,
>> +           "Initialized an internal MFX session using %s implementation\n",
>> +           desc);
>> +
>> +    return 0;
>> +}
>> +
>>   static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession session,
>>                               AVBufferRef *hw_frames_ref)
>>   {
>> @@ -70,13 +122,13 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
>>           q->session = q->internal_session;
>>       } else {
>>           if (!q->internal_session) {
>> -            ret = ff_qsv_init_internal_session(avctx, &q->internal_session,
>> +            ret = qsv_init_internal_session(avctx, &q->session, q,
>>                                                  q->load_plugins);
>>               if (ret < 0)
>>                   return ret;
>>           }
>> -
>> -        q->session = q->internal_session;
>> +        /* the session will close when unref the hw_device_ctx */
>> +       // q->session = q->internal_session;
>>       }
>>   
>>       /* make sure the decoder is uninitialized */
>> @@ -428,6 +480,7 @@ int ff_qsv_decode_close(QSVContext *q)
>>           MFXClose(q->internal_session);
>>   
>>       av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
>> +    av_buffer_unref(&q->device_ctx.hw_device_ctx);
>>       av_freep(&q->frames_ctx.mids);
>>       q->frames_ctx.nb_mids = 0;
>>   
>> diff --git a/libavcodec/qsvdec.h b/libavcodec/qsvdec.h
>> index 41fb716..ae810bc 100644
>> --- a/libavcodec/qsvdec.h
>> +++ b/libavcodec/qsvdec.h
>> @@ -44,6 +44,7 @@ typedef struct QSVContext {
>>       mfxSession internal_session;
>>   
>>       QSVFramesContext frames_ctx;
>> +    QSVDeviceContext device_ctx;
>>   
>>       /**
>>        * a linked list of frames currently being used by QSV
>> -- 
>> 1.8.3.1
>>
> _______________________________________________
> ffmpeg-devel mailing list
> ffmpeg-devel at ffmpeg.org
> http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

-------------- next part --------------
From d9f178725ec42fb73fad0cfd389a9720711ffb42 Mon Sep 17 00:00:00 2001
From: Zhengxu <zhengxu.maxwell at gmail.com>
Date: Mon, 19 Dec 2016 01:27:06 -0500
Subject: [PATCH 1/2] libavcodec/qsvdec: Fix the QSV decoder can't work when
 using system memory

Description: ./ffmpeg -c:v h264_qsv -i in -c:v h264 output.h264 does not
work because the qsv decode will failed.

Root cuase: when using the system rather than the hwaccel,ff_qsv_init_internal_session
does not set the vaDisplay handle for the session. The qsv decode will failed.

Solution: when creating the internal session , call the HwContext API to get session and
release all resource when close the decoder.

Signed-off-by: ChaoX A Liu <chaox.a.liu at gmail.com>
Signed-off-by: Huang, Zhengxu <zhengxu.maxwell at gmail.com>
Signed-off-by: Andrew, Zhang <huazh407 at gmail.com>
---
 libavcodec/qsv.c          |  6 ++---
 libavcodec/qsv_internal.h |  5 ++++
 libavcodec/qsvdec.c       | 69 ++++++++++++++++++++++++++++++++++++++++++++---
 libavcodec/qsvdec.h       |  6 +++++
 libavcodec/qsvdec_h2645.c |  4 +--
 libavcodec/qsvdec_mpeg2.c |  2 +-
 libavcodec/qsvdec_vc1.c   |  2 +-
 7 files changed, 84 insertions(+), 10 deletions(-)

diff --git a/libavcodec/qsv.c b/libavcodec/qsv.c
index aac6ce6..a932fc3 100644
--- a/libavcodec/qsv.c
+++ b/libavcodec/qsv.c
@@ -120,7 +120,7 @@ int ff_qsv_map_pixfmt(enum AVPixelFormat format, uint32_t *fourcc)
     }
 }
 
-static int qsv_load_plugins(mfxSession session, const char *load_plugins,
+int ff_qsv_load_plugins(mfxSession session, const char *load_plugins,
                             void *logctx)
 {
     if (!load_plugins || !*load_plugins)
@@ -185,7 +185,7 @@ int ff_qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
         return ff_qsv_error(ret);
     }
 
-    ret = qsv_load_plugins(*session, load_plugins, avctx);
+    ret = ff_qsv_load_plugins(*session, load_plugins, avctx);
     if (ret < 0) {
         av_log(avctx, AV_LOG_ERROR, "Error loading plugins\n");
         return ret;
@@ -329,7 +329,7 @@ int ff_qsv_init_session_hwcontext(AVCodecContext *avctx, mfxSession *psession,
         }
     }
 
-    ret = qsv_load_plugins(session, load_plugins, avctx);
+    ret = ff_qsv_load_plugins(session, load_plugins, avctx);
     if (ret < 0) {
         av_log(avctx, AV_LOG_ERROR, "Error loading plugins\n");
         return ret;
diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h
index 5d2a216..13f23ef 100644
--- a/libavcodec/qsv_internal.h
+++ b/libavcodec/qsv_internal.h
@@ -50,6 +50,10 @@ typedef struct QSVFrame {
     struct QSVFrame *next;
 } QSVFrame;
 
+typedef struct QSVDeviceContext {
+    AVBufferRef *hw_device_ctx;
+} QSVDeviceContext;
+
 typedef struct QSVFramesContext {
     AVBufferRef *hw_frames_ctx;
     mfxFrameInfo info;
@@ -73,5 +77,6 @@ int ff_qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
 int ff_qsv_init_session_hwcontext(AVCodecContext *avctx, mfxSession *session,
                                   QSVFramesContext *qsv_frames_ctx,
                                   const char *load_plugins, int opaque);
+int ff_qsv_load_plugins(mfxSession session, const char *load_plugins, void *logctx);
 
 #endif /* AVCODEC_QSV_INTERNAL_H */
diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c
index 258042d..2b6e20b 100644
--- a/libavcodec/qsvdec.c
+++ b/libavcodec/qsvdec.c
@@ -41,6 +41,68 @@
 #include "qsv_internal.h"
 #include "qsvdec.h"
 
+static int qsv_init_internal_session(AVCodecContext *avctx, mfxSession *session,
+                                 QSVContext *q, const char *load_plugins)
+{
+    mfxIMPL impl   = MFX_IMPL_AUTO_ANY;
+    mfxVersion ver = { { QSV_VERSION_MINOR, QSV_VERSION_MAJOR } };
+
+    const char *desc;
+    int ret;
+    AVHWDeviceContext *device_hw;
+    AVQSVDeviceContext *hwctx;
+    AVBufferRef *hw_device_ctx;
+    AVDictionary *dev_dict = NULL;
+
+    if(q->dev_name) {
+        ret = av_dict_set(&dev_dict, "child_device", q->dev_name, 0);
+        if (ret < 0)
+            return ret;
+    }
+
+    ret = av_hwdevice_ctx_create(&hw_device_ctx, AV_HWDEVICE_TYPE_QSV, NULL, dev_dict, 0);
+    if (ret < 0) {
+        av_log(NULL, AV_LOG_ERROR, "Failed to create a QSV device\n");
+        av_dict_free(&dev_dict);
+        return ret;
+    }
+    av_dict_free(&dev_dict);
+
+    device_hw = (AVHWDeviceContext*)hw_device_ctx->data;
+    hwctx = device_hw->hwctx;
+
+    *session = hwctx->session;
+
+    q->device_ctx.hw_device_ctx = hw_device_ctx;
+
+    ret = ff_qsv_load_plugins(*session, load_plugins, avctx);
+    if (ret < 0) {
+        av_log(avctx, AV_LOG_ERROR, "Error loading plugins\n");
+        return ret;
+    }
+    MFXQueryIMPL(*session, &impl);
+
+    switch (MFX_IMPL_BASETYPE(impl)) {
+    case MFX_IMPL_SOFTWARE:
+        desc = "software";
+        break;
+    case MFX_IMPL_HARDWARE:
+    case MFX_IMPL_HARDWARE2:
+    case MFX_IMPL_HARDWARE3:
+    case MFX_IMPL_HARDWARE4:
+        desc = "hardware accelerated";
+        break;
+    default:
+        desc = "unknown";
+    }
+
+    av_log(avctx, AV_LOG_VERBOSE,
+           "Initialized an internal MFX session using %s implementation\n",
+           desc);
+
+    return 0;
+}
+
 static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession session,
                             AVBufferRef *hw_frames_ref)
 {
@@ -70,13 +132,13 @@ static int qsv_init_session(AVCodecContext *avctx, QSVContext *q, mfxSession ses
         q->session = q->internal_session;
     } else {
         if (!q->internal_session) {
-            ret = ff_qsv_init_internal_session(avctx, &q->internal_session,
+            ret = qsv_init_internal_session(avctx, &q->session, q,
                                                q->load_plugins);
             if (ret < 0)
                 return ret;
         }
-
-        q->session = q->internal_session;
+        /* the session will close when unref the hw_device_ctx */
+       // q->session = q->internal_session;
     }
 
     /* make sure the decoder is uninitialized */
@@ -428,6 +490,7 @@ int ff_qsv_decode_close(QSVContext *q)
         MFXClose(q->internal_session);
 
     av_buffer_unref(&q->frames_ctx.hw_frames_ctx);
+    av_buffer_unref(&q->device_ctx.hw_device_ctx);
     av_freep(&q->frames_ctx.mids);
     q->frames_ctx.nb_mids = 0;
 
diff --git a/libavcodec/qsvdec.h b/libavcodec/qsvdec.h
index 41fb716..32f960f 100644
--- a/libavcodec/qsvdec.h
+++ b/libavcodec/qsvdec.h
@@ -35,6 +35,10 @@
 #include "avcodec.h"
 #include "qsv_internal.h"
 
+#define QSV_COMMON_OPTS \
+    { "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(qsv.async_depth), AV_OPT_TYPE_INT, { .i64 = ASYNC_DEPTH_DEFAULT }, 0, INT_MAX, VD }, \
+    { "qsv_dev", "Set QSV hardware device (DirectX adapter index, DRM path or X11 display name)", OFFSET(qsv.dev_name), AV_OPT_TYPE_STRING, { .str = "" }, 0, 128, VD },
+
 typedef struct QSVContext {
     // the session used for decoding
     mfxSession session;
@@ -44,6 +48,7 @@ typedef struct QSVContext {
     mfxSession internal_session;
 
     QSVFramesContext frames_ctx;
+    QSVDeviceContext device_ctx;
 
     /**
      * a linked list of frames currently being used by QSV
@@ -62,6 +67,7 @@ typedef struct QSVContext {
     // options set by the caller
     int async_depth;
     int iopattern;
+    char *dev_name;
 
     char *load_plugins;
 
diff --git a/libavcodec/qsvdec_h2645.c b/libavcodec/qsvdec_h2645.c
index aa7aded..1666c19 100644
--- a/libavcodec/qsvdec_h2645.c
+++ b/libavcodec/qsvdec_h2645.c
@@ -237,7 +237,7 @@ AVHWAccel ff_hevc_qsv_hwaccel = {
 };
 
 static const AVOption hevc_options[] = {
-    { "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(qsv.async_depth), AV_OPT_TYPE_INT, { .i64 = ASYNC_DEPTH_DEFAULT }, 0, INT_MAX, VD },
+    QSV_COMMON_OPTS
 
     { "load_plugin", "A user plugin to load in an internal session", OFFSET(load_plugin), AV_OPT_TYPE_INT, { .i64 = LOAD_PLUGIN_HEVC_SW }, LOAD_PLUGIN_NONE, LOAD_PLUGIN_HEVC_HW, VD, "load_plugin" },
     { "none",     NULL, 0, AV_OPT_TYPE_CONST, { .i64 = LOAD_PLUGIN_NONE },    0, 0, VD, "load_plugin" },
@@ -284,7 +284,7 @@ AVHWAccel ff_h264_qsv_hwaccel = {
 };
 
 static const AVOption options[] = {
-    { "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(qsv.async_depth), AV_OPT_TYPE_INT, { .i64 = ASYNC_DEPTH_DEFAULT }, 0, INT_MAX, VD },
+    QSV_COMMON_OPTS
     { NULL },
 };
 
diff --git a/libavcodec/qsvdec_mpeg2.c b/libavcodec/qsvdec_mpeg2.c
index e558ca0..32019f3 100644
--- a/libavcodec/qsvdec_mpeg2.c
+++ b/libavcodec/qsvdec_mpeg2.c
@@ -150,7 +150,7 @@ AVHWAccel ff_mpeg2_qsv_hwaccel = {
 #define OFFSET(x) offsetof(QSVMPEG2Context, x)
 #define VD AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
 static const AVOption options[] = {
-    { "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(qsv.async_depth), AV_OPT_TYPE_INT, { .i64 = ASYNC_DEPTH_DEFAULT }, 0, INT_MAX, VD },
+    QSV_COMMON_OPTS
     { NULL },
 };
 
diff --git a/libavcodec/qsvdec_vc1.c b/libavcodec/qsvdec_vc1.c
index 70a47b1..a7bcaa9 100644
--- a/libavcodec/qsvdec_vc1.c
+++ b/libavcodec/qsvdec_vc1.c
@@ -147,7 +147,7 @@ AVHWAccel ff_vc1_qsv_hwaccel = {
 #define OFFSET(x) offsetof(QSVVC1Context, x)
 #define VD AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
 static const AVOption options[] = {
-    { "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(qsv.async_depth), AV_OPT_TYPE_INT, { .i64 = ASYNC_DEPTH_DEFAULT }, 0, INT_MAX, VD },
+    QSV_COMMON_OPTS
     { NULL },
 };
 
-- 
1.8.3.1



More information about the ffmpeg-devel mailing list