[FFmpeg-devel] [PATCH] lavc/cuvid: fail early if GPU can't handle video resolution (v5)

[pkoshevoy at gmail.com]

From: Pavel Koshevoy <pkoshevoy at gmail.com>

CUVID on GeForce GT 730 and GeForce GTX 1060 does not report any when
decoding 8K h264 packets.  However, it does return an error during
cuvidCreateDecoder call if the indicated video resolution is not
supported.

Given that stream resolution is typically known as a result of probing
it is better to use this information during avcodec_open2 call to fail
immediately, rather than proceeding to decode and never receiving any
frames from the decoder nor receiving any indication of decode failure.
---
 libavcodec/cuvid.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
index 8fc713d..9b35476 100644
--- a/libavcodec/cuvid.c
+++ b/libavcodec/cuvid.c
@@ -612,7 +612,10 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx)
     return 0;
 }
 
-static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo)
+static int cuvid_test_dummy_decoder(AVCodecContext *avctx,
+                                    const CUVIDPARSERPARAMS *cuparseinfo,
+                                    int probed_width,
+                                    int probed_height)
 {
     CuvidContext *ctx = avctx->priv_data;
     CUVIDDECODECREATEINFO cuinfo;
@@ -625,8 +628,8 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
     cuinfo.ChromaFormat = cudaVideoChromaFormat_420;
     cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12;
 
-    cuinfo.ulWidth = 1280;
-    cuinfo.ulHeight = 720;
+    cuinfo.ulWidth = probed_width;
+    cuinfo.ulHeight = probed_height;
     cuinfo.ulTargetWidth = cuinfo.ulWidth;
     cuinfo.ulTargetHeight = cuinfo.ulHeight;
 
@@ -669,6 +672,9 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
                                        AV_PIX_FMT_NV12,
                                        AV_PIX_FMT_NONE };
 
+    int probed_width = avctx->coded_width ? avctx->coded_width : 1280;
+    int probed_height = avctx->coded_height ? avctx->coded_height : 720;
+
     // Accelerated transcoding scenarios with 'ffmpeg' require that the
     // pix_fmt be set to AV_PIX_FMT_CUDA early. The sw_pix_fmt, and the
     // pix_fmt for non-accelerated transcoding, do not need to be correct
@@ -824,7 +830,9 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
     if (ret < 0)
         goto error;
 
-    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo);
+    ret = cuvid_test_dummy_decoder(avctx, &ctx->cuparseinfo,
+                                   probed_width,
+                                   probed_height);
     if (ret < 0)
         goto error;
 
-- 
2.6.6