[FFmpeg-devel] [PATCH] lavc/cuvid: fail early if GPU can't handle given video parameters

[Hendrik Leppkes]

On Tue, Jan 3, 2017 at 8:26 AM,  <pkoshevoy at gmail.com> wrote:
> From: Pavel Koshevoy <pkoshevoy at gmail.com>
>
> Evidently CUVID doesn't support decoding 422 or 444 chroma formats,
> and only a limited set of resolutions per codec are supported.
>
> Given that stream resolution and pixel format are typically known as a
> result of probing it is better to use this info during avcodec_open2
> call and fail early in case the video parameters are not supported,
> rather than failing later during avcodec_send_packet calls.
>
> This problem surfaced when trying to decode 5120x2700 h246 video on
> Geforce GT 730, or when decoding 422 mpeg2 stream on same GPU --
> avcodec_open2 succeeded but decoding failed.
> ---
>  libavcodec/cuvid.c | 58 +++++++++++++++++++++++++++++++++++++++++++++++++-----
>  1 file changed, 53 insertions(+), 5 deletions(-)
>
> diff --git a/libavcodec/cuvid.c b/libavcodec/cuvid.c
> index 8fc713d..febdd71 100644
> --- a/libavcodec/cuvid.c
> +++ b/libavcodec/cuvid.c
> @@ -612,7 +612,11 @@ static av_cold int cuvid_decode_end(AVCodecContext *avctx)
>      return 0;
>  }
>
> -static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cuparseinfo)
> +static int cuvid_test_dummy_decoder(AVCodecContext *avctx,
> +                                    const CUVIDPARSERPARAMS *cuparseinfo,
> +                                    cudaVideoChromaFormat probed_chroma_format,
> +                                    int probed_width,
> +                                    int probed_height)
>  {
>      CuvidContext *ctx = avctx->priv_data;
>      CUVIDDECODECREATEINFO cuinfo;
> @@ -622,11 +626,11 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
>      memset(&cuinfo, 0, sizeof(cuinfo));
>
>      cuinfo.CodecType = cuparseinfo->CodecType;
> -    cuinfo.ChromaFormat = cudaVideoChromaFormat_420;
> +    cuinfo.ChromaFormat = probed_chroma_format;
>      cuinfo.OutputFormat = cudaVideoSurfaceFormat_NV12;
>
> -    cuinfo.ulWidth = 1280;
> -    cuinfo.ulHeight = 720;
> +    cuinfo.ulWidth = probed_width;
> +    cuinfo.ulHeight = probed_height;
>      cuinfo.ulTargetWidth = cuinfo.ulWidth;
>      cuinfo.ulTargetHeight = cuinfo.ulHeight;
>
> @@ -653,6 +657,36 @@ static int cuvid_test_dummy_decoder(AVCodecContext *avctx, CUVIDPARSERPARAMS *cu
>      return 0;
>  }
>
> +static int convert_to_cuda_video_chroma_format(enum AVPixelFormat pix_fmt,
> +                                               cudaVideoChromaFormat *out)
> +{
> +    const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
> +    if (!(out && desc &&
> +          (desc->nb_components == 1 || desc->nb_components == 3) &&
> +          (desc->log2_chroma_w < 2 && desc->log2_chroma_h < 2)))
> +    {
> +        return AVERROR(EINVAL);
> +    }
> +
> +    if (desc->nb_components == 1)
> +    {
> +        *out = cudaVideoChromaFormat_Monochrome;
> +    }
> +    else if (desc->flags == AV_PIX_FMT_FLAG_PLANAR)
> +    {
> +        *out = ((desc->log2_chroma_w == 0) ? cudaVideoChromaFormat_444 :
> +                (desc->log2_chroma_h == 0) ? cudaVideoChromaFormat_422 :
> +                cudaVideoChromaFormat_420);
> +    }
> +    else
> +    {
> +        return AVERROR(EINVAL);
> +    }
> +
> +    // unfortunately, 420 is the only one that works:
> +    return (*out == cudaVideoChromaFormat_420) ? 0 : AVERROR_EXTERNAL;
> +}
> +
>  static av_cold int cuvid_decode_init(AVCodecContext *avctx)
>  {
>      CuvidContext *ctx = avctx->priv_data;
> @@ -663,12 +697,23 @@ static av_cold int cuvid_decode_init(AVCodecContext *avctx)
>      CUcontext cuda_ctx = NULL;
>      CUcontext dummy;
>      const AVBitStreamFilter *bsf;
> +    cudaVideoChromaFormat probed_chroma_format;
> +    int probed_width;
> +    int probed_height;
>      int ret = 0;
>
>      enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_CUDA,
>                                         AV_PIX_FMT_NV12,
>                                         AV_PIX_FMT_NONE };
>
> +    ret = convert_to_cuda_video_chroma_format(avctx->pix_fmt, &probed_chroma_format);
> +    if (ret < 0) {
> +        // pixel format is not supported:
> +        return ret;
> +    }

Generally, there is no guarantee that any of these properties are
already set when init is called. When you use avformat->avcodec, it'll
generally try to probe them from the stream, however if someone just
uses avcodec, the h264 software decoder for example works independent
of dimensions or pixfmt set - because it can determine this from the
bitstream.

So what I'm trying to say, failing when you detect an explicitly
incompatible value might be OK, but failing when its unset seems like
it might be annoying.

- Hendrik