[FFmpeg-devel] [PATCH] lavc/videotoolbox: fix H.264 hwaccel init issue

[Richard Kern]

Ping. This fixes #5595. 

> On Jun 1, 2016, at 10:06 PM, Rick Kern <kernrj at gmail.com> wrote:
> 
> Fixes VTDecompressionSessionCreate() error.
> 
> Signed-off-by: Rick Kern <kernrj at gmail.com>
> ---
> libavcodec/videotoolbox.c | 59 ++++++++++++++++++++++++++++++++++++-----------
> 1 file changed, 45 insertions(+), 14 deletions(-)
> 
> diff --git a/libavcodec/videotoolbox.c b/libavcodec/videotoolbox.c
> index 2f4d531..cadfb23 100644
> --- a/libavcodec/videotoolbox.c
> +++ b/libavcodec/videotoolbox.c
> @@ -487,23 +487,53 @@ static CFDictionaryRef videotoolbox_buffer_attributes_create(int width,
>     return buffer_attributes;
> }
> 
> -static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType codec_type,
> +static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(AVCodecContext *avctx,
> +                                                                   CMVideoCodecType codec_type,
>                                                                    CFDictionaryRef decoder_spec,
>                                                                    int width,
>                                                                    int height)
> {
> -    CMFormatDescriptionRef cm_fmt_desc;
> -    OSStatus status;
> -
> -    status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
> -                                            codec_type,
> -                                            width,
> -                                            height,
> -                                            decoder_spec, // Dictionary of extension
> -                                            &cm_fmt_desc);
> -
> -    if (status)
> -        return NULL;
> +    CMFormatDescriptionRef cm_fmt_desc = NULL;
> +    int status;
> +    H264Context *h = codec_type == kCMVideoCodecType_H264 ? avctx->priv_data : NULL;
> +
> +    if (h && h->sps.data_size && h->pps.data_size) {
> +        int ps_count = 2;
> +        const uint8_t **ps_data = av_malloc(sizeof(uint8_t*) * ps_count);
> +        size_t *ps_sizes = av_malloc(sizeof(size_t)  * ps_count);
> +
> +        ps_data[0]  = h->sps.data;
> +        ps_sizes[0] = h->sps.data_size;
> +
> +        ps_data[1]  = h->pps.data;
> +        ps_sizes[1] = h->pps.data_size;
> +
> +        status = CMVideoFormatDescriptionCreateFromH264ParameterSets(NULL,
> +                                                                     ps_count,
> +                                                                     ps_data,
> +                                                                     ps_sizes,
> +                                                                     4,
> +                                                                     &cm_fmt_desc);
> +        av_freep(&ps_sizes);
> +        av_freep(&ps_data);
> +
> +        if (status) {
> +            av_log(avctx, AV_LOG_ERROR, "Error creating H.264 format description: %d\n", status);
> +            return NULL;
> +        }
> +    } else {
> +        status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
> +                                                codec_type,
> +                                                width,
> +                                                height,
> +                                                decoder_spec, // Dictionary of extension
> +                                                &cm_fmt_desc);
> +
> +        if (status) {
> +            av_log(avctx, AV_LOG_ERROR, "Error creating format description: %d\n", status);
> +            return NULL;
> +        }
> +    }
> 
>     return cm_fmt_desc;
> }
> @@ -543,7 +573,8 @@ static int videotoolbox_default_init(AVCodecContext *avctx)
> 
>     decoder_spec = videotoolbox_decoder_config_create(videotoolbox->cm_codec_type, avctx);
> 
> -    videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(videotoolbox->cm_codec_type,
> +    videotoolbox->cm_fmt_desc = videotoolbox_format_desc_create(avctx,
> +                                                                videotoolbox->cm_codec_type,
>                                                                 decoder_spec,
>                                                                 avctx->width,
>                                                                 avctx->height);
> -- 
> 2.7.4
>