[FFmpeg-devel] [PATCH] Adds decode support for formats other than 420

[James Almer]

On 22/08/14 5:31 PM, Deb Mukherjee wrote:
> Handles new VP9 profiles 1-3 with different color sampling and
> bit-depths.
> ---
>  libavcodec/libvpxdec.c | 69 +++++++++++++++++++++++++++++++++++++++++++-------
>  1 file changed, 60 insertions(+), 9 deletions(-)
> 
> diff --git a/libavcodec/libvpxdec.c b/libavcodec/libvpxdec.c
> index 94e1e4d..7c397fb 100644
> --- a/libavcodec/libvpxdec.c
> +++ b/libavcodec/libvpxdec.c
> @@ -60,7 +60,58 @@ static av_cold int vpx_init(AVCodecContext *avctx,
>      return 0;
>  }
>  
> -static int vp8_decode(AVCodecContext *avctx,
> +// returns 0 on success, 1 on unsupported

0 on success, AVERROR_INVALIDDATA otherwise.

> +static int set_pix_fmt(AVCodecContext *avctx, struct vpx_image *img) {
> +    int ret = 0;

No need for this variable. Just return the corresponding value directly.

> +    if (avctx->codec_id == AV_CODEC_ID_VP8) {
> +        if (img->fmt != VPX_IMG_FMT_I420)
> +            return 1;
> +    }
> +    switch (img->fmt) {
> +        case VPX_IMG_FMT_I420:
> +            avctx->pix_fmt = AV_PIX_FMT_YUV420P;
> +            break;
> +        case VPX_IMG_FMT_I422:
> +            avctx->pix_fmt = AV_PIX_FMT_YUV422P;
> +            break;
> +        case VPX_IMG_FMT_I444:
> +            avctx->pix_fmt = AV_PIX_FMT_YUV444P;
> +            break;
> +        case VPX_IMG_FMT_I42016:

Fails to compile with libvpx 1.3.0 and older.

A quick solution would be to check if VPX_IMG_FMT_HIGH is defined and put all 
these inside pre-processor guards.

> +            if (img->bit_depth == 10) {
> +                avctx->pix_fmt = AV_PIX_FMT_YUV420P10LE;
> +            } else if (img->bit_depth == 12) {
> +                avctx->pix_fmt = AV_PIX_FMT_YUV420P12LE;
> +            } else {
> +                ret = 1;
> +            }
> +            break;
> +        case VPX_IMG_FMT_I42216:
> +            if (img->bit_depth == 10) {
> +                avctx->pix_fmt = AV_PIX_FMT_YUV422P10LE;
> +            } else if (img->bit_depth == 12) {
> +                avctx->pix_fmt = AV_PIX_FMT_YUV422P12LE;
> +            } else {
> +                ret = 1;
> +            }
> +            break;
> +        case VPX_IMG_FMT_I44416:
> +            if (img->bit_depth == 10) {
> +                avctx->pix_fmt = AV_PIX_FMT_YUV444P10LE;
> +            } else if (img->bit_depth == 12) {
> +                avctx->pix_fmt = AV_PIX_FMT_YUV444P12LE;
> +            } else {
> +                ret = 1;
> +            }
> +            break;
> +        default:
> +            ret = 1;
> +            break;
> +    }
> +    return ret;
> +}
> +
> +static int vpx_decode(AVCodecContext *avctx,
>                        void *data, int *got_frame, AVPacket *avpkt)
>  {
>      VP8Context *ctx = avctx->priv_data;
> @@ -82,9 +133,9 @@ static int vp8_decode(AVCodecContext *avctx,
>      }
>  
>      if ((img = vpx_codec_get_frame(&ctx->decoder, &iter))) {
> -        if (img->fmt != VPX_IMG_FMT_I420) {
> -            av_log(avctx, AV_LOG_ERROR, "Unsupported output colorspace (%d)\n",
> -                   img->fmt);
> +        if (set_pix_fmt(avctx, img)) {

if ((ret = set_pix_fmt(avctx, img)) < 0)

Then use "return ret;" below to propagate the value returned by set_pix_fmt();

> +            av_log(avctx, AV_LOG_ERROR, "Unsupported output colorspace (%d) / bit_depth (%d)\n",
> +                   img->fmt, img->bit_depth);

Same here, bit_depth is not available in the vpx_image struct in libvpx <= 1.3.0.
Either remove it, or hardcode an 8 if VPX_IMG_FMT_HIGH is not defined.