[FFmpeg-devel] [PATCH]Fix pix_fmt for vc1 decoding with flags +gray

[Carl Eugen Hoyos]

Hi!

Attached patch fixes the initial pix_fmt when decoding 
vc1 with flags +gray, regression since this commit:
http://git.videolan.org/?p=ffmpeg.git;a=commitdiff;h=16af29a7

Not tested with hardware acceleration, does not fix 
ticket #4542.

Please review, Carl Eugen
-------------- next part --------------
diff --git a/libavcodec/vc1dec.c b/libavcodec/vc1dec.c
index 86e214a..1d45cb1 100644
--- a/libavcodec/vc1dec.c
+++ b/libavcodec/vc1dec.c
@@ -427,10 +427,6 @@ static av_cold int vc1_decode_init(AVCodecContext *avctx)
 
     if (!avctx->extradata_size || !avctx->extradata)
         return -1;
-    if (!(avctx->flags & CODEC_FLAG_GRAY))
-        avctx->pix_fmt = ff_get_format(avctx, avctx->codec->pix_fmts);
-    else
-        avctx->pix_fmt = AV_PIX_FMT_GRAY8;
     v->s.avctx = avctx;
 
     if ((ret = ff_vc1_init_common(v)) < 0)
@@ -445,6 +441,12 @@ static av_cold int vc1_decode_init(AVCodecContext *avctx)
     // That this is necessary might indicate a bug.
     ff_vc1_decode_end(avctx);
 
+    if (!(avctx->flags & CODEC_FLAG_GRAY)) {
+        avctx->pix_fmt = ff_get_format(avctx, avctx->codec->pix_fmts);
+    } else {
+        avctx->pix_fmt = AV_PIX_FMT_GRAY8;
+    }
+
     ff_blockdsp_init(&s->bdsp, avctx);
     ff_h264chroma_init(&v->h264chroma, 8);
     ff_qpeldsp_init(&s->qdsp);