[FFmpeg-devel] [PATCH] Quick ugly hack to debug VDPAU.

[Reimar Döffinger]

Obviously this is not acceptable as-is, the question is
if this seems to be an approach that might be worth experiment
with more.
Either way this hopefully will work at least for debugging the issues with
packed B-frame MPEG-4.

Signed-off-by: Reimar Döffinger <Reimar.Doeffinger at gmx.de>
---
 ffmpeg.c           |  4 +++
 libavcodec/vdpau.c | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 libavcodec/vdpau.h |  2 ++
 3 files changed, 89 insertions(+)

diff --git a/ffmpeg.c b/ffmpeg.c
index 2e084a4..caaea62 100644
--- a/ffmpeg.c
+++ b/ffmpeg.c
@@ -102,6 +102,8 @@
 
 #include "libavutil/avassert.h"
 
+#include "libavcodec/vdpau.h"
+
 const char program_name[] = "ffmpeg";
 const int program_birth_year = 2000;
 
@@ -1668,6 +1670,7 @@ static int decode_video(InputStream *ist, AVPacket *pkt, int *got_output)
     ret = avcodec_decode_video2(ist->st->codec,
                                 decoded_frame, got_output, pkt);
     update_benchmark("decode_video %d.%d", ist->file_index, ist->st->index);
+    *got_output = 0;
 
     if (*got_output || ret<0 || pkt->size)
         decode_error_stat[ret<0] ++;
@@ -1980,6 +1983,7 @@ static int init_input_stream(int ist_index, char *error, int error_len)
 
         if (!av_dict_get(ist->opts, "threads", NULL, 0))
             av_dict_set(&ist->opts, "threads", "auto", 0);
+        av_setup_debug_vdpau(ist->st->codec, codec);
         if ((ret = avcodec_open2(ist->st->codec, codec, &ist->opts)) < 0) {
             char errbuf[128];
             if (ret == AVERROR_EXPERIMENTAL)
diff --git a/libavcodec/vdpau.c b/libavcodec/vdpau.c
index 7a4d145..a0b8963 100644
--- a/libavcodec/vdpau.c
+++ b/libavcodec/vdpau.c
@@ -25,6 +25,7 @@
 #include "avcodec.h"
 #include "h264.h"
 #include "vc1.h"
+#include "libavutil/md5.h"
 
 #undef NDEBUG
 #include <assert.h>
@@ -32,12 +33,94 @@
 #include "vdpau.h"
 #include "vdpau_internal.h"
 
+
 /**
  * @addtogroup VDPAU_Decoding
  *
  * @{
  */
 
+static int vdpau_debug_render2(AVCodecContext *s, AVFrame *f,
+                               const VdpPictureInfo *info, uint32_t count,
+                               const VdpBitstreamBuffer *buffers)
+{
+#undef printf
+    int i;
+    printf("VDPAU render\n");
+    for (i = 0; i < count; i++) {
+        uint8_t md5val[16];
+        av_md5_sum(md5val, buffers[i].bitstream, buffers[i].bitstream_bytes);
+        printf("buffer %i with %i bytes, MD5 %08x%08x%08x%08x\n", i,
+               buffers[i].bitstream_bytes,
+               AV_RB32(md5val), AV_RB32(md5val + 4), AV_RB32(md5val + 8), AV_RB32(md5val + 12));
+    }
+    return 0;
+}
+
+static void vdpau_debug_slice(struct AVCodecContext *s, const AVFrame *src,
+                              int offset[AV_NUM_DATA_POINTERS], int y, int type, int height)
+{
+    struct vdpau_render_state *render = (struct vdpau_render_state *)src->data[0];
+    vdpau_debug_render2(s, (AVFrame *)src, (void *)&render->info, render->bitstream_buffers_used, render->bitstream_buffers);
+}
+
+
+static int vdpau_debug_get_buffer(AVCodecContext *avctx, AVFrame *pic)
+{
+    struct vdpau_render_state *render = av_mallocz(sizeof(*render));
+    render->surface = (intptr_t)render;
+    pic->data[0] = (void *)render;
+    pic->data[3] = (void *)(intptr_t)render->surface;
+    return 0;
+}
+
+static void vdpau_debug_release_buffer(AVCodecContext *avctx, AVFrame *pic)
+{
+    av_freep(pic->data[0]);
+}
+
+static enum AVPixelFormat vdpau_debug_get_format(struct AVCodecContext *avctx,
+                                                 const enum AVPixelFormat *fmt)
+{
+    int i;
+    for (i = 0; fmt[i] != PIX_FMT_NONE; i++) {
+        if (!(avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU) &&
+            (fmt[i] == AV_PIX_FMT_VDPAU_MPEG1 || fmt[i] == AV_PIX_FMT_VDPAU_MPEG2))
+            continue;
+        switch (fmt[i]) {
+        case AV_PIX_FMT_VDPAU:
+        case AV_PIX_FMT_VDPAU_H264:
+        case AV_PIX_FMT_VDPAU_MPEG1:
+        case AV_PIX_FMT_VDPAU_MPEG2:
+        case AV_PIX_FMT_VDPAU_MPEG4:
+        case AV_PIX_FMT_VDPAU_VC1:
+        case AV_PIX_FMT_VDPAU_WMV3:
+            break;
+        default:
+            continue;
+        }
+        return fmt[i];
+    }
+    return PIX_FMT_NONE;
+}
+
+void av_setup_debug_vdpau(AVCodecContext *avctx, const AVCodec *codec)
+{
+    av_freep(&avctx->hwaccel_context);
+    avctx->get_format = vdpau_debug_get_format;
+    avctx->get_buffer = vdpau_debug_get_buffer;
+    avctx->release_buffer = vdpau_debug_release_buffer;
+    if (codec->capabilities & CODEC_CAP_HWACCEL_VDPAU) {
+        avctx->draw_horiz_band = vdpau_debug_slice;
+        avctx->slice_flags = SLICE_FLAG_CODED_ORDER|SLICE_FLAG_ALLOW_FIELD;
+    } else {
+        AVVDPAUContext *vdpc;
+        avctx->hwaccel_context = av_alloc_vdpaucontext();
+        vdpc = avctx->hwaccel_context;
+        vdpc->render2 = vdpau_debug_render2;
+    }
+}
+
 AVVDPAUContext *av_alloc_vdpaucontext(void)
 {
     return av_mallocz(sizeof(AVVDPAUContext));
diff --git a/libavcodec/vdpau.h b/libavcodec/vdpau.h
index b1c836c..375f2c1 100644
--- a/libavcodec/vdpau.h
+++ b/libavcodec/vdpau.h
@@ -190,6 +190,8 @@ struct vdpau_render_state {
 };
 #endif
 
+void av_setup_debug_vdpau(AVCodecContext *avctx, const AVCodec *codec);
+
 /* @}*/
 
 #endif /* AVCODEC_VDPAU_H */
-- 
1.8.4.rc3