[FFmpeg-devel] [PATCH 1/4] lavu/frame: add mb_types side data

[Hendrik Leppkes]

On Sun, Jun 17, 2018 at 2:45 PM Rostislav Pehlivanov
<atomnuker at gmail.com> wrote:
>
> On 17 June 2018 at 05:21, Ramiro Polla <ramiro.polla at gmail.com> wrote:
>
> > ---
> >  libavcodec/avcodec.h       |  4 ++++
> >  libavcodec/mpegutils.c     | 20 ++++++++++++++++++++
> >  libavcodec/options_table.h |  1 +
> >  libavutil/frame.c          |  1 +
> >  libavutil/frame.h          |  9 +++++++++
> >  5 files changed, 35 insertions(+)
> >
> > diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h
> > index c90166deb6..7fe4fc9347 100644
> > --- a/libavcodec/avcodec.h
> > +++ b/libavcodec/avcodec.h
> > @@ -929,6 +929,10 @@ typedef struct RcOverride{
> >   */
> >  #define AV_CODEC_FLAG2_SHOW_ALL       (1 << 22)
> >  /**
> > + * Export macroblock types through frame side data
> > + */
> > +#define AV_CODEC_FLAG2_EXPORT_MB_TYPES (1 << 27)
> > +/**
> >   * Export motion vectors through frame side data
> >   */
> >  #define AV_CODEC_FLAG2_EXPORT_MVS     (1 << 28)
> > diff --git a/libavcodec/mpegutils.c b/libavcodec/mpegutils.c
> > index 3f94540616..0fbe5f8c9d 100644
> > --- a/libavcodec/mpegutils.c
> > +++ b/libavcodec/mpegutils.c
> > @@ -188,6 +188,26 @@ void ff_print_debug_info2(AVCodecContext *avctx,
> > AVFrame *pict, uint8_t *mbskip_
> >          av_freep(&mvs);
> >      }
> >
> > +    if ((avctx->flags2 & AV_CODEC_FLAG2_EXPORT_MB_TYPES) &&
> > mbtype_table) {
> > +        int size = (2 + mb_height * mb_width) * sizeof(uint32_t);
> > +        int mb_x, mb_y;
> > +
> > +        AVFrameSideData *sd;
> > +        uint32_t *out;
> > +
> > +        sd = av_frame_new_side_data(pict, AV_FRAME_DATA_MB_TYPES, size);
> > +        if (!sd)
> > +            return;
> > +
> > +        out = (uint32_t *) sd->data;
> > +        *out++ = mb_height;
> > +        *out++ = mb_width;
> > +
> > +        for (mb_y = 0; mb_y < mb_height; mb_y++)
> > +            for (mb_x = 0; mb_x < mb_width; mb_x++)
> > +                *out++ = mbtype_table[mb_x + mb_y * mb_stride];
> > +    }
> > +
> >      /* TODO: export all the following to make them accessible for users
> > (and filters) */
> >      if (avctx->hwaccel || !mbtype_table)
> >          return;
> > diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h
> > index 099261e168..25c84de321 100644
> > --- a/libavcodec/options_table.h
> > +++ b/libavcodec/options_table.h
> > @@ -76,6 +76,7 @@ static const AVOption avcodec_options[] = {
> >  {"export_mvs", "export motion vectors through frame side data", 0,
> > AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_EXPORT_MVS}, INT_MIN, INT_MAX,
> > V|D, "flags2"},
> >  {"skip_manual", "do not skip samples and export skip information as frame
> > side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_SKIP_MANUAL},
> > INT_MIN, INT_MAX, V|D, "flags2"},
> >  {"ass_ro_flush_noop", "do not reset ASS ReadOrder field on flush", 0,
> > AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_RO_FLUSH_NOOP}, INT_MIN, INT_MAX,
> > S|D, "flags2"},
> > +{"export_mb_types", "export macroblock types through frame side data", 0,
> > AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_FLAG2_EXPORT_MB_TYPES}, INT_MIN,
> > INT_MAX, V|D, "flags2"},
> >  {"time_base", NULL, OFFSET(time_base), AV_OPT_TYPE_RATIONAL, {.dbl = 0},
> > 0, INT_MAX},
> >  {"g", "set the group of picture (GOP) size", OFFSET(gop_size),
> > AV_OPT_TYPE_INT, {.i64 = 12 }, INT_MIN, INT_MAX, V|E},
> >  {"ar", "set audio sampling rate (in Hz)", OFFSET(sample_rate),
> > AV_OPT_TYPE_INT, {.i64 = DEFAULT }, 0, INT_MAX, A|D|E},
> > diff --git a/libavutil/frame.c b/libavutil/frame.c
> > index deb9b6f334..577d4f6e6d 100644
> > --- a/libavutil/frame.c
> > +++ b/libavutil/frame.c
> > @@ -834,6 +834,7 @@ const char *av_frame_side_data_name(enum
> > AVFrameSideDataType type)
> >      case AV_FRAME_DATA_ICC_PROFILE:                 return "ICC profile";
> >      case AV_FRAME_DATA_QP_TABLE_PROPERTIES:         return "QP table
> > properties";
> >      case AV_FRAME_DATA_QP_TABLE_DATA:               return "QP table
> > data";
> > +    case AV_FRAME_DATA_MB_TYPES:                    return "Macroblock
> > types";
> >      }
> >      return NULL;
> >  }
> > diff --git a/libavutil/frame.h b/libavutil/frame.h
> > index 9d57d6ce66..ce1231b03b 100644
> > --- a/libavutil/frame.h
> > +++ b/libavutil/frame.h
> > @@ -158,6 +158,15 @@ enum AVFrameSideDataType {
> >       */
> >      AV_FRAME_DATA_QP_TABLE_DATA,
> >  #endif
> > +
> > +    /**
> > +     * Macroblock types exported by some codecs (on demand through the
> > +     * export_mb_types flag set in the libavcodec AVCodecContext flags2
> > option).
> > +     * The data is composed by a header consisting of uint32_t mb_height
> > and
> > +     * uint32_t mb_width, followed by a uint32_t
> > mb_types[mb_height][mb_width]
> > +     * array.
> > +     */
> > +    AV_FRAME_DATA_MB_TYPES,
> >  };
> >
> >  enum AVActiveFormatDescription {
> > --
> > 2.11.0
> >
> > _______________________________________________
> > ffmpeg-devel mailing list
> > ffmpeg-devel at ffmpeg.org
> > http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
> >
>
> NAK
>
> I'd really rather have a json or xml output which would be codec
> independent rather than this codec specific format. Which could then be fed
> into an analyzer (like the AV1 analyzer). We discussed this before.

Producing some human/tool readable textual format is not something
that should be put on the decoders to do, its something a tool like
ffprobe could do, or at worst put as a formatting helper function in
avutil.
Internally, we should have a common binary format (because thats the
easiest to handle from C) that expresses the MB types. Does our
current format qualify for that?  I don't know, maybe not, can it deal
with varying block sizes modern codecs use?

- Hendrik