[FFmpeg-devel] [PATCH 5/7] lavf: add automatic bitstream filtering

[Rodger Combs]

This solves the problem discussed in https://ffmpeg.org/pipermail/ffmpeg-devel/2015-September/179238.html
by allowing AVCodec::write_header to be delayed until after packets have been
run through required bitstream filters in order to generate global extradata.

It also provides a mechanism by which a muxer can add a bitstream filter to a
stream automatically, rather than prompting the user to do so.
---
 libavformat/avformat.h | 29 +++++++++++++++++++++++++++++
 libavformat/mux.c      | 42 +++++++++++++++++++++++++++++++++++++++---
 2 files changed, 68 insertions(+), 3 deletions(-)

diff --git a/libavformat/avformat.h b/libavformat/avformat.h
index 5226b0a..f3c8260 100644
--- a/libavformat/avformat.h
+++ b/libavformat/avformat.h
@@ -598,6 +598,21 @@ typedef struct AVOutputFormat {
      */
     int (*free_device_capabilities)(struct AVFormatContext *s, struct AVDeviceCapabilitiesQuery *caps);
     enum AVCodecID data_codec; /**< default data codec */
+    /**
+     * Initialize format. May allocate data here, and set any AVFormatContext or
+     * AVStream parameters that need to be set before packets are sent.
+     * Must not write output.
+     *
+     * FIXME: Data allocated here would be leaked if there's a failure before
+     * write_header is called. Ban allocations? Add a `deinit` cleanup function?
+     */
+    int (*init)(struct AVFormatContext *);
+    /**
+     * Set up any necessary bitstream filtering and extract any extra data needed
+     * for the global header.
+     * Return 0 if more packets from this stream must be checked; 1 if not.
+     */
+    int (*check_bitstream)(struct AVFormatContext *, const AVPacket *pkt);
 } AVOutputFormat;
 /**
  * @}
@@ -1167,6 +1182,18 @@ typedef struct AVStream {
     AVRational display_aspect_ratio;
 
     struct FFFrac *priv_pts;
+
+    /**
+     * bitstream filter to run on stream
+     * - encoding: Set by muxer or user using av_add_bitstream_filter
+     * - decoding: unused
+     */
+    AVBitStreamFilterContext *bsfc;
+
+    /**
+     * internal check if check_bitstream should still be run on each packet
+     */
+    int bitstream_checked;
 } AVStream;
 
 AVRational av_stream_get_r_frame_rate(const AVStream *s);
@@ -1782,6 +1809,8 @@ typedef struct AVFormatContext {
      * Demuxing: Set by user.
      */
     int (*open_cb)(struct AVFormatContext *s, AVIOContext **p, const char *url, int flags, const AVIOInterruptCB *int_cb, AVDictionary **options);
+
+    int header_written;
 } AVFormatContext;
 
 int av_format_get_probe_score(const AVFormatContext *s);
diff --git a/libavformat/mux.c b/libavformat/mux.c
index c9ef490..ed2e543 100644
--- a/libavformat/mux.c
+++ b/libavformat/mux.c
@@ -400,7 +400,7 @@ FF_ENABLE_DEPRECATION_WARNINGS
          *options = tmp;
     }
 
-    return 0;
+    return s->oformat->init ? s->oformat->init(s) : 0;
 
 fail:
     av_dict_free(&tmp);
@@ -451,7 +451,7 @@ int avformat_write_header(AVFormatContext *s, AVDictionary **options)
     if ((ret = init_muxer(s, options)) < 0)
         return ret;
 
-    if (s->oformat->write_header) {
+    if (s->oformat->write_header && !s->oformat->check_bitstream) {
         ret = s->oformat->write_header(s);
         if (ret >= 0 && s->pb && s->pb->error < 0)
             ret = s->pb->error;
@@ -459,6 +459,7 @@ int avformat_write_header(AVFormatContext *s, AVDictionary **options)
             return ret;
         if (s->flush_packets && s->pb && s->pb->error >= 0 && s->flags & AVFMT_FLAG_FLUSH_PACKETS)
             avio_flush(s->pb);
+        s->header_written = 1;
     }
 
     if ((ret = init_pts(s)) < 0)
@@ -951,6 +952,18 @@ int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
             ret = AVERROR(EINVAL);
             goto fail;
         }
+
+        if (s->oformat->check_bitstream) {
+            if (!st->bitstream_checked) {
+                if ((ret = s->oformat->check_bitstream(s, pkt)) < 0)
+                    goto fail;
+                else if (ret == 1)
+                    st->bitstream_checked = 1;
+            }
+        }
+
+        if ((ret = av_apply_bitstream_filters(s, pkt, st->bsfc)) < 0)
+            goto fail;
     } else {
         av_log(s, AV_LOG_TRACE, "av_interleaved_write_frame FLUSH\n");
         flush = 1;
@@ -967,10 +980,22 @@ int av_interleaved_write_frame(AVFormatContext *s, AVPacket *pkt)
         if (ret <= 0) //FIXME cleanup needed for ret<0 ?
             return ret;
 
+        if (!s->header_written && s->oformat->write_header) {
+            ret = s->oformat->write_header(s);
+            if (ret >= 0 && s->pb && s->pb->error < 0)
+                ret = s->pb->error;
+            if (ret < 0)
+                goto fail2;
+            if (s->flush_packets && s->pb && s->pb->error >= 0 && s->flags & AVFMT_FLAG_FLUSH_PACKETS)
+                avio_flush(s->pb);
+            s->header_written = 1;
+        }
+
         ret = write_packet(s, &opkt);
         if (ret >= 0)
             s->streams[opkt.stream_index]->nb_frames++;
 
+fail2:
         av_free_packet(&opkt);
 
         if (ret < 0)
@@ -1007,8 +1032,19 @@ int av_write_trailer(AVFormatContext *s)
             goto fail;
     }
 
+    if (!s->header_written && s->oformat->write_header) {
+        ret = s->oformat->write_header(s);
+        if (ret >= 0 && s->pb && s->pb->error < 0)
+            ret = s->pb->error;
+        if (ret < 0)
+            goto fail;
+        if (s->flush_packets && s->pb && s->pb->error >= 0 && s->flags & AVFMT_FLAG_FLUSH_PACKETS)
+            avio_flush(s->pb);
+        s->header_written = 1;
+    }
+
 fail:
-    if (s->oformat->write_trailer)
+    if (s->header_written && s->oformat->write_trailer)
         if (ret >= 0) {
         ret = s->oformat->write_trailer(s);
         } else {
-- 
2.6.0