[FFmpeg-devel] [PATCH] ffmpeg: insert bitmap subtitles as video in filters.

Nicolas George nicolas.george at normalesup.org
Tue Jul 31 16:54:34 CEST 2012


With this feature, it becomes possible to perform commonly
requested tasks, such as hardcoding bitmap subtitles.

This will be reverted once libavfilter has proper support
for subtitles. All the changes have the string "sub2video"
in them, it makes it easy to spot the parts.

Signed-off-by: Nicolas George <nicolas.george at normalesup.org>
---
 Changelog       |    1 +
 doc/ffmpeg.texi |   14 +++++
 ffmpeg.c        |  165 +++++++++++++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 176 insertions(+), 4 deletions(-)


Added a comment, as requested by Clément, and rebased on top of current Git.
Otherwise unchanged.

Does anyone want to comment?


diff --git a/Changelog b/Changelog
index 79a8f4c..64676a7 100644
--- a/Changelog
+++ b/Changelog
@@ -38,6 +38,7 @@ version next:
 - alphaextract and alphamerge filters
 - concat filter
 - flite filter
+- bitmap subtitles in filters (experimental and temporary)
 
 
 version 0.11:
diff --git a/doc/ffmpeg.texi b/doc/ffmpeg.texi
index 904a505..6d71532 100644
--- a/doc/ffmpeg.texi
+++ b/doc/ffmpeg.texi
@@ -989,6 +989,20 @@ ffmpeg -i video.mkv -i image.png -filter_complex 'overlay' out.mkv
 @end example
 @end table
 
+As a special exception, you can use a bitmap subtitle stream as input: it
+will be converted into a video with the same size as the largest video in
+the file, or 720×576 if no video is present. Note that this is an
+experimental and temporary solution. It will be removed once libavfilter has
+proper support for subtitles.
+
+For example, to hardcode subtitles on top of a DVB-T recording stored in
+MPEG-TS format, delaying the subtitles by 1 second:
+ at example
+ffmpeg -i input.ts -filter_complex \
+  '[#0x2ef] setpts=PTS+1/TB [sub] ; [#0x2d0] [sub] overlay' \
+  -sn -map '#0x2dc' output.mkv
+ at end example
+
 @section Preset files
 A preset file contains a sequence of @var{option}=@var{value} pairs,
 one for each line, specifying a sequence of options which would be
diff --git a/ffmpeg.c b/ffmpeg.c
index c2ea5bd..206580b 100644
--- a/ffmpeg.c
+++ b/ffmpeg.c
@@ -249,6 +249,12 @@ typedef struct InputStream {
     int      resample_channels;
     uint64_t resample_channel_layout;
 
+    struct sub2video {
+        int64_t last_pts;
+        AVFilterBufferRef *ref;
+        int w, h;
+    } sub2video;
+
     /* a pool of free buffers for decoded data */
     FrameBuffer *buffer_pool;
     int dr1;
@@ -504,6 +510,143 @@ static void update_benchmark(const char *fmt, ...)
     }
 }
 
+/* sub2video hack:
+   Convert subtitles to video with alpha to insert them in filter graphs.
+   This is a temporary solution until libavfilter gets real subtitles support.
+ */
+
+
+static int sub2video_prepare(InputStream *ist)
+{
+    AVFormatContext *avf = input_files[ist->file_index]->ctx;
+    int i, ret, w, h;
+    uint8_t *image[4];
+    int linesize[4];
+
+    w = ist->st->codec->width;
+    h = ist->st->codec->height;
+    if (!(w && h)) {
+        for (i = 0; i < avf->nb_streams; i++) {
+            if (avf->streams[i]->codec->codec_type == AVMEDIA_TYPE_VIDEO) {
+                w = FFMAX(w, avf->streams[i]->codec->width);
+                h = FFMAX(h, avf->streams[i]->codec->height);
+            }
+        }
+        if (!(w && h)) {
+            w = FFMAX(w, 720);
+            h = FFMAX(h, 576);
+        }
+        av_log(avf, AV_LOG_INFO, "sub2video: using %dx%d canvas\n", w, h);
+    }
+    ist->sub2video.w = ist->st->codec->width  = w;
+    ist->sub2video.h = ist->st->codec->height = h;
+
+    /* rectangles are PIX_FMT_PAL8, but we have no guarantee that the
+       palettes for all rectangles are identical or compatible */
+    ist->st->codec->pix_fmt = PIX_FMT_RGB32;
+
+    ret = av_image_alloc(image, linesize, w, h, PIX_FMT_RGB32, 32);
+    if (ret < 0)
+        return ret;
+    memset(image[0], 0, h * linesize[0]);
+    ist->sub2video.ref = avfilter_get_video_buffer_ref_from_arrays(
+            image, linesize, AV_PERM_READ | AV_PERM_PRESERVE,
+            w, h, PIX_FMT_RGB32);
+    if (!ist->sub2video.ref) {
+        av_free(image[0]);
+        return AVERROR(ENOMEM);
+    }
+    return 0;
+}
+
+static void sub2video_copy_rect(uint8_t *dst, int dst_linesize, int w, int h,
+                                AVSubtitleRect *r)
+{
+    uint32_t *pal, *dst2;
+    uint8_t *src, *src2;
+    int x, y;
+
+    if (r->type != SUBTITLE_BITMAP) {
+        av_log(NULL, AV_LOG_WARNING, "sub2video: non-bitmap subtitle\n");
+        return;
+    }
+    if (r->x < 0 || r->x + r->w > w || r->y < 0 || r->y + r->h > h) {
+        av_log(NULL, AV_LOG_WARNING, "sub2video: rectangle overflowing\n");
+        return;
+    }
+
+    dst += r->y * dst_linesize + r->x * 4;
+    src = r->pict.data[0];
+    pal = (uint32_t *)r->pict.data[1];
+    for (y = 0; y < r->h; y++) {
+        dst2 = (uint32_t *)dst;
+        src2 = src;
+        for (x = 0; x < r->w; x++)
+            *(dst2++) = pal[*(src2++)];
+        dst += dst_linesize;
+        src += r->pict.linesize[0];
+    }
+}
+
+static void sub2video_push_ref(InputStream *ist, int64_t pts)
+{
+    AVFilterBufferRef *ref = ist->sub2video.ref;
+    int i;
+
+    ist->sub2video.last_pts = ref->pts = pts;
+    for (i = 0; i < ist->nb_filters; i++)
+        av_buffersrc_add_ref(ist->filters[i]->filter,
+                             avfilter_ref_buffer(ref, ~0),
+                             AV_BUFFERSRC_FLAG_NO_CHECK_FORMAT |
+                             AV_BUFFERSRC_FLAG_NO_COPY);
+}
+
+static void sub2video_update(InputStream *ist, AVSubtitle *sub, int64_t pts)
+{
+    int w = ist->sub2video.w, h = ist->sub2video.h;
+    AVFilterBufferRef *ref = ist->sub2video.ref;
+    int8_t *dst          = ref->data    [0];
+    int     dst_linesize = ref->linesize[0];
+    int i;
+
+    memset(dst, 0, h * dst_linesize);
+    for (i = 0; i < sub->num_rects; i++)
+        sub2video_copy_rect(dst, dst_linesize, w, h, sub->rects[i]);
+    sub2video_push_ref(ist, pts);
+}
+
+static void sub2video_heartbeat(InputStream *ist, int64_t pts)
+{
+    InputFile *inf = input_files[ist->file_index];
+    int i, j, nb_req;
+    int64_t pts2;
+
+    for (i = 0; i < inf->nb_streams; i++) {
+        InputStream *ist2 = input_streams[inf->ist_index + i];
+        if (!ist2->sub2video.ref)
+            continue;
+        /* subtitles seem to be usually muxed ahead of other streams;
+           if not, substracting a reasonable time here is necessary */
+        pts2 = av_rescale_q(pts, ist->st->time_base, ist2->st->time_base);
+        if (pts2 <= ist2->sub2video.last_pts)
+            continue;
+        for (j = 0, nb_req = 0; j < ist2->nb_filters; j++)
+            nb_req += av_buffersrc_get_nb_failed_requests(ist2->filters[j]->filter);
+        if (nb_req)
+            sub2video_push_ref(ist2, pts2);
+    }
+}
+
+static void sub2video_flush(InputStream *ist)
+{
+    int i;
+
+    for (i = 0; i < ist->nb_filters; i++)
+        av_buffersrc_add_ref(ist->filters[i]->filter, NULL, 0);
+}
+
+/* end of sub2video hack */
+
 static void reset_options(OptionsContext *o, int is_input)
 {
     const OptionDef *po = options;
@@ -745,7 +888,9 @@ static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
         s = input_files[file_idx]->ctx;
 
         for (i = 0; i < s->nb_streams; i++) {
-            if (s->streams[i]->codec->codec_type != type)
+            if (s->streams[i]->codec->codec_type != type &&
+                !(s->streams[i]->codec->codec_type == AVMEDIA_TYPE_SUBTITLE &&
+                  type == AVMEDIA_TYPE_VIDEO /* sub2video hack */))
                 continue;
             if (check_stream_specifier(s, s->streams[i], *p == ':' ? p + 1 : p) == 1) {
                 st = s->streams[i];
@@ -1025,6 +1170,12 @@ static int configure_input_video_filter(FilterGraph *fg, InputFilter *ifilter,
     int pad_idx = in->pad_idx;
     int ret;
 
+    if (ist->st->codec->codec_type == AVMEDIA_TYPE_SUBTITLE) {
+        ret = sub2video_prepare(ist);
+        if (ret < 0)
+            return ret;
+    }
+
     sar = ist->st->sample_aspect_ratio.num ?
           ist->st->sample_aspect_ratio :
           ist->st->codec->sample_aspect_ratio;
@@ -1413,6 +1564,7 @@ void av_noreturn exit_program(int ret)
         av_freep(&input_streams[i]->decoded_frame);
         av_dict_free(&input_streams[i]->opts);
         free_buffer_pool(&input_streams[i]->buffer_pool);
+        avfilter_unref_bufferp(&input_streams[i]->sub2video.ref);
         av_freep(&input_streams[i]->filters);
         av_freep(&input_streams[i]);
     }
@@ -2632,13 +2784,16 @@ static int transcode_subtitles(InputStream *ist, AVPacket *pkt, int *got_output)
     AVSubtitle subtitle;
     int i, ret = avcodec_decode_subtitle2(ist->st->codec,
                                           &subtitle, got_output, pkt);
-    if (ret < 0)
-        return ret;
-    if (!*got_output)
+    if (ret < 0 || !*got_output) {
+        if (!pkt->size)
+            sub2video_flush(ist);
         return ret;
+    }
 
     rate_emu_sleep(ist);
 
+    sub2video_update(ist, &subtitle, pkt->pts);
+
     for (i = 0; i < nb_output_streams; i++) {
         OutputStream *ost = output_streams[i];
 
@@ -3843,6 +3998,8 @@ static int transcode(void)
             }
         }
 
+        sub2video_heartbeat(ist, pkt.pts);
+
         // fprintf(stderr,"read #%d.%d size=%d\n", ist->file_index, ist->st->index, pkt.size);
         if ((ret = output_packet(ist, &pkt)) < 0 ||
             ((ret = poll_filters()) < 0 && ret != AVERROR_EOF)) {
-- 
1.7.10.4



More information about the ffmpeg-devel mailing list