[FFmpeg-devel] [PATCH] lavf/matroskaenc.c: use dyn_buf to write header elements for non-seekable outputs.
Neil Birkbeck
neil.birkbeck at gmail.com
Mon Oct 10 04:13:41 EEST 2016
For non-seekable output files, larger elements written in write_header
(like larger attachments, or possibly many tags) can go over
IO_BUFFER_SIZE meaning seeking back to write valid seek head may fail.
Fate test checksums are the same when "-write_crc32=0". Adding dyn_buf
causes buffer to be seekable and so crc32 is written now (causing the
diffs).
Example to reproduce:
mkfifo /tmp/a.fifo
dd if=/dev/zero bs=1024 count=40 > /tmp/data
ffmpeg -nostats -nostdin -y -f lavfi -i testsrc -vframes 1 \
-attach /tmp/data -metadata:s:1 mimetype=test/data \
-f matroska /tmp/a.fifo & cat /tmp/a.fifo > /tmp/a.mkv
ffprobe /tmp/a.mkv
Previously would generate file like:
Duration: N/A, start: 0.000000, bitrate: N/A
Stream #0:0: Video: mpeg4 (Simple Profile), yuv420p, 320x240 [SAR 1:1 DAR 4:3], 25 fps, 25 tbr, 1k tbn, 25 tbc (default)
Metadata:
ENCODER : Lavc57.51.100 mpeg4
Stream #0:1: Video: mpeg4 (Simple Profile), none, 320x240 [SAR 1:1 DAR 4:3], 25 fps, 25 tbr, 1k tbn, 25 tbc (default)
Metadata:
ENCODER : Lavc57.51.100 mpeg4
Stream #0:2: Attachment: none
Metadata:
filename : data
mimetype : test/data
Stream #0:3: Attachment: none
Metadata:
filename : data
mimetype : test/data
Signed-off-by: Neil Birkbeck <neil.birkbeck at gmail.com>
---
libavformat/matroskaenc.c | 101 +++++++++++++++++++++++++++----------------
tests/fate/matroska.mak | 10 ++++-
tests/fate/wavpack.mak | 4 +-
tests/ref/fate/binsub-mksenc | 2 +-
4 files changed, 75 insertions(+), 42 deletions(-)
diff --git a/libavformat/matroskaenc.c b/libavformat/matroskaenc.c
index 593ddd1..26e2012 100644
--- a/libavformat/matroskaenc.c
+++ b/libavformat/matroskaenc.c
@@ -1244,10 +1244,10 @@ static int mkv_write_track(AVFormatContext *s, MatroskaMuxContext *mkv,
return 0;
}
-static int mkv_write_tracks(AVFormatContext *s)
+static int mkv_write_tracks(AVIOContext *pb, AVFormatContext *s)
{
MatroskaMuxContext *mkv = s->priv_data;
- AVIOContext *dyn_cp, *pb = s->pb;
+ AVIOContext *dyn_cp;
ebml_master tracks;
int i, ret, default_stream_exists = 0;
@@ -1272,10 +1272,10 @@ static int mkv_write_tracks(AVFormatContext *s)
return 0;
}
-static int mkv_write_chapters(AVFormatContext *s)
+static int mkv_write_chapters(AVIOContext *pb, AVFormatContext *s)
{
MatroskaMuxContext *mkv = s->priv_data;
- AVIOContext *dyn_cp, *pb = s->pb;
+ AVIOContext *dyn_cp;
ebml_master chapters, editionentry;
AVRational scale = {1, 1E9};
int i, ret;
@@ -1360,20 +1360,19 @@ static int mkv_write_simpletag(AVIOContext *pb, AVDictionaryEntry *t)
return 0;
}
-static int mkv_write_tag_targets(AVFormatContext *s,
+static int mkv_write_tag_targets(AVIOContext *pb,
+ MatroskaMuxContext *mkv,
unsigned int elementid, unsigned int uid,
ebml_master *tags, ebml_master* tag)
{
- AVIOContext *pb;
- MatroskaMuxContext *mkv = s->priv_data;
ebml_master targets;
int ret;
if (!tags->pos) {
- ret = mkv_add_seekhead_entry(mkv->main_seekhead, MATROSKA_ID_TAGS, avio_tell(s->pb));
+ ret = mkv_add_seekhead_entry(mkv->main_seekhead, MATROSKA_ID_TAGS, avio_tell(pb));
if (ret < 0) return ret;
- start_ebml_master_crc32(s->pb, &mkv->tags_bc, tags, MATROSKA_ID_TAGS, 0);
+ start_ebml_master_crc32(pb, &mkv->tags_bc, tags, MATROSKA_ID_TAGS, 0);
}
pb = mkv->tags_bc;
@@ -1396,15 +1395,15 @@ static int mkv_check_tag_name(const char *name, unsigned int elementid)
av_strcasecmp(name, "language"));
}
-static int mkv_write_tag(AVFormatContext *s, AVDictionary *m, unsigned int elementid,
+static int mkv_write_tag(AVIOContext *pb, MatroskaMuxContext *mkv,
+ AVDictionary *m, unsigned int elementid,
unsigned int uid, ebml_master *tags)
{
- MatroskaMuxContext *mkv = s->priv_data;
ebml_master tag;
int ret;
AVDictionaryEntry *t = NULL;
- ret = mkv_write_tag_targets(s, elementid, uid, tags, &tag);
+ ret = mkv_write_tag_targets(pb, mkv, elementid, uid, tags, &tag);
if (ret < 0)
return ret;
@@ -1431,7 +1430,7 @@ static int mkv_check_tag(AVDictionary *m, unsigned int elementid)
return 0;
}
-static int mkv_write_tags(AVFormatContext *s)
+static int mkv_write_tags(AVIOContext *pb, AVFormatContext *s)
{
MatroskaMuxContext *mkv = s->priv_data;
int i, ret;
@@ -1439,7 +1438,7 @@ static int mkv_write_tags(AVFormatContext *s)
ff_metadata_conv_ctx(s, ff_mkv_metadata_conv, NULL);
if (mkv_check_tag(s->metadata, 0)) {
- ret = mkv_write_tag(s, s->metadata, 0, 0, &mkv->tags);
+ ret = mkv_write_tag(pb, mkv, s->metadata, 0, 0, &mkv->tags);
if (ret < 0) return ret;
}
@@ -1449,28 +1448,28 @@ static int mkv_write_tags(AVFormatContext *s)
if (!mkv_check_tag(st->metadata, MATROSKA_ID_TAGTARGETS_TRACKUID))
continue;
- ret = mkv_write_tag(s, st->metadata, MATROSKA_ID_TAGTARGETS_TRACKUID, i + 1, &mkv->tags);
+ ret = mkv_write_tag(pb, mkv, st->metadata, MATROSKA_ID_TAGTARGETS_TRACKUID, i + 1, &mkv->tags);
if (ret < 0) return ret;
}
if (s->pb->seekable && !mkv->is_live) {
for (i = 0; i < s->nb_streams; i++) {
- AVIOContext *pb;
+ AVIOContext *tag_pb;
ebml_master tag_target;
ebml_master tag;
- mkv_write_tag_targets(s, MATROSKA_ID_TAGTARGETS_TRACKUID, i + 1, &mkv->tags, &tag_target);
- pb = mkv->tags_bc;
+ mkv_write_tag_targets(pb, mkv, MATROSKA_ID_TAGTARGETS_TRACKUID, i + 1, &mkv->tags, &tag_target);
+ tag_pb = mkv->tags_bc;
- tag = start_ebml_master(pb, MATROSKA_ID_SIMPLETAG, 0);
- put_ebml_string(pb, MATROSKA_ID_TAGNAME, "DURATION");
- mkv->stream_duration_offsets[i] = avio_tell(pb);
+ tag = start_ebml_master(tag_pb, MATROSKA_ID_SIMPLETAG, 0);
+ put_ebml_string(tag_pb, MATROSKA_ID_TAGNAME, "DURATION");
+ mkv->stream_duration_offsets[i] = avio_tell(tag_pb);
// Reserve space to write duration as a 20-byte string.
// 2 (ebml id) + 1 (data size) + 20 (data)
- put_ebml_void(pb, 23);
- end_ebml_master(pb, tag);
- end_ebml_master(pb, tag_target);
+ put_ebml_void(tag_pb, 23);
+ end_ebml_master(tag_pb, tag);
+ end_ebml_master(tag_pb, tag_target);
}
}
@@ -1480,23 +1479,23 @@ static int mkv_write_tags(AVFormatContext *s)
if (!mkv_check_tag(ch->metadata, MATROSKA_ID_TAGTARGETS_CHAPTERUID))
continue;
- ret = mkv_write_tag(s, ch->metadata, MATROSKA_ID_TAGTARGETS_CHAPTERUID, ch->id + mkv->chapter_id_offset, &mkv->tags);
+ ret = mkv_write_tag(pb, mkv, ch->metadata, MATROSKA_ID_TAGTARGETS_CHAPTERUID, ch->id + mkv->chapter_id_offset, &mkv->tags);
if (ret < 0) return ret;
}
if (mkv->tags.pos) {
if (s->pb->seekable && !mkv->is_live)
- put_ebml_void(s->pb, avio_tell(mkv->tags_bc) + ((mkv->write_crc && mkv->mode != MODE_WEBM) ? 2 /* ebml id + data size */ + 4 /* CRC32 */ : 0));
+ put_ebml_void(pb, avio_tell(mkv->tags_bc) + ((mkv->write_crc && mkv->mode != MODE_WEBM) ? 2 /* ebml id + data size */ + 4 /* CRC32 */ : 0));
else
- end_ebml_master_crc32(s->pb, &mkv->tags_bc, mkv, mkv->tags);
+ end_ebml_master_crc32(pb, &mkv->tags_bc, mkv, mkv->tags);
}
return 0;
}
-static int mkv_write_attachments(AVFormatContext *s)
+static int mkv_write_attachments(AVIOContext *pb, AVFormatContext *s)
{
MatroskaMuxContext *mkv = s->priv_data;
- AVIOContext *dyn_cp, *pb = s->pb;
+ AVIOContext *dyn_cp;
ebml_master attachments;
AVLFG c;
int i, ret;
@@ -1605,7 +1604,7 @@ static int64_t get_metadata_duration(AVFormatContext *s)
static int mkv_write_header(AVFormatContext *s)
{
MatroskaMuxContext *mkv = s->priv_data;
- AVIOContext *pb = s->pb;
+ AVIOContext *dyn_bc = 0, *pb = s->pb;
ebml_master ebml_header;
AVDictionaryEntry *tag;
int ret, i, version = 2;
@@ -1621,6 +1620,18 @@ static int mkv_write_header(AVFormatContext *s)
av_dict_get(s->metadata, "alpha_mode", NULL, 0))
version = 4;
+ // The IO_BUFFER_SIZE of s->pb may not be big enough to hold all header
+ // data, meaning we wouldn't be able to seek back to output a correct
+ // seek head for streaming outputs; so allocate a dyn_bc for non-seekable
+ // outputs.
+ if (!pb->seekable) {
+ if ((ret = avio_open_dyn_buf(&dyn_bc)) < 0) {
+ av_log(s, AV_LOG_ERROR, "Failed to open dynamic buffer\n");
+ return ret;
+ }
+ pb = dyn_bc;
+ }
+
for (i = 0; i < s->nb_streams; i++) {
if (s->streams[i]->codecpar->codec_id == AV_CODEC_ID_ATRAC3 ||
s->streams[i]->codecpar->codec_id == AV_CODEC_ID_COOK ||
@@ -1732,14 +1743,14 @@ static int mkv_write_header(AVFormatContext *s)
if (s->pb->seekable)
put_ebml_void(s->pb, avio_tell(pb) + ((mkv->write_crc && mkv->mode != MODE_WEBM) ? 2 /* ebml id + data size */ + 4 /* CRC32 */ : 0));
else
- end_ebml_master_crc32(s->pb, &mkv->info_bc, mkv, mkv->info);
- pb = s->pb;
+ end_ebml_master_crc32(dyn_bc, &mkv->info_bc, mkv, mkv->info);
+ pb = dyn_bc ? dyn_bc : s->pb;
// initialize stream_duration fields
mkv->stream_durations = av_mallocz(s->nb_streams * sizeof(int64_t));
mkv->stream_duration_offsets = av_mallocz(s->nb_streams * sizeof(int64_t));
- ret = mkv_write_tracks(s);
+ ret = mkv_write_tracks(pb, s);
if (ret < 0)
goto fail;
@@ -1747,15 +1758,15 @@ static int mkv_write_header(AVFormatContext *s)
mkv->chapter_id_offset = FFMAX(mkv->chapter_id_offset, 1LL - s->chapters[i]->id);
if (mkv->mode != MODE_WEBM) {
- ret = mkv_write_chapters(s);
+ ret = mkv_write_chapters(pb, s);
if (ret < 0)
goto fail;
- ret = mkv_write_tags(s);
+ ret = mkv_write_tags(pb, s);
if (ret < 0)
goto fail;
- ret = mkv_write_attachments(s);
+ ret = mkv_write_attachments(pb, s);
if (ret < 0)
goto fail;
}
@@ -1768,7 +1779,7 @@ static int mkv_write_header(AVFormatContext *s)
ret = AVERROR(ENOMEM);
goto fail;
}
- if (pb->seekable && mkv->reserve_cues_space) {
+ if (s->pb->seekable && mkv->reserve_cues_space) {
mkv->cues_pos = avio_tell(pb);
put_ebml_void(pb, mkv->reserve_cues_space);
}
@@ -1777,6 +1788,15 @@ static int mkv_write_header(AVFormatContext *s)
mkv->cur_audio_pkt.size = 0;
mkv->cluster_pos = -1;
+ if (dyn_bc) {
+ uint8_t *buf = NULL;
+ int size = avio_close_dyn_buf(dyn_bc, &buf);
+ avio_write(s->pb, buf, size);
+ av_free(buf);
+
+ pb = s->pb;
+ dyn_bc = NULL;
+ }
avio_flush(pb);
// start a new cluster every 5 MB or 5 sec, or 32k / 1 sec for streaming or
@@ -1795,6 +1815,11 @@ static int mkv_write_header(AVFormatContext *s)
return 0;
fail:
+ if (dyn_bc) {
+ uint8_t *buf = NULL;
+ avio_close_dyn_buf(dyn_bc, &buf);
+ av_free(buf);
+ }
mkv_free(mkv);
return ret;
}
@@ -2215,7 +2240,7 @@ static int mkv_write_trailer(AVFormatContext *s)
}
if (mkv->mode != MODE_WEBM) {
- ret = mkv_write_chapters(s);
+ ret = mkv_write_chapters(pb, s);
if (ret < 0)
return ret;
}
diff --git a/tests/fate/matroska.mak b/tests/fate/matroska.mak
index 7de9a59..b2dea08 100644
--- a/tests/fate/matroska.mak
+++ b/tests/fate/matroska.mak
@@ -4,6 +4,14 @@
FATE_MATROSKA-$(call DEMMUX, MATROSKA, MATROSKA) += fate-matroska-remux
fate-matroska-remux: CMD = md5 -i $(TARGET_SAMPLES)/vp9-test-vectors/vp90-2-2pass-akiyo.webm -color_trc 4 -c:v copy -fflags +bitexact -strict -2 -f matroska
fate-matroska-remux: CMP = oneline
-fate-matroska-remux: REF = d1a5fc15908ba10ca3efa282059ca79f
+fate-matroska-remux: REF = 66cd120834e017bf38524dc3dfa0f04c
+
+# Tests that matroska muxing with larger attachments to non-seekable
+# outputs (via md5) is correct.
+FATE_MATROSKA-$(call DEMMUX, MATROSKA, MATROSKA) += fate-matroska-attach
+fate-matroska-attach: CMD = md5 -i $(TARGET_SAMPLES)/vp9-test-vectors/vp90-2-2pass-akiyo.webm -attach $(TARGET_SAMPLES)/png1/lena-rgb24.png -metadata:s:1 mimetype=image/png -c:v copy -fflags +bitexact -strict -2 -f matroska
+fate-matroska-attach: CMP = oneline
+fate-matroska-attach: REF = 58ab699c898e0ad39bbae7ce3cada51c
+
FATE_SAMPLES_AVCONV += $(FATE_MATROSKA-yes)
diff --git a/tests/fate/wavpack.mak b/tests/fate/wavpack.mak
index 32ae3f6..11ddbaf 100644
--- a/tests/fate/wavpack.mak
+++ b/tests/fate/wavpack.mak
@@ -91,12 +91,12 @@ fate-wavpack-matroskamode: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/special/matros
FATE_WAVPACK-$(call DEMMUX, WV, MATROSKA) += fate-wavpack-matroska_mux-mono
fate-wavpack-matroska_mux-mono: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/num_channels/mono_16bit_int.wv -c copy -fflags +bitexact -f matroska
fate-wavpack-matroska_mux-mono: CMP = oneline
-fate-wavpack-matroska_mux-mono: REF = 11773e2a518edc788475f3880d849230
+fate-wavpack-matroska_mux-mono: REF = 51df056b305bef77147e11ead9cd36fa
FATE_WAVPACK-$(call DEMMUX, WV, MATROSKA) += fate-wavpack-matroska_mux-61
fate-wavpack-matroska_mux-61: CMD = md5 -i $(TARGET_SAMPLES)/wavpack/num_channels/eva_2.22_6.1_16bit-partial.wv -c copy -fflags +bitexact -f matroska
fate-wavpack-matroska_mux-61: CMP = oneline
-fate-wavpack-matroska_mux-61: REF = 9641abdf596c10c2e21bd9b026d4bade
+fate-wavpack-matroska_mux-61: REF = 506d23d9061b3d7b1d48d04ef9a86a09
FATE_SAMPLES_AVCONV += $(FATE_WAVPACK-yes)
fate-wavpack: $(FATE_WAVPACK-yes)
diff --git a/tests/ref/fate/binsub-mksenc b/tests/ref/fate/binsub-mksenc
index f247d9d..6b5588c 100644
--- a/tests/ref/fate/binsub-mksenc
+++ b/tests/ref/fate/binsub-mksenc
@@ -1 +1 @@
-f80f42e646fce972e73aa6d99dcfa470
+538bbad138e9c08a445ca7c586233d9b
--
2.8.0.rc3.226.g39d4020
More information about the ffmpeg-devel
mailing list