[FFmpeg-devel] [PATCH v2 17/18] fftools/ffmpeg: add loopback decoding
Anton Khirnov
anton at khirnov.net
Thu Mar 7 16:42:36 EET 2024
This allows to send an encoder's output back to decoding and feed the
result into a complex filtergraph.
---
Now using [dec:X] instead of [decX] in filtergraph link labels, as
suggested by Marvin on IRC.
---
Changelog | 1 +
doc/ffmpeg.texi | 49 ++++++++++--
fftools/cmdutils.c | 2 +-
fftools/cmdutils.h | 7 +-
fftools/ffmpeg.c | 26 +++++++
fftools/ffmpeg.h | 22 ++++++
fftools/ffmpeg_dec.c | 164 +++++++++++++++++++++++++++++++++++++++-
fftools/ffmpeg_enc.c | 19 +++++
fftools/ffmpeg_filter.c | 47 +++++++++++-
fftools/ffmpeg_opt.c | 13 +++-
10 files changed, 336 insertions(+), 14 deletions(-)
diff --git a/Changelog b/Changelog
index 0ba3a77c08..22186697fb 100644
--- a/Changelog
+++ b/Changelog
@@ -31,6 +31,7 @@ version <next>:
- removed deprecated ffmpeg CLI options -psnr and -map_channel
- DVD-Video demuxer, powered by libdvdnav and libdvdread
- ffprobe -show_stream_groups option
+- ffmpeg CLI loopback decoders
version 6.1:
diff --git a/doc/ffmpeg.texi b/doc/ffmpeg.texi
index bee3cd4c70..a38ef834e1 100644
--- a/doc/ffmpeg.texi
+++ b/doc/ffmpeg.texi
@@ -219,6 +219,40 @@ Since there is no decoding or encoding, it is very fast and there is no quality
loss. However, it might not work in some cases because of many factors. Applying
filters is obviously also impossible, since filters work on uncompressed data.
+ at section Loopback decoders
+While decoders are normally associated with demuxer streams, it is also possible
+to create "loopback" decoders that decode the output from some encoder and allow
+it to be fed back to complex filtergraphs. This is done with the @code{-dec}
+directive, which takes as a parameter the index of the output stream that should
+be decoded. Every such directive creates a new loopback decoder, indexed with
+successive integers starting at zero. These indices should then be used to refer
+to loopback decoders in complex filtergraph link labels, as described in the
+documentation for @option{-filter_complex}.
+
+E.g. the following example:
+
+ at example
+ffmpeg -i INPUT \
+ -map 0:v:0 -c:v libx264 -crf 45 -f null - \
+ -dec 0:0 -filter_complex '[0:v][dec:0]hstack[stack]' \
+ -map '[stack]' -c:v ffv1 OUTPUT
+ at end example
+
+reads an input video and
+ at itemize
+ at item
+(line 2) encodes it with @code{libx264} at low quality;
+
+ at item
+(line 3) decodes this encoded stream and places it side by side with the
+original input video;
+
+ at item
+(line 4) combined video is then losslessly encoded and written into
+ at file{OUTPUT}.
+
+ at end itemize
+
@c man end DETAILED DESCRIPTION
@chapter Stream selection
@@ -2105,11 +2139,16 @@ type -- see the @option{-filter} options. @var{filtergraph} is a description of
the filtergraph, as described in the ``Filtergraph syntax'' section of the
ffmpeg-filters manual.
-Input link labels must refer to input streams using the
- at code{[file_index:stream_specifier]} syntax (i.e. the same as @option{-map}
-uses). If @var{stream_specifier} matches multiple streams, the first one will be
-used. An unlabeled input will be connected to the first unused input stream of
-the matching type.
+Input link labels must refer to either input streams or loopback decoders. For
+input streams, use the @code{[file_index:stream_specifier]} syntax (i.e. the
+same as @option{-map} uses). If @var{stream_specifier} matches multiple streams,
+the first one will be used.
+
+For decoders, the link label must be [dec:@var{dec_idx}], where @var{dec_idx} is
+the index of the loopback decoder to be connected to given input.
+
+An unlabeled input will be connected to the first unused input stream of the
+matching type.
Output link labels are referred to with @option{-map}. Unlabeled outputs are
added to the first output file.
diff --git a/fftools/cmdutils.c b/fftools/cmdutils.c
index 8efec942bd..f3c258bb99 100644
--- a/fftools/cmdutils.c
+++ b/fftools/cmdutils.c
@@ -528,7 +528,7 @@ static void check_options(const OptionDef *po)
{
while (po->name) {
if (po->flags & OPT_PERFILE)
- av_assert0(po->flags & (OPT_INPUT | OPT_OUTPUT));
+ av_assert0(po->flags & (OPT_INPUT | OPT_OUTPUT | OPT_DECODER));
if (po->type == OPT_TYPE_FUNC)
av_assert0(!(po->flags & (OPT_FLAG_OFFSET | OPT_FLAG_SPEC)));
diff --git a/fftools/cmdutils.h b/fftools/cmdutils.h
index 86428b3fa4..d0c773663b 100644
--- a/fftools/cmdutils.h
+++ b/fftools/cmdutils.h
@@ -144,8 +144,8 @@ typedef struct OptionDef {
#define OPT_AUDIO (1 << 4)
#define OPT_SUBTITLE (1 << 5)
#define OPT_DATA (1 << 6)
-/* The option is per-file (currently ffmpeg-only). At least one of OPT_INPUT or
- * OPT_OUTPUT must be set when this flag is in use.
+/* The option is per-file (currently ffmpeg-only). At least one of OPT_INPUT,
+ * OPT_OUTPUT, OPT_DECODER must be set when this flag is in use.
*/
#define OPT_PERFILE (1 << 7)
@@ -175,6 +175,9 @@ typedef struct OptionDef {
* name is stored in u1.name_canon */
#define OPT_HAS_CANON (1 << 14)
+/* ffmpeg-only - OPT_PERFILE may apply to standalone decoders */
+#define OPT_DECODER (1 << 15)
+
union {
void *dst_ptr;
int (*func_arg)(void *, const char *, const char *);
diff --git a/fftools/ffmpeg.c b/fftools/ffmpeg.c
index ffd25f4c5b..d4e5f978f1 100644
--- a/fftools/ffmpeg.c
+++ b/fftools/ffmpeg.c
@@ -131,6 +131,9 @@ int nb_output_files = 0;
FilterGraph **filtergraphs;
int nb_filtergraphs;
+Decoder **decoders;
+int nb_decoders;
+
#if HAVE_TERMIOS_H
/* init terminal so that we can grab keys */
@@ -340,6 +343,10 @@ static void ffmpeg_cleanup(int ret)
for (int i = 0; i < nb_input_files; i++)
ifile_close(&input_files[i]);
+ for (int i = 0; i < nb_decoders; i++)
+ dec_free(&decoders[i]);
+ av_freep(&decoders);
+
if (vstats_file) {
if (fclose(vstats_file))
av_log(NULL, AV_LOG_ERROR,
@@ -404,6 +411,10 @@ InputStream *ist_iter(InputStream *prev)
static void frame_data_free(void *opaque, uint8_t *data)
{
+ FrameData *fd = (FrameData *)data;
+
+ avcodec_parameters_free(&fd->par_enc);
+
av_free(data);
}
@@ -430,6 +441,21 @@ static int frame_data_ensure(AVBufferRef **dst, int writable)
const FrameData *fd_src = (const FrameData *)src->data;
memcpy(fd, fd_src, sizeof(*fd));
+ fd->par_enc = NULL;
+
+ if (fd_src->par_enc) {
+ int ret = 0;
+
+ fd->par_enc = avcodec_parameters_alloc();
+ ret = fd->par_enc ?
+ avcodec_parameters_copy(fd->par_enc, fd_src->par_enc) :
+ AVERROR(ENOMEM);
+ if (ret < 0) {
+ av_buffer_unref(dst);
+ av_buffer_unref(&src);
+ return ret;
+ }
+ }
av_buffer_unref(&src);
} else {
diff --git a/fftools/ffmpeg.h b/fftools/ffmpeg.h
index 1966817bc3..7454089c2d 100644
--- a/fftools/ffmpeg.h
+++ b/fftools/ffmpeg.h
@@ -331,6 +331,8 @@ typedef struct DecoderOpts {
typedef struct Decoder {
const AVClass *class;
+ enum AVMediaType type;
+
const uint8_t *subtitle_header;
int subtitle_header_size;
@@ -606,6 +608,8 @@ typedef struct FrameData {
int bits_per_raw_sample;
int64_t wallclock[LATENCY_PROBE_NB];
+
+ AVCodecParameters *par_enc;
} FrameData;
extern InputFile **input_files;
@@ -617,6 +621,10 @@ extern int nb_output_files;
extern FilterGraph **filtergraphs;
extern int nb_filtergraphs;
+// standalone decoders (not tied to demuxed streams)
+extern Decoder **decoders;
+extern int nb_decoders;
+
extern char *vstats_filename;
extern float dts_delta_threshold;
@@ -734,6 +742,11 @@ void hw_device_free_all(void);
*/
AVBufferRef *hw_device_for_filter(void);
+/**
+ * Create a standalone decoder.
+ */
+int dec_create(const OptionsContext *o, const char *arg, Scheduler *sch);
+
/**
* @param dec_opts Dictionary filled with decoder options. Its ownership
* is transferred to the decoder.
@@ -748,12 +761,21 @@ int dec_init(Decoder **pdec, Scheduler *sch,
AVFrame *param_out);
void dec_free(Decoder **pdec);
+/*
+ * Called by filters to connect decoder's output to given filtergraph input.
+ *
+ * @param opts filtergraph input options, to be filled by this function
+ */
+int dec_filter_add(Decoder *dec, InputFilter *ifilter, InputFilterOptions *opts);
+
int enc_alloc(Encoder **penc, const AVCodec *codec,
Scheduler *sch, unsigned sch_idx);
void enc_free(Encoder **penc);
int enc_open(void *opaque, const AVFrame *frame);
+int enc_loopback(Encoder *enc);
+
/*
* Initialize muxing state for the given stream, should be called
* after the codec/streamcopy setup has been done.
diff --git a/fftools/ffmpeg_dec.c b/fftools/ffmpeg_dec.c
index 3bf7ab4960..c41c5748e5 100644
--- a/fftools/ffmpeg_dec.c
+++ b/fftools/ffmpeg_dec.c
@@ -17,6 +17,7 @@
*/
#include "libavutil/avassert.h"
+#include "libavutil/avstring.h"
#include "libavutil/dict.h"
#include "libavutil/error.h"
#include "libavutil/log.h"
@@ -71,9 +72,16 @@ typedef struct DecoderPriv {
Scheduler *sch;
unsigned sch_idx;
+ // this decoder's index in decoders or -1
+ int index;
void *log_parent;
char log_name[32];
char *parent_name;
+
+ struct {
+ AVDictionary *opts;
+ const AVCodec *codec;
+ } standalone_init;
} DecoderPriv;
static DecoderPriv *dp_from_dec(Decoder *d)
@@ -101,6 +109,8 @@ void dec_free(Decoder **pdec)
av_frame_free(&dp->frame);
av_packet_free(&dp->pkt);
+ av_dict_free(&dp->standalone_init.opts);
+
for (int i = 0; i < FF_ARRAY_ELEMS(dp->sub_prev); i++)
av_frame_free(&dp->sub_prev[i]);
av_frame_free(&dp->sub_heartbeat);
@@ -145,6 +155,7 @@ static int dec_alloc(DecoderPriv **pdec, Scheduler *sch, int send_end_ts)
if (!dp->pkt)
goto fail;
+ dp->index = -1;
dp->dec.class = &dec_class;
dp->last_filter_in_rescale_delta = AV_NOPTS_VALUE;
dp->last_frame_pts = AV_NOPTS_VALUE;
@@ -754,11 +765,56 @@ static int packet_decode(DecoderPriv *dp, AVPacket *pkt, AVFrame *frame)
}
}
+static int dec_open(DecoderPriv *dp, AVDictionary **dec_opts,
+ const DecoderOpts *o, AVFrame *param_out);
+
+static int dec_standalone_open(DecoderPriv *dp, const AVPacket *pkt)
+{
+ DecoderOpts o;
+ const FrameData *fd;
+ char name[16];
+
+ if (!pkt->opaque_ref)
+ return AVERROR_BUG;
+ fd = (FrameData *)pkt->opaque_ref->data;
+
+ if (!fd->par_enc)
+ return AVERROR_BUG;
+
+ memset(&o, 0, sizeof(o));
+
+ o.par = fd->par_enc;
+ o.time_base = pkt->time_base;
+
+ o.codec = dp->standalone_init.codec;
+ if (!o.codec)
+ o.codec = avcodec_find_decoder(o.par->codec_id);
+ if (!o.codec) {
+ const AVCodecDescriptor *desc = avcodec_descriptor_get(o.par->codec_id);
+
+ av_log(dp, AV_LOG_ERROR, "Cannot find a decoder for codec ID '%s'\n",
+ desc ? desc->name : "?");
+ return AVERROR_DECODER_NOT_FOUND;
+ }
+
+ snprintf(name, sizeof(name), "dec%d", dp->index);
+ o.name = name;
+
+ return dec_open(dp, &dp->standalone_init.opts, &o, NULL);
+}
+
static void dec_thread_set_name(const DecoderPriv *dp)
{
- char name[16];
- snprintf(name, sizeof(name), "dec%s:%s", dp->parent_name,
- dp->dec_ctx->codec->name);
+ char name[16] = "dec";
+
+ if (dp->index >= 0)
+ av_strlcatf(name, sizeof(name), "%d", dp->index);
+ else if (dp->parent_name)
+ av_strlcat(name, dp->parent_name, sizeof(name));
+
+ if (dp->dec_ctx)
+ av_strlcatf(name, sizeof(name), ":%s", dp->dec_ctx->codec->name);
+
ff_thread_setname(name);
}
@@ -814,6 +870,22 @@ static int decoder_thread(void *arg)
av_log(dp, AV_LOG_VERBOSE, "Decoder thread received %s packet\n",
flush_buffers ? "flush" : "EOF");
+ // this is a standalone decoder that has not been initialized yet
+ if (!dp->dec_ctx) {
+ if (flush_buffers)
+ continue;
+ if (input_status < 0) {
+ av_log(dp, AV_LOG_ERROR,
+ "Cannot initialize a standalone decoder\n");
+ ret = input_status;
+ goto finish;
+ }
+
+ ret = dec_standalone_open(dp, dt.pkt);
+ if (ret < 0)
+ goto finish;
+ }
+
ret = packet_decode(dp, have_data ? dt.pkt : NULL, dt.frame);
av_packet_unref(dt.pkt);
@@ -1075,6 +1147,7 @@ static int dec_open(DecoderPriv *dp, AVDictionary **dec_opts,
dp->flags = o->flags;
dp->log_parent = o->log_parent;
+ dp->dec.type = codec->type;
dp->framerate_in = o->framerate;
dp->hwaccel_id = o->hwaccel_id;
@@ -1188,3 +1261,88 @@ fail:
dec_free((Decoder**)&dp);
return ret;
}
+
+int dec_create(const OptionsContext *o, const char *arg, Scheduler *sch)
+{
+ DecoderPriv *dp;
+
+ OutputFile *of;
+ OutputStream *ost;
+ int of_index, ost_index;
+ char *p;
+
+ unsigned enc_idx;
+ int ret;
+
+ ret = dec_alloc(&dp, sch, 0);
+ if (ret < 0)
+ return ret;
+
+ dp->index = nb_decoders;
+
+ ret = GROW_ARRAY(decoders, nb_decoders);
+ if (ret < 0) {
+ dec_free((Decoder **)&dp);
+ return ret;
+ }
+
+ decoders[nb_decoders - 1] = (Decoder *)dp;
+
+ of_index = strtol(arg, &p, 0);
+ if (of_index < 0 || of_index >= nb_output_files) {
+ av_log(dp, AV_LOG_ERROR, "Invalid output file index '%d' in %s\n", of_index, arg);
+ return AVERROR(EINVAL);
+ }
+ of = output_files[of_index];
+
+ ost_index = strtol(p + 1, NULL, 0);
+ if (ost_index < 0 || ost_index >= of->nb_streams) {
+ av_log(dp, AV_LOG_ERROR, "Invalid output stream index '%d' in %s\n", ost_index, arg);
+ return AVERROR(EINVAL);
+ }
+ ost = of->streams[ost_index];
+
+ if (!ost->enc) {
+ av_log(dp, AV_LOG_ERROR, "Output stream %s has no encoder\n", arg);
+ return AVERROR(EINVAL);
+ }
+
+ dp->dec.type = ost->type;
+
+ ret = enc_loopback(ost->enc);
+ if (ret < 0)
+ return ret;
+ enc_idx = ret;
+
+ ret = sch_connect(sch, SCH_ENC(enc_idx), SCH_DEC(dp->sch_idx));
+ if (ret < 0)
+ return ret;
+
+ ret = av_dict_copy(&dp->standalone_init.opts, o->g->codec_opts, 0);
+ if (ret < 0)
+ return ret;
+
+ if (o->codec_names.nb_opt) {
+ const char *name = o->codec_names.opt[o->codec_names.nb_opt - 1].u.str;
+ dp->standalone_init.codec = avcodec_find_decoder_by_name(name);
+ if (!dp->standalone_init.codec) {
+ av_log(dp, AV_LOG_ERROR, "No such decoder: %s\n", name);
+ return AVERROR_DECODER_NOT_FOUND;
+ }
+ }
+
+ return 0;
+}
+
+int dec_filter_add(Decoder *d, InputFilter *ifilter, InputFilterOptions *opts)
+{
+ DecoderPriv *dp = dp_from_dec(d);
+ char name[16];
+
+ snprintf(name, sizeof(name), "dec%d", dp->index);
+ opts->name = av_strdup(name);
+ if (!opts->name)
+ return AVERROR(ENOMEM);
+
+ return dp->sch_idx;
+}
diff --git a/fftools/ffmpeg_enc.c b/fftools/ffmpeg_enc.c
index 88635629f7..d1dae558e1 100644
--- a/fftools/ffmpeg_enc.c
+++ b/fftools/ffmpeg_enc.c
@@ -49,6 +49,7 @@ struct Encoder {
uint64_t packets_encoded;
int opened;
+ int attach_par;
Scheduler *sch;
unsigned sch_idx;
@@ -693,6 +694,18 @@ static int encode_frame(OutputFile *of, OutputStream *ost, AVFrame *frame,
return AVERROR(ENOMEM);
fd->wallclock[LATENCY_PROBE_ENC_POST] = av_gettime_relative();
+ // attach stream parameters to first packet if requested
+ avcodec_parameters_free(&fd->par_enc);
+ if (e->attach_par && !e->packets_encoded) {
+ fd->par_enc = avcodec_parameters_alloc();
+ if (!fd->par_enc)
+ return AVERROR(ENOMEM);
+
+ ret = avcodec_parameters_from_context(fd->par_enc, enc);
+ if (ret < 0)
+ return ret;
+ }
+
if (enc->codec_type == AVMEDIA_TYPE_VIDEO) {
ret = update_video_stats(ost, pkt, !!vstats_filename);
if (ret < 0)
@@ -929,3 +942,9 @@ finish:
return ret;
}
+
+int enc_loopback(Encoder *enc)
+{
+ enc->attach_par = 1;
+ return enc->sch_idx;
+}
diff --git a/fftools/ffmpeg_filter.c b/fftools/ffmpeg_filter.c
index a29008387a..47530f13a7 100644
--- a/fftools/ffmpeg_filter.c
+++ b/fftools/ffmpeg_filter.c
@@ -710,6 +710,34 @@ static int ifilter_bind_ist(InputFilter *ifilter, InputStream *ist)
return 0;
}
+static int ifilter_bind_dec(InputFilterPriv *ifp, Decoder *dec)
+{
+ FilterGraphPriv *fgp = fgp_from_fg(ifp->ifilter.graph);
+ int ret, dec_idx;
+
+ av_assert0(!ifp->bound);
+ ifp->bound = 1;
+
+ if (ifp->type != dec->type) {
+ av_log(fgp, AV_LOG_ERROR, "Tried to connect %s decoder to %s filtergraph input\n",
+ av_get_media_type_string(dec->type), av_get_media_type_string(ifp->type));
+ return AVERROR(EINVAL);
+ }
+
+ ifp->type_src = ifp->type;
+
+ dec_idx = dec_filter_add(dec, &ifp->ifilter, &ifp->opts);
+ if (dec_idx < 0)
+ return dec_idx;
+
+ ret = sch_connect(fgp->sch, SCH_DEC(dec_idx),
+ SCH_FILTER_IN(fgp->sch_idx, ifp->index));
+ if (ret < 0)
+ return ret;
+
+ return 0;
+}
+
static int set_channel_layout(OutputFilterPriv *f, OutputStream *ost)
{
const AVCodec *c = ost->enc_ctx->codec;
@@ -1114,7 +1142,24 @@ static int fg_complex_bind_input(FilterGraph *fg, InputFilter *ifilter)
enum AVMediaType type = ifp->type;
int i, ret;
- if (ifp->linklabel) {
+ if (ifp->linklabel && !strncmp(ifp->linklabel, "dec:", 4)) {
+ // bind to a standalone decoder
+ int dec_idx;
+
+ dec_idx = strtol(ifp->linklabel + 4, NULL, 0);
+ if (dec_idx < 0 || dec_idx >= nb_decoders) {
+ av_log(fg, AV_LOG_ERROR, "Invalid decoder index %d in filtergraph description %s\n",
+ dec_idx, fgp->graph_desc);
+ return AVERROR(EINVAL);
+ }
+
+ ret = ifilter_bind_dec(ifp, decoders[dec_idx]);
+ if (ret < 0)
+ av_log(fg, AV_LOG_ERROR, "Error binding a decoder to filtergraph input %s\n",
+ ifilter->name);
+ return ret;
+ } else if (ifp->linklabel) {
+ // bind to an explicitly specified demuxer stream
AVFormatContext *s;
AVStream *st = NULL;
char *p;
diff --git a/fftools/ffmpeg_opt.c b/fftools/ffmpeg_opt.c
index b550382b4c..6cc549ddb4 100644
--- a/fftools/ffmpeg_opt.c
+++ b/fftools/ffmpeg_opt.c
@@ -1177,11 +1177,13 @@ void show_usage(void)
enum OptGroup {
GROUP_OUTFILE,
GROUP_INFILE,
+ GROUP_DECODER,
};
static const OptionGroupDef groups[] = {
[GROUP_OUTFILE] = { "output url", NULL, OPT_OUTPUT },
[GROUP_INFILE] = { "input url", "i", OPT_INPUT },
+ [GROUP_DECODER] = { "loopback decoder", "dec", OPT_DECODER },
};
static int open_files(OptionGroupList *l, const char *inout, Scheduler *sch,
@@ -1259,6 +1261,13 @@ int ffmpeg_parse_options(int argc, char **argv, Scheduler *sch)
goto fail;
}
+ /* create loopback decoders */
+ ret = open_files(&octx.groups[GROUP_DECODER], "decoder", sch, dec_create);
+ if (ret < 0) {
+ errmsg = "creating loopback decoders";
+ goto fail;
+ }
+
// bind unbound filtegraph inputs/outputs and check consistency
for (int i = 0; i < nb_filtergraphs; i++) {
ret = fg_finalise_bindings(filtergraphs[i]);
@@ -1367,11 +1376,11 @@ const OptionDef options[] = {
{ "recast_media", OPT_TYPE_BOOL, OPT_EXPERT,
{ &recast_media },
"allow recasting stream type in order to force a decoder of different media type" },
- { "c", OPT_TYPE_STRING, OPT_PERSTREAM | OPT_INPUT | OPT_OUTPUT | OPT_HAS_CANON,
+ { "c", OPT_TYPE_STRING, OPT_PERSTREAM | OPT_INPUT | OPT_OUTPUT | OPT_DECODER | OPT_HAS_CANON,
{ .off = OFFSET(codec_names) },
"select encoder/decoder ('copy' to copy stream without reencoding)", "codec",
.u1.name_canon = "codec", },
- { "codec", OPT_TYPE_STRING, OPT_PERSTREAM | OPT_INPUT | OPT_OUTPUT | OPT_EXPERT | OPT_HAS_ALT,
+ { "codec", OPT_TYPE_STRING, OPT_PERSTREAM | OPT_INPUT | OPT_OUTPUT | OPT_DECODER | OPT_EXPERT | OPT_HAS_ALT,
{ .off = OFFSET(codec_names) },
"alias for -c (select encoder/decoder)", "codec",
.u1.names_alt = alt_codec, },
--
2.43.0
More information about the ffmpeg-devel
mailing list