FFmpeg
vf_multiply.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2022 Paul B Mahol
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "libavutil/imgutils.h"
22 #include "libavutil/pixdesc.h"
23 #include "libavutil/opt.h"
24 #include "avfilter.h"
25 #include "internal.h"
26 #include "video.h"
27 #include "framesync.h"
28 
29 typedef struct MultiplyContext {
30  const AVClass *class;
31 
32  float offset;
33  float scale;
34  int planes;
35 
36  int linesize[4];
37  int nb_planes;
38 
41 
42 #define OFFSET(x) offsetof(MultiplyContext, x)
43 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
44 
45 typedef struct ThreadData {
46  AVFrame *src, *ref, *dst;
47 } ThreadData;
48 
49 static const AVOption multiply_options[] = {
50  { "scale", "set scale", OFFSET(scale), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0., 9., FLAGS },
51  { "offset", "set offset", OFFSET(offset), AV_OPT_TYPE_FLOAT, {.dbl=0.5}, -1., 1., FLAGS },
52  { "planes", "set planes", OFFSET(planes), AV_OPT_TYPE_FLAGS, {.i64=0xF}, 0., 0xF, FLAGS },
53  { NULL }
54 };
55 
56 static const enum AVPixelFormat pix_fmts[] = {
59 };
60 
62 {
63  AVFilterContext *ctx = inlink->dst;
64  MultiplyContext *s = ctx->priv;
65  int ret;
66 
67  s->nb_planes = av_pix_fmt_count_planes(inlink->format);
68  if ((ret = av_image_fill_linesizes(s->linesize, inlink->format, inlink->w)) < 0)
69  return ret;
70 
71  return 0;
72 }
73 
74 static void multiply(const uint8_t *ssrc, const uint8_t *rref, uint8_t *ddst,
75  float scale, float offset, int w)
76 {
77  const float *src = (const float *)ssrc;
78  const float *ref = (const float *)rref;
79  float *dst = (float *)ddst;
80 
81  for (int x = 0; x < w; x++) {
82  const float factor = (ref[x] + offset) * scale;
83 
84  dst[x] = src[x] * factor;
85  }
86 }
87 
88 static int multiply_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
89 {
90  MultiplyContext *s = ctx->priv;
91  const float offset = s->offset;
92  const float scale = s->scale;
93  ThreadData *td = arg;
94 
95  for (int p = 0; p < s->nb_planes; p++) {
96  const ptrdiff_t src_linesize = td->src->linesize[p];
97  const ptrdiff_t ref_linesize = td->ref->linesize[p];
98  const ptrdiff_t dst_linesize = td->dst->linesize[p];
99  const int w = td->src->width;
100  const int h = td->src->height;
101  const int slice_start = (h * jobnr) / nb_jobs;
102  const int slice_end = (h * (jobnr+1)) / nb_jobs;
103  const uint8_t *src = td->src->data[p] + slice_start * src_linesize;
104  const uint8_t *ref = td->ref->data[p] + slice_start * ref_linesize;
105  uint8_t *dst = td->dst->data[p] + slice_start * dst_linesize;
106 
107  if (!((1 << p) & s->planes)) {
108  av_image_copy_plane(dst, dst_linesize, ref, ref_linesize,
109  s->linesize[p], slice_end - slice_start);
110  continue;
111  }
112 
113  for (int y = slice_start; y < slice_end; y++) {
114  multiply(src, ref, dst, scale, offset, w);
115 
116  dst += dst_linesize;
117  src += src_linesize;
118  ref += ref_linesize;
119  }
120  }
121 
122  return 0;
123 }
124 
126 {
127  AVFilterContext *ctx = fs->parent;
128  MultiplyContext *s = fs->opaque;
129  AVFilterLink *outlink = ctx->outputs[0];
130  AVFrame *out, *src, *ref;
131  int ret;
132 
133  if ((ret = ff_framesync_get_frame(&s->fs, 0, &src, 0)) < 0 ||
134  (ret = ff_framesync_get_frame(&s->fs, 1, &ref, 0)) < 0)
135  return ret;
136 
137  if (ctx->is_disabled) {
139  if (!out)
140  return AVERROR(ENOMEM);
141  } else {
142  ThreadData td;
143 
144  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
145  if (!out)
146  return AVERROR(ENOMEM);
148 
149  td.src = src;
150  td.ref = ref;
151  td.dst = out;
152 
154  FFMIN(outlink->h, ff_filter_get_nb_threads(ctx)));
155  }
156  out->pts = av_rescale_q(s->fs.pts, s->fs.time_base, outlink->time_base);
157 
158  return ff_filter_frame(outlink, out);
159 }
160 
161 static int config_output(AVFilterLink *outlink)
162 {
163  AVFilterContext *ctx = outlink->src;
164  MultiplyContext *s = ctx->priv;
165  AVFilterLink *source = ctx->inputs[0];
166  AVFilterLink *ref = ctx->inputs[1];
167  FFFrameSyncIn *in;
168  int ret;
169 
170  if (source->w != ref->w || source->h != ref->h) {
171  av_log(ctx, AV_LOG_ERROR, "First input link %s parameters "
172  "(size %dx%d) do not match the corresponding "
173  "second input link %s parameters (%dx%d)\n",
174  ctx->input_pads[0].name, source->w, source->h,
175  ctx->input_pads[1].name, ref->w, ref->h);
176  return AVERROR(EINVAL);
177  }
178 
179  outlink->w = source->w;
180  outlink->h = source->h;
181  outlink->sample_aspect_ratio = source->sample_aspect_ratio;
182  outlink->frame_rate = source->frame_rate;
183 
184  if ((ret = ff_framesync_init(&s->fs, ctx, 2)) < 0)
185  return ret;
186 
187  in = s->fs.in;
188  in[0].time_base = source->time_base;
189  in[1].time_base = ref->time_base;
190  in[0].sync = 1;
191  in[0].before = EXT_STOP;
192  in[0].after = EXT_INFINITY;
193  in[1].sync = 1;
194  in[1].before = EXT_STOP;
195  in[1].after = EXT_INFINITY;
196  s->fs.opaque = s;
197  s->fs.on_event = process_frame;
198 
199  ret = ff_framesync_configure(&s->fs);
200  outlink->time_base = s->fs.time_base;
201 
202  return ret;
203 }
204 
206 {
207  MultiplyContext *s = ctx->priv;
208  return ff_framesync_activate(&s->fs);
209 }
210 
212 {
213  MultiplyContext *s = ctx->priv;
214 
215  ff_framesync_uninit(&s->fs);
216 }
217 
218 static const AVFilterPad multiply_inputs[] = {
219  {
220  .name = "source",
221  .type = AVMEDIA_TYPE_VIDEO,
222  .config_props = config_input,
223  },
224  {
225  .name = "factor",
226  .type = AVMEDIA_TYPE_VIDEO,
227  },
228 };
229 
230 static const AVFilterPad multiply_outputs[] = {
231  {
232  .name = "default",
233  .type = AVMEDIA_TYPE_VIDEO,
234  .config_props = config_output,
235  },
236 };
237 
239 
241  .name = "multiply",
242  .description = NULL_IF_CONFIG_SMALL("Multiply first video stream with second video stream."),
243  .priv_class = &multiply_class,
244  .priv_size = sizeof(MultiplyContext),
245  .uninit = uninit,
246  .activate = activate,
251  .process_command = ff_filter_process_command,
252 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:112
FFFrameSyncIn::time_base
AVRational time_base
Time base for the incoming frames.
Definition: framesync.h:117
ff_framesync_configure
int ff_framesync_configure(FFFrameSync *fs)
Configure a frame sync structure.
Definition: framesync.c:137
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
MultiplyContext::linesize
int linesize[4]
Definition: vf_multiply.c:36
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ff_framesync_uninit
void ff_framesync_uninit(FFFrameSync *fs)
Free all memory currently allocated.
Definition: framesync.c:307
out
FILE * out
Definition: movenc.c:55
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1015
multiply_slice
static int multiply_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Definition: vf_multiply.c:88
ff_framesync_get_frame
int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, unsigned get)
Get the current frame in an input.
Definition: framesync.c:270
FILTER_PIXFMTS_ARRAY
#define FILTER_PIXFMTS_ARRAY(array)
Definition: internal.h:162
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(multiply)
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:374
pixdesc.h
w
uint8_t w
Definition: llviddspenc.c:38
AVOption
AVOption.
Definition: opt.h:357
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:170
FFFrameSync
Frame sync structure.
Definition: framesync.h:168
EXT_INFINITY
@ EXT_INFINITY
Extend the frame to infinity.
Definition: framesync.h:75
video.h
OFFSET
#define OFFSET(x)
Definition: vf_multiply.c:42
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:395
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3005
EXT_STOP
@ EXT_STOP
Completely stop all streams with this one.
Definition: framesync.h:65
slice_end
static int slice_end(AVCodecContext *avctx, AVFrame *pict, int *got_output)
Handle slice ends.
Definition: mpeg12dec.c:1747
FFFrameSyncIn
Input stream structure.
Definition: framesync.h:102
FFFrameSyncIn::sync
unsigned sync
Synchronization level: frames on input at the highest sync level will generate output frame events.
Definition: framesync.h:160
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:33
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
av_image_fill_linesizes
int av_image_fill_linesizes(int linesizes[4], enum AVPixelFormat pix_fmt, int width)
Fill plane linesizes for an image with pixel format pix_fmt and width width.
Definition: imgutils.c:89
s
#define s(width, name)
Definition: cbs_vp9.c:198
process_frame
static int process_frame(FFFrameSync *fs)
Definition: vf_multiply.c:125
MultiplyContext::fs
FFFrameSync fs
Definition: vf_multiply.c:39
multiply_inputs
static const AVFilterPad multiply_inputs[]
Definition: vf_multiply.c:218
ctx
AVFormatContext * ctx
Definition: movenc.c:49
av_frame_clone
AVFrame * av_frame_clone(const AVFrame *src)
Create a new frame that references the same data as src.
Definition: frame.c:593
av_rescale_q
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:182
arg
const char * arg
Definition: jacosubdec.c:67
ThreadData::dst
AVFrame * dst
Definition: vf_blend.c:58
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:709
MultiplyContext
Definition: vf_multiply.c:29
fs
#define fs(width, name, subs,...)
Definition: cbs_vp9.c:200
ThreadData::src
const uint8_t * src
Definition: vf_bm3d.c:55
multiply_outputs
static const AVFilterPad multiply_outputs[]
Definition: vf_multiply.c:230
ff_vf_multiply
const AVFilter ff_vf_multiply
Definition: vf_multiply.c:240
FLAGS
#define FLAGS
Definition: vf_multiply.c:43
uninit
static av_cold void uninit(AVFilterContext *ctx)
Definition: vf_multiply.c:211
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:255
pix_fmts
static enum AVPixelFormat pix_fmts[]
Definition: vf_multiply.c:56
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:94
planes
static const struct @434 planes[]
AV_PIX_FMT_GBRPF32
#define AV_PIX_FMT_GBRPF32
Definition: pixfmt.h:508
MultiplyContext::planes
int planes
Definition: vf_multiply.c:34
ff_filter_process_command
int ff_filter_process_command(AVFilterContext *ctx, const char *cmd, const char *arg, char *res, int res_len, int flags)
Generic processing of user supplied commands that are set in the same way as the filter options.
Definition: avfilter.c:887
offset
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
Definition: writing_filters.txt:86
internal.h
AV_OPT_TYPE_FLOAT
@ AV_OPT_TYPE_FLOAT
Definition: opt.h:248
MultiplyContext::nb_planes
int nb_planes
Definition: vf_multiply.c:37
ff_filter_get_nb_threads
int ff_filter_get_nb_threads(AVFilterContext *ctx)
Get number of threads for current filter instance.
Definition: avfilter.c:827
ThreadData
Used for passing data between threads.
Definition: dsddec.c:71
multiply
static void multiply(const uint8_t *ssrc, const uint8_t *rref, uint8_t *ddst, float scale, float offset, int w)
Definition: vf_multiply.c:74
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:39
AVFilter
Filter definition.
Definition: avfilter.h:166
ret
ret
Definition: filter_design.txt:187
ThreadData::ref
const uint8_t * ref
Definition: vf_bm3d.c:57
ff_framesync_init
int ff_framesync_init(FFFrameSync *fs, AVFilterContext *parent, unsigned nb_in)
Initialize a frame sync structure.
Definition: framesync.c:86
FFFrameSyncIn::before
enum FFFrameSyncExtMode before
Extrapolation mode for timestamps before the first frame.
Definition: framesync.h:107
framesync.h
MultiplyContext::offset
float offset
Definition: vf_multiply.c:32
MultiplyContext::scale
float scale
Definition: vf_multiply.c:33
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
avfilter.h
AV_PIX_FMT_GBRAPF32
#define AV_PIX_FMT_GBRAPF32
Definition: pixfmt.h:509
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:112
slice_start
static int slice_start(SliceContext *sc, VVCContext *s, VVCFrameContext *fc, const CodedBitstreamUnit *unit, const int is_first_slice)
Definition: dec.c:700
config_output
static int config_output(AVFilterLink *outlink)
Definition: vf_multiply.c:161
activate
static int activate(AVFilterContext *ctx)
Definition: vf_multiply.c:205
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
factor
static const int factor[16]
Definition: vf_pp7.c:79
AVFILTER_FLAG_SLICE_THREADS
#define AVFILTER_FLAG_SLICE_THREADS
The filter supports multithreading by splitting frames into multiple parts and processing them concur...
Definition: avfilter.h:117
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
scale
static void scale(int *out, const int *in, const int w, const int h, const int shift)
Definition: intra.c:291
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:183
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
AV_OPT_TYPE_FLAGS
@ AV_OPT_TYPE_FLAGS
Definition: opt.h:244
AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL
#define AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL
Same as AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC, except that the filter will have its filter_frame() c...
Definition: avfilter.h:155
imgutils.h
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, a positive or negative value, which is typically indicating the size in bytes of each pict...
Definition: frame.h:419
multiply_options
static const AVOption multiply_options[]
Definition: vf_multiply.c:49
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FFFrameSyncIn::after
enum FFFrameSyncExtMode after
Extrapolation mode for timestamps after the last frame.
Definition: framesync.h:112
h
h
Definition: vp9dsp_template.c:2038
ff_framesync_activate
int ff_framesync_activate(FFFrameSync *fs)
Examine the frames in the filter's input and try to produce output.
Definition: framesync.c:358
ff_filter_execute
static av_always_inline int ff_filter_execute(AVFilterContext *ctx, avfilter_action_func *func, void *arg, int *ret, int nb_jobs)
Definition: internal.h:134
config_input
static int config_input(AVFilterLink *inlink)
Definition: vf_multiply.c:61