FFmpeg
vf_multiply.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2022 Paul B Mahol
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "libavutil/imgutils.h"
22 #include "libavutil/pixdesc.h"
23 #include "libavutil/opt.h"
24 #include "avfilter.h"
25 #include "filters.h"
26 #include "video.h"
27 #include "framesync.h"
28 
29 typedef struct MultiplyContext {
30  const AVClass *class;
31 
32  float offset;
33  float scale;
34  int planes;
35 
36  int linesize[4];
37  int nb_planes;
38 
41 
42 #define OFFSET(x) offsetof(MultiplyContext, x)
43 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
44 
45 typedef struct ThreadData {
46  AVFrame *src, *ref, *dst;
47 } ThreadData;
48 
49 static const AVOption multiply_options[] = {
50  { "scale", "set scale", OFFSET(scale), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0., 9., FLAGS },
51  { "offset", "set offset", OFFSET(offset), AV_OPT_TYPE_FLOAT, {.dbl=0.5}, -1., 1., FLAGS },
52  { "planes", "set planes", OFFSET(planes), AV_OPT_TYPE_FLAGS, {.i64=0xF}, 0., 0xF, FLAGS },
53  { NULL }
54 };
55 
56 static const enum AVPixelFormat pix_fmts[] = {
59 };
60 
62 {
63  AVFilterContext *ctx = inlink->dst;
64  MultiplyContext *s = ctx->priv;
65  int ret;
66 
67  s->nb_planes = av_pix_fmt_count_planes(inlink->format);
68  if ((ret = av_image_fill_linesizes(s->linesize, inlink->format, inlink->w)) < 0)
69  return ret;
70 
71  return 0;
72 }
73 
74 static void multiply(const uint8_t *ssrc, const uint8_t *rref, uint8_t *ddst,
75  float scale, float offset, int w)
76 {
77  const float *src = (const float *)ssrc;
78  const float *ref = (const float *)rref;
79  float *dst = (float *)ddst;
80 
81  for (int x = 0; x < w; x++) {
82  const float factor = (ref[x] + offset) * scale;
83 
84  dst[x] = src[x] * factor;
85  }
86 }
87 
88 static int multiply_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
89 {
90  MultiplyContext *s = ctx->priv;
91  const float offset = s->offset;
92  const float scale = s->scale;
93  ThreadData *td = arg;
94 
95  for (int p = 0; p < s->nb_planes; p++) {
96  const ptrdiff_t src_linesize = td->src->linesize[p];
97  const ptrdiff_t ref_linesize = td->ref->linesize[p];
98  const ptrdiff_t dst_linesize = td->dst->linesize[p];
99  const int w = td->src->width;
100  const int h = td->src->height;
101  const int slice_start = (h * jobnr) / nb_jobs;
102  const int slice_end = (h * (jobnr+1)) / nb_jobs;
103  const uint8_t *src = td->src->data[p] + slice_start * src_linesize;
104  const uint8_t *ref = td->ref->data[p] + slice_start * ref_linesize;
105  uint8_t *dst = td->dst->data[p] + slice_start * dst_linesize;
106 
107  if (!((1 << p) & s->planes)) {
108  av_image_copy_plane(dst, dst_linesize, ref, ref_linesize,
109  s->linesize[p], slice_end - slice_start);
110  continue;
111  }
112 
113  for (int y = slice_start; y < slice_end; y++) {
114  multiply(src, ref, dst, scale, offset, w);
115 
116  dst += dst_linesize;
117  src += src_linesize;
118  ref += ref_linesize;
119  }
120  }
121 
122  return 0;
123 }
124 
126 {
127  AVFilterContext *ctx = fs->parent;
128  MultiplyContext *s = fs->opaque;
129  AVFilterLink *outlink = ctx->outputs[0];
130  AVFrame *out, *src, *ref;
131  int ret;
132 
133  if ((ret = ff_framesync_get_frame(&s->fs, 0, &src, 0)) < 0 ||
134  (ret = ff_framesync_get_frame(&s->fs, 1, &ref, 0)) < 0)
135  return ret;
136 
137  if (ctx->is_disabled) {
139  if (!out)
140  return AVERROR(ENOMEM);
141  } else {
142  ThreadData td;
143 
144  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
145  if (!out)
146  return AVERROR(ENOMEM);
148 
149  td.src = src;
150  td.ref = ref;
151  td.dst = out;
152 
154  FFMIN(outlink->h, ff_filter_get_nb_threads(ctx)));
155  }
156  out->pts = av_rescale_q(s->fs.pts, s->fs.time_base, outlink->time_base);
157 
158  return ff_filter_frame(outlink, out);
159 }
160 
161 static int config_output(AVFilterLink *outlink)
162 {
163  AVFilterContext *ctx = outlink->src;
164  MultiplyContext *s = ctx->priv;
165  AVFilterLink *source = ctx->inputs[0];
166  AVFilterLink *ref = ctx->inputs[1];
168  FilterLink *ol = ff_filter_link(outlink);
169  FFFrameSyncIn *in;
170  int ret;
171 
172  if (source->w != ref->w || source->h != ref->h) {
173  av_log(ctx, AV_LOG_ERROR, "First input link %s parameters "
174  "(size %dx%d) do not match the corresponding "
175  "second input link %s parameters (%dx%d)\n",
176  ctx->input_pads[0].name, source->w, source->h,
177  ctx->input_pads[1].name, ref->w, ref->h);
178  return AVERROR(EINVAL);
179  }
180 
181  outlink->w = source->w;
182  outlink->h = source->h;
183  outlink->sample_aspect_ratio = source->sample_aspect_ratio;
184  ol->frame_rate = il->frame_rate;
185 
186  if ((ret = ff_framesync_init(&s->fs, ctx, 2)) < 0)
187  return ret;
188 
189  in = s->fs.in;
190  in[0].time_base = source->time_base;
191  in[1].time_base = ref->time_base;
192  in[0].sync = 1;
193  in[0].before = EXT_STOP;
194  in[0].after = EXT_INFINITY;
195  in[1].sync = 1;
196  in[1].before = EXT_STOP;
197  in[1].after = EXT_INFINITY;
198  s->fs.opaque = s;
199  s->fs.on_event = process_frame;
200 
201  ret = ff_framesync_configure(&s->fs);
202  outlink->time_base = s->fs.time_base;
203 
204  return ret;
205 }
206 
208 {
209  MultiplyContext *s = ctx->priv;
210  return ff_framesync_activate(&s->fs);
211 }
212 
214 {
215  MultiplyContext *s = ctx->priv;
216 
217  ff_framesync_uninit(&s->fs);
218 }
219 
220 static const AVFilterPad multiply_inputs[] = {
221  {
222  .name = "source",
223  .type = AVMEDIA_TYPE_VIDEO,
224  .config_props = config_input,
225  },
226  {
227  .name = "factor",
228  .type = AVMEDIA_TYPE_VIDEO,
229  },
230 };
231 
232 static const AVFilterPad multiply_outputs[] = {
233  {
234  .name = "default",
235  .type = AVMEDIA_TYPE_VIDEO,
236  .config_props = config_output,
237  },
238 };
239 
241 
243  .name = "multiply",
244  .description = NULL_IF_CONFIG_SMALL("Multiply first video stream with second video stream."),
245  .priv_class = &multiply_class,
246  .priv_size = sizeof(MultiplyContext),
247  .uninit = uninit,
248  .activate = activate,
253  .process_command = ff_filter_process_command,
254 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:116
FFFrameSyncIn::time_base
AVRational time_base
Time base for the incoming frames.
Definition: framesync.h:117
ff_framesync_configure
int ff_framesync_configure(FFFrameSync *fs)
Configure a frame sync structure.
Definition: framesync.c:137
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
MultiplyContext::linesize
int linesize[4]
Definition: vf_multiply.c:36
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
FILTER_PIXFMTS_ARRAY
#define FILTER_PIXFMTS_ARRAY(array)
Definition: filters.h:242
ff_framesync_uninit
void ff_framesync_uninit(FFFrameSync *fs)
Free all memory currently allocated.
Definition: framesync.c:301
out
FILE * out
Definition: movenc.c:55
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1062
multiply_slice
static int multiply_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Definition: vf_multiply.c:88
ff_framesync_get_frame
int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, unsigned get)
Get the current frame in an input.
Definition: framesync.c:269
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(multiply)
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: filters.h:262
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
pixdesc.h
w
uint8_t w
Definition: llviddspenc.c:38
AVOption
AVOption.
Definition: opt.h:429
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:205
FFFrameSync
Frame sync structure.
Definition: framesync.h:168
EXT_INFINITY
@ EXT_INFINITY
Extend the frame to infinity.
Definition: framesync.h:75
video.h
OFFSET
#define OFFSET(x)
Definition: vf_multiply.c:42
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:410
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3210
EXT_STOP
@ EXT_STOP
Completely stop all streams with this one.
Definition: framesync.h:65
slice_end
static int slice_end(AVCodecContext *avctx, AVFrame *pict, int *got_output)
Handle slice ends.
Definition: mpeg12dec.c:1719
FFFrameSyncIn
Input stream structure.
Definition: framesync.h:102
FFFrameSyncIn::sync
unsigned sync
Synchronization level: frames on input at the highest sync level will generate output frame events.
Definition: framesync.h:160
AVFilterPad
A filter pad used for either input or output.
Definition: filters.h:38
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
av_cold
#define av_cold
Definition: attributes.h:90
av_image_fill_linesizes
int av_image_fill_linesizes(int linesizes[4], enum AVPixelFormat pix_fmt, int width)
Fill plane linesizes for an image with pixel format pix_fmt and width width.
Definition: imgutils.c:89
s
#define s(width, name)
Definition: cbs_vp9.c:198
process_frame
static int process_frame(FFFrameSync *fs)
Definition: vf_multiply.c:125
MultiplyContext::fs
FFFrameSync fs
Definition: vf_multiply.c:39
filters.h
multiply_inputs
static const AVFilterPad multiply_inputs[]
Definition: vf_multiply.c:220
ctx
AVFormatContext * ctx
Definition: movenc.c:49
av_frame_clone
AVFrame * av_frame_clone(const AVFrame *src)
Create a new frame that references the same data as src.
Definition: frame.c:597
av_rescale_q
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: filters.h:263
arg
const char * arg
Definition: jacosubdec.c:67
ThreadData::dst
AVFrame * dst
Definition: vf_blend.c:58
planes
static const struct @465 planes[]
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:75
NULL
#define NULL
Definition: coverity.c:32
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:713
MultiplyContext
Definition: vf_multiply.c:29
fs
#define fs(width, name, subs,...)
Definition: cbs_vp9.c:200
ThreadData::src
const uint8_t * src
Definition: vf_bm3d.c:54
multiply_outputs
static const AVFilterPad multiply_outputs[]
Definition: vf_multiply.c:232
ff_vf_multiply
const AVFilter ff_vf_multiply
Definition: vf_multiply.c:242
FLAGS
#define FLAGS
Definition: vf_multiply.c:43
uninit
static av_cold void uninit(AVFilterContext *ctx)
Definition: vf_multiply.c:213
ff_filter_link
static FilterLink * ff_filter_link(AVFilterLink *link)
Definition: filters.h:197
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:255
pix_fmts
static enum AVPixelFormat pix_fmts[]
Definition: vf_multiply.c:56
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:94
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:83
AV_PIX_FMT_GBRPF32
#define AV_PIX_FMT_GBRPF32
Definition: pixfmt.h:532
MultiplyContext::planes
int planes
Definition: vf_multiply.c:34
ff_filter_process_command
int ff_filter_process_command(AVFilterContext *ctx, const char *cmd, const char *arg, char *res, int res_len, int flags)
Generic processing of user supplied commands that are set in the same way as the filter options.
Definition: avfilter.c:901
offset
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
Definition: writing_filters.txt:86
AV_OPT_TYPE_FLOAT
@ AV_OPT_TYPE_FLOAT
Underlying C type is float.
Definition: opt.h:271
MultiplyContext::nb_planes
int nb_planes
Definition: vf_multiply.c:37
ff_filter_get_nb_threads
int ff_filter_get_nb_threads(AVFilterContext *ctx)
Get number of threads for current filter instance.
Definition: avfilter.c:841
ThreadData
Used for passing data between threads.
Definition: dsddec.c:71
multiply
static void multiply(const uint8_t *ssrc, const uint8_t *rref, uint8_t *ddst, float scale, float offset, int w)
Definition: vf_multiply.c:74
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
AVFilterPad::name
const char * name
Pad name.
Definition: filters.h:44
slice_start
static int slice_start(SliceContext *sc, VVCContext *s, VVCFrameContext *fc, const CodedBitstreamUnit *unit, const int is_first_slice)
Definition: dec.c:738
AVFilter
Filter definition.
Definition: avfilter.h:201
ret
ret
Definition: filter_design.txt:187
ThreadData::ref
const uint8_t * ref
Definition: vf_bm3d.c:56
ff_framesync_init
int ff_framesync_init(FFFrameSync *fs, AVFilterContext *parent, unsigned nb_in)
Initialize a frame sync structure.
Definition: framesync.c:86
FFFrameSyncIn::before
enum FFFrameSyncExtMode before
Extrapolation mode for timestamps before the first frame.
Definition: framesync.h:107
framesync.h
ff_filter_execute
int ff_filter_execute(AVFilterContext *ctx, avfilter_action_func *func, void *arg, int *ret, int nb_jobs)
Definition: avfilter.c:1667
MultiplyContext::offset
float offset
Definition: vf_multiply.c:32
MultiplyContext::scale
float scale
Definition: vf_multiply.c:33
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
avfilter.h
AV_PIX_FMT_GBRAPF32
#define AV_PIX_FMT_GBRAPF32
Definition: pixfmt.h:533
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:112
config_output
static int config_output(AVFilterLink *outlink)
Definition: vf_multiply.c:161
activate
static int activate(AVFilterContext *ctx)
Definition: vf_multiply.c:207
AVFilterContext
An instance of a filter.
Definition: avfilter.h:457
factor
static const int factor[16]
Definition: vf_pp7.c:80
AVFILTER_FLAG_SLICE_THREADS
#define AVFILTER_FLAG_SLICE_THREADS
The filter supports multithreading by splitting frames into multiple parts and processing them concur...
Definition: avfilter.h:152
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
scale
static void scale(int *out, const int *in, const int w, const int h, const int shift)
Definition: intra.c:291
AV_OPT_TYPE_FLAGS
@ AV_OPT_TYPE_FLAGS
Underlying C type is unsigned int.
Definition: opt.h:255
AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL
#define AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL
Same as AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC, except that the filter will have its filter_frame() c...
Definition: avfilter.h:190
imgutils.h
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, a positive or negative value, which is typically indicating the size in bytes of each pict...
Definition: frame.h:434
multiply_options
static const AVOption multiply_options[]
Definition: vf_multiply.c:49
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FFFrameSyncIn::after
enum FFFrameSyncExtMode after
Extrapolation mode for timestamps after the last frame.
Definition: framesync.h:112
h
h
Definition: vp9dsp_template.c:2070
ff_framesync_activate
int ff_framesync_activate(FFFrameSync *fs)
Examine the frames in the filter's input and try to produce output.
Definition: framesync.c:352
src
#define src
Definition: vp8dsp.c:248
config_input
static int config_input(AVFilterLink *inlink)
Definition: vf_multiply.c:61