FFmpeg
vf_multiply.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2022 Paul B Mahol
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "libavutil/imgutils.h"
22 #include "libavutil/pixdesc.h"
23 #include "libavutil/opt.h"
24 #include "avfilter.h"
25 #include "formats.h"
26 #include "internal.h"
27 #include "video.h"
28 #include "framesync.h"
29 
30 typedef struct MultiplyContext {
31  const AVClass *class;
32 
33  float offset;
34  float scale;
35  int planes;
36 
37  int linesize[4];
38  int nb_planes;
39 
42 
43 #define OFFSET(x) offsetof(MultiplyContext, x)
44 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
45 
46 typedef struct ThreadData {
47  AVFrame *src, *ref, *dst;
48 } ThreadData;
49 
50 static const AVOption multiply_options[] = {
51  { "scale", "set scale", OFFSET(scale), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0., 9., FLAGS },
52  { "offset", "set offset", OFFSET(offset), AV_OPT_TYPE_FLOAT, {.dbl=0.5}, -1., 1., FLAGS },
53  { "planes", "set planes", OFFSET(planes), AV_OPT_TYPE_FLAGS, {.i64=0xF}, 0., 0xF, FLAGS },
54  { NULL }
55 };
56 
57 static const enum AVPixelFormat pix_fmts[] = {
60 };
61 
63 {
64  AVFilterContext *ctx = inlink->dst;
65  MultiplyContext *s = ctx->priv;
66  int ret;
67 
68  s->nb_planes = av_pix_fmt_count_planes(inlink->format);
69  if ((ret = av_image_fill_linesizes(s->linesize, inlink->format, inlink->w)) < 0)
70  return ret;
71 
72  return 0;
73 }
74 
75 static void multiply(const uint8_t *ssrc, const uint8_t *rref, uint8_t *ddst,
76  float scale, float offset, int w)
77 {
78  const float *src = (const float *)ssrc;
79  const float *ref = (const float *)rref;
80  float *dst = (float *)ddst;
81 
82  for (int x = 0; x < w; x++) {
83  const float factor = (ref[x] + offset) * scale;
84 
85  dst[x] = src[x] * factor;
86  }
87 }
88 
89 static int multiply_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
90 {
91  MultiplyContext *s = ctx->priv;
92  const float offset = s->offset;
93  const float scale = s->scale;
94  ThreadData *td = arg;
95 
96  for (int p = 0; p < s->nb_planes; p++) {
97  const ptrdiff_t src_linesize = td->src->linesize[p];
98  const ptrdiff_t ref_linesize = td->ref->linesize[p];
99  const ptrdiff_t dst_linesize = td->dst->linesize[p];
100  const int w = td->src->width;
101  const int h = td->src->height;
102  const int slice_start = (h * jobnr) / nb_jobs;
103  const int slice_end = (h * (jobnr+1)) / nb_jobs;
104  const uint8_t *src = td->src->data[p] + slice_start * src_linesize;
105  const uint8_t *ref = td->ref->data[p] + slice_start * ref_linesize;
106  uint8_t *dst = td->dst->data[p] + slice_start * dst_linesize;
107 
108  if (!((1 << p) & s->planes)) {
109  av_image_copy_plane(dst, dst_linesize, ref, ref_linesize,
110  s->linesize[p], slice_end - slice_start);
111  continue;
112  }
113 
114  for (int y = slice_start; y < slice_end; y++) {
115  multiply(src, ref, dst, scale, offset, w);
116 
117  dst += dst_linesize;
118  src += src_linesize;
119  ref += ref_linesize;
120  }
121  }
122 
123  return 0;
124 }
125 
127 {
128  AVFilterContext *ctx = fs->parent;
129  MultiplyContext *s = fs->opaque;
130  AVFilterLink *outlink = ctx->outputs[0];
131  AVFrame *out, *src, *ref;
132  int ret;
133 
134  if ((ret = ff_framesync_get_frame(&s->fs, 0, &src, 0)) < 0 ||
135  (ret = ff_framesync_get_frame(&s->fs, 1, &ref, 0)) < 0)
136  return ret;
137 
138  if (ctx->is_disabled) {
140  if (!out)
141  return AVERROR(ENOMEM);
142  } else {
143  ThreadData td;
144 
145  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
146  if (!out)
147  return AVERROR(ENOMEM);
149 
150  td.src = src;
151  td.ref = ref;
152  td.dst = out;
153 
155  FFMIN(outlink->h, ff_filter_get_nb_threads(ctx)));
156  }
157  out->pts = av_rescale_q(s->fs.pts, s->fs.time_base, outlink->time_base);
158 
159  return ff_filter_frame(outlink, out);
160 }
161 
162 static int config_output(AVFilterLink *outlink)
163 {
164  AVFilterContext *ctx = outlink->src;
165  MultiplyContext *s = ctx->priv;
166  AVFilterLink *source = ctx->inputs[0];
167  AVFilterLink *ref = ctx->inputs[1];
168  FFFrameSyncIn *in;
169  int ret;
170 
171  if (source->w != ref->w || source->h != ref->h) {
172  av_log(ctx, AV_LOG_ERROR, "First input link %s parameters "
173  "(size %dx%d) do not match the corresponding "
174  "second input link %s parameters (%dx%d)\n",
175  ctx->input_pads[0].name, source->w, source->h,
176  ctx->input_pads[1].name, ref->w, ref->h);
177  return AVERROR(EINVAL);
178  }
179 
180  outlink->w = source->w;
181  outlink->h = source->h;
182  outlink->sample_aspect_ratio = source->sample_aspect_ratio;
183  outlink->frame_rate = source->frame_rate;
184 
185  if ((ret = ff_framesync_init(&s->fs, ctx, 2)) < 0)
186  return ret;
187 
188  in = s->fs.in;
189  in[0].time_base = source->time_base;
190  in[1].time_base = ref->time_base;
191  in[0].sync = 1;
192  in[0].before = EXT_STOP;
193  in[0].after = EXT_INFINITY;
194  in[1].sync = 1;
195  in[1].before = EXT_STOP;
196  in[1].after = EXT_INFINITY;
197  s->fs.opaque = s;
198  s->fs.on_event = process_frame;
199 
200  ret = ff_framesync_configure(&s->fs);
201  outlink->time_base = s->fs.time_base;
202 
203  return ret;
204 }
205 
207 {
208  MultiplyContext *s = ctx->priv;
209  return ff_framesync_activate(&s->fs);
210 }
211 
213 {
214  MultiplyContext *s = ctx->priv;
215 
216  ff_framesync_uninit(&s->fs);
217 }
218 
219 static const AVFilterPad multiply_inputs[] = {
220  {
221  .name = "source",
222  .type = AVMEDIA_TYPE_VIDEO,
223  .config_props = config_input,
224  },
225  {
226  .name = "factor",
227  .type = AVMEDIA_TYPE_VIDEO,
228  },
229 };
230 
231 static const AVFilterPad multiply_outputs[] = {
232  {
233  .name = "default",
234  .type = AVMEDIA_TYPE_VIDEO,
235  .config_props = config_output,
236  },
237 };
238 
240 
242  .name = "multiply",
243  .description = NULL_IF_CONFIG_SMALL("Multiply first video stream with second video stream."),
244  .priv_class = &multiply_class,
245  .priv_size = sizeof(MultiplyContext),
246  .uninit = uninit,
247  .activate = activate,
252  .process_command = ff_filter_process_command,
253 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:101
FFFrameSyncIn::time_base
AVRational time_base
Time base for the incoming frames.
Definition: framesync.h:117
ff_framesync_configure
int ff_framesync_configure(FFFrameSync *fs)
Configure a frame sync structure.
Definition: framesync.c:134
td
#define td
Definition: regdef.h:70
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
MultiplyContext::linesize
int linesize[4]
Definition: vf_multiply.c:37
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ff_framesync_uninit
void ff_framesync_uninit(FFFrameSync *fs)
Free all memory currently allocated.
Definition: framesync.c:304
out
FILE * out
Definition: movenc.c:54
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1009
multiply_slice
static int multiply_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
Definition: vf_multiply.c:89
ff_framesync_get_frame
int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, unsigned get)
Get the current frame in an input.
Definition: framesync.c:267
FILTER_PIXFMTS_ARRAY
#define FILTER_PIXFMTS_ARRAY(array)
Definition: internal.h:170
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(multiply)
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:325
pixdesc.h
w
uint8_t w
Definition: llviddspenc.c:38
AVOption
AVOption.
Definition: opt.h:251
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:175
FFFrameSync
Frame sync structure.
Definition: framesync.h:168
EXT_INFINITY
@ EXT_INFINITY
Extend the frame to infinity.
Definition: framesync.h:75
video.h
OFFSET
#define OFFSET(x)
Definition: vf_multiply.c:43
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
formats.h
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2847
EXT_STOP
@ EXT_STOP
Completely stop all streams with this one.
Definition: framesync.h:65
FFFrameSyncIn
Input stream structure.
Definition: framesync.h:102
scale
static av_always_inline float scale(float x, float s)
Definition: vf_v360.c:1389
FFFrameSyncIn::sync
unsigned sync
Synchronization level: frames on input at the highest sync level will generate output frame events.
Definition: framesync.h:160
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:49
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
av_image_fill_linesizes
int av_image_fill_linesizes(int linesizes[4], enum AVPixelFormat pix_fmt, int width)
Fill plane linesizes for an image with pixel format pix_fmt and width width.
Definition: imgutils.c:89
s
#define s(width, name)
Definition: cbs_vp9.c:256
process_frame
static int process_frame(FFFrameSync *fs)
Definition: vf_multiply.c:126
MultiplyContext::fs
FFFrameSync fs
Definition: vf_multiply.c:40
slice_end
static int slice_end(AVCodecContext *avctx, AVFrame *pict)
Handle slice ends.
Definition: mpeg12dec.c:2014
multiply_inputs
static const AVFilterPad multiply_inputs[]
Definition: vf_multiply.c:219
ctx
AVFormatContext * ctx
Definition: movenc.c:48
av_frame_clone
AVFrame * av_frame_clone(const AVFrame *src)
Create a new frame that references the same data as src.
Definition: frame.c:474
av_rescale_q
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:190
arg
const char * arg
Definition: jacosubdec.c:67
ThreadData::dst
AVFrame * dst
Definition: vf_blend.c:56
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:603
MultiplyContext
Definition: vf_multiply.c:30
fs
#define fs(width, name, subs,...)
Definition: cbs_vp9.c:258
ThreadData::src
const uint8_t * src
Definition: vf_bm3d.c:55
multiply_outputs
static const AVFilterPad multiply_outputs[]
Definition: vf_multiply.c:231
ff_vf_multiply
const AVFilter ff_vf_multiply
Definition: vf_multiply.c:241
FLAGS
#define FLAGS
Definition: vf_multiply.c:44
planes
static const struct @336 planes[]
uninit
static av_cold void uninit(AVFilterContext *ctx)
Definition: vf_multiply.c:212
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:255
pix_fmts
static enum AVPixelFormat pix_fmts[]
Definition: vf_multiply.c:57
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:115
AV_PIX_FMT_GBRPF32
#define AV_PIX_FMT_GBRPF32
Definition: pixfmt.h:454
MultiplyContext::planes
int planes
Definition: vf_multiply.c:35
ff_filter_process_command
int ff_filter_process_command(AVFilterContext *ctx, const char *cmd, const char *arg, char *res, int res_len, int flags)
Generic processing of user supplied commands that are set in the same way as the filter options.
Definition: avfilter.c:873
offset
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
Definition: writing_filters.txt:86
internal.h
AV_OPT_TYPE_FLOAT
@ AV_OPT_TYPE_FLOAT
Definition: opt.h:228
MultiplyContext::nb_planes
int nb_planes
Definition: vf_multiply.c:38
ff_filter_get_nb_threads
int ff_filter_get_nb_threads(AVFilterContext *ctx)
Get number of threads for current filter instance.
Definition: avfilter.c:793
ThreadData
Used for passing data between threads.
Definition: dsddec.c:69
multiply
static void multiply(const uint8_t *ssrc, const uint8_t *rref, uint8_t *ddst, float scale, float offset, int w)
Definition: vf_multiply.c:75
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:55
AVFilter
Filter definition.
Definition: avfilter.h:171
ret
ret
Definition: filter_design.txt:187
ThreadData::ref
const uint8_t * ref
Definition: vf_bm3d.c:57
ff_framesync_init
int ff_framesync_init(FFFrameSync *fs, AVFilterContext *parent, unsigned nb_in)
Initialize a frame sync structure.
Definition: framesync.c:86
FFFrameSyncIn::before
enum FFFrameSyncExtMode before
Extrapolation mode for timestamps before the first frame.
Definition: framesync.h:107
framesync.h
MultiplyContext::offset
float offset
Definition: vf_multiply.c:33
MultiplyContext::scale
float scale
Definition: vf_multiply.c:34
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:65
avfilter.h
AV_PIX_FMT_GBRAPF32
#define AV_PIX_FMT_GBRAPF32
Definition: pixfmt.h:455
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:112
config_output
static int config_output(AVFilterLink *outlink)
Definition: vf_multiply.c:162
activate
static int activate(AVFilterContext *ctx)
Definition: vf_multiply.c:206
AVFilterContext
An instance of a filter.
Definition: avfilter.h:408
factor
static const int factor[16]
Definition: vf_pp7.c:76
AVFILTER_FLAG_SLICE_THREADS
#define AVFILTER_FLAG_SLICE_THREADS
The filter supports multithreading by splitting frames into multiple parts and processing them concur...
Definition: avfilter.h:127
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:191
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
AV_OPT_TYPE_FLAGS
@ AV_OPT_TYPE_FLAGS
Definition: opt.h:224
AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL
#define AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL
Same as AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC, except that the filter will have its filter_frame() c...
Definition: avfilter.h:160
imgutils.h
multiply_options
static const AVOption multiply_options[]
Definition: vf_multiply.c:50
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
FFFrameSyncIn::after
enum FFFrameSyncExtMode after
Extrapolation mode for timestamps after the last frame.
Definition: framesync.h:112
h
h
Definition: vp9dsp_template.c:2038
ff_framesync_activate
int ff_framesync_activate(FFFrameSync *fs)
Examine the frames in the filter's input and try to produce output.
Definition: framesync.c:355
ff_filter_execute
static av_always_inline int ff_filter_execute(AVFilterContext *ctx, avfilter_action_func *func, void *arg, int *ret, int nb_jobs)
Definition: internal.h:142
config_input
static int config_input(AVFilterLink *inlink)
Definition: vf_multiply.c:62