FFmpeg
vf_dnn_processing.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019 Guo Yejun
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * implementing a generic image processing filter using deep learning networks.
24  */
25 
26 #include "libavformat/avio.h"
27 #include "libavutil/opt.h"
28 #include "libavutil/pixdesc.h"
29 #include "libavutil/avassert.h"
30 #include "libavutil/imgutils.h"
31 #include "avfilter.h"
32 #include "dnn_interface.h"
33 #include "formats.h"
34 #include "internal.h"
35 #include "libswscale/swscale.h"
36 
37 typedef struct DnnProcessingContext {
38  const AVClass *class;
39 
45 
48 
49  // input & output of the model at execution time
52 
58 
59 #define OFFSET(x) offsetof(DnnProcessingContext, x)
60 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
61 static const AVOption dnn_processing_options[] = {
62  { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS, "backend" },
63  { "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
64 #if (CONFIG_LIBTENSORFLOW == 1)
65  { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
66 #endif
67 #if (CONFIG_LIBOPENVINO == 1)
68  { "openvino", "openvino backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, "backend" },
69 #endif
70  { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
71  { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
72  { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
73  { "options", "backend options", OFFSET(backend_options), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
74  { NULL }
75 };
76 
77 AVFILTER_DEFINE_CLASS(dnn_processing);
78 
80 {
81  DnnProcessingContext *ctx = context->priv;
82 
83  if (!ctx->model_filename) {
84  av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n");
85  return AVERROR(EINVAL);
86  }
87  if (!ctx->model_inputname) {
88  av_log(ctx, AV_LOG_ERROR, "input name of the model network is not specified\n");
89  return AVERROR(EINVAL);
90  }
91  if (!ctx->model_outputname) {
92  av_log(ctx, AV_LOG_ERROR, "output name of the model network is not specified\n");
93  return AVERROR(EINVAL);
94  }
95 
97  if (!ctx->dnn_module) {
98  av_log(ctx, AV_LOG_ERROR, "could not create DNN module for requested backend\n");
99  return AVERROR(ENOMEM);
100  }
101  if (!ctx->dnn_module->load_model) {
102  av_log(ctx, AV_LOG_ERROR, "load_model for network is not specified\n");
103  return AVERROR(EINVAL);
104  }
105 
106  ctx->model = (ctx->dnn_module->load_model)(ctx->model_filename, ctx->backend_options);
107  if (!ctx->model) {
108  av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n");
109  return AVERROR(EINVAL);
110  }
111 
112  return 0;
113 }
114 
116 {
117  static const enum AVPixelFormat pix_fmts[] = {
123  };
124  AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
125  return ff_set_common_formats(context, fmts_list);
126 }
127 
128 #define LOG_FORMAT_CHANNEL_MISMATCH() \
129  av_log(ctx, AV_LOG_ERROR, \
130  "the frame's format %s does not match " \
131  "the model input channel %d\n", \
132  av_get_pix_fmt_name(fmt), \
133  model_input->channels);
134 
135 static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)
136 {
137  AVFilterContext *ctx = inlink->dst;
138  enum AVPixelFormat fmt = inlink->format;
139 
140  // the design is to add explicit scale filter before this filter
141  if (model_input->height != -1 && model_input->height != inlink->h) {
142  av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n",
143  model_input->height, inlink->h);
144  return AVERROR(EIO);
145  }
146  if (model_input->width != -1 && model_input->width != inlink->w) {
147  av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n",
148  model_input->width, inlink->w);
149  return AVERROR(EIO);
150  }
151 
152  switch (fmt) {
153  case AV_PIX_FMT_RGB24:
154  case AV_PIX_FMT_BGR24:
155  if (model_input->channels != 3) {
157  return AVERROR(EIO);
158  }
159  if (model_input->dt != DNN_FLOAT && model_input->dt != DNN_UINT8) {
160  av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n");
161  return AVERROR(EIO);
162  }
163  return 0;
164  case AV_PIX_FMT_GRAY8:
165  if (model_input->channels != 1) {
167  return AVERROR(EIO);
168  }
169  if (model_input->dt != DNN_UINT8) {
170  av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type uint8.\n");
171  return AVERROR(EIO);
172  }
173  return 0;
174  case AV_PIX_FMT_GRAYF32:
175  case AV_PIX_FMT_YUV420P:
176  case AV_PIX_FMT_YUV422P:
177  case AV_PIX_FMT_YUV444P:
178  case AV_PIX_FMT_YUV410P:
179  case AV_PIX_FMT_YUV411P:
180  if (model_input->channels != 1) {
182  return AVERROR(EIO);
183  }
184  if (model_input->dt != DNN_FLOAT) {
185  av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type float32.\n");
186  return AVERROR(EIO);
187  }
188  return 0;
189  default:
190  av_log(ctx, AV_LOG_ERROR, "%s not supported.\n", av_get_pix_fmt_name(fmt));
191  return AVERROR(EIO);
192  }
193 
194  return 0;
195 }
196 
198 {
199  AVFilterContext *context = inlink->dst;
200  DnnProcessingContext *ctx = context->priv;
202  DNNData model_input;
203  int check;
204 
205  result = ctx->model->get_input(ctx->model->model, &model_input, ctx->model_inputname);
206  if (result != DNN_SUCCESS) {
207  av_log(ctx, AV_LOG_ERROR, "could not get input from the model\n");
208  return AVERROR(EIO);
209  }
210 
211  check = check_modelinput_inlink(&model_input, inlink);
212  if (check != 0) {
213  return check;
214  }
215 
216  ctx->input.width = inlink->w;
217  ctx->input.height = inlink->h;
218  ctx->input.channels = model_input.channels;
219  ctx->input.dt = model_input.dt;
220 
221  result = (ctx->model->set_input)(ctx->model->model,
222  &ctx->input, ctx->model_inputname);
223  if (result != DNN_SUCCESS) {
224  av_log(ctx, AV_LOG_ERROR, "could not set input and output for the model\n");
225  return AVERROR(EIO);
226  }
227 
228  return 0;
229 }
230 
231 static int prepare_sws_context(AVFilterLink *outlink)
232 {
233  AVFilterContext *context = outlink->src;
234  DnnProcessingContext *ctx = context->priv;
235  AVFilterLink *inlink = context->inputs[0];
236  enum AVPixelFormat fmt = inlink->format;
237  DNNDataType input_dt = ctx->input.dt;
238  DNNDataType output_dt = ctx->output.dt;
239 
240  switch (fmt) {
241  case AV_PIX_FMT_RGB24:
242  case AV_PIX_FMT_BGR24:
243  if (input_dt == DNN_FLOAT) {
244  ctx->sws_gray8_to_grayf32 = sws_getContext(inlink->w * 3,
245  inlink->h,
247  inlink->w * 3,
248  inlink->h,
250  0, NULL, NULL, NULL);
251  }
252  if (output_dt == DNN_FLOAT) {
253  ctx->sws_grayf32_to_gray8 = sws_getContext(outlink->w * 3,
254  outlink->h,
256  outlink->w * 3,
257  outlink->h,
259  0, NULL, NULL, NULL);
260  }
261  return 0;
262  case AV_PIX_FMT_YUV420P:
263  case AV_PIX_FMT_YUV422P:
264  case AV_PIX_FMT_YUV444P:
265  case AV_PIX_FMT_YUV410P:
266  case AV_PIX_FMT_YUV411P:
267  av_assert0(input_dt == DNN_FLOAT);
268  av_assert0(output_dt == DNN_FLOAT);
269  ctx->sws_gray8_to_grayf32 = sws_getContext(inlink->w,
270  inlink->h,
272  inlink->w,
273  inlink->h,
275  0, NULL, NULL, NULL);
276  ctx->sws_grayf32_to_gray8 = sws_getContext(outlink->w,
277  outlink->h,
279  outlink->w,
280  outlink->h,
282  0, NULL, NULL, NULL);
283 
284  if (inlink->w != outlink->w || inlink->h != outlink->h) {
286  int sws_src_h = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
287  int sws_src_w = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
288  int sws_dst_h = AV_CEIL_RSHIFT(outlink->h, desc->log2_chroma_h);
289  int sws_dst_w = AV_CEIL_RSHIFT(outlink->w, desc->log2_chroma_w);
290  ctx->sws_uv_scale = sws_getContext(sws_src_w, sws_src_h, AV_PIX_FMT_GRAY8,
291  sws_dst_w, sws_dst_h, AV_PIX_FMT_GRAY8,
293  ctx->sws_uv_height = sws_src_h;
294  }
295  return 0;
296  default:
297  //do nothing
298  break;
299  }
300 
301  return 0;
302 }
303 
304 static int config_output(AVFilterLink *outlink)
305 {
306  AVFilterContext *context = outlink->src;
307  DnnProcessingContext *ctx = context->priv;
309 
310  // have a try run in case that the dnn model resize the frame
311  result = (ctx->dnn_module->execute_model)(ctx->model, &ctx->output, (const char **)&ctx->model_outputname, 1);
312  if (result != DNN_SUCCESS){
313  av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
314  return AVERROR(EIO);
315  }
316 
317  outlink->w = ctx->output.width;
318  outlink->h = ctx->output.height;
319 
320  prepare_sws_context(outlink);
321 
322  return 0;
323 }
324 
326 {
327  int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
328  DNNData *dnn_input = &ctx->input;
329 
330  switch (frame->format) {
331  case AV_PIX_FMT_RGB24:
332  case AV_PIX_FMT_BGR24:
333  if (dnn_input->dt == DNN_FLOAT) {
334  sws_scale(ctx->sws_gray8_to_grayf32, (const uint8_t **)frame->data, frame->linesize,
335  0, frame->height, (uint8_t * const*)(&dnn_input->data),
336  (const int [4]){frame->width * 3 * sizeof(float), 0, 0, 0});
337  } else {
338  av_assert0(dnn_input->dt == DNN_UINT8);
339  av_image_copy_plane(dnn_input->data, bytewidth,
340  frame->data[0], frame->linesize[0],
341  bytewidth, frame->height);
342  }
343  return 0;
344  case AV_PIX_FMT_GRAY8:
345  case AV_PIX_FMT_GRAYF32:
346  av_image_copy_plane(dnn_input->data, bytewidth,
347  frame->data[0], frame->linesize[0],
348  bytewidth, frame->height);
349  return 0;
350  case AV_PIX_FMT_YUV420P:
351  case AV_PIX_FMT_YUV422P:
352  case AV_PIX_FMT_YUV444P:
353  case AV_PIX_FMT_YUV410P:
354  case AV_PIX_FMT_YUV411P:
355  sws_scale(ctx->sws_gray8_to_grayf32, (const uint8_t **)frame->data, frame->linesize,
356  0, frame->height, (uint8_t * const*)(&dnn_input->data),
357  (const int [4]){frame->width * sizeof(float), 0, 0, 0});
358  return 0;
359  default:
360  return AVERROR(EIO);
361  }
362 
363  return 0;
364 }
365 
367 {
368  int bytewidth = av_image_get_linesize(frame->format, frame->width, 0);
369  DNNData *dnn_output = &ctx->output;
370 
371  switch (frame->format) {
372  case AV_PIX_FMT_RGB24:
373  case AV_PIX_FMT_BGR24:
374  if (dnn_output->dt == DNN_FLOAT) {
375  sws_scale(ctx->sws_grayf32_to_gray8, (const uint8_t *[4]){(const uint8_t *)dnn_output->data, 0, 0, 0},
376  (const int[4]){frame->width * 3 * sizeof(float), 0, 0, 0},
377  0, frame->height, (uint8_t * const*)frame->data, frame->linesize);
378 
379  } else {
380  av_assert0(dnn_output->dt == DNN_UINT8);
381  av_image_copy_plane(frame->data[0], frame->linesize[0],
382  dnn_output->data, bytewidth,
383  bytewidth, frame->height);
384  }
385  return 0;
386  case AV_PIX_FMT_GRAY8:
387  // it is possible that data type of dnn output is float32,
388  // need to add support for such case when needed.
389  av_assert0(dnn_output->dt == DNN_UINT8);
390  av_image_copy_plane(frame->data[0], frame->linesize[0],
391  dnn_output->data, bytewidth,
392  bytewidth, frame->height);
393  return 0;
394  case AV_PIX_FMT_GRAYF32:
395  av_assert0(dnn_output->dt == DNN_FLOAT);
396  av_image_copy_plane(frame->data[0], frame->linesize[0],
397  dnn_output->data, bytewidth,
398  bytewidth, frame->height);
399  return 0;
400  case AV_PIX_FMT_YUV420P:
401  case AV_PIX_FMT_YUV422P:
402  case AV_PIX_FMT_YUV444P:
403  case AV_PIX_FMT_YUV410P:
404  case AV_PIX_FMT_YUV411P:
405  sws_scale(ctx->sws_grayf32_to_gray8, (const uint8_t *[4]){(const uint8_t *)dnn_output->data, 0, 0, 0},
406  (const int[4]){frame->width * sizeof(float), 0, 0, 0},
407  0, frame->height, (uint8_t * const*)frame->data, frame->linesize);
408  return 0;
409  default:
410  return AVERROR(EIO);
411  }
412 
413  return 0;
414 }
415 
417 {
418  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
419  av_assert0(desc);
420  return !(desc->flags & AV_PIX_FMT_FLAG_RGB) && desc->nb_components == 3;
421 }
422 
424 {
425  const AVPixFmtDescriptor *desc;
426  int uv_height;
427 
428  if (!ctx->sws_uv_scale) {
429  av_assert0(in->height == out->height && in->width == out->width);
430  desc = av_pix_fmt_desc_get(in->format);
431  uv_height = AV_CEIL_RSHIFT(in->height, desc->log2_chroma_h);
432  for (int i = 1; i < 3; ++i) {
433  int bytewidth = av_image_get_linesize(in->format, in->width, i);
434  av_image_copy_plane(out->data[i], out->linesize[i],
435  in->data[i], in->linesize[i],
436  bytewidth, uv_height);
437  }
438  } else {
439  sws_scale(ctx->sws_uv_scale, (const uint8_t **)(in->data + 1), in->linesize + 1,
440  0, ctx->sws_uv_height, out->data + 1, out->linesize + 1);
441  sws_scale(ctx->sws_uv_scale, (const uint8_t **)(in->data + 2), in->linesize + 2,
442  0, ctx->sws_uv_height, out->data + 2, out->linesize + 2);
443  }
444 
445  return 0;
446 }
447 
449 {
450  AVFilterContext *context = inlink->dst;
451  AVFilterLink *outlink = context->outputs[0];
452  DnnProcessingContext *ctx = context->priv;
453  DNNReturnType dnn_result;
454  AVFrame *out;
455 
456  copy_from_frame_to_dnn(ctx, in);
457 
458  dnn_result = (ctx->dnn_module->execute_model)(ctx->model, &ctx->output, (const char **)&ctx->model_outputname, 1);
459  if (dnn_result != DNN_SUCCESS){
460  av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
461  av_frame_free(&in);
462  return AVERROR(EIO);
463  }
464 
465  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
466  if (!out) {
467  av_frame_free(&in);
468  return AVERROR(ENOMEM);
469  }
470 
471  av_frame_copy_props(out, in);
472  copy_from_dnn_to_frame(ctx, out);
473 
474  if (isPlanarYUV(in->format))
475  copy_uv_planes(ctx, out, in);
476 
477  av_frame_free(&in);
478  return ff_filter_frame(outlink, out);
479 }
480 
482 {
484 
487  sws_freeContext(context->sws_uv_scale);
488 
489  if (context->dnn_module)
490  (context->dnn_module->free_model)(&context->model);
491 
492  av_freep(&context->dnn_module);
493 }
494 
496  {
497  .name = "default",
498  .type = AVMEDIA_TYPE_VIDEO,
499  .config_props = config_input,
500  .filter_frame = filter_frame,
501  },
502  { NULL }
503 };
504 
506  {
507  .name = "default",
508  .type = AVMEDIA_TYPE_VIDEO,
509  .config_props = config_output,
510  },
511  { NULL }
512 };
513 
515  .name = "dnn_processing",
516  .description = NULL_IF_CONFIG_SMALL("Apply DNN processing filter to the input."),
517  .priv_size = sizeof(DnnProcessingContext),
518  .init = init,
519  .uninit = uninit,
521  .inputs = dnn_processing_inputs,
522  .outputs = dnn_processing_outputs,
523  .priv_class = &dnn_processing_class,
524 };
void * model
Definition: dnn_interface.h:45
#define NULL
Definition: coverity.c:32
int av_image_get_linesize(enum AVPixelFormat pix_fmt, int width, int plane)
Compute the size of an image line with format pix_fmt and width width for the plane plane...
Definition: imgutils.c:76
static enum AVPixelFormat pix_fmt
Buffered I/O operations.
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2573
This structure describes decoded (raw) audio or video data.
Definition: frame.h:308
#define SWS_BICUBIC
Definition: swscale.h:60
AVOption.
Definition: opt.h:248
static const AVOption dnn_processing_options[]
DNNReturnType(* execute_model)(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output)
Definition: dnn_interface.h:61
void(* free_model)(DNNModel **model)
Definition: dnn_interface.h:63
const char * desc
Definition: libsvtav1.c:79
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:71
misc image utilities
Main libavfilter public API header.
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
struct SwsContext * sws_uv_scale
#define OFFSET(x)
int channels
Definition: dnn_interface.h:40
static const AVFilterPad dnn_processing_outputs[]
struct SwsContext * sws_grayf32_to_gray8
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:99
static av_cold void uninit(AVFilterContext *ctx)
uint8_t log2_chroma_w
Amount to shift the luma width right to find the chroma width.
Definition: pixdesc.h:92
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:287
const char * name
Pad name.
Definition: internal.h:60
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:347
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
AVFilter ff_vf_dnn_processing
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1091
uint8_t
#define av_cold
Definition: attributes.h:88
AVOptions.
static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)
struct SwsContext * sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat, int dstW, int dstH, enum AVPixelFormat dstFormat, int flags, SwsFilter *srcFilter, SwsFilter *dstFilter, const double *param)
Allocate and return an SwsContext.
Definition: utils.c:1912
static int config_input(AVFilterLink *inlink)
DNNBackendType
Definition: dnn_interface.h:33
external API header
#define av_log(a,...)
DNNDataType
Definition: dnn_interface.h:35
A filter pad used for either input or output.
Definition: internal.h:54
static int copy_from_frame_to_dnn(DnnProcessingContext *ctx, const AVFrame *frame)
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:50
int width
Definition: frame.h:366
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:588
uint8_t log2_chroma_h
Amount to shift the luma height right to find the chroma height.
Definition: pixdesc.h:101
int height
Definition: dnn_interface.h:40
static av_cold int init(AVFilterContext *context)
#define AV_PIX_FMT_FLAG_RGB
The pixel format contains RGB-like data (as opposed to YUV/grayscale).
Definition: pixdesc.h:148
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:203
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:153
void * data
Definition: dnn_interface.h:38
void * priv
private data for use by the filter
Definition: avfilter.h:354
simple assert() macros that are a bit more flexible than ISO C assert().
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:70
uint64_t flags
Combination of AV_PIX_FMT_FLAG_...
Definition: pixdesc.h:106
uint8_t nb_components
The number of components each pixel has, (1-4)
Definition: pixdesc.h:83
static int copy_uv_planes(DnnProcessingContext *ctx, AVFrame *out, const AVFrame *in)
AVFormatContext * ctx
Definition: movenc.c:48
void sws_freeContext(struct SwsContext *swsContext)
Free the swscaler context swsContext.
Definition: utils.c:2332
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:69
static const AVFilterPad dnn_processing_inputs[]
static int prepare_sws_context(AVFilterLink *outlink)
AVFILTER_DEFINE_CLASS(dnn_processing)
DNN inference engine interface.
DNNReturnType
Definition: dnn_interface.h:31
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:381
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:431
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:339
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:81
int attribute_align_arg sws_scale(struct SwsContext *c, const uint8_t *const srcSlice[], const int srcStride[], int srcSliceY, int srcSliceH, uint8_t *const dst[], const int dstStride[])
swscale wrapper, so we don&#39;t need to export the SwsContext.
Definition: swscale.c:744
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
planar YUV 4:1:0, 9bpp, (1 Cr & Cb sample per 4x4 Y samples)
Definition: pixfmt.h:72
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:145
static int query_formats(AVFilterContext *context)
static int copy_from_dnn_to_frame(DnnProcessingContext *ctx, AVFrame *frame)
static av_always_inline int isPlanarYUV(enum AVPixelFormat pix_fmt)
const char * name
Filter name.
Definition: avfilter.h:149
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:351
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:300
DNNModule * ff_get_dnn_module(DNNBackendType backend_type)
Definition: dnn_interface.c:32
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:322
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
Y , 8bpp.
Definition: pixfmt.h:74
static int filter_frame(AVFilterLink *inlink, AVFrame *in)
planar YUV 4:1:1, 12bpp, (1 Cr & Cb sample per 4x1 Y samples)
Definition: pixfmt.h:73
DNNReturnType(* set_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:53
#define LOG_FORMAT_CHANNEL_MISMATCH()
A list of supported formats for one end of a filter link.
Definition: formats.h:65
An instance of a filter.
Definition: avfilter.h:339
and forward the result(frame or status change) to the corresponding input.If nothing is possible
int height
Definition: frame.h:366
FILE * out
Definition: movenc.c:54
struct SwsContext * sws_gray8_to_grayf32
#define av_freep(p)
DNNModel *(* load_model)(const char *model_filename, const char *options)
Definition: dnn_interface.h:59
#define av_always_inline
Definition: attributes.h:45
DNNBackendType backend_type
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:373
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
internal API functions
static int config_output(AVFilterLink *outlink)
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your local context
DNNDataType dt
Definition: dnn_interface.h:39
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:671
int i
Definition: input.c:407
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:58
#define FLAGS
#define check(x, y, S, v)