FFmpeg
vf_dnn_processing.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019 Guo Yejun
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * implementing a generic image processing filter using deep learning networks.
24  */
25 
26 #include "libavformat/avio.h"
27 #include "libavutil/opt.h"
28 #include "libavutil/pixdesc.h"
29 #include "libavutil/avassert.h"
30 #include "libavutil/imgutils.h"
31 #include "filters.h"
32 #include "dnn_interface.h"
33 #include "formats.h"
34 #include "internal.h"
35 #include "libswscale/swscale.h"
36 #include "libavutil/time.h"
37 
38 typedef struct DnnProcessingContext {
39  const AVClass *class;
40 
46  int async;
47 
50 
54 
55 #define OFFSET(x) offsetof(DnnProcessingContext, x)
56 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
57 static const AVOption dnn_processing_options[] = {
58  { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS, "backend" },
59  { "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
60 #if (CONFIG_LIBTENSORFLOW == 1)
61  { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
62 #endif
63 #if (CONFIG_LIBOPENVINO == 1)
64  { "openvino", "openvino backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 2 }, 0, 0, FLAGS, "backend" },
65 #endif
66  { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
67  { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
68  { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
69  { "options", "backend options", OFFSET(backend_options), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
70  { "async", "use DNN async inference", OFFSET(async), AV_OPT_TYPE_BOOL, { .i64 = 1}, 0, 1, FLAGS},
71  { NULL }
72 };
73 
74 AVFILTER_DEFINE_CLASS(dnn_processing);
75 
77 {
78  DnnProcessingContext *ctx = context->priv;
79 
80  if (!ctx->model_filename) {
81  av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n");
82  return AVERROR(EINVAL);
83  }
84  if (!ctx->model_inputname) {
85  av_log(ctx, AV_LOG_ERROR, "input name of the model network is not specified\n");
86  return AVERROR(EINVAL);
87  }
88  if (!ctx->model_outputname) {
89  av_log(ctx, AV_LOG_ERROR, "output name of the model network is not specified\n");
90  return AVERROR(EINVAL);
91  }
92 
94  if (!ctx->dnn_module) {
95  av_log(ctx, AV_LOG_ERROR, "could not create DNN module for requested backend\n");
96  return AVERROR(ENOMEM);
97  }
98  if (!ctx->dnn_module->load_model) {
99  av_log(ctx, AV_LOG_ERROR, "load_model for network is not specified\n");
100  return AVERROR(EINVAL);
101  }
102 
103  ctx->model = (ctx->dnn_module->load_model)(ctx->model_filename, ctx->backend_options, context);
104  if (!ctx->model) {
105  av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n");
106  return AVERROR(EINVAL);
107  }
108 
109  if (!ctx->dnn_module->execute_model_async && ctx->async) {
110  ctx->async = 0;
111  av_log(ctx, AV_LOG_WARNING, "this backend does not support async execution, roll back to sync.\n");
112  }
113 
114 #if !HAVE_PTHREAD_CANCEL
115  if (ctx->async) {
116  ctx->async = 0;
117  av_log(ctx, AV_LOG_WARNING, "pthread is not supported, roll back to sync.\n");
118  }
119 #endif
120 
121  return 0;
122 }
123 
125 {
126  static const enum AVPixelFormat pix_fmts[] = {
133  };
134  AVFilterFormats *fmts_list = ff_make_format_list(pix_fmts);
135  return ff_set_common_formats(context, fmts_list);
136 }
137 
138 #define LOG_FORMAT_CHANNEL_MISMATCH() \
139  av_log(ctx, AV_LOG_ERROR, \
140  "the frame's format %s does not match " \
141  "the model input channel %d\n", \
142  av_get_pix_fmt_name(fmt), \
143  model_input->channels);
144 
145 static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)
146 {
147  AVFilterContext *ctx = inlink->dst;
148  enum AVPixelFormat fmt = inlink->format;
149 
150  // the design is to add explicit scale filter before this filter
151  if (model_input->height != -1 && model_input->height != inlink->h) {
152  av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n",
153  model_input->height, inlink->h);
154  return AVERROR(EIO);
155  }
156  if (model_input->width != -1 && model_input->width != inlink->w) {
157  av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n",
158  model_input->width, inlink->w);
159  return AVERROR(EIO);
160  }
161  if (model_input->dt != DNN_FLOAT) {
162  avpriv_report_missing_feature(ctx, "data type rather than DNN_FLOAT");
163  return AVERROR(EIO);
164  }
165 
166  switch (fmt) {
167  case AV_PIX_FMT_RGB24:
168  case AV_PIX_FMT_BGR24:
169  if (model_input->channels != 3) {
171  return AVERROR(EIO);
172  }
173  return 0;
174  case AV_PIX_FMT_GRAYF32:
175  case AV_PIX_FMT_YUV420P:
176  case AV_PIX_FMT_YUV422P:
177  case AV_PIX_FMT_YUV444P:
178  case AV_PIX_FMT_YUV410P:
179  case AV_PIX_FMT_YUV411P:
180  case AV_PIX_FMT_NV12:
181  if (model_input->channels != 1) {
183  return AVERROR(EIO);
184  }
185  return 0;
186  default:
188  return AVERROR(EIO);
189  }
190 
191  return 0;
192 }
193 
195 {
196  AVFilterContext *context = inlink->dst;
197  DnnProcessingContext *ctx = context->priv;
199  DNNData model_input;
200  int check;
201 
202  result = ctx->model->get_input(ctx->model->model, &model_input, ctx->model_inputname);
203  if (result != DNN_SUCCESS) {
204  av_log(ctx, AV_LOG_ERROR, "could not get input from the model\n");
205  return AVERROR(EIO);
206  }
207 
208  check = check_modelinput_inlink(&model_input, inlink);
209  if (check != 0) {
210  return check;
211  }
212 
213  return 0;
214 }
215 
217 {
218  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(pix_fmt);
219  av_assert0(desc);
220  return !(desc->flags & AV_PIX_FMT_FLAG_RGB) && desc->nb_components == 3;
221 }
222 
223 static int prepare_uv_scale(AVFilterLink *outlink)
224 {
225  AVFilterContext *context = outlink->src;
226  DnnProcessingContext *ctx = context->priv;
227  AVFilterLink *inlink = context->inputs[0];
228  enum AVPixelFormat fmt = inlink->format;
229 
230  if (isPlanarYUV(fmt)) {
231  if (inlink->w != outlink->w || inlink->h != outlink->h) {
232  if (fmt == AV_PIX_FMT_NV12) {
233  ctx->sws_uv_scale = sws_getContext(inlink->w >> 1, inlink->h >> 1, AV_PIX_FMT_YA8,
234  outlink->w >> 1, outlink->h >> 1, AV_PIX_FMT_YA8,
236  ctx->sws_uv_height = inlink->h >> 1;
237  } else {
239  int sws_src_h = AV_CEIL_RSHIFT(inlink->h, desc->log2_chroma_h);
240  int sws_src_w = AV_CEIL_RSHIFT(inlink->w, desc->log2_chroma_w);
241  int sws_dst_h = AV_CEIL_RSHIFT(outlink->h, desc->log2_chroma_h);
242  int sws_dst_w = AV_CEIL_RSHIFT(outlink->w, desc->log2_chroma_w);
243  ctx->sws_uv_scale = sws_getContext(sws_src_w, sws_src_h, AV_PIX_FMT_GRAY8,
244  sws_dst_w, sws_dst_h, AV_PIX_FMT_GRAY8,
246  ctx->sws_uv_height = sws_src_h;
247  }
248  }
249  }
250 
251  return 0;
252 }
253 
254 static int config_output(AVFilterLink *outlink)
255 {
256  AVFilterContext *context = outlink->src;
257  DnnProcessingContext *ctx = context->priv;
259  AVFilterLink *inlink = context->inputs[0];
260 
261  // have a try run in case that the dnn model resize the frame
262  result = ctx->model->get_output(ctx->model->model, ctx->model_inputname, inlink->w, inlink->h,
263  ctx->model_outputname, &outlink->w, &outlink->h);
264  if (result != DNN_SUCCESS) {
265  av_log(ctx, AV_LOG_ERROR, "could not get output from the model\n");
266  return AVERROR(EIO);
267  }
268 
269  prepare_uv_scale(outlink);
270 
271  return 0;
272 }
273 
275 {
276  const AVPixFmtDescriptor *desc;
277  int uv_height;
278 
279  if (!ctx->sws_uv_scale) {
280  av_assert0(in->height == out->height && in->width == out->width);
281  desc = av_pix_fmt_desc_get(in->format);
282  uv_height = AV_CEIL_RSHIFT(in->height, desc->log2_chroma_h);
283  for (int i = 1; i < 3; ++i) {
284  int bytewidth = av_image_get_linesize(in->format, in->width, i);
285  av_image_copy_plane(out->data[i], out->linesize[i],
286  in->data[i], in->linesize[i],
287  bytewidth, uv_height);
288  }
289  } else if (in->format == AV_PIX_FMT_NV12) {
290  sws_scale(ctx->sws_uv_scale, (const uint8_t **)(in->data + 1), in->linesize + 1,
291  0, ctx->sws_uv_height, out->data + 1, out->linesize + 1);
292  } else {
293  sws_scale(ctx->sws_uv_scale, (const uint8_t **)(in->data + 1), in->linesize + 1,
294  0, ctx->sws_uv_height, out->data + 1, out->linesize + 1);
295  sws_scale(ctx->sws_uv_scale, (const uint8_t **)(in->data + 2), in->linesize + 2,
296  0, ctx->sws_uv_height, out->data + 2, out->linesize + 2);
297  }
298 
299  return 0;
300 }
301 
303 {
304  AVFilterContext *context = inlink->dst;
305  AVFilterLink *outlink = context->outputs[0];
306  DnnProcessingContext *ctx = context->priv;
307  DNNReturnType dnn_result;
308  AVFrame *out;
309 
310  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
311  if (!out) {
312  av_frame_free(&in);
313  return AVERROR(ENOMEM);
314  }
315  av_frame_copy_props(out, in);
316 
317  dnn_result = (ctx->dnn_module->execute_model)(ctx->model, ctx->model_inputname, in,
318  (const char **)&ctx->model_outputname, 1, out);
319  if (dnn_result != DNN_SUCCESS){
320  av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
321  av_frame_free(&in);
322  av_frame_free(&out);
323  return AVERROR(EIO);
324  }
325 
326  if (isPlanarYUV(in->format))
327  copy_uv_planes(ctx, out, in);
328 
329  av_frame_free(&in);
330  return ff_filter_frame(outlink, out);
331 }
332 
334 {
335  AVFilterLink *inlink = filter_ctx->inputs[0];
336  AVFilterLink *outlink = filter_ctx->outputs[0];
337  AVFrame *in = NULL;
338  int64_t pts;
339  int ret, status;
340  int got_frame = 0;
341 
342  FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
343 
344  do {
345  // drain all input frames
346  ret = ff_inlink_consume_frame(inlink, &in);
347  if (ret < 0)
348  return ret;
349  if (ret > 0) {
350  ret = filter_frame(inlink, in);
351  if (ret < 0)
352  return ret;
353  got_frame = 1;
354  }
355  } while (ret > 0);
356 
357  // if frame got, schedule to next filter
358  if (got_frame)
359  return 0;
360 
361  if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
362  if (status == AVERROR_EOF) {
363  ff_outlink_set_status(outlink, status, pts);
364  return ret;
365  }
366  }
367 
368  FF_FILTER_FORWARD_WANTED(outlink, inlink);
369 
370  return FFERROR_NOT_READY;
371 }
372 
373 static int flush_frame(AVFilterLink *outlink, int64_t pts, int64_t *out_pts)
374 {
375  DnnProcessingContext *ctx = outlink->src->priv;
376  int ret;
377  DNNAsyncStatusType async_state;
378 
379  ret = (ctx->dnn_module->flush)(ctx->model);
380  if (ret != DNN_SUCCESS) {
381  return -1;
382  }
383 
384  do {
385  AVFrame *in_frame = NULL;
386  AVFrame *out_frame = NULL;
387  async_state = (ctx->dnn_module->get_async_result)(ctx->model, &in_frame, &out_frame);
388  if (out_frame) {
389  if (isPlanarYUV(in_frame->format))
390  copy_uv_planes(ctx, out_frame, in_frame);
391  av_frame_free(&in_frame);
392  ret = ff_filter_frame(outlink, out_frame);
393  if (ret < 0)
394  return ret;
395  if (out_pts)
396  *out_pts = out_frame->pts + pts;
397  }
398  av_usleep(5000);
399  } while (async_state >= DAST_NOT_READY);
400 
401  return 0;
402 }
403 
405 {
406  AVFilterLink *inlink = filter_ctx->inputs[0];
407  AVFilterLink *outlink = filter_ctx->outputs[0];
409  AVFrame *in = NULL, *out = NULL;
410  int64_t pts;
411  int ret, status;
412  int got_frame = 0;
413  int async_state;
414 
415  FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink);
416 
417  do {
418  // drain all input frames
419  ret = ff_inlink_consume_frame(inlink, &in);
420  if (ret < 0)
421  return ret;
422  if (ret > 0) {
423  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
424  if (!out) {
425  av_frame_free(&in);
426  return AVERROR(ENOMEM);
427  }
429  if ((ctx->dnn_module->execute_model_async)(ctx->model, ctx->model_inputname, in,
430  (const char **)&ctx->model_outputname, 1, out) != DNN_SUCCESS) {
431  return AVERROR(EIO);
432  }
433  }
434  } while (ret > 0);
435 
436  // drain all processed frames
437  do {
438  AVFrame *in_frame = NULL;
439  AVFrame *out_frame = NULL;
440  async_state = (ctx->dnn_module->get_async_result)(ctx->model, &in_frame, &out_frame);
441  if (out_frame) {
442  if (isPlanarYUV(in_frame->format))
443  copy_uv_planes(ctx, out_frame, in_frame);
444  av_frame_free(&in_frame);
445  ret = ff_filter_frame(outlink, out_frame);
446  if (ret < 0)
447  return ret;
448  got_frame = 1;
449  }
450  } while (async_state == DAST_SUCCESS);
451 
452  // if frame got, schedule to next filter
453  if (got_frame)
454  return 0;
455 
456  if (ff_inlink_acknowledge_status(inlink, &status, &pts)) {
457  if (status == AVERROR_EOF) {
458  int64_t out_pts = pts;
459  ret = flush_frame(outlink, pts, &out_pts);
460  ff_outlink_set_status(outlink, status, out_pts);
461  return ret;
462  }
463  }
464 
465  FF_FILTER_FORWARD_WANTED(outlink, inlink);
466 
467  return 0;
468 }
469 
471 {
472  DnnProcessingContext *ctx = filter_ctx->priv;
473 
474  if (ctx->async)
475  return activate_async(filter_ctx);
476  else
477  return activate_sync(filter_ctx);
478 }
479 
481 {
483 
484  sws_freeContext(context->sws_uv_scale);
485 
486  if (context->dnn_module)
487  (context->dnn_module->free_model)(&context->model);
488 
489  av_freep(&context->dnn_module);
490 }
491 
493  {
494  .name = "default",
495  .type = AVMEDIA_TYPE_VIDEO,
496  .config_props = config_input,
497  },
498  { NULL }
499 };
500 
502  {
503  .name = "default",
504  .type = AVMEDIA_TYPE_VIDEO,
505  .config_props = config_output,
506  },
507  { NULL }
508 };
509 
511  .name = "dnn_processing",
512  .description = NULL_IF_CONFIG_SMALL("Apply DNN processing filter to the input."),
513  .priv_size = sizeof(DnnProcessingContext),
514  .init = init,
515  .uninit = uninit,
517  .inputs = dnn_processing_inputs,
518  .outputs = dnn_processing_outputs,
519  .priv_class = &dnn_processing_class,
520  .activate = activate,
521 };
void * model
Definition: dnn_interface.h:54
int ff_inlink_consume_frame(AVFilterLink *link, AVFrame **rframe)
Take a frame from the link&#39;s FIFO and update the link&#39;s stats.
Definition: avfilter.c:1489
#define NULL
Definition: coverity.c:32
int av_image_get_linesize(enum AVPixelFormat pix_fmt, int width, int plane)
Compute the size of an image line with format pix_fmt and width width for the plane plane...
Definition: imgutils.c:76
static enum AVPixelFormat pix_fmt
Buffered I/O operations.
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2573
This structure describes decoded (raw) audio or video data.
Definition: frame.h:314
#define SWS_BICUBIC
Definition: swscale.h:60
AVOption.
Definition: opt.h:248
static int prepare_uv_scale(AVFilterLink *outlink)
static const AVOption dnn_processing_options[]
8 bits gray, 8 bits alpha
Definition: pixfmt.h:143
void(* free_model)(DNNModel **model)
Definition: dnn_interface.h:88
const char * desc
Definition: libsvtav1.c:79
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:71
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
struct SwsContext * sws_uv_scale
#define OFFSET(x)
int channels
Definition: dnn_interface.h:49
static const AVFilterPad dnn_processing_outputs[]
static int flush_frame(AVFilterLink *outlink, int64_t pts, int64_t *out_pts)
int av_usleep(unsigned usec)
Sleep for a period of time.
Definition: time.c:84
return FFERROR_NOT_READY
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:99
static av_cold void uninit(AVFilterContext *ctx)
uint8_t log2_chroma_w
Amount to shift the luma width right to find the chroma width.
Definition: pixdesc.h:92
static void ff_outlink_set_status(AVFilterLink *link, int status, int64_t pts)
Set the status field of a link from the source filter.
Definition: filters.h:189
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:287
const char * name
Pad name.
Definition: internal.h:60
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:349
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
AVFilter ff_vf_dnn_processing
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1091
uint8_t
#define av_cold
Definition: attributes.h:88
AVOptions.
DNNReturnType(* execute_model)(const DNNModel *model, const char *input_name, AVFrame *in_frame, const char **output_names, uint32_t nb_output, AVFrame *out_frame)
Definition: dnn_interface.h:78
DNNReturnType(* execute_model_async)(const DNNModel *model, const char *input_name, AVFrame *in_frame, const char **output_names, uint32_t nb_output, AVFrame *out_frame)
Definition: dnn_interface.h:81
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:407
static int check_modelinput_inlink(const DNNData *model_input, const AVFilterLink *inlink)
struct SwsContext * sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat, int dstW, int dstH, enum AVPixelFormat dstFormat, int flags, SwsFilter *srcFilter, SwsFilter *dstFilter, const double *param)
Allocate and return an SwsContext.
Definition: utils.c:1917
#define AVERROR_EOF
End of file.
Definition: error.h:55
static int config_input(AVFilterLink *inlink)
DNNBackendType
Definition: dnn_interface.h:35
external API header
#define av_log(a,...)
#define FF_FILTER_FORWARD_STATUS_BACK(outlink, inlink)
Forward the status on an output link to an input link.
Definition: filters.h:199
A filter pad used for either input or output.
Definition: internal.h:54
int ff_inlink_acknowledge_status(AVFilterLink *link, int *rstatus, int64_t *rpts)
Test and acknowledge the change of status on the link.
Definition: avfilter.c:1444
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:61
int width
Definition: frame.h:372
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:588
uint8_t log2_chroma_h
Amount to shift the luma height right to find the chroma height.
Definition: pixdesc.h:101
int height
Definition: dnn_interface.h:49
DNNModel *(* load_model)(const char *model_filename, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:76
static av_cold int init(AVFilterContext *context)
#define AV_PIX_FMT_FLAG_RGB
The pixel format contains RGB-like data (as opposed to YUV/grayscale).
Definition: pixdesc.h:148
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:203
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:115
void * priv
private data for use by the filter
Definition: avfilter.h:356
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
simple assert() macros that are a bit more flexible than ISO C assert().
static FilteringContext * filter_ctx
Definition: transcoding.c:47
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:70
uint64_t flags
Combination of AV_PIX_FMT_FLAG_...
Definition: pixdesc.h:106
uint8_t nb_components
The number of components each pixel has, (1-4)
Definition: pixdesc.h:83
DNNAsyncStatusType
Definition: dnn_interface.h:39
static int copy_uv_planes(DnnProcessingContext *ctx, AVFrame *out, const AVFrame *in)
AVFormatContext * ctx
Definition: movenc.c:48
void sws_freeContext(struct SwsContext *swsContext)
Free the swscaler context swsContext.
Definition: utils.c:2337
DNNReturnType(* flush)(const DNNModel *model)
Definition: dnn_interface.h:86
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:69
static const AVFilterPad dnn_processing_inputs[]
AVFILTER_DEFINE_CLASS(dnn_processing)
DNN inference engine interface.
DNNReturnType
Definition: dnn_interface.h:33
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:387
static int activate(AVFilterContext *filter_ctx)
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:431
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:345
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:81
int attribute_align_arg sws_scale(struct SwsContext *c, const uint8_t *const srcSlice[], const int srcStride[], int srcSliceY, int srcSliceH, uint8_t *const dst[], const int dstStride[])
swscale wrapper, so we don&#39;t need to export the SwsContext.
Definition: swscale.c:745
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
planar YUV 4:1:0, 9bpp, (1 Cr & Cb sample per 4x4 Y samples)
Definition: pixfmt.h:72
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:145
static int query_formats(AVFilterContext *context)
static int activate_async(AVFilterContext *filter_ctx)
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
static av_always_inline int isPlanarYUV(enum AVPixelFormat pix_fmt)
static int activate_sync(AVFilterContext *filter_ctx)
const char * name
Filter name.
Definition: avfilter.h:149
DNNAsyncStatusType(* get_async_result)(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_interface.h:84
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:353
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:303
DNNModule * ff_get_dnn_module(DNNBackendType backend_type)
Definition: dnn_interface.c:32
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
static int64_t pts
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:328
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
Y , 8bpp.
Definition: pixfmt.h:74
DNNReturnType(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:63
static int filter_frame(AVFilterLink *inlink, AVFrame *in)
planar YUV 4:1:1, 12bpp, (1 Cr & Cb sample per 4x1 Y samples)
Definition: pixfmt.h:73
FF_FILTER_FORWARD_WANTED(outlink, inlink)
#define LOG_FORMAT_CHANNEL_MISMATCH()
A list of supported formats for one end of a filter link.
Definition: formats.h:65
An instance of a filter.
Definition: avfilter.h:341
and forward the result(frame or status change) to the corresponding input.If nothing is possible
int height
Definition: frame.h:372
FILE * out
Definition: movenc.c:54
#define av_freep(p)
#define av_always_inline
Definition: attributes.h:45
DNNBackendType backend_type
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:373
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
internal API functions
static int config_output(AVFilterLink *outlink)
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your local context
DNNDataType dt
Definition: dnn_interface.h:48
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:658
int i
Definition: input.c:407
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:58
#define FLAGS
#define check(x, y, S, v)