FFmpeg
vf_dnn_processing.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019 Guo Yejun
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * implementing a generic image processing filter using deep learning networks.
24  */
25 
26 #include "libavformat/avio.h"
27 #include "libavutil/opt.h"
28 #include "libavutil/pixdesc.h"
29 #include "libavutil/avassert.h"
30 #include "avfilter.h"
31 #include "dnn_interface.h"
32 #include "formats.h"
33 #include "internal.h"
34 
35 typedef struct DnnProcessingContext {
36  const AVClass *class;
37 
43 
46 
47  // input & output of the model at execution time
51 
52 #define OFFSET(x) offsetof(DnnProcessingContext, x)
53 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM
54 static const AVOption dnn_processing_options[] = {
55  { "dnn_backend", "DNN backend", OFFSET(backend_type), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS, "backend" },
56  { "native", "native backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 0 }, 0, 0, FLAGS, "backend" },
57 #if (CONFIG_LIBTENSORFLOW == 1)
58  { "tensorflow", "tensorflow backend flag", 0, AV_OPT_TYPE_CONST, { .i64 = 1 }, 0, 0, FLAGS, "backend" },
59 #endif
60  { "model", "path to model file", OFFSET(model_filename), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
61  { "input", "input name of the model", OFFSET(model_inputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
62  { "output", "output name of the model", OFFSET(model_outputname), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
63  { "fmt", "AVPixelFormat of the frame", OFFSET(fmt), AV_OPT_TYPE_PIXEL_FMT, { .i64=AV_PIX_FMT_RGB24 }, AV_PIX_FMT_NONE, AV_PIX_FMT_NB - 1, FLAGS },
64  { NULL }
65 };
66 
67 AVFILTER_DEFINE_CLASS(dnn_processing);
68 
70 {
71  DnnProcessingContext *ctx = context->priv;
72  int supported = 0;
73  // as the first step, only rgb24 and bgr24 are supported
74  const enum AVPixelFormat supported_pixel_fmts[] = {
77  };
78  for (int i = 0; i < sizeof(supported_pixel_fmts) / sizeof(enum AVPixelFormat); ++i) {
79  if (supported_pixel_fmts[i] == ctx->fmt) {
80  supported = 1;
81  break;
82  }
83  }
84  if (!supported) {
85  av_log(context, AV_LOG_ERROR, "pixel fmt %s not supported yet\n",
86  av_get_pix_fmt_name(ctx->fmt));
88  }
89 
90  if (!ctx->model_filename) {
91  av_log(ctx, AV_LOG_ERROR, "model file for network is not specified\n");
92  return AVERROR(EINVAL);
93  }
94  if (!ctx->model_inputname) {
95  av_log(ctx, AV_LOG_ERROR, "input name of the model network is not specified\n");
96  return AVERROR(EINVAL);
97  }
98  if (!ctx->model_outputname) {
99  av_log(ctx, AV_LOG_ERROR, "output name of the model network is not specified\n");
100  return AVERROR(EINVAL);
101  }
102 
104  if (!ctx->dnn_module) {
105  av_log(ctx, AV_LOG_ERROR, "could not create DNN module for requested backend\n");
106  return AVERROR(ENOMEM);
107  }
108  if (!ctx->dnn_module->load_model) {
109  av_log(ctx, AV_LOG_ERROR, "load_model for network is not specified\n");
110  return AVERROR(EINVAL);
111  }
112 
113  ctx->model = (ctx->dnn_module->load_model)(ctx->model_filename);
114  if (!ctx->model) {
115  av_log(ctx, AV_LOG_ERROR, "could not load DNN model\n");
116  return AVERROR(EINVAL);
117  }
118 
119  return 0;
120 }
121 
123 {
125  DnnProcessingContext *ctx = context->priv;
126  enum AVPixelFormat pixel_fmts[2];
127  pixel_fmts[0] = ctx->fmt;
128  pixel_fmts[1] = AV_PIX_FMT_NONE;
129 
130  formats = ff_make_format_list(pixel_fmts);
131  return ff_set_common_formats(context, formats);
132 }
133 
135 {
136  AVFilterContext *context = inlink->dst;
137  DnnProcessingContext *ctx = context->priv;
139  DNNData dnn_data;
140 
141  result = ctx->model->get_input(ctx->model->model, &dnn_data, ctx->model_inputname);
142  if (result != DNN_SUCCESS) {
143  av_log(ctx, AV_LOG_ERROR, "could not get input from the model\n");
144  return AVERROR(EIO);
145  }
146 
147  // the design is to add explicit scale filter before this filter
148  if (dnn_data.height != -1 && dnn_data.height != inlink->h) {
149  av_log(ctx, AV_LOG_ERROR, "the model requires frame height %d but got %d\n",
150  dnn_data.height, inlink->h);
151  return AVERROR(EIO);
152  }
153  if (dnn_data.width != -1 && dnn_data.width != inlink->w) {
154  av_log(ctx, AV_LOG_ERROR, "the model requires frame width %d but got %d\n",
155  dnn_data.width, inlink->w);
156  return AVERROR(EIO);
157  }
158 
159  if (dnn_data.channels != 3) {
160  av_log(ctx, AV_LOG_ERROR, "the model requires input channels %d\n",
161  dnn_data.channels);
162  return AVERROR(EIO);
163  }
164  if (dnn_data.dt != DNN_FLOAT && dnn_data.dt != DNN_UINT8) {
165  av_log(ctx, AV_LOG_ERROR, "only support dnn models with input data type as float32 and uint8.\n");
166  return AVERROR(EIO);
167  }
168 
169  ctx->input.width = inlink->w;
170  ctx->input.height = inlink->h;
171  ctx->input.channels = dnn_data.channels;
172  ctx->input.dt = dnn_data.dt;
173 
174  result = (ctx->model->set_input_output)(ctx->model->model,
175  &ctx->input, ctx->model_inputname,
176  (const char **)&ctx->model_outputname, 1);
177  if (result != DNN_SUCCESS) {
178  av_log(ctx, AV_LOG_ERROR, "could not set input and output for the model\n");
179  return AVERROR(EIO);
180  }
181 
182  return 0;
183 }
184 
185 static int config_output(AVFilterLink *outlink)
186 {
187  AVFilterContext *context = outlink->src;
188  DnnProcessingContext *ctx = context->priv;
190 
191  // have a try run in case that the dnn model resize the frame
192  result = (ctx->dnn_module->execute_model)(ctx->model, &ctx->output, 1);
193  if (result != DNN_SUCCESS){
194  av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
195  return AVERROR(EIO);
196  }
197 
198  outlink->w = ctx->output.width;
199  outlink->h = ctx->output.height;
200 
201  return 0;
202 }
203 
204 static int copy_from_frame_to_dnn(DNNData *dnn_data, const AVFrame *in)
205 {
206  // extend this function to support more formats
208 
209  if (dnn_data->dt == DNN_FLOAT) {
210  float *dnn_input = dnn_data->data;
211  for (int i = 0; i < in->height; i++) {
212  for(int j = 0; j < in->width * 3; j++) {
213  int k = i * in->linesize[0] + j;
214  int t = i * in->width * 3 + j;
215  dnn_input[t] = in->data[0][k] / 255.0f;
216  }
217  }
218  } else {
219  uint8_t *dnn_input = dnn_data->data;
220  av_assert0(dnn_data->dt == DNN_UINT8);
221  for (int i = 0; i < in->height; i++) {
222  for(int j = 0; j < in->width * 3; j++) {
223  int k = i * in->linesize[0] + j;
224  int t = i * in->width * 3 + j;
225  dnn_input[t] = in->data[0][k];
226  }
227  }
228  }
229 
230  return 0;
231 }
232 
233 static int copy_from_dnn_to_frame(AVFrame *out, const DNNData *dnn_data)
234 {
235  // extend this function to support more formats
237 
238  if (dnn_data->dt == DNN_FLOAT) {
239  float *dnn_output = dnn_data->data;
240  for (int i = 0; i < out->height; i++) {
241  for(int j = 0; j < out->width * 3; j++) {
242  int k = i * out->linesize[0] + j;
243  int t = i * out->width * 3 + j;
244  out->data[0][k] = av_clip_uintp2((int)(dnn_output[t] * 255.0f), 8);
245  }
246  }
247  } else {
248  uint8_t *dnn_output = dnn_data->data;
249  av_assert0(dnn_data->dt == DNN_UINT8);
250  for (int i = 0; i < out->height; i++) {
251  for(int j = 0; j < out->width * 3; j++) {
252  int k = i * out->linesize[0] + j;
253  int t = i * out->width * 3 + j;
254  out->data[0][k] = dnn_output[t];
255  }
256  }
257  }
258 
259  return 0;
260 }
261 
263 {
264  AVFilterContext *context = inlink->dst;
265  AVFilterLink *outlink = context->outputs[0];
266  DnnProcessingContext *ctx = context->priv;
267  DNNReturnType dnn_result;
268  AVFrame *out;
269 
270  copy_from_frame_to_dnn(&ctx->input, in);
271 
272  dnn_result = (ctx->dnn_module->execute_model)(ctx->model, &ctx->output, 1);
273  if (dnn_result != DNN_SUCCESS){
274  av_log(ctx, AV_LOG_ERROR, "failed to execute model\n");
275  av_frame_free(&in);
276  return AVERROR(EIO);
277  }
278  av_assert0(ctx->output.channels == 3);
279 
280  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
281  if (!out) {
282  av_frame_free(&in);
283  return AVERROR(ENOMEM);
284  }
285 
286  av_frame_copy_props(out, in);
287  copy_from_dnn_to_frame(out, &ctx->output);
288  av_frame_free(&in);
289  return ff_filter_frame(outlink, out);
290 }
291 
293 {
295 
296  if (context->dnn_module)
297  (context->dnn_module->free_model)(&context->model);
298 
299  av_freep(&context->dnn_module);
300 }
301 
303  {
304  .name = "default",
305  .type = AVMEDIA_TYPE_VIDEO,
306  .config_props = config_input,
307  .filter_frame = filter_frame,
308  },
309  { NULL }
310 };
311 
313  {
314  .name = "default",
315  .type = AVMEDIA_TYPE_VIDEO,
316  .config_props = config_output,
317  },
318  { NULL }
319 };
320 
322  .name = "dnn_processing",
323  .description = NULL_IF_CONFIG_SMALL("Apply DNN processing filter to the input."),
324  .priv_size = sizeof(DnnProcessingContext),
325  .init = init,
326  .uninit = uninit,
328  .inputs = dnn_processing_inputs,
329  .outputs = dnn_processing_outputs,
330  .priv_class = &dnn_processing_class,
331 };
void * model
Definition: dnn_interface.h:45
#define NULL
Definition: coverity.c:32
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
Buffered I/O operations.
This structure describes decoded (raw) audio or video data.
Definition: frame.h:295
AVOption.
Definition: opt.h:246
static const AVOption dnn_processing_options[]
void(* free_model)(DNNModel **model)
Definition: dnn_interface.h:61
Main libavfilter public API header.
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:68
#define OFFSET(x)
int channels
Definition: dnn_interface.h:40
static const AVFilterPad dnn_processing_outputs[]
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:99
static av_cold void uninit(AVFilterContext *ctx)
enum AVPixelFormat fmt
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:283
const char * name
Pad name.
Definition: internal.h:60
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DNNModel *(* load_model)(const char *model_filename)
Definition: dnn_interface.h:57
AVFilter ff_vf_dnn_processing
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1093
uint8_t
#define av_cold
Definition: attributes.h:82
AVOptions.
#define f(width, name)
Definition: cbs_vp9.c:255
static int config_input(AVFilterLink *inlink)
DNNBackendType
Definition: dnn_interface.h:33
#define av_log(a,...)
A filter pad used for either input or output.
Definition: internal.h:54
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:48
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
int width
Definition: frame.h:353
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:569
int height
Definition: dnn_interface.h:40
static av_cold int init(AVFilterContext *context)
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:202
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
void * data
Definition: dnn_interface.h:38
void * priv
private data for use by the filter
Definition: avfilter.h:353
simple assert() macros that are a bit more flexible than ISO C assert().
AVFormatContext * ctx
Definition: movenc.c:48
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:69
static const AVFilterPad dnn_processing_inputs[]
AVFILTER_DEFINE_CLASS(dnn_processing)
DNN inference engine interface.
DNNReturnType
Definition: dnn_interface.h:31
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:368
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:326
static int copy_from_dnn_to_frame(AVFrame *out, const DNNData *dnn_data)
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:144
static int query_formats(AVFilterContext *context)
const char * name
Filter name.
Definition: avfilter.h:148
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:350
DNNModule * ff_get_dnn_module(DNNBackendType backend_type)
Definition: dnn_interface.c:31
DNNReturnType(* execute_model)(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
Definition: dnn_interface.h:59
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:309
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
static int filter_frame(AVFilterLink *inlink, AVFrame *in)
A list of supported formats for one end of a filter link.
Definition: formats.h:64
static int copy_from_frame_to_dnn(DNNData *dnn_data, const AVFrame *in)
An instance of a filter.
Definition: avfilter.h:338
and forward the result(frame or status change) to the corresponding input.If nothing is possible
number of pixel formats, DO NOT USE THIS if you want to link with shared libav* because the number of...
Definition: pixfmt.h:351
int height
Definition: frame.h:353
FILE * out
Definition: movenc.c:54
#define av_freep(p)
DNNReturnType(* set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
Definition: dnn_interface.h:51
DNNBackendType backend_type
formats
Definition: signature.h:48
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2438
internal API functions
static int config_output(AVFilterLink *outlink)
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your local context
DNNDataType dt
Definition: dnn_interface.h:39
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:654
#define FLAGS