FFmpeg
dnn_backend_native.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN native backend implementation.
24  */
25 
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
30 
31 #define OFFSET(x) offsetof(NativeContext, x)
32 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
33 static const AVOption dnn_native_options[] = {
34  { "conv2d_threads", "threads num for conv2d layer", OFFSET(options.conv2d_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS },
35  { NULL },
36 };
37 
39  .class_name = "dnn_native",
40  .item_name = av_default_item_name,
41  .option = dnn_native_options,
42  .version = LIBAVUTIL_VERSION_INT,
43  .category = AV_CLASS_CATEGORY_FILTER,
44 };
45 
46 static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
47 {
48  NativeModel *native_model = (NativeModel *)model;
49  NativeContext *ctx = &native_model->ctx;
50 
51  for (int i = 0; i < native_model->operands_num; ++i) {
52  DnnOperand *oprd = &native_model->operands[i];
53  if (strcmp(oprd->name, input_name) == 0) {
54  if (oprd->type != DOT_INPUT) {
55  av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
56  return DNN_ERROR;
57  }
58  input->dt = oprd->data_type;
59  av_assert0(oprd->dims[0] == 1);
60  input->height = oprd->dims[1];
61  input->width = oprd->dims[2];
62  input->channels = oprd->dims[3];
63  return DNN_SUCCESS;
64  }
65  }
66 
67  // do not find the input operand
68  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
69  return DNN_ERROR;
70 }
71 
72 static DNNReturnType set_input_native(void *model, DNNData *input, const char *input_name)
73 {
74  NativeModel *native_model = (NativeModel *)model;
75  NativeContext *ctx = &native_model->ctx;
76  DnnOperand *oprd = NULL;
77 
78  if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
79  av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
80  return DNN_ERROR;
81  }
82 
83  /* inputs */
84  for (int i = 0; i < native_model->operands_num; ++i) {
85  oprd = &native_model->operands[i];
86  if (strcmp(oprd->name, input_name) == 0) {
87  if (oprd->type != DOT_INPUT) {
88  av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
89  return DNN_ERROR;
90  }
91  break;
92  }
93  oprd = NULL;
94  }
95  if (!oprd) {
96  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
97  return DNN_ERROR;
98  }
99 
100  oprd->dims[0] = 1;
101  oprd->dims[1] = input->height;
102  oprd->dims[2] = input->width;
103  oprd->dims[3] = input->channels;
104 
105  av_freep(&oprd->data);
107  if (oprd->length <= 0) {
108  av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n");
109  return DNN_ERROR;
110  }
111  oprd->data = av_malloc(oprd->length);
112  if (!oprd->data) {
113  av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n");
114  return DNN_ERROR;
115  }
116 
117  input->data = oprd->data;
118 
119  return DNN_SUCCESS;
120 }
121 
122 // Loads model and its parameters that are stored in a binary file with following structure:
123 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
124 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
125 // For DEPTH_TO_SPACE layer: block_size
126 DNNModel *ff_dnn_load_model_native(const char *model_filename, const char *options)
127 {
128  DNNModel *model = NULL;
129  char header_expected[] = "FFMPEGDNNNATIVE";
130  char *buf;
131  size_t size;
132  int version, header_size, major_version_expected = 1;
133  NativeModel *native_model = NULL;
134  AVIOContext *model_file_context;
135  int file_size, dnn_size, parsed_size;
136  int32_t layer;
137  DNNLayerType layer_type;
138 
139  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
140  return NULL;
141  }
142  file_size = avio_size(model_file_context);
143 
144  model = av_mallocz(sizeof(DNNModel));
145  if (!model){
146  goto fail;
147  }
148 
149  /**
150  * check file header with string and version
151  */
152  size = sizeof(header_expected);
153  buf = av_malloc(size);
154  if (!buf) {
155  goto fail;
156  }
157 
158  // size - 1 to skip the ending '\0' which is not saved in file
159  avio_get_str(model_file_context, size - 1, buf, size);
160  dnn_size = size - 1;
161  if (strncmp(buf, header_expected, size) != 0) {
162  av_freep(&buf);
163  goto fail;
164  }
165  av_freep(&buf);
166 
167  version = (int32_t)avio_rl32(model_file_context);
168  dnn_size += 4;
169  if (version != major_version_expected) {
170  goto fail;
171  }
172 
173  // currently no need to check minor version
174  version = (int32_t)avio_rl32(model_file_context);
175  dnn_size += 4;
176  header_size = dnn_size;
177 
178  native_model = av_mallocz(sizeof(NativeModel));
179  if (!native_model){
180  goto fail;
181  }
182 
183  native_model->ctx.class = &dnn_native_class;
184  model->options = options;
185  if (av_opt_set_from_string(&native_model->ctx, model->options, NULL, "=", "&") < 0)
186  goto fail;
187  model->model = (void *)native_model;
188 
189 #if !HAVE_PTHREAD_CANCEL
190  if (native_model->ctx.options.conv2d_threads > 1){
191  av_log(&native_model->ctx, AV_LOG_WARNING, "'conv2d_threads' option was set but it is not supported "
192  "on this build (pthread support is required)\n");
193  }
194 #endif
195 
196  avio_seek(model_file_context, file_size - 8, SEEK_SET);
197  native_model->layers_num = (int32_t)avio_rl32(model_file_context);
198  native_model->operands_num = (int32_t)avio_rl32(model_file_context);
199  dnn_size += 8;
200  avio_seek(model_file_context, header_size, SEEK_SET);
201 
202  native_model->layers = av_mallocz(native_model->layers_num * sizeof(Layer));
203  if (!native_model->layers){
204  goto fail;
205  }
206 
207  native_model->operands = av_mallocz(native_model->operands_num * sizeof(DnnOperand));
208  if (!native_model->operands){
209  goto fail;
210  }
211 
212  for (layer = 0; layer < native_model->layers_num; ++layer){
213  layer_type = (int32_t)avio_rl32(model_file_context);
214  dnn_size += 4;
215 
216  if (layer_type >= DLT_COUNT) {
217  goto fail;
218  }
219 
220  native_model->layers[layer].type = layer_type;
221  parsed_size = layer_funcs[layer_type].pf_load(&native_model->layers[layer], model_file_context, file_size, native_model->operands_num);
222  if (!parsed_size) {
223  goto fail;
224  }
225  dnn_size += parsed_size;
226  }
227 
228  for (int32_t i = 0; i < native_model->operands_num; ++i){
229  DnnOperand *oprd;
230  int32_t name_len;
231  int32_t operand_index = (int32_t)avio_rl32(model_file_context);
232  dnn_size += 4;
233 
234  if (operand_index >= native_model->operands_num) {
235  goto fail;
236  }
237 
238  oprd = &native_model->operands[operand_index];
239  name_len = (int32_t)avio_rl32(model_file_context);
240  dnn_size += 4;
241 
242  avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
243  dnn_size += name_len;
244 
245  oprd->type = (int32_t)avio_rl32(model_file_context);
246  dnn_size += 4;
247 
248  oprd->data_type = (int32_t)avio_rl32(model_file_context);
249  dnn_size += 4;
250 
251  for (int32_t dim = 0; dim < 4; ++dim) {
252  oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
253  dnn_size += 4;
254  }
255 
256  oprd->isNHWC = 1;
257  }
258 
259  avio_closep(&model_file_context);
260 
261  if (dnn_size != file_size){
262  ff_dnn_free_model_native(&model);
263  return NULL;
264  }
265 
266  model->set_input = &set_input_native;
267  model->get_input = &get_input_native;
268 
269  return model;
270 
271 fail:
272  ff_dnn_free_model_native(&model);
273  avio_closep(&model_file_context);
274  return NULL;
275 }
276 
277 DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output)
278 {
279  NativeModel *native_model = (NativeModel *)model->model;
280  NativeContext *ctx = &native_model->ctx;
281  int32_t layer;
282 
283  if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
284  av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
285  return DNN_ERROR;
286  }
287  if (!native_model->operands[0].data) {
288  av_log(ctx, AV_LOG_ERROR, "Empty model input data\n");
289  return DNN_ERROR;
290  }
291 
292  for (layer = 0; layer < native_model->layers_num; ++layer){
293  DNNLayerType layer_type = native_model->layers[layer].type;
294  if (layer_funcs[layer_type].pf_exec(native_model->operands,
295  native_model->layers[layer].input_operand_indexes,
296  native_model->layers[layer].output_operand_index,
297  native_model->layers[layer].params,
298  &native_model->ctx) == DNN_ERROR) {
299  av_log(ctx, AV_LOG_ERROR, "Failed to execuet model\n");
300  return DNN_ERROR;
301  }
302  }
303 
304  for (uint32_t i = 0; i < nb_output; ++i) {
305  DnnOperand *oprd = NULL;
306  const char *output_name = output_names[i];
307  for (int j = 0; j < native_model->operands_num; ++j) {
308  if (strcmp(native_model->operands[j].name, output_name) == 0) {
309  oprd = &native_model->operands[j];
310  break;
311  }
312  }
313 
314  if (oprd == NULL) {
315  av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n");
316  return DNN_ERROR;
317  }
318 
319  outputs[i].data = oprd->data;
320  outputs[i].height = oprd->dims[1];
321  outputs[i].width = oprd->dims[2];
322  outputs[i].channels = oprd->dims[3];
323  outputs[i].dt = oprd->data_type;
324  }
325 
326  return DNN_SUCCESS;
327 }
328 
330 {
331  int32_t result = 1;
332  for (int i = 0; i < 4; ++i)
333  result *= oprd->dims[i];
334 
335  return result;
336 }
337 
339 {
340  // currently, we just support DNN_FLOAT
341  uint64_t len = sizeof(float);
342  for (int i = 0; i < 4; i++) {
343  len *= oprd->dims[i];
344  if (len > INT32_MAX)
345  return 0;
346  }
347  return len;
348 }
349 
351 {
352  NativeModel *native_model;
353  ConvolutionalParams *conv_params;
354  int32_t layer;
355 
356  if (*model)
357  {
358  if ((*model)->model) {
359  native_model = (NativeModel *)(*model)->model;
360  if (native_model->layers) {
361  for (layer = 0; layer < native_model->layers_num; ++layer){
362  if (native_model->layers[layer].type == DLT_CONV2D){
363  conv_params = (ConvolutionalParams *)native_model->layers[layer].params;
364  av_freep(&conv_params->kernel);
365  av_freep(&conv_params->biases);
366  }
367  av_freep(&native_model->layers[layer].params);
368  }
369  av_freep(&native_model->layers);
370  }
371 
372  if (native_model->operands) {
373  for (uint32_t operand = 0; operand < native_model->operands_num; ++operand)
374  av_freep(&native_model->operands[operand].data);
375  av_freep(&native_model->operands);
376  }
377 
378  av_freep(&native_model);
379  }
380  av_freep(model);
381  }
382 }
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1127
void * model
Definition: dnn_interface.h:45
#define NULL
Definition: coverity.c:32
Bytestream IO Context.
Definition: avio.h:161
int8_t isNHWC
NHWC if 1, otherwise NCHW.
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:346
version
Definition: libkvazaar.c:317
AVOption.
Definition: opt.h:248
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, const char **output_names, uint32_t nb_output)
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
int channels
Definition: dnn_interface.h:40
DNN inference functions interface for native backend.
int64_t avio_seek(AVIOContext *s, int64_t offset, int whence)
fseek() equivalent for AVIOContext.
Definition: aviobuf.c:253
#define AVIO_FLAG_READ
read-only
Definition: avio.h:674
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
int32_t calculate_operand_dims_count(const DnnOperand *oprd)
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:237
DNNOperandType type
input/output/intermediate operand of the network
const char * options
Definition: dnn_interface.h:47
DnnOperand * operands
int32_t input_operand_indexes[4]
a layer can have multiple inputs and one output.
LayerFunc layer_funcs[DLT_COUNT]
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition: opt.c:1558
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
#define av_malloc(s)
static DNNReturnType set_input_native(void *model, DNNData *input, const char *input_name)
const AVClass * class
DNNDataType data_type
support different kinds of data type such as float, half float, int8 etc, first support float now...
ptrdiff_t size
Definition: opengl_enc.c:100
NativeContext ctx
#define av_log(a,...)
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:50
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
int height
Definition: dnn_interface.h:40
unsigned int avio_rl32(AVIOContext *s)
Definition: aviobuf.c:759
void * data
Definition: dnn_interface.h:38
void * data
data pointer with data length in bytes.
simple assert() macros that are a bit more flexible than ISO C assert().
int32_t dims[4]
there are two memory layouts, NHWC or NCHW, so we use dims, dims[0] is Number.
#define fail()
Definition: checkasm.h:123
DNNModel * ff_dnn_load_model_native(const char *model_filename, const char *options)
char name[128]
to avoid possible memory leak, do not use char *name
int32_t
AVFormatContext * ctx
Definition: movenc.c:48
#define OFFSET(x)
DNNReturnType
Definition: dnn_interface.h:31
void ff_dnn_free_model_native(DNNModel **model)
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
if(ret)
const AVClass dnn_native_class
static const AVOption dnn_native_options[]
static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
Describe the class of an AVClass context structure.
Definition: log.h:67
int dim
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
DNNLayerType type
DNNLayerType
the enum value of DNNLayerType should not be changed, the same values are used in convert_from_tensor...
#define FLAGS
const OptionDef options[]
Definition: ffmpeg_opt.c:3393
DNNReturnType(* set_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:53
NativeOptions options
int32_t calculate_operand_data_length(const DnnOperand *oprd)
int len
void * params
and forward the result(frame or status change) to the corresponding input.If nothing is possible
#define av_freep(p)
int avio_get_str(AVIOContext *pb, int maxlen, char *buf, int buflen)
Read a string from pb into buf.
Definition: aviobuf.c:869
DNNDataType dt
Definition: dnn_interface.h:39
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1182
int i
Definition: input.c:407
int32_t output_operand_index
LAYER_LOAD_FUNC pf_load