FFmpeg
dnn_backend_native.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN native backend implementation.
24  */
25 
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
30 
31 static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
32 {
33  ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
34 
35  for (int i = 0; i < network->operands_num; ++i) {
36  DnnOperand *oprd = &network->operands[i];
37  if (strcmp(oprd->name, input_name) == 0) {
38  if (oprd->type != DOT_INPUT)
39  return DNN_ERROR;
40  input->dt = oprd->data_type;
41  av_assert0(oprd->dims[0] == 1);
42  input->height = oprd->dims[1];
43  input->width = oprd->dims[2];
44  input->channels = oprd->dims[3];
45  return DNN_SUCCESS;
46  }
47  }
48 
49  // do not find the input operand
50  return DNN_ERROR;
51 }
52 
53 static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
54 {
55  ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
56  DnnOperand *oprd = NULL;
57 
58  if (network->layers_num <= 0 || network->operands_num <= 0)
59  return DNN_ERROR;
60 
61  /* inputs */
62  for (int i = 0; i < network->operands_num; ++i) {
63  oprd = &network->operands[i];
64  if (strcmp(oprd->name, input_name) == 0) {
65  if (oprd->type != DOT_INPUT)
66  return DNN_ERROR;
67  break;
68  }
69  oprd = NULL;
70  }
71 
72  if (!oprd)
73  return DNN_ERROR;
74 
75  oprd->dims[0] = 1;
76  oprd->dims[1] = input->height;
77  oprd->dims[2] = input->width;
78  oprd->dims[3] = input->channels;
79 
80  av_freep(&oprd->data);
82  oprd->data = av_malloc(oprd->length);
83  if (!oprd->data)
84  return DNN_ERROR;
85 
86  input->data = oprd->data;
87 
88  /* outputs */
89  network->nb_output = 0;
90  av_freep(&network->output_indexes);
91  network->output_indexes = av_mallocz_array(nb_output, sizeof(*network->output_indexes));
92  if (!network->output_indexes)
93  return DNN_ERROR;
94 
95  for (uint32_t i = 0; i < nb_output; ++i) {
96  const char *output_name = output_names[i];
97  for (int j = 0; j < network->operands_num; ++j) {
98  oprd = &network->operands[j];
99  if (strcmp(oprd->name, output_name) == 0) {
100  network->output_indexes[network->nb_output++] = j;
101  break;
102  }
103  }
104  }
105 
106  if (network->nb_output != nb_output)
107  return DNN_ERROR;
108 
109  return DNN_SUCCESS;
110 }
111 
112 // Loads model and its parameters that are stored in a binary file with following structure:
113 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
114 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
115 // For DEPTH_TO_SPACE layer: block_size
116 DNNModel *ff_dnn_load_model_native(const char *model_filename)
117 {
118  DNNModel *model = NULL;
119  char header_expected[] = "FFMPEGDNNNATIVE";
120  char *buf;
121  size_t size;
122  int version, header_size, major_version_expected = 1;
123  ConvolutionalNetwork *network = NULL;
124  AVIOContext *model_file_context;
125  int file_size, dnn_size, parsed_size;
126  int32_t layer;
127  DNNLayerType layer_type;
128 
129  model = av_malloc(sizeof(DNNModel));
130  if (!model){
131  return NULL;
132  }
133 
134  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
135  av_freep(&model);
136  return NULL;
137  }
138  file_size = avio_size(model_file_context);
139 
140  /**
141  * check file header with string and version
142  */
143  size = sizeof(header_expected);
144  buf = av_malloc(size);
145  if (!buf) {
146  avio_closep(&model_file_context);
147  av_freep(&model);
148  return NULL;
149  }
150 
151  // size - 1 to skip the ending '\0' which is not saved in file
152  avio_get_str(model_file_context, size - 1, buf, size);
153  dnn_size = size - 1;
154  if (strncmp(buf, header_expected, size) != 0) {
155  av_freep(&buf);
156  avio_closep(&model_file_context);
157  av_freep(&model);
158  return NULL;
159  }
160  av_freep(&buf);
161 
162  version = (int32_t)avio_rl32(model_file_context);
163  dnn_size += 4;
164  if (version != major_version_expected) {
165  avio_closep(&model_file_context);
166  av_freep(&model);
167  return NULL;
168  }
169 
170  // currently no need to check minor version
171  version = (int32_t)avio_rl32(model_file_context);
172  dnn_size += 4;
173  header_size = dnn_size;
174 
175  network = av_mallocz(sizeof(ConvolutionalNetwork));
176  if (!network){
177  avio_closep(&model_file_context);
178  av_freep(&model);
179  return NULL;
180  }
181  model->model = (void *)network;
182 
183  avio_seek(model_file_context, file_size - 8, SEEK_SET);
184  network->layers_num = (int32_t)avio_rl32(model_file_context);
185  network->operands_num = (int32_t)avio_rl32(model_file_context);
186  dnn_size += 8;
187  avio_seek(model_file_context, header_size, SEEK_SET);
188 
189  network->layers = av_mallocz(network->layers_num * sizeof(Layer));
190  if (!network->layers){
191  avio_closep(&model_file_context);
192  ff_dnn_free_model_native(&model);
193  return NULL;
194  }
195 
196  network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand));
197  if (!network->operands){
198  avio_closep(&model_file_context);
199  ff_dnn_free_model_native(&model);
200  return NULL;
201  }
202 
203  for (layer = 0; layer < network->layers_num; ++layer){
204  layer_type = (int32_t)avio_rl32(model_file_context);
205  dnn_size += 4;
206 
207  if (layer_type >= DLT_COUNT) {
208  avio_closep(&model_file_context);
209  ff_dnn_free_model_native(&model);
210  return NULL;
211  }
212 
213  network->layers[layer].type = layer_type;
214  parsed_size = layer_funcs[layer_type].pf_load(&network->layers[layer], model_file_context, file_size);
215  if (!parsed_size) {
216  avio_closep(&model_file_context);
217  ff_dnn_free_model_native(&model);
218  return NULL;
219  }
220  dnn_size += parsed_size;
221  }
222 
223  for (int32_t i = 0; i < network->operands_num; ++i){
224  DnnOperand *oprd;
225  int32_t name_len;
226  int32_t operand_index = (int32_t)avio_rl32(model_file_context);
227  dnn_size += 4;
228 
229  oprd = &network->operands[operand_index];
230  name_len = (int32_t)avio_rl32(model_file_context);
231  dnn_size += 4;
232 
233  avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
234  dnn_size += name_len;
235 
236  oprd->type = (int32_t)avio_rl32(model_file_context);
237  dnn_size += 4;
238 
239  oprd->data_type = (int32_t)avio_rl32(model_file_context);
240  dnn_size += 4;
241 
242  for (int32_t dim = 0; dim < 4; ++dim) {
243  oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
244  dnn_size += 4;
245  }
246 
247  oprd->isNHWC = 1;
248  }
249 
250  avio_closep(&model_file_context);
251 
252  if (dnn_size != file_size){
253  ff_dnn_free_model_native(&model);
254  return NULL;
255  }
256 
258  model->get_input = &get_input_native;
259 
260  return model;
261 }
262 
264 {
265  ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
266  int32_t layer;
267  uint32_t nb = FFMIN(nb_output, network->nb_output);
268 
269  if (network->layers_num <= 0 || network->operands_num <= 0)
270  return DNN_ERROR;
271  if (!network->operands[0].data)
272  return DNN_ERROR;
273 
274  for (layer = 0; layer < network->layers_num; ++layer){
275  DNNLayerType layer_type = network->layers[layer].type;
276  layer_funcs[layer_type].pf_exec(network->operands,
277  network->layers[layer].input_operand_indexes,
278  network->layers[layer].output_operand_index,
279  network->layers[layer].params);
280  }
281 
282  for (uint32_t i = 0; i < nb; ++i) {
283  DnnOperand *oprd = &network->operands[network->output_indexes[i]];
284  outputs[i].data = oprd->data;
285  outputs[i].height = oprd->dims[1];
286  outputs[i].width = oprd->dims[2];
287  outputs[i].channels = oprd->dims[3];
288  outputs[i].dt = oprd->data_type;
289  }
290 
291  return DNN_SUCCESS;
292 }
293 
295 {
296  int32_t result = 1;
297  for (int i = 0; i < 4; ++i)
298  result *= oprd->dims[i];
299 
300  return result;
301 }
302 
304 {
305  // currently, we just support DNN_FLOAT
306  return oprd->dims[0] * oprd->dims[1] * oprd->dims[2] * oprd->dims[3] * sizeof(float);
307 }
308 
310 {
311  ConvolutionalNetwork *network;
312  ConvolutionalParams *conv_params;
313  int32_t layer;
314 
315  if (*model)
316  {
317  network = (ConvolutionalNetwork *)(*model)->model;
318  for (layer = 0; layer < network->layers_num; ++layer){
319  if (network->layers[layer].type == DLT_CONV2D){
320  conv_params = (ConvolutionalParams *)network->layers[layer].params;
321  av_freep(&conv_params->kernel);
322  av_freep(&conv_params->biases);
323  }
324  av_freep(&network->layers[layer].params);
325  }
326  av_freep(&network->layers);
327 
328  for (uint32_t operand = 0; operand < network->operands_num; ++operand)
329  av_freep(&network->operands[operand].data);
330  av_freep(&network->operands);
331 
332  av_freep(&network->output_indexes);
333  av_freep(&network);
334  av_freep(model);
335  }
336 }
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1187
void * model
Definition: dnn_interface.h:45
#define NULL
Definition: coverity.c:32
Bytestream IO Context.
Definition: avio.h:161
int8_t isNHWC
NHWC if 1, otherwise NCHW.
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:339
int channels
Definition: dnn_interface.h:40
DNN inference functions interface for native backend.
int64_t avio_seek(AVIOContext *s, int64_t offset, int whence)
fseek() equivalent for AVIOContext.
Definition: aviobuf.c:246
#define AVIO_FLAG_READ
read-only
Definition: avio.h:674
int32_t calculate_operand_dims_count(const DnnOperand *oprd)
int version
Definition: avisynth_c.h:858
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:236
DNNOperandType type
input/output/intermediate operand of the network
int32_t input_operand_indexes[4]
a layer can have multiple inputs and one output.
LayerFunc layer_funcs[DLT_COUNT]
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
#define av_malloc(s)
DNNDataType data_type
support different kinds of data type such as float, half float, int8 etc, first support float now...
ptrdiff_t size
Definition: opengl_enc.c:100
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:48
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
int height
Definition: dnn_interface.h:40
unsigned int avio_rl32(AVIOContext *s)
Definition: aviobuf.c:772
void * data
Definition: dnn_interface.h:38
void * data
data pointer with data length in bytes.
simple assert() macros that are a bit more flexible than ISO C assert().
int32_t dims[4]
there are two memory layouts, NHWC or NCHW, so we use dims, dims[0] is Number.
char name[128]
to avoid possible memory leak, do not use char *name
#define FFMIN(a, b)
Definition: common.h:96
int32_t
LAYER_EXEC_FUNC pf_exec
static DNNReturnType set_input_output_native(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
DNNReturnType
Definition: dnn_interface.h:31
void ff_dnn_free_model_native(DNNModel **model)
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
void * buf
Definition: avisynth_c.h:766
int dim
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
DNNLayerType type
DNNLayerType
the enum value of DNNLayerType should not be changed, the same values are used in convert_from_tensor...
int32_t calculate_operand_data_length(const DnnOperand *oprd)
void * params
and forward the result(frame or status change) to the corresponding input.If nothing is possible
DNNModel * ff_dnn_load_model_native(const char *model_filename)
#define av_freep(p)
DNNReturnType(* set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
Definition: dnn_interface.h:51
int avio_get_str(AVIOContext *pb, int maxlen, char *buf, int buflen)
Read a string from pb into buf.
Definition: aviobuf.c:882
DNNDataType dt
Definition: dnn_interface.h:39
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1242
for(j=16;j >0;--j)
int32_t output_operand_index
void * av_mallocz_array(size_t nmemb, size_t size)
Definition: mem.c:191
LAYER_LOAD_FUNC pf_load