FFmpeg
dnn_backend_native.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN native backend implementation.
24  */
25 
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
30 
31 static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
32 {
33  ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
34  DnnOperand *oprd = NULL;
35 
36  if (network->layers_num <= 0 || network->operands_num <= 0)
37  return DNN_ERROR;
38 
39  /* inputs */
40  av_assert0(input->dt == DNN_FLOAT);
41  for (int i = 0; i < network->operands_num; ++i) {
42  oprd = &network->operands[i];
43  if (strcmp(oprd->name, input_name) == 0) {
44  if (oprd->type != DOT_INPUT)
45  return DNN_ERROR;
46  break;
47  }
48  oprd = NULL;
49  }
50 
51  if (!oprd)
52  return DNN_ERROR;
53 
54  oprd->dims[0] = 1;
55  oprd->dims[1] = input->height;
56  oprd->dims[2] = input->width;
57  oprd->dims[3] = input->channels;
58 
59  av_freep(&oprd->data);
61  oprd->data = av_malloc(oprd->length);
62  if (!oprd->data)
63  return DNN_ERROR;
64 
65  input->data = oprd->data;
66 
67  /* outputs */
68  network->nb_output = 0;
69  av_freep(&network->output_indexes);
70  network->output_indexes = av_mallocz_array(nb_output, sizeof(*network->output_indexes));
71  if (!network->output_indexes)
72  return DNN_ERROR;
73 
74  for (uint32_t i = 0; i < nb_output; ++i) {
75  const char *output_name = output_names[i];
76  for (int j = 0; j < network->operands_num; ++j) {
77  oprd = &network->operands[j];
78  if (strcmp(oprd->name, output_name) == 0) {
79  network->output_indexes[network->nb_output++] = j;
80  break;
81  }
82  }
83  }
84 
85  if (network->nb_output != nb_output)
86  return DNN_ERROR;
87 
88  return DNN_SUCCESS;
89 }
90 
91 // Loads model and its parameters that are stored in a binary file with following structure:
92 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
93 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
94 // For DEPTH_TO_SPACE layer: block_size
95 DNNModel *ff_dnn_load_model_native(const char *model_filename)
96 {
97  DNNModel *model = NULL;
98  char header_expected[] = "FFMPEGDNNNATIVE";
99  char *buf;
100  size_t size;
101  int version, header_size, major_version_expected = 0;
102  ConvolutionalNetwork *network = NULL;
103  AVIOContext *model_file_context;
104  int file_size, dnn_size, parsed_size;
105  int32_t layer;
106  DNNLayerType layer_type;
107 
108  model = av_malloc(sizeof(DNNModel));
109  if (!model){
110  return NULL;
111  }
112 
113  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
114  av_freep(&model);
115  return NULL;
116  }
117  file_size = avio_size(model_file_context);
118 
119  /**
120  * check file header with string and version
121  */
122  size = sizeof(header_expected);
123  buf = av_malloc(size);
124  if (!buf) {
125  avio_closep(&model_file_context);
126  av_freep(&model);
127  return NULL;
128  }
129 
130  // size - 1 to skip the ending '\0' which is not saved in file
131  avio_get_str(model_file_context, size - 1, buf, size);
132  dnn_size = size - 1;
133  if (strncmp(buf, header_expected, size) != 0) {
134  av_freep(&buf);
135  avio_closep(&model_file_context);
136  av_freep(&model);
137  return NULL;
138  }
139  av_freep(&buf);
140 
141  version = (int32_t)avio_rl32(model_file_context);
142  dnn_size += 4;
143  if (version != major_version_expected) {
144  avio_closep(&model_file_context);
145  av_freep(&model);
146  return NULL;
147  }
148 
149  // currently no need to check minor version
150  version = (int32_t)avio_rl32(model_file_context);
151  dnn_size += 4;
152  header_size = dnn_size;
153 
154  network = av_mallocz(sizeof(ConvolutionalNetwork));
155  if (!network){
156  avio_closep(&model_file_context);
157  av_freep(&model);
158  return NULL;
159  }
160  model->model = (void *)network;
161 
162  avio_seek(model_file_context, file_size - 8, SEEK_SET);
163  network->layers_num = (int32_t)avio_rl32(model_file_context);
164  network->operands_num = (int32_t)avio_rl32(model_file_context);
165  dnn_size += 8;
166  avio_seek(model_file_context, header_size, SEEK_SET);
167 
168  network->layers = av_mallocz(network->layers_num * sizeof(Layer));
169  if (!network->layers){
170  avio_closep(&model_file_context);
171  ff_dnn_free_model_native(&model);
172  return NULL;
173  }
174 
175  network->operands = av_mallocz(network->operands_num * sizeof(DnnOperand));
176  if (!network->operands){
177  avio_closep(&model_file_context);
178  ff_dnn_free_model_native(&model);
179  return NULL;
180  }
181 
182  for (layer = 0; layer < network->layers_num; ++layer){
183  layer_type = (int32_t)avio_rl32(model_file_context);
184  dnn_size += 4;
185 
186  if (layer_type >= DLT_COUNT) {
187  avio_closep(&model_file_context);
188  ff_dnn_free_model_native(&model);
189  return NULL;
190  }
191 
192  network->layers[layer].type = layer_type;
193  parsed_size = layer_funcs[layer_type].pf_load(&network->layers[layer], model_file_context, file_size);
194  if (!parsed_size) {
195  avio_closep(&model_file_context);
196  ff_dnn_free_model_native(&model);
197  return NULL;
198  }
199  dnn_size += parsed_size;
200  }
201 
202  for (int32_t i = 0; i < network->operands_num; ++i){
203  DnnOperand *oprd;
204  int32_t name_len;
205  int32_t operand_index = (int32_t)avio_rl32(model_file_context);
206  dnn_size += 4;
207 
208  oprd = &network->operands[operand_index];
209  name_len = (int32_t)avio_rl32(model_file_context);
210  dnn_size += 4;
211 
212  avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
213  dnn_size += name_len;
214 
215  oprd->type = (int32_t)avio_rl32(model_file_context);
216  dnn_size += 4;
217 
218  oprd->data_type = (int32_t)avio_rl32(model_file_context);
219  dnn_size += 4;
220 
221  for (int32_t dim = 0; dim < 4; ++dim) {
222  oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
223  dnn_size += 4;
224  }
225 
226  oprd->isNHWC = 1;
227  }
228 
229  avio_closep(&model_file_context);
230 
231  if (dnn_size != file_size){
232  ff_dnn_free_model_native(&model);
233  return NULL;
234  }
235 
237 
238  return model;
239 }
240 
242 {
243  ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
244  int32_t layer;
245  uint32_t nb = FFMIN(nb_output, network->nb_output);
246 
247  if (network->layers_num <= 0 || network->operands_num <= 0)
248  return DNN_ERROR;
249  if (!network->operands[0].data)
250  return DNN_ERROR;
251 
252  for (layer = 0; layer < network->layers_num; ++layer){
253  DNNLayerType layer_type = network->layers[layer].type;
254  layer_funcs[layer_type].pf_exec(network->operands,
255  network->layers[layer].input_operand_indexes,
256  network->layers[layer].output_operand_index,
257  network->layers[layer].params);
258  }
259 
260  for (uint32_t i = 0; i < nb; ++i) {
261  DnnOperand *oprd = &network->operands[network->output_indexes[i]];
262  outputs[i].data = oprd->data;
263  outputs[i].height = oprd->dims[1];
264  outputs[i].width = oprd->dims[2];
265  outputs[i].channels = oprd->dims[3];
266  }
267 
268  return DNN_SUCCESS;
269 }
270 
272 {
273  int32_t result = 1;
274  for (int i = 0; i < 4; ++i)
275  result *= oprd->dims[i];
276 
277  return result;
278 }
279 
281 {
282  // currently, we just support DNN_FLOAT
283  return oprd->dims[0] * oprd->dims[1] * oprd->dims[2] * oprd->dims[3] * sizeof(float);
284 }
285 
287 {
288  ConvolutionalNetwork *network;
289  ConvolutionalParams *conv_params;
290  int32_t layer;
291 
292  if (*model)
293  {
294  network = (ConvolutionalNetwork *)(*model)->model;
295  for (layer = 0; layer < network->layers_num; ++layer){
296  if (network->layers[layer].type == DLT_CONV2D){
297  conv_params = (ConvolutionalParams *)network->layers[layer].params;
298  av_freep(&conv_params->kernel);
299  av_freep(&conv_params->biases);
300  }
301  av_freep(&network->layers[layer].params);
302  }
303  av_freep(&network->layers);
304 
305  for (uint32_t operand = 0; operand < network->operands_num; ++operand)
306  av_freep(&network->operands[operand].data);
307  av_freep(&network->operands);
308 
309  av_freep(&network->output_indexes);
310  av_freep(&network);
311  av_freep(model);
312  }
313 }
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1187
void * model
Definition: dnn_interface.h:50
#define NULL
Definition: coverity.c:32
Bytestream IO Context.
Definition: avio.h:161
int8_t isNHWC
NHWC if 1, otherwise NCHW.
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:339
int channels
Definition: dnn_interface.h:45
DNN inference functions interface for native backend.
int64_t avio_seek(AVIOContext *s, int64_t offset, int whence)
fseek() equivalent for AVIOContext.
Definition: aviobuf.c:246
#define AVIO_FLAG_READ
read-only
Definition: avio.h:674
int32_t calculate_operand_dims_count(const DnnOperand *oprd)
int version
Definition: avisynth_c.h:858
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:236
DNNDataType dt
Definition: dnn_interface.h:39
DNNOperandType type
input/output/intermediate operand of the network
int32_t input_operand_indexes[4]
a layer can have multiple inputs and one output.
LayerFunc layer_funcs[DLT_COUNT]
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
#define av_malloc(s)
DNNDataType data_type
support different kinds of data type such as float, half float, int8 etc, first support float now...
ptrdiff_t size
Definition: opengl_enc.c:100
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
int height
Definition: dnn_interface.h:45
unsigned int avio_rl32(AVIOContext *s)
Definition: aviobuf.c:772
void * data
data pointer with data length in bytes.
simple assert() macros that are a bit more flexible than ISO C assert().
int32_t dims[4]
there are two memory layouts, NHWC or NCHW, so we use dims, dims[0] is Number.
char name[128]
to avoid possible memory leak, do not use char *name
static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
#define FFMIN(a, b)
Definition: common.h:96
int32_t
LAYER_EXEC_FUNC pf_exec
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
DNNReturnType
Definition: dnn_interface.h:31
void ff_dnn_free_model_native(DNNModel **model)
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
void * buf
Definition: avisynth_c.h:766
DNNReturnType(* set_input_output)(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
Definition: dnn_interface.h:53
int dim
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
DNNLayerType type
DNNLayerType
the enum value of DNNLayerType should not be changed, the same values are used in convert_from_tensor...
int32_t calculate_operand_data_length(const DnnOperand *oprd)
float * data
Definition: dnn_interface.h:44
void * params
and forward the result(frame or status change) to the corresponding input.If nothing is possible
DNNModel * ff_dnn_load_model_native(const char *model_filename)
#define av_freep(p)
int avio_get_str(AVIOContext *pb, int maxlen, char *buf, int buflen)
Read a string from pb into buf.
Definition: aviobuf.c:882
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1242
for(j=16;j >0;--j)
int32_t output_operand_index
void * av_mallocz_array(size_t nmemb, size_t size)
Definition: mem.c:191
LAYER_LOAD_FUNC pf_load