FFmpeg
dnn_backend_native.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN native backend implementation.
24  */
25 
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
30 #include "dnn_io_proc.h"
31 #include "dnn_backend_common.h"
32 
33 #define OFFSET(x) offsetof(NativeContext, x)
34 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
35 static const AVOption dnn_native_options[] = {
36  { "conv2d_threads", "threads num for conv2d layer", OFFSET(options.conv2d_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS },
37  { "async", "use DNN async inference", OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
38  { NULL },
39 };
40 
41 static const AVClass dnn_native_class = {
42  .class_name = "dnn_native",
43  .item_name = av_default_item_name,
44  .option = dnn_native_options,
45  .version = LIBAVUTIL_VERSION_INT,
46  .category = AV_CLASS_CATEGORY_FILTER,
47 };
48 
49 static DNNReturnType execute_model_native(Queue *lltask_queue);
50 
52 {
53  NativeModel *native_model = task->model;
54  NativeContext *ctx = &native_model->ctx;
55  LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask));
56 
57  if (!lltask) {
58  av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for LastLevelTaskItem\n");
59  return DNN_ERROR;
60  }
61  task->inference_todo = 1;
62  task->inference_done = 0;
63  lltask->task = task;
64 
65  if (ff_queue_push_back(lltask_queue, lltask) < 0) {
66  av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
67  av_freep(&lltask);
68  return DNN_ERROR;
69  }
70  return DNN_SUCCESS;
71 }
72 
73 static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
74 {
75  NativeModel *native_model = model;
76  NativeContext *ctx = &native_model->ctx;
77 
78  for (int i = 0; i < native_model->operands_num; ++i) {
79  DnnOperand *oprd = &native_model->operands[i];
80  if (strcmp(oprd->name, input_name) == 0) {
81  if (oprd->type != DOT_INPUT) {
82  av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
83  return DNN_ERROR;
84  }
85  input->dt = oprd->data_type;
86  av_assert0(oprd->dims[0] == 1);
87  input->height = oprd->dims[1];
88  input->width = oprd->dims[2];
89  input->channels = oprd->dims[3];
90  return DNN_SUCCESS;
91  }
92  }
93 
94  // do not find the input operand
95  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
96  return DNN_ERROR;
97 }
98 
99 static DNNReturnType get_output_native(void *model, const char *input_name, int input_width, int input_height,
100  const char *output_name, int *output_width, int *output_height)
101 {
102  DNNReturnType ret = 0;
103  NativeModel *native_model = model;
104  NativeContext *ctx = &native_model->ctx;
105  TaskItem task;
106  DNNExecBaseParams exec_params = {
107  .input_name = input_name,
108  .output_names = &output_name,
109  .nb_output = 1,
110  .in_frame = NULL,
111  .out_frame = NULL,
112  };
113 
114  if (ff_dnn_fill_gettingoutput_task(&task, &exec_params, native_model, input_height, input_width, ctx) != DNN_SUCCESS) {
115  ret = DNN_ERROR;
116  goto err;
117  }
118 
119  if (extract_lltask_from_task(&task, native_model->lltask_queue) != DNN_SUCCESS) {
120  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
121  ret = DNN_ERROR;
122  goto err;
123  }
124 
125  ret = execute_model_native(native_model->lltask_queue);
126  *output_width = task.out_frame->width;
127  *output_height = task.out_frame->height;
128 
129 err:
130  av_frame_free(&task.out_frame);
131  av_frame_free(&task.in_frame);
132  return ret;
133 }
134 
135 // Loads model and its parameters that are stored in a binary file with following structure:
136 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
137 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
138 // For DEPTH_TO_SPACE layer: block_size
139 DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
140 {
141 #define DNN_NATIVE_MAGIC "FFMPEGDNNNATIVE"
142  DNNModel *model = NULL;
143  // sizeof - 1 to skip the terminating '\0' which is not written in the file
144  char buf[sizeof(DNN_NATIVE_MAGIC) - 1];
145  int version, header_size, major_version_expected = 1;
146  NativeModel *native_model = NULL;
147  AVIOContext *model_file_context;
148  int file_size, dnn_size, parsed_size;
149  int32_t layer;
150  DNNLayerType layer_type;
151 
152  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
153  return NULL;
154  }
155  file_size = avio_size(model_file_context);
156 
157  model = av_mallocz(sizeof(DNNModel));
158  if (!model){
159  goto fail;
160  }
161 
162  /**
163  * check file header with string and version
164  */
165  if (avio_read(model_file_context, buf, sizeof(buf)) != sizeof(buf) ||
166  memcmp(buf, DNN_NATIVE_MAGIC, sizeof(buf)))
167  goto fail;
168  dnn_size = sizeof(buf);
169 
170  version = (int32_t)avio_rl32(model_file_context);
171  dnn_size += 4;
172  if (version != major_version_expected) {
173  goto fail;
174  }
175 
176  // currently no need to check minor version
177  version = (int32_t)avio_rl32(model_file_context);
178  dnn_size += 4;
179  header_size = dnn_size;
180 
181  native_model = av_mallocz(sizeof(NativeModel));
182  if (!native_model){
183  goto fail;
184  }
185  model->model = native_model;
186 
187  native_model->ctx.class = &dnn_native_class;
188  model->options = options;
189  if (av_opt_set_from_string(&native_model->ctx, model->options, NULL, "=", "&") < 0)
190  goto fail;
191  native_model->model = model;
192 
193  if (native_model->ctx.options.async) {
194  av_log(&native_model->ctx, AV_LOG_WARNING, "Async not supported. Rolling back to sync\n");
195  native_model->ctx.options.async = 0;
196  }
197 
198 #if !HAVE_PTHREAD_CANCEL
199  if (native_model->ctx.options.conv2d_threads > 1){
200  av_log(&native_model->ctx, AV_LOG_WARNING, "'conv2d_threads' option was set but it is not supported "
201  "on this build (pthread support is required)\n");
202  }
203 #endif
204 
205  avio_seek(model_file_context, file_size - 8, SEEK_SET);
206  native_model->layers_num = (int32_t)avio_rl32(model_file_context);
207  native_model->operands_num = (int32_t)avio_rl32(model_file_context);
208  dnn_size += 8;
209  avio_seek(model_file_context, header_size, SEEK_SET);
210 
211  native_model->layers = av_mallocz(native_model->layers_num * sizeof(Layer));
212  if (!native_model->layers){
213  goto fail;
214  }
215 
216  native_model->operands = av_mallocz(native_model->operands_num * sizeof(DnnOperand));
217  if (!native_model->operands){
218  goto fail;
219  }
220 
221  native_model->task_queue = ff_queue_create();
222  if (!native_model->task_queue) {
223  goto fail;
224  }
225 
226  native_model->lltask_queue = ff_queue_create();
227  if (!native_model->lltask_queue) {
228  goto fail;
229  }
230 
231  for (layer = 0; layer < native_model->layers_num; ++layer){
232  layer_type = (int32_t)avio_rl32(model_file_context);
233  dnn_size += 4;
234 
235  if (layer_type >= DLT_COUNT) {
236  goto fail;
237  }
238 
239  native_model->layers[layer].type = layer_type;
240  parsed_size = ff_layer_funcs[layer_type].pf_load(&native_model->layers[layer], model_file_context, file_size, native_model->operands_num);
241  if (!parsed_size) {
242  goto fail;
243  }
244  dnn_size += parsed_size;
245  }
246 
247  for (int32_t i = 0; i < native_model->operands_num; ++i){
248  DnnOperand *oprd;
249  int32_t name_len;
250  int32_t operand_index = (int32_t)avio_rl32(model_file_context);
251  dnn_size += 4;
252 
253  if (operand_index >= native_model->operands_num) {
254  goto fail;
255  }
256 
257  oprd = &native_model->operands[operand_index];
258  name_len = (int32_t)avio_rl32(model_file_context);
259  dnn_size += 4;
260 
261  avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
262  dnn_size += name_len;
263 
264  oprd->type = (int32_t)avio_rl32(model_file_context);
265  dnn_size += 4;
266 
267  oprd->data_type = (int32_t)avio_rl32(model_file_context);
268  dnn_size += 4;
269 
270  for (int32_t dim = 0; dim < 4; ++dim) {
271  oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
272  dnn_size += 4;
273  }
274  if (oprd->type == DOT_INPUT && oprd->dims[0] != 1)
275  goto fail;
276 
277  oprd->isNHWC = 1;
278  }
279 
280  avio_closep(&model_file_context);
281 
282  if (dnn_size != file_size){
283  ff_dnn_free_model_native(&model);
284  return NULL;
285  }
286 
287  model->get_input = &get_input_native;
288  model->get_output = &get_output_native;
289  model->filter_ctx = filter_ctx;
290  model->func_type = func_type;
291 
292  return model;
293 
294 fail:
295  ff_dnn_free_model_native(&model);
296  avio_closep(&model_file_context);
297  return NULL;
298 }
299 
301 {
302  NativeModel *native_model = NULL;
304  int32_t layer;
306  DnnOperand *oprd = NULL;
307  LastLevelTaskItem *lltask = NULL;
308  TaskItem *task = NULL;
309  DNNReturnType ret = 0;
310 
311  lltask = ff_queue_pop_front(lltask_queue);
312  if (!lltask) {
313  av_log(NULL, AV_LOG_ERROR, "Failed to get LastLevelTaskItem\n");
314  ret = DNN_ERROR;
315  goto err;
316  }
317  task = lltask->task;
318  native_model = task->model;
319  ctx = &native_model->ctx;
320 
321  if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
322  av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
323  ret = DNN_ERROR;
324  goto err;
325  }
326 
327  for (int i = 0; i < native_model->operands_num; ++i) {
328  oprd = &native_model->operands[i];
329  if (strcmp(oprd->name, task->input_name) == 0) {
330  if (oprd->type != DOT_INPUT) {
331  av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", task->input_name);
332  ret = DNN_ERROR;
333  goto err;
334  }
335  break;
336  }
337  oprd = NULL;
338  }
339  if (!oprd) {
340  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);
341  ret = DNN_ERROR;
342  goto err;
343  }
344 
345  oprd->dims[1] = task->in_frame->height;
346  oprd->dims[2] = task->in_frame->width;
347 
348  av_freep(&oprd->data);
350  if (oprd->length <= 0) {
351  av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n");
352  ret = DNN_ERROR;
353  goto err;
354  }
355  oprd->data = av_malloc(oprd->length);
356  if (!oprd->data) {
357  av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n");
358  ret = DNN_ERROR;
359  goto err;
360  }
361 
362  input.height = oprd->dims[1];
363  input.width = oprd->dims[2];
364  input.channels = oprd->dims[3];
365  input.data = oprd->data;
366  input.dt = oprd->data_type;
367  if (task->do_ioproc) {
368  if (native_model->model->frame_pre_proc != NULL) {
369  native_model->model->frame_pre_proc(task->in_frame, &input, native_model->model->filter_ctx);
370  } else {
372  }
373  }
374 
375  if (task->nb_output != 1) {
376  // currently, the filter does not need multiple outputs,
377  // so we just pending the support until we really need it.
378  avpriv_report_missing_feature(ctx, "multiple outputs");
379  ret = DNN_ERROR;
380  goto err;
381  }
382 
383  for (layer = 0; layer < native_model->layers_num; ++layer){
384  DNNLayerType layer_type = native_model->layers[layer].type;
385  if (ff_layer_funcs[layer_type].pf_exec(native_model->operands,
386  native_model->layers[layer].input_operand_indexes,
387  native_model->layers[layer].output_operand_index,
388  native_model->layers[layer].params,
389  &native_model->ctx) == DNN_ERROR) {
390  av_log(ctx, AV_LOG_ERROR, "Failed to execute model\n");
391  ret = DNN_ERROR;
392  goto err;
393  }
394  }
395 
396  for (uint32_t i = 0; i < task->nb_output; ++i) {
397  DnnOperand *oprd = NULL;
398  const char *output_name = task->output_names[i];
399  for (int j = 0; j < native_model->operands_num; ++j) {
400  if (strcmp(native_model->operands[j].name, output_name) == 0) {
401  oprd = &native_model->operands[j];
402  break;
403  }
404  }
405 
406  if (oprd == NULL) {
407  av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n");
408  ret = DNN_ERROR;
409  goto err;
410  }
411 
412  output.data = oprd->data;
413  output.height = oprd->dims[1];
414  output.width = oprd->dims[2];
415  output.channels = oprd->dims[3];
416  output.dt = oprd->data_type;
417 
418  if (task->do_ioproc) {
419  if (native_model->model->frame_post_proc != NULL) {
420  native_model->model->frame_post_proc(task->out_frame, &output, native_model->model->filter_ctx);
421  } else {
423  }
424  } else {
425  task->out_frame->width = output.width;
426  task->out_frame->height = output.height;
427  }
428  }
429  task->inference_done++;
430 err:
431  av_freep(&lltask);
432  return ret;
433 }
434 
436 {
437  NativeModel *native_model = model->model;
438  NativeContext *ctx = &native_model->ctx;
439  TaskItem *task;
440 
441  if (ff_check_exec_params(ctx, DNN_NATIVE, model->func_type, exec_params) != 0) {
442  return DNN_ERROR;
443  }
444 
445  task = av_malloc(sizeof(*task));
446  if (!task) {
447  av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
448  return DNN_ERROR;
449  }
450 
451  if (ff_dnn_fill_task(task, exec_params, native_model, ctx->options.async, 1) != DNN_SUCCESS) {
452  av_freep(&task);
453  return DNN_ERROR;
454  }
455 
456  if (ff_queue_push_back(native_model->task_queue, task) < 0) {
457  av_freep(&task);
458  av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
459  return DNN_ERROR;
460  }
461 
462  if (extract_lltask_from_task(task, native_model->lltask_queue) != DNN_SUCCESS) {
463  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
464  return DNN_ERROR;
465  }
466 
467  return execute_model_native(native_model->lltask_queue);
468 }
469 
471 {
472  NativeModel *native_model = model->model;
473 
474  if (ff_queue_size(native_model->lltask_queue) == 0) {
475  // no pending task need to flush
476  return DNN_SUCCESS;
477  }
478 
479  // for now, use sync node with flush operation
480  // Switch to async when it is supported
481  return execute_model_native(native_model->lltask_queue);
482 }
483 
485 {
486  NativeModel *native_model = model->model;
487  return ff_dnn_get_result_common(native_model->task_queue, in, out);
488 }
489 
491 {
492  int32_t result = 1;
493  for (int i = 0; i < 4; ++i)
494  result *= oprd->dims[i];
495 
496  return result;
497 }
498 
500 {
501  // currently, we just support DNN_FLOAT
502  uint64_t len = sizeof(float);
503  for (int i = 0; i < 4; i++) {
504  len *= oprd->dims[i];
505  if (len > INT32_MAX)
506  return 0;
507  }
508  return len;
509 }
510 
512 {
513  NativeModel *native_model;
514  ConvolutionalParams *conv_params;
515  int32_t layer;
516 
517  if (*model)
518  {
519  if ((*model)->model) {
520  native_model = (*model)->model;
521  if (native_model->layers) {
522  for (layer = 0; layer < native_model->layers_num; ++layer){
523  if (native_model->layers[layer].type == DLT_CONV2D){
524  conv_params = (ConvolutionalParams *)native_model->layers[layer].params;
525  av_freep(&conv_params->kernel);
526  av_freep(&conv_params->biases);
527  }
528  av_freep(&native_model->layers[layer].params);
529  }
530  av_freep(&native_model->layers);
531  }
532 
533  if (native_model->operands) {
534  for (uint32_t operand = 0; operand < native_model->operands_num; ++operand)
535  av_freep(&native_model->operands[operand].data);
536  av_freep(&native_model->operands);
537  }
538 
539  while (ff_queue_size(native_model->lltask_queue) != 0) {
540  LastLevelTaskItem *item = ff_queue_pop_front(native_model->lltask_queue);
541  av_freep(&item);
542  }
543  ff_queue_destroy(native_model->lltask_queue);
544 
545  while (ff_queue_size(native_model->task_queue) != 0) {
546  TaskItem *item = ff_queue_pop_front(native_model->task_queue);
547  av_frame_free(&item->in_frame);
548  av_frame_free(&item->out_frame);
549  av_freep(&item);
550  }
551  ff_queue_destroy(native_model->task_queue);
552 
553  av_freep(&native_model);
554  }
555  av_freep(model);
556  }
557 }
DLT_COUNT
@ DLT_COUNT
Definition: dnn_backend_native.h:50
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
ff_dnn_fill_task
DNNReturnType ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:56
filter_ctx
static FilteringContext * filter_ctx
Definition: transcoding.c:49
out
FILE * out
Definition: movenc.c:54
DnnOperand::isNHWC
int8_t isNHWC
NHWC if 1, otherwise NCHW.
Definition: dnn_backend_native.h:91
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:29
ff_queue_size
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition: queue.c:88
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:112
LastLevelTaskItem
Definition: dnn_backend_common.h:50
ConvolutionalParams::kernel
float * kernel
Definition: dnn_backend_native_layer_conv2d.h:33
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:303
AVFrame::width
int width
Definition: frame.h:361
ff_dnn_load_model_native
DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_native.c:139
AVOption
AVOption.
Definition: opt.h:247
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:101
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:68
dnn_backend_native_layers.h
get_input_native
static DNNReturnType get_input_native(void *model, DNNData *input, const char *input_name)
Definition: dnn_backend_native.c:73
dnn_io_proc.h
TaskItem
Definition: dnn_backend_common.h:36
avio_size
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:338
ff_dnn_get_result_native
DNNAsyncStatusType ff_dnn_get_result_native(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_backend_native.c:484
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:31
NativeModel::operands
DnnOperand * operands
Definition: dnn_backend_native.h:129
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:90
ff_queue_create
Queue * ff_queue_create(void)
Create a Queue instance.
Definition: queue.c:47
ff_layer_funcs
const LayerFunc ff_layer_funcs[DLT_COUNT]
Definition: dnn_backend_native_layers.c:32
ff_proc_from_dnn_to_frame
DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:27
ff_proc_from_frame_to_dnn
DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:100
TaskItem::model
void * model
Definition: dnn_backend_common.h:37
fail
#define fail()
Definition: checkasm.h:127
DNN_SUCCESS
@ DNN_SUCCESS
Definition: dnn_interface.h:33
NativeModel::task_queue
Queue * task_queue
Definition: dnn_backend_native.h:131
DNNModel::get_output
DNNReturnType(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:97
ff_calculate_operand_data_length
int32_t ff_calculate_operand_data_length(const DnnOperand *oprd)
Definition: dnn_backend_native.c:499
DNNLayerType
DNNLayerType
the enum value of DNNLayerType should not be changed, the same values are used in convert_from_tensor...
Definition: dnn_backend_native.h:40
Queue
Linear double-ended data structure.
Definition: queue.c:33
ff_queue_push_back
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition: queue.c:130
DnnOperand::type
DNNOperandType type
input/output/intermediate operand of the network
Definition: dnn_backend_native.h:79
avassert.h
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
NativeModel::ctx
NativeContext ctx
Definition: dnn_backend_native.h:125
NativeModel::layers_num
int32_t layers_num
Definition: dnn_backend_native.h:128
OFFSET
#define OFFSET(x)
Definition: dnn_backend_native.c:33
LastLevelTaskItem::task
TaskItem * task
Definition: dnn_backend_common.h:51
Layer::type
DNNLayerType type
Definition: dnn_backend_native.h:58
DLT_CONV2D
@ DLT_CONV2D
Definition: dnn_backend_native.h:42
extract_lltask_from_task
static DNNReturnType extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
Definition: dnn_backend_native.c:51
ff_queue_destroy
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition: queue.c:72
DnnOperand::name
char name[128]
to avoid possible memory leak, do not use char *name
Definition: dnn_backend_native.h:96
DnnOperand::data
void * data
data pointer with data length in bytes.
Definition: dnn_backend_native.h:104
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DNNReturnType
DNNReturnType
Definition: dnn_interface.h:33
DnnOperand::data_type
DNNDataType data_type
support different kinds of data type such as float, half float, int8 etc, first support float now.
Definition: dnn_backend_native.h:85
DNNData
Definition: dnn_interface.h:59
ctx
AVFormatContext * ctx
Definition: movenc.c:48
DNNModel::get_input
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:95
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:45
NativeModel::lltask_queue
Queue * lltask_queue
Definition: dnn_backend_native.h:132
ff_dnn_free_model_native
void ff_dnn_free_model_native(DNNModel **model)
Definition: dnn_backend_native.c:511
LIBAVUTIL_VERSION_INT
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
Layer::params
void * params
Definition: dnn_backend_native.h:66
result
and forward the result(frame or status change) to the corresponding input. If nothing is possible
NULL
#define NULL
Definition: coverity.c:32
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:104
NativeModel
Definition: dnn_backend_native.h:124
av_opt_set_from_string
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition: opt.c:1559
DnnOperand::dims
int32_t dims[4]
there are two memory layouts, NHWC or NCHW, so we use dims, dims[0] is Number.
Definition: dnn_backend_native.h:74
av_default_item_name
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:38
dnn_native_options
static const AVOption dnn_native_options[]
Definition: dnn_backend_native.c:35
AV_CLASS_CATEGORY_FILTER
@ AV_CLASS_CATEGORY_FILTER
Definition: log.h:36
DnnOperand::length
int32_t length
Definition: dnn_backend_native.h:105
DOT_INPUT
@ DOT_INPUT
Definition: dnn_backend_native.h:53
NativeModel::model
DNNModel * model
Definition: dnn_backend_native.h:126
options
const OptionDef options[]
avio_rl32
unsigned int avio_rl32(AVIOContext *s)
Definition: aviobuf.c:742
AVIOContext
Bytestream IO Context.
Definition: avio.h:161
Layer::output_operand_index
int32_t output_operand_index
Definition: dnn_backend_native.h:65
NativeContext
Definition: dnn_backend_native.h:118
ff_dnn_fill_gettingoutput_task
DNNReturnType ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:161
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:46
Layer
Definition: dnn_backend_native.h:57
Layer::input_operand_indexes
int32_t input_operand_indexes[4]
a layer can have multiple inputs and one output.
Definition: dnn_backend_native.h:64
avio_get_str
int avio_get_str(AVIOContext *pb, int maxlen, char *buf, int buflen)
Read a string from pb into buf.
Definition: aviobuf.c:852
NativeModel::layers
Layer * layers
Definition: dnn_backend_native.h:127
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:92
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
dnn_backend_native_layer_conv2d.h
version
version
Definition: libkvazaar.c:307
dnn_backend_native.h
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
LayerFunc::pf_load
LAYER_LOAD_FUNC pf_load
Definition: dnn_backend_native_layers.h:33
get_output_native
static DNNReturnType get_output_native(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_native.c:99
avio_closep
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1213
NativeModel::operands_num
int32_t operands_num
Definition: dnn_backend_native.h:130
i
int i
Definition: input.c:406
NativeContext::options
NativeOptions options
Definition: dnn_backend_native.h:120
DNN_ERROR
@ DNN_ERROR
Definition: dnn_interface.h:33
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:261
TaskItem::output_names
const char ** output_names
Definition: dnn_backend_common.h:41
len
int len
Definition: vorbis_enc_data.h:426
DNN_NATIVE_MAGIC
#define DNN_NATIVE_MAGIC
dim
int dim
Definition: vorbis_enc_data.h:425
dnn_native_class
static const AVClass dnn_native_class
Definition: dnn_backend_native.c:41
ret
ret
Definition: filter_design.txt:187
avio_seek
int64_t avio_seek(AVIOContext *s, int64_t offset, int whence)
fseek() equivalent for AVIOContext.
Definition: aviobuf.c:246
AVClass::class_name
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:71
NativeContext::class
const AVClass * class
Definition: dnn_backend_native.h:119
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:39
AVFrame::height
int height
Definition: frame.h:361
ff_dnn_execute_model_native
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_native.c:435
NativeOptions::conv2d_threads
uint32_t conv2d_threads
Definition: dnn_backend_native.h:115
DnnOperand
Definition: dnn_backend_native.h:69
dnn_backend_common.h
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Definition: opt.h:224
avio_read
int avio_read(AVIOContext *s, unsigned char *buf, int size)
Read size bytes from AVIOContext into buf.
Definition: aviobuf.c:625
ff_calculate_operand_dims_count
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
Definition: dnn_backend_native.c:490
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:141
avio_open
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1155
AVFilterContext
An instance of a filter.
Definition: avfilter.h:346
DNNModel
Definition: dnn_interface.h:84
AVIO_FLAG_READ
#define AVIO_FLAG_READ
read-only
Definition: avio.h:603
DNN_NATIVE
@ DNN_NATIVE
Definition: dnn_interface.h:35
FLAGS
#define FLAGS
Definition: dnn_backend_native.c:34
TaskItem::input_name
const char * input_name
Definition: dnn_backend_common.h:40
DNNModel::options
const char * options
Definition: dnn_interface.h:88
NativeOptions::async
uint8_t async
Definition: dnn_backend_native.h:114
AV_OPT_TYPE_BOOL
@ AV_OPT_TYPE_BOOL
Definition: opt.h:241
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
int32_t
int32_t
Definition: audioconvert.c:56
DNNExecBaseParams
Definition: dnn_interface.h:67
execute_model_native
static DNNReturnType execute_model_native(Queue *lltask_queue)
Definition: dnn_backend_native.c:300
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:28
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:43
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
ff_dnn_flush_native
DNNReturnType ff_dnn_flush_native(const DNNModel *model)
Definition: dnn_backend_native.c:470
TaskItem::nb_output
uint32_t nb_output
Definition: dnn_backend_common.h:44
ConvolutionalParams
Definition: dnn_backend_native_layer_conv2d.h:27
DNNModel::model
void * model
Definition: dnn_interface.h:86
ConvolutionalParams::biases
float * biases
Definition: dnn_backend_native_layer_conv2d.h:34