FFmpeg
dnn_backend_native.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN inference functions interface for native backend.
24  */
25 
26 
27 #ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_H
28 #define AVFILTER_DNN_DNN_BACKEND_NATIVE_H
29 
30 #include "../dnn_interface.h"
31 #include "libavformat/avio.h"
32 #include "libavutil/opt.h"
33 #include "queue.h"
34 
35 /**
36  * the enum value of DNNLayerType should not be changed,
37  * the same values are used in convert_from_tensorflow.py
38  * and, it is used to index the layer execution/load function pointer.
39  */
40 typedef enum {
41  DLT_INPUT = 0,
49  DLT_DENSE = 8,
51 } DNNLayerType;
52 
56 
57 typedef struct Layer{
59  /**
60  * a layer can have multiple inputs and one output.
61  * 4 is just a big enough number for input operands (increase it if necessary),
62  * do not use 'int32_t *input_operand_indexes', so we don't worry about mem leaks.
63  */
66  void *params;
67 } Layer;
68 
69 typedef struct DnnOperand{
70  /**
71  * there are two memory layouts, NHWC or NCHW, so we use dims,
72  * dims[0] is Number.
73  */
75 
76  /**
77  * input/output/intermediate operand of the network
78  */
80 
81  /**
82  * support different kinds of data type such as float, half float, int8 etc,
83  * first support float now.
84  */
86 
87  /**
88  * NHWC if 1, otherwise NCHW.
89  * let's first support NHWC only, this flag is for extensive usage.
90  */
91  int8_t isNHWC;
92 
93  /**
94  * to avoid possible memory leak, do not use char *name
95  */
96  char name[128];
97 
98  /**
99  * data pointer with data length in bytes.
100  * usedNumbersLeft is only valid for intermediate operand,
101  * it means how many layers still depend on this operand,
102  * todo: the memory can be reused when usedNumbersLeft is zero.
103  */
104  void *data;
107 }DnnOperand;
108 
109 typedef struct InputParams{
111 } InputParams;
112 
113 typedef struct NativeOptions{
114  uint8_t async;
115  uint32_t conv2d_threads;
116 } NativeOptions;
117 
118 typedef struct NativeContext {
119  const AVClass *class;
121 } NativeContext;
122 
123 // Represents simple feed-forward convolutional network.
124 typedef struct NativeModel{
133 } NativeModel;
134 
135 DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
136 
137 int ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params);
138 
140 
141 int ff_dnn_flush_native(const DNNModel *model);
142 
143 void ff_dnn_free_model_native(DNNModel **model);
144 
145 // NOTE: User must check for error (return value <= 0) to handle
146 // case like integer overflow.
149 #endif
DLT_COUNT
@ DLT_COUNT
Definition: dnn_backend_native.h:50
InputParams
Definition: dnn_backend_native.h:109
opt.h
out
FILE * out
Definition: movenc.c:54
ff_calculate_operand_dims_count
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
Definition: dnn_backend_native.c:494
DnnOperand::isNHWC
int8_t isNHWC
NHWC if 1, otherwise NCHW.
Definition: dnn_backend_native.h:91
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:330
DLT_AVG_POOL
@ DLT_AVG_POOL
Definition: dnn_backend_native.h:48
NONE
@ NONE
Definition: dnn_backend_native.h:55
NativeModel::operands
DnnOperand * operands
Definition: dnn_backend_native.h:129
DLT_INPUT
@ DLT_INPUT
Definition: dnn_backend_native.h:41
NativeModel::task_queue
Queue * task_queue
Definition: dnn_backend_native.h:131
filter_ctx
static FilteringContext * filter_ctx
Definition: transcode.c:51
DLT_MATH_BINARY
@ DLT_MATH_BINARY
Definition: dnn_backend_native.h:46
SIGMOID
@ SIGMOID
Definition: dnn_backend_native.h:55
DNNLayerType
DNNLayerType
the enum value of DNNLayerType should not be changed, the same values are used in convert_from_tensor...
Definition: dnn_backend_native.h:40
Queue
Linear double-ended data structure.
Definition: queue.c:33
DnnOperand::type
DNNOperandType type
input/output/intermediate operand of the network
Definition: dnn_backend_native.h:79
NativeModel::ctx
NativeContext ctx
Definition: dnn_backend_native.h:125
NativeModel::layers_num
int32_t layers_num
Definition: dnn_backend_native.h:128
DLT_MAXIMUM
@ DLT_MAXIMUM
Definition: dnn_backend_native.h:45
TANH
@ TANH
Definition: dnn_backend_native.h:55
ff_dnn_load_model_native
DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_native.c:139
Layer::type
DNNLayerType type
Definition: dnn_backend_native.h:58
DLT_CONV2D
@ DLT_CONV2D
Definition: dnn_backend_native.h:42
DnnOperand::name
char name[128]
to avoid possible memory leak, do not use char *name
Definition: dnn_backend_native.h:96
DnnOperand::data
void * data
data pointer with data length in bytes.
Definition: dnn_backend_native.h:104
DnnOperand::data_type
DNNDataType data_type
support different kinds of data type such as float, half float, int8 etc, first support float now.
Definition: dnn_backend_native.h:85
InputParams::height
int height
Definition: dnn_backend_native.h:110
SAME_CLAMP_TO_EDGE
@ SAME_CLAMP_TO_EDGE
Definition: dnn_backend_native.h:54
NativeModel::lltask_queue
Queue * lltask_queue
Definition: dnn_backend_native.h:132
DOT_OUTPUT
@ DOT_OUTPUT
Definition: dnn_backend_native.h:53
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
Layer::params
void * params
Definition: dnn_backend_native.h:66
NativeModel
Definition: dnn_backend_native.h:124
DnnOperand::dims
int32_t dims[4]
there are two memory layouts, NHWC or NCHW, so we use dims, dims[0] is Number.
Definition: dnn_backend_native.h:74
SAME
@ SAME
Definition: dnn_backend_native.h:54
DOT_INTERMEDIATE
@ DOT_INTERMEDIATE
Definition: dnn_backend_native.h:53
DNNOperandType
DNNOperandType
Definition: dnn_backend_native.h:53
DnnOperand::length
int32_t length
Definition: dnn_backend_native.h:105
DOT_INPUT
@ DOT_INPUT
Definition: dnn_backend_native.h:53
NativeModel::model
DNNModel * model
Definition: dnn_backend_native.h:126
options
const OptionDef options[]
Layer::output_operand_index
int32_t output_operand_index
Definition: dnn_backend_native.h:65
DLT_MIRROR_PAD
@ DLT_MIRROR_PAD
Definition: dnn_backend_native.h:44
NativeContext
Definition: dnn_backend_native.h:118
Layer
Definition: dnn_backend_native.h:57
Layer::input_operand_indexes
int32_t input_operand_indexes[4]
a layer can have multiple inputs and one output.
Definition: dnn_backend_native.h:64
queue.h
DnnOperand::usedNumbersLeft
int32_t usedNumbersLeft
Definition: dnn_backend_native.h:106
NativeModel::layers
Layer * layers
Definition: dnn_backend_native.h:127
avio.h
VALID
@ VALID
Definition: dnn_backend_native.h:54
DNNPaddingParam
DNNPaddingParam
Definition: dnn_backend_native.h:54
DNNDataType
DNNDataType
Definition: dnn_interface.h:37
RELU
@ RELU
Definition: dnn_backend_native.h:55
NativeModel::operands_num
int32_t operands_num
Definition: dnn_backend_native.h:130
ff_calculate_operand_data_length
int32_t ff_calculate_operand_data_length(const DnnOperand *oprd)
Definition: dnn_backend_native.c:503
DNNActivationFunc
DNNActivationFunc
Definition: dnn_backend_native.h:55
NativeContext::options
NativeOptions options
Definition: dnn_backend_native.h:120
ff_dnn_execute_model_native
int ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_native.c:435
NativeOptions
Definition: dnn_backend_native.h:113
ff_dnn_flush_native
int ff_dnn_flush_native(const DNNModel *model)
Definition: dnn_backend_native.c:474
ff_dnn_get_result_native
DNNAsyncStatusType ff_dnn_get_result_native(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_backend_native.c:488
NativeOptions::conv2d_threads
uint32_t conv2d_threads
Definition: dnn_backend_native.h:115
DnnOperand
Definition: dnn_backend_native.h:69
DLT_MATH_UNARY
@ DLT_MATH_UNARY
Definition: dnn_backend_native.h:47
ff_dnn_free_model_native
void ff_dnn_free_model_native(DNNModel **model)
Definition: dnn_backend_native.c:515
AVFilterContext
An instance of a filter.
Definition: avfilter.h:392
DNNModel
Definition: dnn_interface.h:84
LEAKY_RELU
@ LEAKY_RELU
Definition: dnn_backend_native.h:55
NativeOptions::async
uint8_t async
Definition: dnn_backend_native.h:114
DLT_DENSE
@ DLT_DENSE
Definition: dnn_backend_native.h:49
int32_t
int32_t
Definition: audioconvert.c:56
DNNExecBaseParams
Definition: dnn_interface.h:67
DLT_DEPTH_TO_SPACE
@ DLT_DEPTH_TO_SPACE
Definition: dnn_backend_native.h:43
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
InputParams::channels
int channels
Definition: dnn_backend_native.h:110
InputParams::width
int width
Definition: dnn_backend_native.h:110