FFmpeg
dnn_interface.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN inference engine interface.
24  */
25 
26 #ifndef AVFILTER_DNN_INTERFACE_H
27 #define AVFILTER_DNN_INTERFACE_H
28 
29 #include <stdint.h>
30 #include "libavutil/frame.h"
31 #include "avfilter.h"
32 
33 #define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')
34 
35 typedef enum {DNN_TF = 1, DNN_OV, DNN_TH} DNNBackendType;
36 
37 typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
38 
39 typedef enum {
44 
45 typedef enum {
46  DAST_FAIL, // something wrong
47  DAST_EMPTY_QUEUE, // no more inference result to get
48  DAST_NOT_READY, // all queued inferences are not finished
49  DAST_SUCCESS // got a result frame successfully
51 
52 typedef enum {
54  DFT_PROCESS_FRAME, // process the whole frame
55  DFT_ANALYTICS_DETECT, // detect from the whole frame
56  DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
58 
59 typedef enum {
63 } DNNLayout;
64 
65 typedef struct DNNData{
66  void *data;
67  int dims[4];
68  // dt and order together decide the color format
72  float scale;
73  float mean;
74 } DNNData;
75 
76 typedef struct DNNExecBaseParams {
77  const char *input_name;
78  const char **output_names;
79  uint32_t nb_output;
83 
86  const char *target;
88 
91 typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);
92 
93 typedef struct DNNModel{
94  // Stores model that can be different for different backends.
95  void *model;
96  // Stores FilterContext used for the interaction between AVFrame and DNNData
98  // Stores function type of the model
100  // Gets model input information
101  // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
102  int (*get_input)(void *model, DNNData *input, const char *input_name);
103  // Gets model output width/height with given input w/h
104  int (*get_output)(void *model, const char *input_name, int input_width, int input_height,
105  const char *output_name, int *output_width, int *output_height);
106  // set the pre process to transfer data from AVFrame to DNNData
107  // the default implementation within DNN is used if it is not provided by the filter
109  // set the post process to transfer data from DNNData to AVFrame
110  // the default implementation within DNN is used if it is not provided by the filter
112  // set the post process to interpret detect result from DNNData
114  // set the post process to interpret classify result from DNNData
116 } DNNModel;
117 
118 typedef struct TFOptions{
119  const AVClass *clazz;
120 
121  char *sess_config;
122 } TFOptions;
123 
124 typedef struct OVOptions {
125  const AVClass *clazz;
126 
130  float scale;
131  float mean;
132 } OVOptions;
133 
134 typedef struct THOptions {
135  const AVClass *clazz;
136  int optimize;
137 } THOptions;
138 
139 typedef struct DNNModule DNNModule;
140 
141 typedef struct DnnContext {
142  const AVClass *clazz;
143 
145 
151  int async;
152 
154  uint32_t nb_outputs;
156 
157  int nireq;
158  char *device;
159 
160 #if CONFIG_LIBTENSORFLOW
161  TFOptions tf_option;
162 #endif
163 
164 #if CONFIG_LIBOPENVINO
165  OVOptions ov_option;
166 #endif
167 #if CONFIG_LIBTORCH
168  THOptions torch_option;
169 #endif
170 } DnnContext;
171 
172 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
173 struct DNNModule {
174  const AVClass clazz;
175  // Loads model and parameters from given file. Returns NULL if it is not possible.
177  // Executes model with specified input and output. Returns the error code otherwise.
178  int (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params);
179  // Retrieve inference result.
181  // Flush all the pending tasks.
182  int (*flush)(const DNNModel *model);
183  // Frees memory allocated for model.
184  void (*free_model)(DNNModel **model);
185 };
186 
187 // Initializes DNNModule depending on chosen backend.
188 const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);
189 
191 void *ff_dnn_child_next(DnnContext *obj, void *prev);
192 const AVClass *ff_dnn_child_class_iterate(void **iter);
193 
195 {
196  return layout == DL_NHWC ? 2 : 3;
197 }
198 
200 {
201  return layout == DL_NHWC ? 1 : 2;
202 }
203 
205 {
206  return layout == DL_NHWC ? 3 : 1;
207 }
208 
209 #endif
TFOptions::sess_config
char * sess_config
Definition: dnn_interface.h:121
DNNColorOrder
DNNColorOrder
Definition: dnn_interface.h:39
out
FILE * out
Definition: movenc.c:55
DNNModule::get_result
DNNAsyncStatusType(* get_result)(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_interface.h:180
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
DnnContext::model
DNNModel * model
Definition: dnn_interface.h:144
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
DNNData::data
void * data
Definition: dnn_interface.h:66
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:374
OVOptions::mean
float mean
Definition: dnn_interface.h:131
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:108
DetectPostProc
int(* DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:90
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:77
DnnContext::clazz
const AVClass * clazz
Definition: dnn_interface.h:142
DNNExecBaseParams::in_frame
AVFrame * in_frame
Definition: dnn_interface.h:80
THOptions
Definition: dnn_interface.h:134
DFT_NONE
@ DFT_NONE
Definition: dnn_interface.h:53
OVOptions::batch_size
int batch_size
Definition: dnn_interface.h:127
DnnContext::dnn_module
const DNNModule * dnn_module
Definition: dnn_interface.h:155
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:97
dnn_get_width_idx_by_layout
static int dnn_get_width_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:194
DnnContext
Definition: dnn_interface.h:141
filter_ctx
static FilteringContext * filter_ctx
Definition: transcode.c:52
DAST_FAIL
@ DAST_FAIL
Definition: dnn_interface.h:46
DL_NHWC
@ DL_NHWC
Definition: dnn_interface.h:62
OVOptions::clazz
const AVClass * clazz
Definition: dnn_interface.h:125
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
DnnContext::async
int async
Definition: dnn_interface.h:151
DnnContext::model_filename
char * model_filename
Definition: dnn_interface.h:146
DCO_NONE
@ DCO_NONE
Definition: dnn_interface.h:40
DNNExecClassificationParams
Definition: dnn_interface.h:84
DnnContext::model_outputnames
char ** model_outputnames
Definition: dnn_interface.h:153
OVOptions::layout
DNNLayout layout
Definition: dnn_interface.h:129
DNNData::order
DNNColorOrder order
Definition: dnn_interface.h:70
DNNData
Definition: dnn_interface.h:65
DNNModule::clazz
const AVClass clazz
Definition: dnn_interface.h:174
ctx
AVFormatContext * ctx
Definition: movenc.c:49
DL_NCHW
@ DL_NCHW
Definition: dnn_interface.h:61
DNN_OV
@ DNN_OV
Definition: dnn_interface.h:35
DNNExecClassificationParams::target
const char * target
Definition: dnn_interface.h:86
DnnContext::backend_options
char * backend_options
Definition: dnn_interface.h:150
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
DNNExecClassificationParams::base
DNNExecBaseParams base
Definition: dnn_interface.h:85
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:111
TFOptions::clazz
const AVClass * clazz
Definition: dnn_interface.h:119
ff_get_dnn_module
const DNNModule * ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
Definition: dnn_interface.c:82
DnnContext::nireq
int nireq
Definition: dnn_interface.h:157
DnnContext::backend_type
DNNBackendType backend_type
Definition: dnn_interface.h:147
ClassifyPostProc
int(* ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:91
DAST_SUCCESS
@ DAST_SUCCESS
Definition: dnn_interface.h:49
DnnContext::model_inputname
char * model_inputname
Definition: dnn_interface.h:148
DNNBackendType
DNNBackendType
Definition: dnn_interface.h:35
DAST_EMPTY_QUEUE
@ DAST_EMPTY_QUEUE
Definition: dnn_interface.h:47
DnnContext::nb_outputs
uint32_t nb_outputs
Definition: dnn_interface.h:154
DNNLayout
DNNLayout
Definition: dnn_interface.h:59
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:113
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:99
DNNDataType
DNNDataType
Definition: dnn_interface.h:37
DNNData::dt
DNNDataType dt
Definition: dnn_interface.h:69
frame.h
DNNData::scale
float scale
Definition: dnn_interface.h:72
DNNData::layout
DNNLayout layout
Definition: dnn_interface.h:71
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
DNNExecBaseParams::out_frame
AVFrame * out_frame
Definition: dnn_interface.h:81
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
OVOptions::input_resizable
int input_resizable
Definition: dnn_interface.h:128
THOptions::optimize
int optimize
Definition: dnn_interface.h:136
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:55
ff_dnn_child_next
void * ff_dnn_child_next(DnnContext *obj, void *prev)
Definition: dnn_interface.c:113
DnnContext::device
char * device
Definition: dnn_interface.h:158
DNNModel::classify_post_proc
ClassifyPostProc classify_post_proc
Definition: dnn_interface.h:115
OVOptions::scale
float scale
Definition: dnn_interface.h:130
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
DNNData::mean
float mean
Definition: dnn_interface.h:73
DNNModel::get_input
int(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:102
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
DFT_ANALYTICS_CLASSIFY
@ DFT_ANALYTICS_CLASSIFY
Definition: dnn_interface.h:56
DNNModule::free_model
void(* free_model)(DNNModel **model)
Definition: dnn_interface.h:184
avfilter.h
THOptions::clazz
const AVClass * clazz
Definition: dnn_interface.h:135
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
DNNExecBaseParams::output_names
const char ** output_names
Definition: dnn_interface.h:78
FramePrePostProc
int(* FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:89
DL_NONE
@ DL_NONE
Definition: dnn_interface.h:60
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
DNNModel
Definition: dnn_interface.h:93
DNNData::dims
int dims[4]
Definition: dnn_interface.h:67
DNN_TH
@ DNN_TH
Definition: dnn_interface.h:35
dnn_get_height_idx_by_layout
static int dnn_get_height_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:199
ff_dnn_init_child_class
void ff_dnn_init_child_class(DnnContext *ctx)
Definition: dnn_interface.c:105
dnn_get_channel_idx_by_layout
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:204
TFOptions
Definition: dnn_interface.h:118
ff_dnn_child_class_iterate
const AVClass * ff_dnn_child_class_iterate(void **iter)
Definition: dnn_interface.c:133
OVOptions
Definition: dnn_interface.h:124
DNNExecBaseParams
Definition: dnn_interface.h:76
DNNModel::get_output
int(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:104
DCO_BGR
@ DCO_BGR
Definition: dnn_interface.h:41
DAST_NOT_READY
@ DAST_NOT_READY
Definition: dnn_interface.h:48
int
int
Definition: ffmpeg_filter.c:424
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
DNNModule
Definition: dnn_interface.h:173
DNNModule::flush
int(* flush)(const DNNModel *model)
Definition: dnn_interface.h:182
DNNExecBaseParams::nb_output
uint32_t nb_output
Definition: dnn_interface.h:79
DNNModel::model
void * model
Definition: dnn_interface.h:95
DnnContext::model_outputnames_string
char * model_outputnames_string
Definition: dnn_interface.h:149
DNNModule::execute_model
int(* execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_interface.h:178