FFmpeg
dnn_interface.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN inference engine interface.
24  */
25 
26 #ifndef AVFILTER_DNN_INTERFACE_H
27 #define AVFILTER_DNN_INTERFACE_H
28 
29 #include <stdint.h>
30 #include "libavutil/frame.h"
31 #include "avfilter.h"
32 
34 
36 
37 typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
38 
39 typedef enum {
44 
45 typedef enum {
46  DAST_FAIL, // something wrong
47  DAST_EMPTY_QUEUE, // no more inference result to get
48  DAST_NOT_READY, // all queued inferences are not finished
49  DAST_SUCCESS // got a result frame successfully
51 
52 typedef enum {
54  DFT_PROCESS_FRAME, // process the whole frame
55  DFT_ANALYTICS_DETECT, // detect from the whole frame
56  DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
58 
59 typedef struct DNNData{
60  void *data;
62  // dt and order together decide the color format
65 } DNNData;
66 
67 typedef struct DNNExecBaseParams {
68  const char *input_name;
69  const char **output_names;
70  uint32_t nb_output;
74 
77  const char *target;
79 
82 typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);
83 
84 typedef struct DNNModel{
85  // Stores model that can be different for different backends.
86  void *model;
87  // Stores options when the model is executed by the backend
88  const char *options;
89  // Stores FilterContext used for the interaction between AVFrame and DNNData
91  // Stores function type of the model
93  // Gets model input information
94  // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
95  DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name);
96  // Gets model output width/height with given input w/h
97  DNNReturnType (*get_output)(void *model, const char *input_name, int input_width, int input_height,
98  const char *output_name, int *output_width, int *output_height);
99  // set the pre process to transfer data from AVFrame to DNNData
100  // the default implementation within DNN is used if it is not provided by the filter
102  // set the post process to transfer data from DNNData to AVFrame
103  // the default implementation within DNN is used if it is not provided by the filter
105  // set the post process to interpret detect result from DNNData
107  // set the post process to interpret classify result from DNNData
109 } DNNModel;
110 
111 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
112 typedef struct DNNModule{
113  // Loads model and parameters from given file. Returns NULL if it is not possible.
114  DNNModel *(*load_model)(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
115  // Executes model with specified input and output. Returns DNN_ERROR otherwise.
116  DNNReturnType (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params);
117  // Executes model with specified input and output asynchronously. Returns DNN_ERROR otherwise.
119  // Retrieve inference result.
121  // Flush all the pending tasks.
122  DNNReturnType (*flush)(const DNNModel *model);
123  // Frees memory allocated for model.
124  void (*free_model)(DNNModel **model);
125 } DNNModule;
126 
127 // Initializes DNNModule depending on chosen backend.
129 
130 #endif
DNNColorOrder
DNNColorOrder
Definition: dnn_interface.h:39
filter_ctx
static FilteringContext * filter_ctx
Definition: transcoding.c:48
out
FILE * out
Definition: movenc.c:54
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
DNNData::data
void * data
Definition: dnn_interface.h:60
DNNModule::execute_model_async
DNNReturnType(* execute_model_async)(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_interface.h:118
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:303
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:101
DetectPostProc
int(* DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:81
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:68
DNNData::height
int height
Definition: dnn_interface.h:61
DNNExecBaseParams::in_frame
AVFrame * in_frame
Definition: dnn_interface.h:71
DFT_NONE
@ DFT_NONE
Definition: dnn_interface.h:53
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:90
DNN_SUCCESS
@ DNN_SUCCESS
Definition: dnn_interface.h:33
DAST_FAIL
@ DAST_FAIL
Definition: dnn_interface.h:46
DNNModel::get_output
DNNReturnType(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:97
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
DCO_NONE
@ DCO_NONE
Definition: dnn_interface.h:40
DNNExecClassificationParams
Definition: dnn_interface.h:75
DNNData::order
DNNColorOrder order
Definition: dnn_interface.h:64
DNNReturnType
DNNReturnType
Definition: dnn_interface.h:33
DNNData
Definition: dnn_interface.h:59
DNNModel::get_input
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:95
DNN_OV
@ DNN_OV
Definition: dnn_interface.h:35
DNNExecClassificationParams::target
const char * target
Definition: dnn_interface.h:77
ff_get_dnn_module
DNNModule * ff_get_dnn_module(DNNBackendType backend_type)
Definition: dnn_interface.c:32
DNNExecClassificationParams::base
DNNExecBaseParams base
Definition: dnn_interface.h:76
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:104
options
const OptionDef options[]
ClassifyPostProc
int(* ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:82
DAST_SUCCESS
@ DAST_SUCCESS
Definition: dnn_interface.h:49
DNNBackendType
DNNBackendType
Definition: dnn_interface.h:35
DAST_EMPTY_QUEUE
@ DAST_EMPTY_QUEUE
Definition: dnn_interface.h:47
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:106
DNNModule::flush
DNNReturnType(* flush)(const DNNModel *model)
Definition: dnn_interface.h:122
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:92
DNNDataType
DNNDataType
Definition: dnn_interface.h:37
DNNData::dt
DNNDataType dt
Definition: dnn_interface.h:63
frame.h
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
DNNModule::execute_model
DNNReturnType(* execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_interface.h:116
DNNExecBaseParams::out_frame
AVFrame * out_frame
Definition: dnn_interface.h:72
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:55
DNNModule::get_async_result
DNNAsyncStatusType(* get_async_result)(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_interface.h:120
DNN_ERROR
@ DNN_ERROR
Definition: dnn_interface.h:33
DNNModel::classify_post_proc
ClassifyPostProc classify_post_proc
Definition: dnn_interface.h:108
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
DFT_ANALYTICS_CLASSIFY
@ DFT_ANALYTICS_CLASSIFY
Definition: dnn_interface.h:56
DNNModule::free_model
void(* free_model)(DNNModel **model)
Definition: dnn_interface.h:124
avfilter.h
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
DNNExecBaseParams::output_names
const char ** output_names
Definition: dnn_interface.h:69
FramePrePostProc
int(* FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:80
AVFilterContext
An instance of a filter.
Definition: avfilter.h:333
DNNModel
Definition: dnn_interface.h:84
DNNData::channels
int channels
Definition: dnn_interface.h:61
DNN_NATIVE
@ DNN_NATIVE
Definition: dnn_interface.h:35
DNNModel::options
const char * options
Definition: dnn_interface.h:88
DNNData::width
int width
Definition: dnn_interface.h:61
DNNExecBaseParams
Definition: dnn_interface.h:67
DCO_BGR
@ DCO_BGR
Definition: dnn_interface.h:41
DAST_NOT_READY
@ DAST_NOT_READY
Definition: dnn_interface.h:48
int
int
Definition: ffmpeg_filter.c:156
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
DNNModule
Definition: dnn_interface.h:112
DNNExecBaseParams::nb_output
uint32_t nb_output
Definition: dnn_interface.h:70
DNNModel::model
void * model
Definition: dnn_interface.h:86