Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
21 /**
22  * @file
23  * DNN inference engine interface.
24  */
29 #include <stdint.h>
30 #include "libavutil/frame.h"
31 #include "avfilter.h"
37 typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
39 typedef enum {
44 typedef enum {
45  DAST_FAIL, // something wrong
46  DAST_EMPTY_QUEUE, // no more inference result to get
47  DAST_NOT_READY, // all queued inferences are not finished
48  DAST_SUCCESS // got a result frame successfully
51 typedef enum {
53  DFT_PROCESS_FRAME, // process the whole frame
54  DFT_ANALYTICS_DETECT, // detect from the whole frame
55  // we can add more such as detect_from_crop, classify_from_bbox, etc.
58 typedef struct DNNData{
59  void *data;
61  // dt and order together decide the color format
64 } DNNData;
66 typedef struct DNNModel{
67  // Stores model that can be different for different backends.
68  void *model;
69  // Stores options when the model is executed by the backend
70  const char *options;
71  // Stores FilterContext used for the interaction between AVFrame and DNNData
73  // Stores function type of the model
75  // Gets model input information
76  // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
77  DNNReturnType (*get_input)(void *model, DNNData *input, const char *input_name);
78  // Gets model output width/height with given input w/h
79  DNNReturnType (*get_output)(void *model, const char *input_name, int input_width, int input_height,
80  const char *output_name, int *output_width, int *output_height);
81  // set the pre process to transfer data from AVFrame to DNNData
82  // the default implementation within DNN is used if it is not provided by the filter
83  int (*pre_proc)(AVFrame *frame_in, DNNData *model_input, AVFilterContext *filter_ctx);
84  // set the post process to transfer data from DNNData to AVFrame
85  // the default implementation within DNN is used if it is not provided by the filter
86  int (*post_proc)(AVFrame *frame_out, DNNData *model_output, AVFilterContext *filter_ctx);
87 } DNNModel;
89 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
90 typedef struct DNNModule{
91  // Loads model and parameters from given file. Returns NULL if it is not possible.
92  DNNModel *(*load_model)(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
93  // Executes model with specified input and output. Returns DNN_ERROR otherwise.
94  DNNReturnType (*execute_model)(const DNNModel *model, const char *input_name, AVFrame *in_frame,
95  const char **output_names, uint32_t nb_output, AVFrame *out_frame);
96  // Executes model with specified input and output asynchronously. Returns DNN_ERROR otherwise.
97  DNNReturnType (*execute_model_async)(const DNNModel *model, const char *input_name, AVFrame *in_frame,
98  const char **output_names, uint32_t nb_output, AVFrame *out_frame);
99  // Retrieve inference result.
100  DNNAsyncStatusType (*get_async_result)(const DNNModel *model, AVFrame **in, AVFrame **out);
101  // Flush all the pending tasks.
102  DNNReturnType (*flush)(const DNNModel *model);
103  // Frees memory allocated for model.
104  void (*free_model)(DNNModel **model);
105 } DNNModule;
107 // Initializes DNNModule depending on chosen backend.
110 #endif
void * model
Definition: dnn_interface.h:68
This structure describes decoded (raw) audio or video data.
Definition: frame.h:314
static void flush(AVCodecContext *avctx)
Main libavfilter public API header.
int channels
Definition: dnn_interface.h:60
static int get_input(AVFrame *frame, int frame_num)
Definition: filter_audio.c:242
const char * options
Definition: dnn_interface.h:70
Definition: dnn_interface.h:35
Definition: dnn_interface.h:37
DNNColorOrder order
Definition: dnn_interface.h:63
int height
Definition: dnn_interface.h:60
void * data
Definition: dnn_interface.h:59
static void free_model(AVFilterContext *ctx, int n)
Definition: af_arnndn.c:1536
static FilteringContext * filter_ctx
Definition: transcoding.c:47
reference-counted frame API
Definition: dnn_interface.h:44
Definition: dnn_interface.h:39
Definition: dnn_interface.h:33
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
DNNFunctionType func_type
Definition: dnn_interface.h:74
Definition: dnn_interface.h:51
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
const OptionDef options[]
Definition: ffmpeg_opt.c:3424
static void get_output(ConvolveContext *s, FFTComplex *input, AVFrame *out, int w, int h, int n, int plane, float scale)
Definition: vf_convolve.c:329
DNNModule * ff_get_dnn_module(DNNBackendType backend_type)
Definition: dnn_interface.c:32
An instance of a filter.
Definition: avfilter.h:341
FILE * out
Definition: movenc.c:54
AVFilterContext * filter_ctx
Definition: dnn_interface.h:72
DNNDataType dt
Definition: dnn_interface.h:62