FFmpeg
dnn_backend_common.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 /**
20  * @file
21  * DNN common functions different backends.
22  */
23 
24 #include "dnn_backend_common.h"
25 
26 #define DNN_ASYNC_SUCCESS (void *)0
27 #define DNN_ASYNC_FAIL (void *)-1
28 
29 int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
30 {
31  if (!exec_params) {
32  av_log(ctx, AV_LOG_ERROR, "exec_params is null when execute model.\n");
33  return AVERROR(EINVAL);
34  }
35 
36  if (!exec_params->in_frame) {
37  av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
38  return AVERROR(EINVAL);
39  }
40 
41  if (!exec_params->out_frame && func_type == DFT_PROCESS_FRAME) {
42  av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
43  return AVERROR(EINVAL);
44  }
45 
46  if (exec_params->nb_output != 1 && backend != DNN_TF) {
47  // currently, the filter does not need multiple outputs,
48  // so we just pending the support until we really need it.
49  avpriv_report_missing_feature(ctx, "multiple outputs");
50  return AVERROR(ENOSYS);
51  }
52 
53  return 0;
54 }
55 
56 int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc) {
57  if (task == NULL || exec_params == NULL || backend_model == NULL)
58  return AVERROR(EINVAL);
59  if (do_ioproc != 0 && do_ioproc != 1)
60  return AVERROR(EINVAL);
61  if (async != 0 && async != 1)
62  return AVERROR(EINVAL);
63 
64  task->do_ioproc = do_ioproc;
65  task->async = async;
66  task->input_name = exec_params->input_name;
67  task->in_frame = exec_params->in_frame;
68  task->out_frame = exec_params->out_frame;
69  task->model = backend_model;
70  task->nb_output = exec_params->nb_output;
71  task->output_names = exec_params->output_names;
72 
73  return 0;
74 }
75 
76 /**
77  * Thread routine for async execution.
78  * @param args pointer to DNNAsyncExecModule module
79  */
80 static void *async_thread_routine(void *args)
81 {
82  DNNAsyncExecModule *async_module = args;
83  void *request = async_module->args;
84 
85  if (async_module->start_inference(request) != 0) {
86  return DNN_ASYNC_FAIL;
87  }
88  async_module->callback(request);
89  return DNN_ASYNC_SUCCESS;
90 }
91 
93 {
94  void *status = 0;
95  if (!async_module) {
96  return AVERROR(EINVAL);
97  }
98 #if HAVE_PTHREAD_CANCEL
99  pthread_join(async_module->thread_id, &status);
100  if (status == DNN_ASYNC_FAIL) {
101  av_log(NULL, AV_LOG_ERROR, "Last Inference Failed.\n");
102  return DNN_GENERIC_ERROR;
103  }
104 #endif
105  async_module->start_inference = NULL;
106  async_module->callback = NULL;
107  async_module->args = NULL;
108  return 0;
109 }
110 
112 {
113  int ret;
114  void *status = 0;
115 
116  if (!async_module) {
117  av_log(ctx, AV_LOG_ERROR, "async_module is null when starting async inference.\n");
118  return AVERROR(EINVAL);
119  }
120 
121 #if HAVE_PTHREAD_CANCEL
122  pthread_join(async_module->thread_id, &status);
123  if (status == DNN_ASYNC_FAIL) {
124  av_log(ctx, AV_LOG_ERROR, "Unable to start inference as previous inference failed.\n");
125  return DNN_GENERIC_ERROR;
126  }
127  ret = pthread_create(&async_module->thread_id, NULL, async_thread_routine, async_module);
128  if (ret != 0) {
129  av_log(ctx, AV_LOG_ERROR, "Unable to start async inference.\n");
130  return ret;
131  }
132 #else
133  ret = async_module->start_inference(async_module->args);
134  if (ret != 0) {
135  return ret;
136  }
137  async_module->callback(async_module->args);
138 #endif
139  return 0;
140 }
141 
143 {
144  TaskItem *task = ff_queue_peek_front(task_queue);
145 
146  if (!task) {
147  return DAST_EMPTY_QUEUE;
148  }
149 
150  if (task->inference_done != task->inference_todo) {
151  return DAST_NOT_READY;
152  }
153 
154  *in = task->in_frame;
155  *out = task->out_frame;
156  ff_queue_pop_front(task_queue);
157  av_freep(&task);
158 
159  return DAST_SUCCESS;
160 }
161 
162 int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
163 {
164  AVFrame *in_frame = NULL;
165  AVFrame *out_frame = NULL;
166 
167  in_frame = av_frame_alloc();
168  if (!in_frame) {
169  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n");
170  return AVERROR(ENOMEM);
171  }
172 
173  out_frame = av_frame_alloc();
174  if (!out_frame) {
175  av_frame_free(&in_frame);
176  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output frame\n");
177  return AVERROR(ENOMEM);
178  }
179 
180  in_frame->width = input_width;
181  in_frame->height = input_height;
182  exec_params->in_frame = in_frame;
183  exec_params->out_frame = out_frame;
184 
185  return ff_dnn_fill_task(task, exec_params, backend_model, 0, 0);
186 }
pthread_join
static av_always_inline int pthread_join(pthread_t thread, void **value_ptr)
Definition: os2threads.h:94
status
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
out
FILE * out
Definition: movenc.c:54
DNNAsyncExecModule
Common Async Execution Mechanism for the DNN Backends.
Definition: dnn_backend_common.h:58
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:29
DNN_GENERIC_ERROR
#define DNN_GENERIC_ERROR
Definition: dnn_interface.h:33
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:116
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:325
AVFrame::width
int width
Definition: frame.h:397
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:68
TaskItem
Definition: dnn_backend_common.h:36
DNNAsyncExecModule::callback
void(* callback)(void *args)
Completion Callback for the backend.
Definition: dnn_backend_common.h:70
DNNExecBaseParams::in_frame
AVFrame * in_frame
Definition: dnn_interface.h:71
DNN_ASYNC_SUCCESS
#define DNN_ASYNC_SUCCESS
Definition: dnn_backend_common.c:26
TaskItem::model
void * model
Definition: dnn_backend_common.h:37
Queue
Linear double-ended data structure.
Definition: queue.c:33
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:104
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
ff_dnn_fill_gettingoutput_task
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:162
ctx
AVFormatContext * ctx
Definition: movenc.c:48
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:45
pthread_create
static av_always_inline int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
Definition: os2threads.h:80
NULL
#define NULL
Definition: coverity.c:32
ff_dnn_async_module_cleanup
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
Definition: dnn_backend_common.c:92
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:38
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:42
DAST_SUCCESS
@ DAST_SUCCESS
Definition: dnn_interface.h:49
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:46
DNNBackendType
DNNBackendType
Definition: dnn_interface.h:35
DAST_EMPTY_QUEUE
@ DAST_EMPTY_QUEUE
Definition: dnn_interface.h:47
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
ff_dnn_fill_task
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:56
DNNExecBaseParams::out_frame
AVFrame * out_frame
Definition: dnn_interface.h:72
DNNAsyncExecModule::start_inference
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
Definition: dnn_backend_common.h:63
DNNAsyncExecModule::args
void * args
Argument for the execution functions.
Definition: dnn_backend_common.h:76
TaskItem::output_names
const char ** output_names
Definition: dnn_backend_common.h:41
async_thread_routine
static void * async_thread_routine(void *args)
Thread routine for async execution.
Definition: dnn_backend_common.c:80
ret
ret
Definition: filter_design.txt:187
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:39
AVFrame::height
int height
Definition: frame.h:397
dnn_backend_common.h
DNN_ASYNC_FAIL
#define DNN_ASYNC_FAIL
Definition: dnn_backend_common.c:27
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:142
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DNNExecBaseParams::output_names
const char ** output_names
Definition: dnn_interface.h:69
ff_dnn_start_inference_async
int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
Definition: dnn_backend_common.c:111
TaskItem::input_name
const char * input_name
Definition: dnn_backend_common.h:40
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
DNNExecBaseParams
Definition: dnn_interface.h:67
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:43
DAST_NOT_READY
@ DAST_NOT_READY
Definition: dnn_interface.h:48
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
TaskItem::nb_output
uint32_t nb_output
Definition: dnn_backend_common.h:44
DNNExecBaseParams::nb_output
uint32_t nb_output
Definition: dnn_interface.h:70