FFmpeg
dnn_backend_common.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 /**
20  * @file
21  * DNN common functions different backends.
22  */
23 
24 #include "dnn_backend_common.h"
25 
26 #define DNN_ASYNC_SUCCESS (void *)0
27 #define DNN_ASYNC_FAIL (void *)-1
28 
29 int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
30 {
31  if (!exec_params) {
32  av_log(ctx, AV_LOG_ERROR, "exec_params is null when execute model.\n");
33  return AVERROR(EINVAL);
34  }
35 
36  if (!exec_params->in_frame) {
37  av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
38  return AVERROR(EINVAL);
39  }
40 
41  if (!exec_params->out_frame && func_type == DFT_PROCESS_FRAME) {
42  av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
43  return AVERROR(EINVAL);
44  }
45 
46  return 0;
47 }
48 
49 int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc) {
50  if (task == NULL || exec_params == NULL || backend_model == NULL)
51  return AVERROR(EINVAL);
52  if (do_ioproc != 0 && do_ioproc != 1)
53  return AVERROR(EINVAL);
54  if (async != 0 && async != 1)
55  return AVERROR(EINVAL);
56 
57  task->do_ioproc = do_ioproc;
58  task->async = async;
59  task->input_name = exec_params->input_name;
60  task->in_frame = exec_params->in_frame;
61  task->out_frame = exec_params->out_frame;
62  task->model = backend_model;
63  task->nb_output = exec_params->nb_output;
64  task->output_names = exec_params->output_names;
65 
66  return 0;
67 }
68 
69 /**
70  * Thread routine for async execution.
71  * @param args pointer to DNNAsyncExecModule module
72  */
73 static void *async_thread_routine(void *args)
74 {
75  DNNAsyncExecModule *async_module = args;
76  void *request = async_module->args;
77 
78  if (async_module->start_inference(request) != 0) {
79  return DNN_ASYNC_FAIL;
80  }
81  async_module->callback(request);
82  return DNN_ASYNC_SUCCESS;
83 }
84 
86 {
87  void *status = 0;
88  if (!async_module) {
89  return AVERROR(EINVAL);
90  }
91 #if HAVE_PTHREAD_CANCEL
92  pthread_join(async_module->thread_id, &status);
93  if (status == DNN_ASYNC_FAIL) {
94  av_log(NULL, AV_LOG_ERROR, "Last Inference Failed.\n");
95  return DNN_GENERIC_ERROR;
96  }
97 #endif
98  async_module->start_inference = NULL;
99  async_module->callback = NULL;
100  async_module->args = NULL;
101  return 0;
102 }
103 
105 {
106  int ret;
107  void *status = 0;
108 
109  if (!async_module) {
110  av_log(ctx, AV_LOG_ERROR, "async_module is null when starting async inference.\n");
111  return AVERROR(EINVAL);
112  }
113 
114 #if HAVE_PTHREAD_CANCEL
115  pthread_join(async_module->thread_id, &status);
116  if (status == DNN_ASYNC_FAIL) {
117  av_log(ctx, AV_LOG_ERROR, "Unable to start inference as previous inference failed.\n");
118  return DNN_GENERIC_ERROR;
119  }
120  ret = pthread_create(&async_module->thread_id, NULL, async_thread_routine, async_module);
121  if (ret != 0) {
122  av_log(ctx, AV_LOG_ERROR, "Unable to start async inference.\n");
123  return ret;
124  }
125 #else
126  ret = async_module->start_inference(async_module->args);
127  if (ret != 0) {
128  return ret;
129  }
130  async_module->callback(async_module->args);
131 #endif
132  return 0;
133 }
134 
136 {
137  TaskItem *task = ff_queue_peek_front(task_queue);
138 
139  if (!task) {
140  return DAST_EMPTY_QUEUE;
141  }
142 
143  if (task->inference_done != task->inference_todo) {
144  return DAST_NOT_READY;
145  }
146 
147  *in = task->in_frame;
148  *out = task->out_frame;
149  ff_queue_pop_front(task_queue);
150  av_freep(&task);
151 
152  return DAST_SUCCESS;
153 }
154 
155 int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
156 {
157  AVFrame *in_frame = NULL;
158  AVFrame *out_frame = NULL;
159 
160  in_frame = av_frame_alloc();
161  if (!in_frame) {
162  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n");
163  return AVERROR(ENOMEM);
164  }
165 
166  out_frame = av_frame_alloc();
167  if (!out_frame) {
168  av_frame_free(&in_frame);
169  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output frame\n");
170  return AVERROR(ENOMEM);
171  }
172 
173  in_frame->width = input_width;
174  in_frame->height = input_height;
175  exec_params->in_frame = in_frame;
176  exec_params->out_frame = out_frame;
177 
178  return ff_dnn_fill_task(task, exec_params, backend_model, 0, 0);
179 }
pthread_join
static av_always_inline int pthread_join(pthread_t thread, void **value_ptr)
Definition: os2threads.h:94
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
out
FILE * out
Definition: movenc.c:54
DNNAsyncExecModule
Common Async Execution Mechanism for the DNN Backends.
Definition: dnn_backend_common.h:58
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:29
DNN_GENERIC_ERROR
#define DNN_GENERIC_ERROR
Definition: dnn_interface.h:33
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:100
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:340
AVFrame::width
int width
Definition: frame.h:412
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:77
TaskItem
Definition: dnn_backend_common.h:36
DNNAsyncExecModule::callback
void(* callback)(void *args)
Completion Callback for the backend.
Definition: dnn_backend_common.h:70
DNNExecBaseParams::in_frame
AVFrame * in_frame
Definition: dnn_interface.h:80
DNN_ASYNC_SUCCESS
#define DNN_ASYNC_SUCCESS
Definition: dnn_backend_common.c:26
TaskItem::model
void * model
Definition: dnn_backend_common.h:37
Queue
Linear double-ended data structure.
Definition: queue.c:33
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:88
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
ff_dnn_fill_gettingoutput_task
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:155
ctx
AVFormatContext * ctx
Definition: movenc.c:48
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:45
pthread_create
static av_always_inline int pthread_create(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg)
Definition: os2threads.h:80
NULL
#define NULL
Definition: coverity.c:32
ff_dnn_async_module_cleanup
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
Definition: dnn_backend_common.c:85
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:38
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:42
DAST_SUCCESS
@ DAST_SUCCESS
Definition: dnn_interface.h:49
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:46
DNNBackendType
DNNBackendType
Definition: dnn_interface.h:35
DAST_EMPTY_QUEUE
@ DAST_EMPTY_QUEUE
Definition: dnn_interface.h:47
ff_dnn_fill_task
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:49
DNNExecBaseParams::out_frame
AVFrame * out_frame
Definition: dnn_interface.h:81
DNNAsyncExecModule::start_inference
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
Definition: dnn_backend_common.h:63
DNNAsyncExecModule::args
void * args
Argument for the execution functions.
Definition: dnn_backend_common.h:76
TaskItem::output_names
const char ** output_names
Definition: dnn_backend_common.h:41
async_thread_routine
static void * async_thread_routine(void *args)
Thread routine for async execution.
Definition: dnn_backend_common.c:73
ret
ret
Definition: filter_design.txt:187
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:39
AVFrame::height
int height
Definition: frame.h:412
status
ov_status_e status
Definition: dnn_backend_openvino.c:120
dnn_backend_common.h
DNN_ASYNC_FAIL
#define DNN_ASYNC_FAIL
Definition: dnn_backend_common.c:27
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:135
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DNNExecBaseParams::output_names
const char ** output_names
Definition: dnn_interface.h:78
ff_dnn_start_inference_async
int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
Definition: dnn_backend_common.c:104
TaskItem::input_name
const char * input_name
Definition: dnn_backend_common.h:40
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
DNNExecBaseParams
Definition: dnn_interface.h:76
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:43
DAST_NOT_READY
@ DAST_NOT_READY
Definition: dnn_interface.h:48
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
TaskItem::nb_output
uint32_t nb_output
Definition: dnn_backend_common.h:44
DNNExecBaseParams::nb_output
uint32_t nb_output
Definition: dnn_interface.h:79