26#include <torch/torch.h>
27#include <torch/script.h>
68#define OFFSET(x) offsetof(THOptions, x)
69#define FLAGS AV_OPT_FLAG_FILTERING_PARAM
127 if (!model || !*model)
130 th_model = (
THModel *)(*model);
178 auto forward_method = th_model->
jit_model->get_method(
"forward");
179 auto graph = forward_method.graph();
180 auto graph_inputs = graph->inputs();
181 for (
auto gi : graph_inputs) {
182 if (
auto tt = gi->type()->cast<c10::TensorType>()) {
183 if (tt->scalarType().has_value()) {
184 if (*tt->scalarType() == c10::kByte) {
187 "Detected UINT8 input tensor dtype from model graph\n");
193 }
catch (
const c10::Error& e) {
195 "Could not inspect model graph for input dtype, defaulting to DNN_FLOAT\n");
209static void cuda_tensor_deleter(
void *
arg)
236 int stride_bytes =
frame->linesize[0];
240 torch::Tensor byte_tensor = torch::from_blob(
242 {1, height, width, channels},
243 {(long)(height * stride_bytes), (long)stride_bytes,
246 torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCUDA));
250 byte_tensor.to(torch::kFloat32).div(255.0f)
251 .permute({0, 3, 1, 2})
258static void fill_model_output_th_cuda(
THModel *th_model,
TaskItem *task, torch::Tensor &out_slice)
266 bool needs_flip =
false;
267 switch (hw_frames_ctx->sw_format) {
269 hw_channels = 3; rgb_start = 0; needs_flip =
false;
272 hw_channels = 3; rgb_start = 0; needs_flip =
true;
275 hw_channels = 4; rgb_start = 0; needs_flip =
false;
278 hw_channels = 4; rgb_start = 0; needs_flip =
true;
281 hw_channels = 4; rgb_start = 1; needs_flip =
false;
284 hw_channels = 4; rgb_start = 1; needs_flip =
true;
288 "Unsupported sw_format for CUDA zero-copy output\n");
294 torch::Tensor out_u8 =
295 out_slice.mul(255.0f)
296 .permute({0, 2, 3, 1})
300 out_u8 = out_u8.flip({3});
302 int out_h = (int)out_u8.size(1);
303 int out_w = (int)out_u8.size(2);
307 torch::Tensor out_frame_tensor = torch::from_blob(
309 {1, out_h, out_w, hw_channels},
310 {(long)(out_h * task->out_frame->linesize[0]),
311 (long)task->out_frame->linesize[0],
312 (long)hw_channels, 1L},
314 torch::TensorOptions()
315 .dtype(torch::kUInt8)
316 .device(torch::kCUDA));
319 out_frame_tensor.slice(3, rgb_start, rgb_start + 3)
323 torch::cuda::synchronize();
334 int ret, width_idx, height_idx, channel_idx;
335 int batch_size =
ctx->batch_size;
336 void *batch_data =
NULL;
337 size_t input_tensor_size;
360 input_tensor_size = (input.
dt ==
DNN_UINT8) ?
sizeof(uint8_t) :
sizeof(
float);
367 for (
int i = 0;
i < batch_size;
i++) {
396 infer_request->
output =
new torch::Tensor();
397 *infer_request->
input_tensor = torch::from_blob(batch_data,
418 std::vector<torch::jit::IValue>
inputs;
419 torch::NoGradGuard no_grad;
431 if (
ctx->torch_option.optimize)
432 torch::jit::setGraphExecutorOptimize(
true);
434 torch::jit::setGraphExecutorOptimize(
false);
441 const char *device_name =
ctx->device ?
ctx->device :
"cpu";
442 c10::Device device(device_name);
457 torch::Tensor *output = infer_request->
output;
460 auto slices = torch::split(*output, 1, 0);
464 torch::Tensor out_slice = slices[
i];
465 c10::IntArrayRef
sizes = out_slice.sizes();
471 if (
sizes.size() == 4) {
490 "UINT8 output is not supported for CUDA frames; the "
491 "model must return a FLOAT tensor for CUDA inference\n");
494 fill_model_output_th_cuda(th_model, task, out_slice);
497 if (out_slice.device() != torch::kCPU)
498 out_slice = out_slice.to(torch::kCPU);
500 outputs.data = out_slice.data_ptr();
533 av_log(th_model->
ctx,
AV_LOG_ERROR,
"Unable to push back request_queue when failed to start inference.\n");
550 if (lltask ==
NULL) {
568 "UINT8 input is not supported for CUDA frames; use software "
569 "frames, or set input_dtype=float for CUDA inference\n");
573 ret = fill_model_input_th_cuda(th_model, request);
609 const char *output_name,
int *output_width,
int *output_height)
617 .input_name = input_name,
618 .output_names = &output_name,
667 const char *device_name =
ctx->device ?
ctx->device :
"cpu";
672 model = &th_model->
model;
675 c10::Device device = c10::Device(device_name);
676 if (device.is_xpu()) {
681#if TORCH_VERSION_MAJOR > 2 || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 6)
682 at::detail::getXPUHooks().init();
684 at::detail::getXPUHooks().initXPU();
686 }
else if (device.is_cuda()) {
688 if (!torch::cuda::is_available()) {
693 }
else if (!device.is_cpu()) {
699 th_model->
jit_model =
new torch::jit::Module;
700 (*th_model->
jit_model) = torch::jit::load(
ctx->model_filename);
702 }
catch (
const c10::Error& e) {
707 if (
ctx->nireq <= 0) {
716 for (
int i = 0;
i <
ctx->nireq;
i++) {
static const AVFilterPad inputs[]
static const AVFilterPad outputs[]
static AVFormatContext * ctx
#define i(width, name, range_min, range_max)
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
void ff_dnn_wait_requests(SafeQueue *request_queue, int nireq)
Wait for all inference requests to complete before teardown.
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
DNN common functions different backends.
#define DNN_DEFINE_CLASS(fname)
static void infer_completion_callback(void *args)
const DNNModule ff_dnn_backend_torch
static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)
static int fill_model_input_th(THModel *th_model, THRequestItem *request)
static DNNAsyncStatusType dnn_get_result_th(const DNNModel *model, AVFrame **in, AVFrame **out)
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
static int th_start_inference(void *args)
static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
static const AVOption dnn_th_options[]
static void th_free_request(THInferRequest *request)
static THInferRequest * th_create_inference_request(void)
static int dnn_flush_th(const DNNModel *model)
static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
static DNNModel * dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
static void deleter(void *arg)
static void destroy_request_item(THRequestItem **arg)
static void dnn_free_model_th(DNNModel **model)
static void infer_completion_callback(void *args)
static int dnn_get_height_idx_by_layout(DNNLayout layout)
static int dnn_get_width_idx_by_layout(DNNLayout layout)
#define DNN_GENERIC_ERROR
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
DNN input&output process between AVFrame and DNNData.
static const uint8_t frame_size[4]
@ AV_OPT_TYPE_INT
Underlying C type is int.
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
#define AV_LOG_VERBOSE
Detailed information.
#define AV_LOG_INFO
Standard information.
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
An API-specific header for AV_HWDEVICE_TYPE_CUDA.
FFmpeg internal API for CUDA.
static const int sizes[][2]
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
Memory handling functions.
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
@ AV_PIX_FMT_0BGR
packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
@ AV_PIX_FMT_0RGB
packed RGB 8:8:8, 32bpp, XRGBXRGB... X=unused/undefined
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Queue * ff_queue_create(void)
Create a Queue instance.
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
uint8_t * data
The data buffer.
This structure describes decoded (raw) audio or video data.
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame.
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
This struct describes a set or pool of "hardware" frames (i.e.
Common Async Execution Mechanism for the DNN Backends.
void * args
Argument for the execution functions.
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
void(* callback)(void *args)
Completion Callback for the backend.
int(* get_input)(struct DNNModel *model, DNNData *input, const char *input_name)
int(* get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
FramePrePostProc frame_pre_proc
FramePrePostProc frame_post_proc
AVFilterContext * filter_ctx
DNNFunctionType func_type
Linear double-ended data structure.
Double-ended queue with mutex locks ensuring data consistency while multithreading.
torch::Tensor * input_tensor
torch::jit::Module * jit_model
SafeQueue * request_queue
LastLevelTaskItem ** lltasks
THInferRequest * infer_request
DNNAsyncExecModule exec_module
#define av_malloc_array(a, b)
static FilteringContext * filter_ctx