FFmpeg
Loading...
Searching...
No Matches
dnn_backend_torch.cpp
Go to the documentation of this file.
1/*
2 * Copyright (c) 2024
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21/**
22 * @file
23 * DNN Torch backend implementation.
24 */
25
26#include <torch/torch.h>
27#include <torch/script.h>
28
29extern "C" {
30#include "config.h"
31#include "dnn_io_proc.h"
32#include "dnn_backend_common.h"
33#include "libavutil/opt.h"
34#include "libavutil/mem.h"
35#include "libavutil/cpu.h"
36#if CONFIG_CUDA
37#include "libavutil/hwcontext.h"
40#include "libavutil/pixfmt.h"
41#endif
42#include "queue.h"
43#include "safe_queue.h"
44}
45
55
56typedef struct THInferRequest {
57 torch::Tensor *output;
58 torch::Tensor *input_tensor;
60
67
68#define OFFSET(x) offsetof(THOptions, x)
69#define FLAGS AV_OPT_FLAG_FILTERING_PARAM
70static const AVOption dnn_th_options[] = {
71 { "optimize", "turn on graph executor optimization", OFFSET(optimize), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS},
72 { NULL }
73};
74
75static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
76{
77 THModel *th_model = (THModel *)task->model;
78 DnnContext *ctx = th_model->ctx;
79 LastLevelTaskItem *lltask = (LastLevelTaskItem *)av_malloc(sizeof(*lltask));
80 if (!lltask) {
81 av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for LastLevelTaskItem\n");
82 return AVERROR(ENOMEM);
83 }
84 task->inference_todo = 1;
85 task->inference_done = 0;
86 lltask->task = task;
87 if (ff_queue_push_back(lltask_queue, lltask) < 0) {
88 av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
89 av_freep(&lltask);
90 return AVERROR(ENOMEM);
91 }
92 return 0;
93}
94
95static void th_free_request(THInferRequest *request)
96{
97 if (!request)
98 return;
99 if (request->output) {
100 delete(request->output);
101 request->output = NULL;
102 }
103 if (request->input_tensor) {
104 delete(request->input_tensor);
105 request->input_tensor = NULL;
106 }
107 return;
108}
109
111{
112 THRequestItem *item;
113 if (!arg || !*arg) {
114 return;
115 }
116 item = *arg;
118 av_freep(&item->infer_request);
119 av_freep(&item->lltasks);
121 av_freep(arg);
122}
123
124static void dnn_free_model_th(DNNModel **model)
125{
126 THModel *th_model;
127 if (!model || !*model)
128 return;
129
130 th_model = (THModel *)(*model);
131
132 if (th_model->request_queue) {
133 ff_dnn_wait_requests(th_model->request_queue, th_model->nb_requests);
134 while (ff_safe_queue_size(th_model->request_queue) != 0) {
137 }
139 }
140
141 if (th_model->lltask_queue)
143 if (th_model->task_queue)
144 ff_queue_destroy(th_model->task_queue);
145
146 if (th_model->jit_model)
147 delete th_model->jit_model;
148
149 av_freep(&th_model);
150 *model = NULL;
151}
152
153static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)
154{
155 THModel *th_model = (THModel *)model;
156
157 input->order = DCO_RGB;
158 input->layout = DL_NCHW;
159 input->dims[0] = 1;
160 input->dims[1] = 3;
161 input->dims[2] = -1;
162 input->dims[3] = -1;
163
164 switch (th_model->ctx->input_dtype) {
165 case DNN_FLOAT:
166 input->dt = DNN_FLOAT;
167 break;
168 case DNN_UINT8:
169 input->dt = DNN_UINT8;
170 break;
171 case DNN_AUTO:
172 default:
173 input->dt = DNN_FLOAT;
174
175 /* save/reload via torch::jit::load() erases graph scalar types, so
176 * detection rarely succeeds; set input_dtype=uint8 when it fails. */
177 try {
178 auto forward_method = th_model->jit_model->get_method("forward");
179 auto graph = forward_method.graph();
180 auto graph_inputs = graph->inputs();
181 for (auto gi : graph_inputs) {
182 if (auto tt = gi->type()->cast<c10::TensorType>()) {
183 if (tt->scalarType().has_value()) {
184 if (*tt->scalarType() == c10::kByte) {
185 input->dt = DNN_UINT8;
186 av_log(th_model->ctx, AV_LOG_VERBOSE,
187 "Detected UINT8 input tensor dtype from model graph\n");
188 }
189 break;
190 }
191 }
192 }
193 } catch (const c10::Error& e) {
194 av_log(th_model->ctx, AV_LOG_VERBOSE,
195 "Could not inspect model graph for input dtype, defaulting to DNN_FLOAT\n");
196 }
197 break;
198 }
199
200 return 0;
201}
202
203static void deleter(void *arg)
204{
205 av_freep(&arg);
206}
207
208#if CONFIG_CUDA
209static void cuda_tensor_deleter(void *arg)
210{
211 /* No-op: GPU memory is owned by FFmpeg AVBuffer ref-counting.
212 * LibTorch must not free it. */
213 (void)arg;
214}
215
216/**
217 * Map a CUDA frame's GPU pointer directly into a LibTorch tensor,
218 * bypassing any host-device memory copy.
219 *
220 * The resulting tensor is a zero-copy view over the frame's VRAM
221 * buffer; the AVBuffer reference keeps the memory alive.
222 */
223static int fill_model_input_th_cuda(THModel *th_model, THRequestItem *request)
224{
225 THInferRequest *infer_request = request->infer_request;
226 LastLevelTaskItem *lltask = request->lltasks[0];
227 TaskItem *task = lltask->task;
228 AVFrame *frame = task->in_frame;
229
230
231 int height = frame->height;
232 int width = frame->width;
233 /* linesize[0] is in bytes; for packed RGB/BGR it equals width * channels
234 * plus alignment padding. Use it as the stride so PyTorch respects the
235 * actual memory layout. */
236 int stride_bytes = frame->linesize[0];
237 int channels = stride_bytes / width; /* 3 for RGB24, 4 for RGB0/BGR0 */
238
239 /* Wrap the GPU device pointer in a LibTorch tensor (no copy). */
240 torch::Tensor byte_tensor = torch::from_blob(
241 frame->data[0],
242 {1, height, width, channels},
243 {(long)(height * stride_bytes), (long)stride_bytes,
244 (long)channels, 1L},
245 cuda_tensor_deleter,
246 torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCUDA));
247
248 /* Convert NHWC uint8 → NCHW float32 in [0, 1] and keep on GPU. */
249 *infer_request->input_tensor =
250 byte_tensor.to(torch::kFloat32).div(255.0f)
251 .permute({0, 3, 1, 2}) /* NHWC → NCHW */
252 .slice(1, 0, 3) /* drop alpha if present */
253 .contiguous();
254
255 return 0;
256}
257
258static void fill_model_output_th_cuda(THModel *th_model, TaskItem *task, torch::Tensor &out_slice)
259{
260 AVHWFramesContext *hw_frames_ctx =
262
263 /* Determine channel layout from sw_format. */
264 int hw_channels = 3;
265 int rgb_start = 0;
266 bool needs_flip = false;
267 switch (hw_frames_ctx->sw_format) {
268 case AV_PIX_FMT_RGB24:
269 hw_channels = 3; rgb_start = 0; needs_flip = false;
270 break;
271 case AV_PIX_FMT_BGR24:
272 hw_channels = 3; rgb_start = 0; needs_flip = true;
273 break;
274 case AV_PIX_FMT_RGB0:
275 hw_channels = 4; rgb_start = 0; needs_flip = false;
276 break;
277 case AV_PIX_FMT_BGR0:
278 hw_channels = 4; rgb_start = 0; needs_flip = true;
279 break;
280 case AV_PIX_FMT_0RGB:
281 hw_channels = 4; rgb_start = 1; needs_flip = false;
282 break;
283 case AV_PIX_FMT_0BGR:
284 hw_channels = 4; rgb_start = 1; needs_flip = true;
285 break;
286 default:
287 av_log(th_model->ctx, AV_LOG_ERROR,
288 "Unsupported sw_format for CUDA zero-copy output\n");
289 hw_channels = 3;
290 break;
291 }
292
293 /* Convert model output: NCHW float [0,1] → NHWC uint8 [0,255]. */
294 torch::Tensor out_u8 =
295 out_slice.mul(255.0f)
296 .permute({0, 2, 3, 1})
297 .to(torch::kUInt8)
298 .contiguous();
299 if (needs_flip)
300 out_u8 = out_u8.flip({3});
301
302 int out_h = (int)out_u8.size(1);
303 int out_w = (int)out_u8.size(2);
304
305 /* Map the output frame's VRAM into a tensor with correct
306 * stride (linesize includes alignment padding). */
307 torch::Tensor out_frame_tensor = torch::from_blob(
308 task->out_frame->data[0],
309 {1, out_h, out_w, hw_channels},
310 {(long)(out_h * task->out_frame->linesize[0]),
311 (long)task->out_frame->linesize[0],
312 (long)hw_channels, 1L},
313 cuda_tensor_deleter,
314 torch::TensorOptions()
315 .dtype(torch::kUInt8)
316 .device(torch::kCUDA));
317
318 /* Device-to-Device copy into the correct channel slice. */
319 out_frame_tensor.slice(3, rgb_start, rgb_start + 3)
320 .copy_(out_u8);
321
322 /* Flush the CUDA stream before the encoder reads the frame. */
323 torch::cuda::synchronize();
324}
325#endif /* CONFIG_CUDA */
326
327static int fill_model_input_th(THModel *th_model, THRequestItem *request)
328{
329 LastLevelTaskItem *lltask = NULL;
330 TaskItem *task = NULL;
331 THInferRequest *infer_request = NULL;
332 DNNData input = { 0 };
333 DnnContext *ctx = th_model->ctx;
334 int ret, width_idx, height_idx, channel_idx;
335 int batch_size = ctx->batch_size;
336 void *batch_data = NULL;
337 size_t input_tensor_size;
338 int frame_size = 0;
339
340 infer_request = request->infer_request;
341
342 ret = get_input_th(&th_model->model, &input, NULL);
343 if (ret != 0) {
344 goto err;
345 }
346 width_idx = dnn_get_width_idx_by_layout(input.layout);
347 height_idx = dnn_get_height_idx_by_layout(input.layout);
348 channel_idx = dnn_get_channel_idx_by_layout(input.layout);
349
351 if (!lltask) {
352 ret = AVERROR(EINVAL);
353 goto err;
354 }
355 task = lltask->task;
356 input.dims[height_idx] = task->in_frame->height;
357 input.dims[width_idx] = task->in_frame->width;
358
359 frame_size = input.dims[height_idx] * input.dims[width_idx] * input.dims[channel_idx];
360 input_tensor_size = (input.dt == DNN_UINT8) ? sizeof(uint8_t) : sizeof(float);
361 batch_data = av_malloc(batch_size * frame_size * input_tensor_size);
362 if (!batch_data) {
363 ret = AVERROR(ENOMEM);
364 goto err;
365 }
366
367 for (int i = 0; i < batch_size; i++) {
369 if (!lltask)
370 break;
371
372 request->lltasks[i] = lltask;
373 request->lltask_count = i + 1;
374 task = lltask->task;
375
376 input.data = (char *)batch_data + i * frame_size * input_tensor_size;
377
378 switch (th_model->model.func_type) {
380 input.scale = (input.dt == DNN_UINT8) ? 1 : 255;
381 if (task->do_ioproc) {
382 if (th_model->model.frame_pre_proc != NULL) {
383 th_model->model.frame_pre_proc(task->in_frame, &input, th_model->model.filter_ctx);
384 } else {
385 ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx);
386 }
387 }
388 break;
389 default:
390 avpriv_report_missing_feature(NULL, "model function type %d", th_model->model.func_type);
391 break;
392 }
393 }
394
395 infer_request->input_tensor = new torch::Tensor();
396 infer_request->output = new torch::Tensor();
397 *infer_request->input_tensor = torch::from_blob(batch_data,
398 {request->lltask_count, input.dims[channel_idx], input.dims[height_idx], input.dims[width_idx]},
399 deleter, (input.dt == DNN_UINT8) ? torch::kUInt8 : torch::kFloat32);
400
401 return 0;
402
403err:
404 if (batch_data)
405 av_freep(&batch_data);
406 th_free_request(infer_request);
407 return ret;
408}
409
410static int th_start_inference(void *args)
411{
412 THRequestItem *request = (THRequestItem *)args;
413 THInferRequest *infer_request = NULL;
414 LastLevelTaskItem *lltask = NULL;
415 TaskItem *task = NULL;
416 THModel *th_model = NULL;
418 std::vector<torch::jit::IValue> inputs;
419 torch::NoGradGuard no_grad;
420
421 if (!request) {
422 av_log(NULL, AV_LOG_ERROR, "THRequestItem is NULL\n");
423 return AVERROR(EINVAL);
424 }
425 infer_request = request->infer_request;
426 lltask = request->lltasks[0];
427 task = lltask->task;
428 th_model = (THModel *)task->model;
429 ctx = th_model->ctx;
430
431 if (ctx->torch_option.optimize)
432 torch::jit::setGraphExecutorOptimize(true);
433 else
434 torch::jit::setGraphExecutorOptimize(false);
435
436 if (!infer_request->input_tensor || !infer_request->output) {
437 av_log(ctx, AV_LOG_ERROR, "input or output tensor is NULL\n");
438 return DNN_GENERIC_ERROR;
439 }
440 // Transfer tensor to the same device as model
441 const char *device_name = ctx->device ? ctx->device : "cpu";
442 c10::Device device(device_name);
443 if (infer_request->input_tensor->device() != device)
444 *infer_request->input_tensor = infer_request->input_tensor->to(device);
445 inputs.push_back(*infer_request->input_tensor);
446
447 *infer_request->output = th_model->jit_model->forward(inputs).toTensor();
448
449 return 0;
450}
451
452static void infer_completion_callback(void *args) {
453 THRequestItem *request = (THRequestItem*)args;
454 THInferRequest *infer_request = request->infer_request;
455 LastLevelTaskItem *lltask = request->lltasks[0];
456 THModel *th_model = (THModel *)lltask->task->model;
457 torch::Tensor *output = infer_request->output;
458 DNNData outputs = { 0 };
459
460 auto slices = torch::split(*output, /*split_size=*/1, /*dim=*/0);
461 for (uint32_t i = 0; i < request->lltask_count; i++) {
462 lltask = request->lltasks[i];
463 TaskItem *task = lltask->task;
464 torch::Tensor out_slice = slices[i];
465 c10::IntArrayRef sizes = out_slice.sizes();
466
467 outputs.order = DCO_RGB;
468 outputs.layout = DL_NCHW;
469 outputs.dt = (out_slice.dtype() == torch::kUInt8) ? DNN_UINT8 : DNN_FLOAT;
470
471 if (sizes.size() == 4) {
472 // 4 dimensions: [batch_size, channel, height, width]
473 // this format of data is normally used for video frame SR
474 outputs.dims[0] = sizes.at(0); // N
475 outputs.dims[1] = sizes.at(1); // C
476 outputs.dims[2] = sizes.at(2); // H
477 outputs.dims[3] = sizes.at(3); // W
478 } else {
479 avpriv_report_missing_feature(th_model->ctx, "Support of this kind of model");
480 goto err;
481 }
482
483 switch (th_model->model.func_type) {
485 if (task->do_ioproc) {
486#if CONFIG_CUDA
487 if (task->out_frame->format == AV_PIX_FMT_CUDA) {
488 if (outputs.dt == DNN_UINT8) {
489 av_log(th_model->ctx, AV_LOG_ERROR,
490 "UINT8 output is not supported for CUDA frames; the "
491 "model must return a FLOAT tensor for CUDA inference\n");
492 goto err;
493 }
494 fill_model_output_th_cuda(th_model, task, out_slice);
495 } else {
496#endif
497 if (out_slice.device() != torch::kCPU)
498 out_slice = out_slice.to(torch::kCPU);
499 outputs.scale = (outputs.dt == DNN_UINT8) ? 1 : 255;
500 outputs.data = out_slice.data_ptr();
501 if (th_model->model.frame_post_proc != NULL) {
502 th_model->model.frame_post_proc(task->out_frame, &outputs,
503 th_model->model.filter_ctx);
504 } else {
506 th_model->ctx);
507 }
508#if CONFIG_CUDA
509 }
510#endif
511 } else {
514 }
515 break;
516 default:
517 avpriv_report_missing_feature(th_model->ctx, "model function type %d", th_model->model.func_type);
518 goto err;
519 }
520 task->inference_done++;
521 }
522
523err:
524 for (uint32_t i = 0; i < request->lltask_count; i++) {
525 av_freep(&request->lltasks[i]);
526 }
527 request->lltask_count = 0;
528
529 th_free_request(infer_request);
530
531 if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {
532 destroy_request_item(&request);
533 av_log(th_model->ctx, AV_LOG_ERROR, "Unable to push back request_queue when failed to start inference.\n");
534 }
535}
536
537static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
538{
539 THModel *th_model = NULL;
540 LastLevelTaskItem *lltask;
541 TaskItem *task = NULL;
542 int ret = 0;
543
544 if (ff_queue_size(lltask_queue) == 0) {
545 destroy_request_item(&request);
546 return 0;
547 }
548
549 lltask = (LastLevelTaskItem *)ff_queue_peek_front(lltask_queue);
550 if (lltask == NULL) {
551 av_log(NULL, AV_LOG_ERROR, "Failed to get LastLevelTaskItem\n");
552 ret = AVERROR(EINVAL);
553 goto err;
554 }
555 task = lltask->task;
556 th_model = (THModel *)task->model;
557
558#if CONFIG_CUDA
559 if (task->in_frame->format == AV_PIX_FMT_CUDA) {
560 DNNData input = { 0 };
561
562 ret = get_input_th(&th_model->model, &input, NULL);
563 if (ret != 0) {
564 goto err;
565 }
566 if (input.dt == DNN_UINT8) {
567 av_log(th_model->ctx, AV_LOG_ERROR,
568 "UINT8 input is not supported for CUDA frames; use software "
569 "frames, or set input_dtype=float for CUDA inference\n");
570 ret = AVERROR(ENOSYS);
571 goto err;
572 }
573 ret = fill_model_input_th_cuda(th_model, request);
574 } else {
575 ret = fill_model_input_th(th_model, request);
576 }
577#else
578 ret = fill_model_input_th(th_model, request);
579#endif
580 if (ret != 0) {
581 goto err;
582 }
583
584 if (task->async) {
585 ret = ff_dnn_start_inference_async(th_model->ctx, &request->exec_module);
586 if (ret != 0) {
587 goto err;
588 }
589 return 0;
590 } else {
591 // Synchronous execution path
592 ret = th_start_inference((void *)(request));
593 if (ret != 0) {
594 goto err;
595 }
597 return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;
598 }
599
600err:
602 if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {
603 destroy_request_item(&request);
604 }
605 return ret;
606}
607
608static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height,
609 const char *output_name, int *output_width, int *output_height)
610{
611 int ret = 0;
612 THModel *th_model = (THModel*) model;
613 DnnContext *ctx = th_model->ctx;
614 TaskItem task = { 0 };
615 THRequestItem *request = NULL;
616 DNNExecBaseParams exec_params = {
617 .input_name = input_name,
618 .output_names = &output_name,
619 .nb_output = 1,
620 .in_frame = NULL,
621 .out_frame = NULL,
622 };
623 ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, th_model, input_height, input_width, ctx);
624 if ( ret != 0) {
625 goto err;
626 }
627
628 ret = extract_lltask_from_task(&task, th_model->lltask_queue);
629 if ( ret != 0) {
630 av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
631 goto err;
632 }
633
635 if (!request) {
636 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
637 ret = AVERROR(EINVAL);
638 goto err;
639 }
640
641 ret = execute_model_th(request, th_model->lltask_queue);
642 *output_width = task.out_frame->width;
643 *output_height = task.out_frame->height;
644
645err:
647 av_frame_free(&task.in_frame);
648 return ret;
649}
650
652{
654 if (!request) {
655 return NULL;
656 }
657 request->input_tensor = NULL;
658 request->output = NULL;
659 return request;
660}
661
663{
664 DNNModel *model = NULL;
665 THModel *th_model = NULL;
666 THRequestItem *item = NULL;
667 const char *device_name = ctx->device ? ctx->device : "cpu";
668
669 th_model = (THModel *)av_mallocz(sizeof(THModel));
670 if (!th_model)
671 return NULL;
672 model = &th_model->model;
673 th_model->ctx = ctx;
674
675 c10::Device device = c10::Device(device_name);
676 if (device.is_xpu()) {
677 if (!at::hasXPU()) {
678 av_log(ctx, AV_LOG_ERROR, "No XPU device found\n");
679 goto fail;
680 }
681#if TORCH_VERSION_MAJOR > 2 || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 6)
682 at::detail::getXPUHooks().init();
683#else
684 at::detail::getXPUHooks().initXPU();
685#endif
686 } else if (device.is_cuda()) {
687 // CUDA device - works for both NVIDIA CUDA and AMD ROCm (which uses CUDA-compatible API)
688 if (!torch::cuda::is_available()) {
689 av_log(ctx, AV_LOG_ERROR, "CUDA/ROCm is not available\n");
690 goto fail;
691 }
692 av_log(ctx, AV_LOG_INFO, "Using CUDA/ROCm device: %s\n", device_name);
693 } else if (!device.is_cpu()) {
694 av_log(ctx, AV_LOG_ERROR, "Not supported device:\"%s\"\n", device_name);
695 goto fail;
696 }
697
698 try {
699 th_model->jit_model = new torch::jit::Module;
700 (*th_model->jit_model) = torch::jit::load(ctx->model_filename);
701 th_model->jit_model->to(device);
702 } catch (const c10::Error& e) {
703 av_log(ctx, AV_LOG_ERROR, "Failed to load torch model\n");
704 goto fail;
705 }
706
707 if (ctx->nireq <= 0) {
708 ctx->nireq = av_cpu_count() / 2 + 1;
709 }
710
712 if (!th_model->request_queue) {
713 goto fail;
714 }
715
716 for (int i = 0; i < ctx->nireq; i++) {
717 item = (THRequestItem *)av_mallocz(sizeof(THRequestItem));
718 if (!item) {
719 goto fail;
720 }
722 if (!item->infer_request) {
723 goto fail;
724 }
725 item->lltasks = (LastLevelTaskItem **)av_malloc_array(ctx->batch_size, sizeof(*item->lltasks));
726 if (!item->lltasks) {
727 goto fail;
728 }
729 item->lltask_count = 0;
730
733 item->exec_module.args = item;
734
735 if (ff_safe_queue_push_back(th_model->request_queue, item) < 0) {
736 goto fail;
737 }
738 th_model->nb_requests++;
739 item = NULL;
740 }
741
742 th_model->task_queue = ff_queue_create();
743 th_model->lltask_queue = ff_queue_create();
744
745 model->get_input = &get_input_th;
746 model->get_output = &get_output_th;
747 model->filter_ctx = filter_ctx;
748 model->func_type = func_type;
749 return model;
750
751fail:
752 if (item) {
754 }
755 dnn_free_model_th(&model);
756 return NULL;
757}
758
759static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
760{
761 THModel *th_model = (THModel *)model;
762 DnnContext *ctx = th_model->ctx;
763 TaskItem *task;
764 THRequestItem *request;
765 int ret = 0;
766
767 ret = ff_check_exec_params(ctx, DNN_TH, model->func_type, exec_params);
768 if (ret != 0) {
769 av_log(ctx, AV_LOG_ERROR, "exec parameter checking fail.\n");
770 return ret;
771 }
772
773 task = (TaskItem *)av_malloc(sizeof(TaskItem));
774 if (!task) {
775 av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
776 return AVERROR(ENOMEM);
777 }
778
779 ret = ff_dnn_fill_task(task, exec_params, th_model, ctx->async, 1);
780 if (ret != 0) {
781 av_freep(&task);
782 av_log(ctx, AV_LOG_ERROR, "unable to fill task.\n");
783 return ret;
784 }
785
786 ret = ff_queue_push_back(th_model->task_queue, task);
787 if (ret < 0) {
788 av_freep(&task);
789 av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
790 return ret;
791 }
792
793 ret = extract_lltask_from_task(task, th_model->lltask_queue);
794 if (ret != 0) {
795 av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
796 return ret;
797 }
798
799 while (ff_queue_size(th_model->lltask_queue) >= ctx->batch_size) {
801 if (!request) {
802 av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
803 return AVERROR(EINVAL);
804 }
805
806 ret = execute_model_th(request, th_model->lltask_queue);
807 if (ret != 0) {
808 return ret;
809 }
810 }
811
812 return 0;
813}
814
816{
817 THModel *th_model = (THModel *)model;
818 return ff_dnn_get_result_common(th_model->task_queue, in, out);
819}
820
821static int dnn_flush_th(const DNNModel *model)
822{
823 THModel *th_model = (THModel *)model;
824 THRequestItem *request;
825
826 if (ff_queue_size(th_model->lltask_queue) == 0)
827 // no pending task need to flush
828 return 0;
829
831 if (!request) {
832 av_log(th_model->ctx, AV_LOG_ERROR, "unable to get infer request.\n");
833 return AVERROR(EINVAL);
834 }
835
836 return execute_model_th(request, th_model->lltask_queue);
837}
838
839extern const DNNModule ff_dnn_backend_torch = {
840 .clazz = DNN_DEFINE_CLASS(dnn_th),
841 .type = DNN_TH,
842 .load_model = dnn_load_model_th,
843 .execute_model = dnn_execute_model_th,
844 .get_result = dnn_get_result_th,
845 .flush = dnn_flush_th,
846 .free_model = dnn_free_model_th,
847};
static const AVFilterPad inputs[]
Definition af_aap.c:299
static const AVFilterPad outputs[]
Definition af_aap.c:310
static FILE * out
static AVFormatContext * ctx
channels
Definition aptx.h:31
#define i(width, name, range_min, range_max)
Definition cbs_h264.c:63
#define FLAGS
Definition cmdutils.c:596
#define NULL
Definition coverity.c:32
static AVFrame * frame
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
void ff_dnn_wait_requests(SafeQueue *request_queue, int nireq)
Wait for all inference requests to complete before teardown.
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
DNN common functions different backends.
#define DNN_DEFINE_CLASS(fname)
static void infer_completion_callback(void *args)
const DNNModule ff_dnn_backend_torch
static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)
static int fill_model_input_th(THModel *th_model, THRequestItem *request)
static DNNAsyncStatusType dnn_get_result_th(const DNNModel *model, AVFrame **in, AVFrame **out)
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
static int th_start_inference(void *args)
static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
static const AVOption dnn_th_options[]
static void th_free_request(THInferRequest *request)
static THInferRequest * th_create_inference_request(void)
static int dnn_flush_th(const DNNModel *model)
static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
static DNNModel * dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
#define OFFSET(x)
static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
static void deleter(void *arg)
static void destroy_request_item(THRequestItem **arg)
static void dnn_free_model_th(DNNModel **model)
static void infer_completion_callback(void *args)
static int dnn_get_height_idx_by_layout(DNNLayout layout)
DNNAsyncStatusType
@ DL_NCHW
@ DNN_TH
DNNFunctionType
@ DFT_PROCESS_FRAME
static int dnn_get_width_idx_by_layout(DNNLayout layout)
#define DNN_GENERIC_ERROR
@ DNN_UINT8
@ DNN_FLOAT
@ DNN_AUTO
@ DCO_RGB
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition dnn_io_proc.c:42
DNN input&output process between AVFrame and DNNData.
int ret
Definition ffplay.c:391
static const uint8_t frame_size[4]
Definition g723_1.h:222
#define fail
Definition test.h:479
@ AV_OPT_TYPE_INT
Underlying C type is int.
Definition opt.h:258
#define AVERROR(e)
Definition error.h:45
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition frame.c:65
#define AV_LOG_VERBOSE
Detailed information.
Definition log.h:226
#define AV_LOG_INFO
Standard information.
Definition log.h:221
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition log.h:210
An API-specific header for AV_HWDEVICE_TYPE_CUDA.
FFmpeg internal API for CUDA.
static const int sizes[][2]
Definition img2dec.c:63
const char * arg
Definition jacosubdec.c:66
const char * to
Definition webvttdec.c:37
int av_cpu_count(void)
Definition cpu.c:230
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
Memory handling functions.
#define av_malloc(s)
Definition ops_static.c:52
AVOptions.
pixel format definitions
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition pixfmt.h:75
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition pixfmt.h:265
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition pixfmt.h:260
@ AV_PIX_FMT_0BGR
packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined
Definition pixfmt.h:264
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition pixfmt.h:263
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition pixfmt.h:76
@ AV_PIX_FMT_0RGB
packed RGB 8:8:8, 32bpp, XRGBXRGB... X=unused/undefined
Definition pixfmt.h:262
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition queue.c:72
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition queue.c:151
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition queue.c:130
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition queue.c:93
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition queue.c:88
Queue * ff_queue_create(void)
Create a Queue instance.
Definition queue.c:47
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
Definition safe_queue.c:87
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Definition safe_queue.c:97
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
Definition safe_queue.c:61
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
Definition safe_queue.c:33
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
Definition safe_queue.c:50
uint8_t * data
The data buffer.
Definition buffer.h:90
An instance of a filter.
Definition avfilter.h:279
This structure describes decoded (raw) audio or video data.
Definition frame.h:479
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition frame.h:500
int width
Definition frame.h:551
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame.
Definition frame.h:779
int height
Definition frame.h:551
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition frame.h:566
This struct describes a set or pool of "hardware" frames (i.e.
Definition hwcontext.h:118
AVOption.
Definition opt.h:428
Common Async Execution Mechanism for the DNN Backends.
void * args
Argument for the execution functions.
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
void(* callback)(void *args)
Completion Callback for the backend.
float scale
DNNDataType dt
int dims[4]
DNNColorOrder order
void * data
DNNLayout layout
int(* get_input)(struct DNNModel *model, DNNData *input, const char *input_name)
int(* get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
FramePrePostProc frame_pre_proc
FramePrePostProc frame_post_proc
AVFilterContext * filter_ctx
DNNFunctionType func_type
Linear double-ended data structure.
Definition executor.c:51
Double-ended queue with mutex locks ensuring data consistency while multithreading.
Definition safe_queue.c:27
torch::Tensor * input_tensor
torch::Tensor * output
torch::jit::Module * jit_model
Queue * lltask_queue
Queue * task_queue
DnnContext * ctx
SafeQueue * request_queue
LastLevelTaskItem ** lltasks
THInferRequest * infer_request
DNNAsyncExecModule exec_module
uint32_t inference_done
AVFrame * in_frame
uint8_t do_ioproc
uint32_t inference_todo
AVFrame * out_frame
#define av_malloc_array(a, b)
#define av_mallocz(s)
#define av_freep(p)
#define av_log(a,...)
#define height
Definition dsp.h:99
#define width
Definition dsp.h:99
static FilteringContext * filter_ctx
Definition transcode.c:52