FFmpeg
dnn_backend_torch.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2024
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN Torch backend implementation.
24  */
25 
26 #include <torch/torch.h>
27 #include <torch/script.h>
28 
29 extern "C" {
30 #include "../internal.h"
31 #include "dnn_io_proc.h"
32 #include "dnn_backend_common.h"
33 #include "libavutil/opt.h"
34 #include "libavutil/mem.h"
35 #include "queue.h"
36 #include "safe_queue.h"
37 }
38 
39 typedef struct THModel {
42  torch::jit::Module *jit_model;
46 } THModel;
47 
48 typedef struct THInferRequest {
49  torch::Tensor *output;
50  torch::Tensor *input_tensor;
52 
53 typedef struct THRequestItem {
58 
59 
60 #define OFFSET(x) offsetof(THOptions, x)
61 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
62 static const AVOption dnn_th_options[] = {
63  { "optimize", "turn on graph executor optimization", OFFSET(optimize), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS},
64  { NULL }
65 };
66 
67 static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
68 {
69  THModel *th_model = (THModel *)task->model;
70  DnnContext *ctx = th_model->ctx;
71  LastLevelTaskItem *lltask = (LastLevelTaskItem *)av_malloc(sizeof(*lltask));
72  if (!lltask) {
73  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for LastLevelTaskItem\n");
74  return AVERROR(ENOMEM);
75  }
76  task->inference_todo = 1;
77  task->inference_done = 0;
78  lltask->task = task;
79  if (ff_queue_push_back(lltask_queue, lltask) < 0) {
80  av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
81  av_freep(&lltask);
82  return AVERROR(ENOMEM);
83  }
84  return 0;
85 }
86 
87 static void th_free_request(THInferRequest *request)
88 {
89  if (!request)
90  return;
91  if (request->output) {
92  delete(request->output);
93  request->output = NULL;
94  }
95  if (request->input_tensor) {
96  delete(request->input_tensor);
97  request->input_tensor = NULL;
98  }
99  return;
100 }
101 
103 {
104  THRequestItem *item;
105  if (!arg || !*arg) {
106  return;
107  }
108  item = *arg;
110  av_freep(&item->infer_request);
111  av_freep(&item->lltask);
113  av_freep(arg);
114 }
115 
116 static void dnn_free_model_th(DNNModel **model)
117 {
118  THModel *th_model;
119  if (!model || !*model)
120  return;
121 
122  th_model = (THModel *) (*model);
123  while (ff_safe_queue_size(th_model->request_queue) != 0) {
125  destroy_request_item(&item);
126  }
128 
129  while (ff_queue_size(th_model->lltask_queue) != 0) {
131  av_freep(&item);
132  }
133  ff_queue_destroy(th_model->lltask_queue);
134 
135  while (ff_queue_size(th_model->task_queue) != 0) {
136  TaskItem *item = (TaskItem *)ff_queue_pop_front(th_model->task_queue);
137  av_frame_free(&item->in_frame);
138  av_frame_free(&item->out_frame);
139  av_freep(&item);
140  }
141  ff_queue_destroy(th_model->task_queue);
142  delete th_model->jit_model;
143  av_freep(&th_model);
144  *model = NULL;
145 }
146 
147 static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)
148 {
149  input->dt = DNN_FLOAT;
150  input->order = DCO_RGB;
151  input->layout = DL_NCHW;
152  input->dims[0] = 1;
153  input->dims[1] = 3;
154  input->dims[2] = -1;
155  input->dims[3] = -1;
156  return 0;
157 }
158 
159 static void deleter(void *arg)
160 {
161  av_freep(&arg);
162 }
163 
164 static int fill_model_input_th(THModel *th_model, THRequestItem *request)
165 {
166  LastLevelTaskItem *lltask = NULL;
167  TaskItem *task = NULL;
168  THInferRequest *infer_request = NULL;
169  DNNData input = { 0 };
170  DnnContext *ctx = th_model->ctx;
171  int ret, width_idx, height_idx, channel_idx;
172 
173  lltask = (LastLevelTaskItem *)ff_queue_pop_front(th_model->lltask_queue);
174  if (!lltask) {
175  ret = AVERROR(EINVAL);
176  goto err;
177  }
178  request->lltask = lltask;
179  task = lltask->task;
180  infer_request = request->infer_request;
181 
182  ret = get_input_th(&th_model->model, &input, NULL);
183  if ( ret != 0) {
184  goto err;
185  }
186  width_idx = dnn_get_width_idx_by_layout(input.layout);
187  height_idx = dnn_get_height_idx_by_layout(input.layout);
188  channel_idx = dnn_get_channel_idx_by_layout(input.layout);
189  input.dims[height_idx] = task->in_frame->height;
190  input.dims[width_idx] = task->in_frame->width;
191  input.data = av_malloc(input.dims[height_idx] * input.dims[width_idx] *
192  input.dims[channel_idx] * sizeof(float));
193  if (!input.data)
194  return AVERROR(ENOMEM);
195  infer_request->input_tensor = new torch::Tensor();
196  infer_request->output = new torch::Tensor();
197 
198  switch (th_model->model.func_type) {
199  case DFT_PROCESS_FRAME:
200  input.scale = 255;
201  if (task->do_ioproc) {
202  if (th_model->model.frame_pre_proc != NULL) {
203  th_model->model.frame_pre_proc(task->in_frame, &input, th_model->model.filter_ctx);
204  } else {
206  }
207  }
208  break;
209  default:
210  avpriv_report_missing_feature(NULL, "model function type %d", th_model->model.func_type);
211  break;
212  }
213  *infer_request->input_tensor = torch::from_blob(input.data,
214  {1, input.dims[channel_idx], input.dims[height_idx], input.dims[width_idx]},
215  deleter, torch::kFloat32);
216  return 0;
217 
218 err:
219  th_free_request(infer_request);
220  return ret;
221 }
222 
223 static int th_start_inference(void *args)
224 {
225  THRequestItem *request = (THRequestItem *)args;
226  THInferRequest *infer_request = NULL;
227  LastLevelTaskItem *lltask = NULL;
228  TaskItem *task = NULL;
229  THModel *th_model = NULL;
230  DnnContext *ctx = NULL;
231  std::vector<torch::jit::IValue> inputs;
232  torch::NoGradGuard no_grad;
233 
234  if (!request) {
235  av_log(NULL, AV_LOG_ERROR, "THRequestItem is NULL\n");
236  return AVERROR(EINVAL);
237  }
238  infer_request = request->infer_request;
239  lltask = request->lltask;
240  task = lltask->task;
241  th_model = (THModel *)task->model;
242  ctx = th_model->ctx;
243 
244  if (ctx->torch_option.optimize)
245  torch::jit::setGraphExecutorOptimize(true);
246  else
247  torch::jit::setGraphExecutorOptimize(false);
248 
249  if (!infer_request->input_tensor || !infer_request->output) {
250  av_log(ctx, AV_LOG_ERROR, "input or output tensor is NULL\n");
251  return DNN_GENERIC_ERROR;
252  }
253  // Transfer tensor to the same device as model
254  c10::Device device = (*th_model->jit_model->parameters().begin()).device();
255  if (infer_request->input_tensor->device() != device)
256  *infer_request->input_tensor = infer_request->input_tensor->to(device);
257  inputs.push_back(*infer_request->input_tensor);
258 
259  *infer_request->output = th_model->jit_model->forward(inputs).toTensor();
260 
261  return 0;
262 }
263 
264 static void infer_completion_callback(void *args) {
265  THRequestItem *request = (THRequestItem*)args;
266  LastLevelTaskItem *lltask = request->lltask;
267  TaskItem *task = lltask->task;
268  DNNData outputs = { 0 };
269  THInferRequest *infer_request = request->infer_request;
270  THModel *th_model = (THModel *)task->model;
271  torch::Tensor *output = infer_request->output;
272 
273  c10::IntArrayRef sizes = output->sizes();
274  outputs.order = DCO_RGB;
275  outputs.layout = DL_NCHW;
276  outputs.dt = DNN_FLOAT;
277  if (sizes.size() == 4) {
278  // 4 dimensions: [batch_size, channel, height, width]
279  // this format of data is normally used for video frame SR
280  outputs.dims[0] = sizes.at(0); // N
281  outputs.dims[1] = sizes.at(1); // C
282  outputs.dims[2] = sizes.at(2); // H
283  outputs.dims[3] = sizes.at(3); // W
284  } else {
285  avpriv_report_missing_feature(th_model->ctx, "Support of this kind of model");
286  goto err;
287  }
288 
289  switch (th_model->model.func_type) {
290  case DFT_PROCESS_FRAME:
291  if (task->do_ioproc) {
292  // Post process can only deal with CPU memory.
293  if (output->device() != torch::kCPU)
294  *output = output->to(torch::kCPU);
295  outputs.scale = 255;
296  outputs.data = output->data_ptr();
297  if (th_model->model.frame_post_proc != NULL) {
298  th_model->model.frame_post_proc(task->out_frame, &outputs, th_model->model.filter_ctx);
299  } else {
300  ff_proc_from_dnn_to_frame(task->out_frame, &outputs, th_model->ctx);
301  }
302  } else {
305  }
306  break;
307  default:
308  avpriv_report_missing_feature(th_model->ctx, "model function type %d", th_model->model.func_type);
309  goto err;
310  }
311  task->inference_done++;
312  av_freep(&request->lltask);
313 err:
314  th_free_request(infer_request);
315 
316  if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {
317  destroy_request_item(&request);
318  av_log(th_model->ctx, AV_LOG_ERROR, "Unable to push back request_queue when failed to start inference.\n");
319  }
320 }
321 
322 static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
323 {
324  THModel *th_model = NULL;
325  LastLevelTaskItem *lltask;
326  TaskItem *task = NULL;
327  int ret = 0;
328 
329  if (ff_queue_size(lltask_queue) == 0) {
330  destroy_request_item(&request);
331  return 0;
332  }
333 
334  lltask = (LastLevelTaskItem *)ff_queue_peek_front(lltask_queue);
335  if (lltask == NULL) {
336  av_log(NULL, AV_LOG_ERROR, "Failed to get LastLevelTaskItem\n");
337  ret = AVERROR(EINVAL);
338  goto err;
339  }
340  task = lltask->task;
341  th_model = (THModel *)task->model;
342 
343  ret = fill_model_input_th(th_model, request);
344  if ( ret != 0) {
345  goto err;
346  }
347  if (task->async) {
348  avpriv_report_missing_feature(th_model->ctx, "LibTorch async");
349  } else {
350  ret = th_start_inference((void *)(request));
351  if (ret != 0) {
352  goto err;
353  }
354  infer_completion_callback(request);
355  return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;
356  }
357 
358 err:
359  th_free_request(request->infer_request);
360  if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {
361  destroy_request_item(&request);
362  }
363  return ret;
364 }
365 
366 static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height,
367  const char *output_name, int *output_width, int *output_height)
368 {
369  int ret = 0;
370  THModel *th_model = (THModel*) model;
371  DnnContext *ctx = th_model->ctx;
372  TaskItem task = { 0 };
373  THRequestItem *request = NULL;
374  DNNExecBaseParams exec_params = {
375  .input_name = input_name,
376  .output_names = &output_name,
377  .nb_output = 1,
378  .in_frame = NULL,
379  .out_frame = NULL,
380  };
381  ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, th_model, input_height, input_width, ctx);
382  if ( ret != 0) {
383  goto err;
384  }
385 
386  ret = extract_lltask_from_task(&task, th_model->lltask_queue);
387  if ( ret != 0) {
388  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
389  goto err;
390  }
391 
392  request = (THRequestItem*) ff_safe_queue_pop_front(th_model->request_queue);
393  if (!request) {
394  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
395  ret = AVERROR(EINVAL);
396  goto err;
397  }
398 
399  ret = execute_model_th(request, th_model->lltask_queue);
400  *output_width = task.out_frame->width;
401  *output_height = task.out_frame->height;
402 
403 err:
404  av_frame_free(&task.out_frame);
405  av_frame_free(&task.in_frame);
406  return ret;
407 }
408 
410 {
411  THInferRequest *request = (THInferRequest *)av_malloc(sizeof(THInferRequest));
412  if (!request) {
413  return NULL;
414  }
415  request->input_tensor = NULL;
416  request->output = NULL;
417  return request;
418 }
419 
421 {
422  DNNModel *model = NULL;
423  THModel *th_model = NULL;
424  THRequestItem *item = NULL;
425  const char *device_name = ctx->device ? ctx->device : "cpu";
426 
427  th_model = (THModel *)av_mallocz(sizeof(THModel));
428  if (!th_model)
429  return NULL;
430  model = &th_model->model;
431  th_model->ctx = ctx;
432 
433  c10::Device device = c10::Device(device_name);
434  if (device.is_xpu()) {
435  if (!at::hasXPU()) {
436  av_log(ctx, AV_LOG_ERROR, "No XPU device found\n");
437  goto fail;
438  }
439  at::detail::getXPUHooks().initXPU();
440  } else if (!device.is_cpu()) {
441  av_log(ctx, AV_LOG_ERROR, "Not supported device:\"%s\"\n", device_name);
442  goto fail;
443  }
444 
445  try {
446  th_model->jit_model = new torch::jit::Module;
447  (*th_model->jit_model) = torch::jit::load(ctx->model_filename);
448  th_model->jit_model->to(device);
449  } catch (const c10::Error& e) {
450  av_log(ctx, AV_LOG_ERROR, "Failed to load torch model\n");
451  goto fail;
452  }
453 
454  th_model->request_queue = ff_safe_queue_create();
455  if (!th_model->request_queue) {
456  goto fail;
457  }
458 
459  item = (THRequestItem *)av_mallocz(sizeof(THRequestItem));
460  if (!item) {
461  goto fail;
462  }
463  item->lltask = NULL;
465  if (!item->infer_request) {
466  av_log(NULL, AV_LOG_ERROR, "Failed to allocate memory for Torch inference request\n");
467  goto fail;
468  }
471  item->exec_module.args = item;
472 
473  if (ff_safe_queue_push_back(th_model->request_queue, item) < 0) {
474  goto fail;
475  }
476  item = NULL;
477 
478  th_model->task_queue = ff_queue_create();
479  if (!th_model->task_queue) {
480  goto fail;
481  }
482 
483  th_model->lltask_queue = ff_queue_create();
484  if (!th_model->lltask_queue) {
485  goto fail;
486  }
487 
488  model->get_input = &get_input_th;
489  model->get_output = &get_output_th;
490  model->filter_ctx = filter_ctx;
491  model->func_type = func_type;
492  return model;
493 
494 fail:
495  if (item) {
496  destroy_request_item(&item);
497  av_freep(&item);
498  }
499  dnn_free_model_th(&model);
500  return NULL;
501 }
502 
503 static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
504 {
505  THModel *th_model = (THModel *)model;
506  DnnContext *ctx = th_model->ctx;
507  TaskItem *task;
508  THRequestItem *request;
509  int ret = 0;
510 
511  ret = ff_check_exec_params(ctx, DNN_TH, model->func_type, exec_params);
512  if (ret != 0) {
513  av_log(ctx, AV_LOG_ERROR, "exec parameter checking fail.\n");
514  return ret;
515  }
516 
517  task = (TaskItem *)av_malloc(sizeof(TaskItem));
518  if (!task) {
519  av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
520  return AVERROR(ENOMEM);
521  }
522 
523  ret = ff_dnn_fill_task(task, exec_params, th_model, 0, 1);
524  if (ret != 0) {
525  av_freep(&task);
526  av_log(ctx, AV_LOG_ERROR, "unable to fill task.\n");
527  return ret;
528  }
529 
530  ret = ff_queue_push_back(th_model->task_queue, task);
531  if (ret < 0) {
532  av_freep(&task);
533  av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
534  return ret;
535  }
536 
537  ret = extract_lltask_from_task(task, th_model->lltask_queue);
538  if (ret != 0) {
539  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
540  return ret;
541  }
542 
543  request = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);
544  if (!request) {
545  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
546  return AVERROR(EINVAL);
547  }
548 
549  return execute_model_th(request, th_model->lltask_queue);
550 }
551 
553 {
554  THModel *th_model = (THModel *)model;
555  return ff_dnn_get_result_common(th_model->task_queue, in, out);
556 }
557 
558 static int dnn_flush_th(const DNNModel *model)
559 {
560  THModel *th_model = (THModel *)model;
561  THRequestItem *request;
562 
563  if (ff_queue_size(th_model->lltask_queue) == 0)
564  // no pending task need to flush
565  return 0;
566 
567  request = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);
568  if (!request) {
569  av_log(th_model->ctx, AV_LOG_ERROR, "unable to get infer request.\n");
570  return AVERROR(EINVAL);
571  }
572 
573  return execute_model_th(request, th_model->lltask_queue);
574 }
575 
576 extern const DNNModule ff_dnn_backend_torch = {
577  .clazz = DNN_DEFINE_CLASS(dnn_th),
578  .type = DNN_TH,
579  .load_model = dnn_load_model_th,
580  .execute_model = dnn_execute_model_th,
581  .get_result = dnn_get_result_th,
582  .flush = dnn_flush_th,
583  .free_model = dnn_free_model_th,
584 };
THRequestItem::lltask
LastLevelTaskItem * lltask
Definition: dnn_backend_torch.cpp:55
THModel::lltask_queue
Queue * lltask_queue
Definition: dnn_backend_torch.cpp:45
THRequestItem::infer_request
THInferRequest * infer_request
Definition: dnn_backend_torch.cpp:54
THModel::ctx
DnnContext * ctx
Definition: dnn_backend_torch.cpp:41
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ff_safe_queue_pop_front
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Definition: safe_queue.c:105
out
FILE * out
Definition: movenc.c:55
deleter
static void deleter(void *arg)
Definition: dnn_backend_torch.cpp:159
FLAGS
#define FLAGS
Definition: dnn_backend_torch.cpp:61
THModel
Definition: dnn_backend_torch.cpp:39
DNNAsyncExecModule
Common Async Execution Mechanism for the DNN Backends.
Definition: dnn_backend_common.h:65
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:56
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:30
ff_queue_size
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition: queue.c:88
DNN_GENERIC_ERROR
#define DNN_GENERIC_ERROR
Definition: dnn_interface.h:33
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:160
LastLevelTaskItem
Definition: dnn_backend_common.h:57
ff_dnn_backend_torch
const DNNModule ff_dnn_backend_torch
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:374
AVFrame::width
int width
Definition: frame.h:446
SafeQueue
Double-ended queue with mutex locks ensuring data consistency while multithreading.
Definition: safe_queue.c:46
dnn_execute_model_th
static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_torch.cpp:503
AVOption
AVOption.
Definition: opt.h:357
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:110
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:81
dnn_io_proc.h
TaskItem
Definition: dnn_backend_common.h:43
DNNAsyncExecModule::callback
void(* callback)(void *args)
Completion Callback for the backend.
Definition: dnn_backend_common.h:77
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:30
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:99
ff_queue_create
Queue * ff_queue_create(void)
Create a Queue instance.
Definition: queue.c:47
dnn_get_width_idx_by_layout
static int dnn_get_width_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:197
TaskItem::model
void * model
Definition: dnn_backend_common.h:44
fail
#define fail()
Definition: checkasm.h:185
DnnContext
Definition: dnn_interface.h:143
filter_ctx
static FilteringContext * filter_ctx
Definition: transcode.c:52
Queue
Linear double-ended data structure.
Definition: queue.c:33
ff_queue_push_back
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition: queue.c:130
THModel::jit_model
torch::jit::Module * jit_model
Definition: dnn_backend_torch.cpp:42
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
LastLevelTaskItem::task
TaskItem * task
Definition: dnn_backend_common.h:58
destroy_request_item
static void destroy_request_item(THRequestItem **arg)
Definition: dnn_backend_torch.cpp:102
th_create_inference_request
static THInferRequest * th_create_inference_request(void)
Definition: dnn_backend_torch.cpp:409
ff_queue_destroy
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition: queue.c:72
DNNData
Definition: dnn_interface.h:69
DNNModule::clazz
const AVClass clazz
Definition: dnn_interface.h:176
ff_dnn_fill_gettingoutput_task
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:156
DNNModel::get_output
int(* get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:106
ctx
AVFormatContext * ctx
Definition: movenc.c:49
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:52
DL_NCHW
@ DL_NCHW
Definition: dnn_interface.h:65
dnn_load_model_th
static DNNModel * dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
Definition: dnn_backend_torch.cpp:420
arg
const char * arg
Definition: jacosubdec.c:67
if
if(ret)
Definition: filter_design.txt:179
ff_safe_queue_size
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
Definition: safe_queue.c:80
ff_proc_from_frame_to_dnn
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:182
THRequestItem::exec_module
DNNAsyncExecModule exec_module
Definition: dnn_backend_torch.cpp:56
NULL
#define NULL
Definition: coverity.c:32
sizes
static const int sizes[][2]
Definition: img2dec.c:59
get_input_th
static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)
Definition: dnn_backend_torch.cpp:147
ff_safe_queue_create
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
Definition: safe_queue.c:52
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:113
get_output_th
static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_torch.cpp:366
ff_dnn_async_module_cleanup
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
Definition: dnn_backend_common.c:86
infer_completion_callback
static void infer_completion_callback(void *args)
Definition: dnn_backend_torch.cpp:264
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:45
extract_lltask_from_task
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
Definition: dnn_backend_torch.cpp:67
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
THInferRequest::output
torch::Tensor * output
Definition: dnn_backend_torch.cpp:49
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:49
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:53
queue.h
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:101
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
ff_safe_queue_destroy
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
Definition: safe_queue.c:69
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:41
dnn_get_result_th
static DNNAsyncStatusType dnn_get_result_th(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_backend_torch.cpp:552
ff_dnn_fill_task
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:50
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
DNN_DEFINE_CLASS
#define DNN_DEFINE_CLASS(fname)
Definition: dnn_backend_common.h:39
THRequestItem
Definition: dnn_backend_torch.cpp:53
ff_safe_queue_push_back
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
Definition: safe_queue.c:95
th_start_inference
static int th_start_inference(void *args)
Definition: dnn_backend_torch.cpp:223
THInferRequest::input_tensor
torch::Tensor * input_tensor
Definition: dnn_backend_torch.cpp:50
DNNAsyncExecModule::start_inference
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
Definition: dnn_backend_common.h:70
DNNAsyncExecModule::args
void * args
Argument for the execution functions.
Definition: dnn_backend_common.h:83
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:256
safe_queue.h
THInferRequest
Definition: dnn_backend_torch.cpp:48
outputs
static const AVFilterPad outputs[]
Definition: af_aap.c:311
ret
ret
Definition: filter_design.txt:187
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:46
AVFrame::height
int height
Definition: frame.h:446
dnn_backend_common.h
THModel::model
DNNModel model
Definition: dnn_backend_torch.cpp:40
dnn_th_options
static const AVOption dnn_th_options[]
Definition: dnn_backend_torch.cpp:62
execute_model_th
static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
Definition: dnn_backend_torch.cpp:322
OFFSET
#define OFFSET(x)
Definition: dnn_backend_torch.cpp:60
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Definition: opt.h:245
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:136
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:46
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
DNNModel
Definition: dnn_interface.h:97
DNN_TH
@ DNN_TH
Definition: dnn_interface.h:38
mem.h
dnn_get_height_idx_by_layout
static int dnn_get_height_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:202
dnn_flush_th
static int dnn_flush_th(const DNNModel *model)
Definition: dnn_backend_torch.cpp:558
THModel::task_queue
Queue * task_queue
Definition: dnn_backend_torch.cpp:44
dnn_get_channel_idx_by_layout
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:207
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
DNNExecBaseParams
Definition: dnn_interface.h:80
DNNModel::get_input
int(* get_input)(struct DNNModel *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:104
dnn_free_model_th
static void dnn_free_model_th(DNNModel **model)
Definition: dnn_backend_torch.cpp:116
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:50
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:49
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:58
DNNModule
Definition: dnn_interface.h:175
fill_model_input_th
static int fill_model_input_th(THModel *th_model, THRequestItem *request)
Definition: dnn_backend_torch.cpp:164
THModel::request_queue
SafeQueue * request_queue
Definition: dnn_backend_torch.cpp:43
ff_proc_from_dnn_to_frame
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:42
th_free_request
static void th_free_request(THInferRequest *request)
Definition: dnn_backend_torch.cpp:87