FFmpeg
dnn_backend_tf.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN tensorflow backend implementation.
24  */
25 
26 #include "dnn_backend_tf.h"
27 #include "dnn_backend_native.h"
30 #include "libavformat/avio.h"
31 #include "libavutil/avassert.h"
32 #include "libavutil/avstring.h"
33 #include "libavutil/cpu.h"
34 #include "libavcodec/defs.h"
35 #include "../internal.h"
38 #include "dnn_io_proc.h"
39 #include "dnn_backend_common.h"
40 #include "safe_queue.h"
41 #include <tensorflow/c/c_api.h>
42 
43 typedef struct TFOptions{
44  char *sess_config;
45  uint8_t async;
46  uint32_t nireq;
47 } TFOptions;
48 
49 typedef struct TFContext {
50  const AVClass *class;
52 } TFContext;
53 
54 typedef struct TFModel{
57  TF_Graph *graph;
58  TF_Session *session;
59  TF_Status *status;
63 } TFModel;
64 
65 /**
66  * Stores execution parameters for single
67  * call to the TensorFlow C API
68  */
69 typedef struct TFInferRequest {
70  TF_Output *tf_outputs;
71  TF_Tensor **output_tensors;
72  TF_Output *tf_input;
73  TF_Tensor *input_tensor;
75 
76 typedef struct TFRequestItem {
79  TF_Status *status;
82 
83 #define OFFSET(x) offsetof(TFContext, x)
84 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
85 static const AVOption dnn_tensorflow_options[] = {
86  { "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
88  { NULL }
89 };
90 
91 AVFILTER_DEFINE_CLASS(dnn_tensorflow);
92 
93 static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue);
94 static void infer_completion_callback(void *args);
95 static inline void destroy_request_item(TFRequestItem **arg);
96 
97 static void free_buffer(void *data, size_t length)
98 {
99  av_freep(&data);
100 }
101 
102 /**
103  * Free the contents of TensorFlow inference request.
104  * It does not free the TFInferRequest instance.
105  *
106  * @param request pointer to TFInferRequest instance.
107  * NULL pointer is allowed.
108  */
109 static void tf_free_request(TFInferRequest *request)
110 {
111  if (!request)
112  return;
113  if (request->input_tensor) {
114  TF_DeleteTensor(request->input_tensor);
115  request->input_tensor = NULL;
116  }
117  av_freep(&request->tf_input);
118  av_freep(&request->tf_outputs);
119  if (request->output_tensors) {
120  int nb_output = sizeof(*request->output_tensors)/sizeof(request->output_tensors[0]);
121  for (uint32_t i = 0; i < nb_output; ++i) {
122  if (request->output_tensors[i]) {
123  TF_DeleteTensor(request->output_tensors[i]);
124  request->output_tensors[i] = NULL;
125  }
126  }
127  av_freep(&request->output_tensors);
128  }
129 }
130 
131 /**
132  * Create a TensorFlow inference request. All properties
133  * are initially unallocated and set as NULL.
134  *
135  * @return pointer to the allocated TFInferRequest instance.
136  */
138 {
139  TFInferRequest *infer_request = av_malloc(sizeof(TFInferRequest));
140  if (!infer_request) {
141  return NULL;
142  }
143  infer_request->tf_outputs = NULL;
144  infer_request->tf_input = NULL;
145  infer_request->input_tensor = NULL;
146  infer_request->output_tensors = NULL;
147  return infer_request;
148 }
149 
150 /**
151  * Start synchronous inference for the TensorFlow model.
152  *
153  * @param request pointer to the TFRequestItem for inference
154  * @retval 0 if execution is successful
155  * @retval AVERROR(EINVAL) if request is NULL
156  * @retval DNN_GENERIC_ERROR if execution fails
157  */
158 static int tf_start_inference(void *args)
159 {
160  TFRequestItem *request = args;
161  TFInferRequest *infer_request = request->infer_request;
162  LastLevelTaskItem *lltask = request->lltask;
163  TaskItem *task = lltask->task;
164  TFModel *tf_model = task->model;
165 
166  if (!request) {
167  av_log(&tf_model->ctx, AV_LOG_ERROR, "TFRequestItem is NULL\n");
168  return AVERROR(EINVAL);
169  }
170 
171  TF_SessionRun(tf_model->session, NULL,
172  infer_request->tf_input, &infer_request->input_tensor, 1,
173  infer_request->tf_outputs, infer_request->output_tensors,
174  task->nb_output, NULL, 0, NULL,
175  request->status);
176  if (TF_GetCode(request->status) != TF_OK) {
177  av_log(&tf_model->ctx, AV_LOG_ERROR, "%s", TF_Message(request->status));
178  return DNN_GENERIC_ERROR;
179  }
180  return 0;
181 }
182 
183 /**
184  * Free the TFRequestItem completely.
185  *
186  * @param arg Address of the TFInferRequest instance.
187  */
188 static inline void destroy_request_item(TFRequestItem **arg) {
189  TFRequestItem *request;
190  if (!arg) {
191  return;
192  }
193  request = *arg;
194  tf_free_request(request->infer_request);
195  av_freep(&request->infer_request);
196  av_freep(&request->lltask);
197  TF_DeleteStatus(request->status);
199  av_freep(arg);
200 }
201 
202 static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
203 {
204  TFModel *tf_model = task->model;
205  TFContext *ctx = &tf_model->ctx;
206  LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask));
207  if (!lltask) {
208  av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for LastLevelTaskItem\n");
209  return AVERROR(ENOMEM);
210  }
211  task->inference_todo = 1;
212  task->inference_done = 0;
213  lltask->task = task;
214  if (ff_queue_push_back(lltask_queue, lltask) < 0) {
215  av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
216  av_freep(&lltask);
217  return AVERROR(ENOMEM);
218  }
219  return 0;
220 }
221 
222 static TF_Buffer *read_graph(const char *model_filename)
223 {
224  TF_Buffer *graph_buf;
225  unsigned char *graph_data = NULL;
226  AVIOContext *model_file_context;
227  long size, bytes_read;
228 
229  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
230  return NULL;
231  }
232 
233  size = avio_size(model_file_context);
234 
235  graph_data = av_malloc(size);
236  if (!graph_data){
237  avio_closep(&model_file_context);
238  return NULL;
239  }
240  bytes_read = avio_read(model_file_context, graph_data, size);
241  avio_closep(&model_file_context);
242  if (bytes_read != size){
243  av_freep(&graph_data);
244  return NULL;
245  }
246 
247  graph_buf = TF_NewBuffer();
248  graph_buf->data = graph_data;
249  graph_buf->length = size;
250  graph_buf->data_deallocator = free_buffer;
251 
252  return graph_buf;
253 }
254 
255 static TF_Tensor *allocate_input_tensor(const DNNData *input)
256 {
257  TF_DataType dt;
258  size_t size;
259  int64_t input_dims[] = {1, input->height, input->width, input->channels};
260  switch (input->dt) {
261  case DNN_FLOAT:
262  dt = TF_FLOAT;
263  size = sizeof(float);
264  break;
265  case DNN_UINT8:
266  dt = TF_UINT8;
267  size = 1;
268  break;
269  default:
270  av_assert0(!"should not reach here");
271  }
272 
273  return TF_AllocateTensor(dt, input_dims, 4,
274  input_dims[1] * input_dims[2] * input_dims[3] * size);
275 }
276 
277 static int get_input_tf(void *model, DNNData *input, const char *input_name)
278 {
279  TFModel *tf_model = model;
280  TFContext *ctx = &tf_model->ctx;
281  TF_Status *status;
282  int64_t dims[4];
283 
284  TF_Output tf_output;
285  tf_output.oper = TF_GraphOperationByName(tf_model->graph, input_name);
286  if (!tf_output.oper) {
287  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
288  return AVERROR(EINVAL);
289  }
290 
291  tf_output.index = 0;
292  input->dt = TF_OperationOutputType(tf_output);
293  input->order = DCO_RGB;
294 
295  status = TF_NewStatus();
296  TF_GraphGetTensorShape(tf_model->graph, tf_output, dims, 4, status);
297  if (TF_GetCode(status) != TF_OK){
298  TF_DeleteStatus(status);
299  av_log(ctx, AV_LOG_ERROR, "Failed to get input tensor shape: number of dimension incorrect\n");
300  return DNN_GENERIC_ERROR;
301  }
302  TF_DeleteStatus(status);
303 
304  // currently only NHWC is supported
305  av_assert0(dims[0] == 1 || dims[0] == -1);
306  input->height = dims[1];
307  input->width = dims[2];
308  input->channels = dims[3];
309 
310  return 0;
311 }
312 
313 static int get_output_tf(void *model, const char *input_name, int input_width, int input_height,
314  const char *output_name, int *output_width, int *output_height)
315 {
316  int ret;
317  TFModel *tf_model = model;
318  TFContext *ctx = &tf_model->ctx;
319  TaskItem task;
320  TFRequestItem *request;
321  DNNExecBaseParams exec_params = {
322  .input_name = input_name,
323  .output_names = &output_name,
324  .nb_output = 1,
325  .in_frame = NULL,
326  .out_frame = NULL,
327  };
328 
329  ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, tf_model, input_height, input_width, ctx);
330  if (ret != 0) {
331  goto err;
332  }
333 
334  ret = extract_lltask_from_task(&task, tf_model->lltask_queue);
335  if (ret != 0) {
336  av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
337  goto err;
338  }
339 
340  request = ff_safe_queue_pop_front(tf_model->request_queue);
341  if (!request) {
342  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
343  ret = AVERROR(EINVAL);
344  goto err;
345  }
346 
347  ret = execute_model_tf(request, tf_model->lltask_queue);
348  *output_width = task.out_frame->width;
349  *output_height = task.out_frame->height;
350 
351 err:
352  av_frame_free(&task.out_frame);
353  av_frame_free(&task.in_frame);
354  return ret;
355 }
356 
357 #define SPACE_CHARS " \t\r\n"
358 static int hex_to_data(uint8_t *data, const char *p)
359 {
360  int c, len, v;
361 
362  len = 0;
363  v = 1;
364  for (;;) {
365  p += strspn(p, SPACE_CHARS);
366  if (*p == '\0')
367  break;
368  c = av_toupper((unsigned char) *p++);
369  if (c >= '0' && c <= '9')
370  c = c - '0';
371  else if (c >= 'A' && c <= 'F')
372  c = c - 'A' + 10;
373  else
374  break;
375  v = (v << 4) | c;
376  if (v & 0x100) {
377  if (data) {
378  data[len] = v;
379  }
380  len++;
381  v = 1;
382  }
383  }
384  return len;
385 }
386 
387 static int load_tf_model(TFModel *tf_model, const char *model_filename)
388 {
389  TFContext *ctx = &tf_model->ctx;
390  TF_Buffer *graph_def;
391  TF_ImportGraphDefOptions *graph_opts;
392  TF_SessionOptions *sess_opts;
393  const TF_Operation *init_op;
394  uint8_t *sess_config = NULL;
395  int sess_config_length = 0;
396 
397  // prepare the sess config data
398  if (tf_model->ctx.options.sess_config != NULL) {
399  const char *config;
400  /*
401  tf_model->ctx.options.sess_config is hex to present the serialized proto
402  required by TF_SetConfig below, so we need to first generate the serialized
403  proto in a python script, tools/python/tf_sess_config.py is a script example
404  to generate the configs of sess_config.
405  */
406  if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) {
407  av_log(ctx, AV_LOG_ERROR, "sess_config should start with '0x'\n");
408  return AVERROR(EINVAL);
409  }
410  config = tf_model->ctx.options.sess_config + 2;
411  sess_config_length = hex_to_data(NULL, config);
412 
413  sess_config = av_mallocz(sess_config_length + AV_INPUT_BUFFER_PADDING_SIZE);
414  if (!sess_config) {
415  av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n");
416  return AVERROR(ENOMEM);
417  }
418  if (hex_to_data(sess_config, config) < 0) {
419  av_log(ctx, AV_LOG_ERROR, "failed to convert hex to data\n");
420  return AVERROR(EINVAL);
421  }
422  }
423 
424  graph_def = read_graph(model_filename);
425  if (!graph_def){
426  av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", model_filename);
427  av_freep(&sess_config);
428  return AVERROR(EINVAL);
429  }
430  tf_model->graph = TF_NewGraph();
431  tf_model->status = TF_NewStatus();
432  graph_opts = TF_NewImportGraphDefOptions();
433  TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);
434  TF_DeleteImportGraphDefOptions(graph_opts);
435  TF_DeleteBuffer(graph_def);
436  if (TF_GetCode(tf_model->status) != TF_OK){
437  av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to model graph\n");
438  av_freep(&sess_config);
439  return DNN_GENERIC_ERROR;
440  }
441 
442  init_op = TF_GraphOperationByName(tf_model->graph, "init");
443  sess_opts = TF_NewSessionOptions();
444 
445  if (sess_config) {
446  TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->status);
447  av_freep(&sess_config);
448  if (TF_GetCode(tf_model->status) != TF_OK) {
449  TF_DeleteSessionOptions(sess_opts);
450  av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess options with %s\n",
451  tf_model->ctx.options.sess_config);
452  return DNN_GENERIC_ERROR;
453  }
454  }
455 
456  tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
457  TF_DeleteSessionOptions(sess_opts);
458  if (TF_GetCode(tf_model->status) != TF_OK)
459  {
460  av_freep(&sess_config);
461  av_log(ctx, AV_LOG_ERROR, "Failed to create new session with model graph\n");
462  return DNN_GENERIC_ERROR;
463  }
464 
465  // Run initialization operation with name "init" if it is present in graph
466  if (init_op){
467  TF_SessionRun(tf_model->session, NULL,
468  NULL, NULL, 0,
469  NULL, NULL, 0,
470  &init_op, 1, NULL, tf_model->status);
471  if (TF_GetCode(tf_model->status) != TF_OK)
472  {
473  av_freep(&sess_config);
474  av_log(ctx, AV_LOG_ERROR, "Failed to run session when initializing\n");
475  return DNN_GENERIC_ERROR;
476  }
477  }
478 
479  return 0;
480 }
481 
482 #define NAME_BUFFER_SIZE 256
483 
484 static int add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
485  ConvolutionalParams* params, const int layer)
486 {
487  TFContext *ctx = &tf_model->ctx;
488  TF_Operation *op;
489  TF_OperationDescription *op_desc;
490  TF_Output input;
491  int64_t strides[] = {1, 1, 1, 1};
492  TF_Tensor *kernel_tensor = NULL, *biases_tensor = NULL;
493  int64_t dims[4];
494  int dims_len;
495  char name_buffer[NAME_BUFFER_SIZE];
496  int32_t size;
497 
498  size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
499  input.index = 0;
500 
501  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
502  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
503  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
504  dims[0] = params->output_num;
505  dims[1] = params->kernel_size;
506  dims[2] = params->kernel_size;
507  dims[3] = params->input_num;
508  dims_len = 4;
509  kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
510  memcpy(TF_TensorData(kernel_tensor), params->kernel, size * sizeof(float));
511  TF_SetAttrTensor(op_desc, "value", kernel_tensor, tf_model->status);
512  if (TF_GetCode(tf_model->status) != TF_OK){
513  goto err;
514  }
515  op = TF_FinishOperation(op_desc, tf_model->status);
516  if (TF_GetCode(tf_model->status) != TF_OK){
517  goto err;
518  }
519 
520  snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
521  op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
522  input.oper = op;
523  TF_AddInput(op_desc, input);
524  input.oper = transpose_op;
525  TF_AddInput(op_desc, input);
526  TF_SetAttrType(op_desc, "T", TF_FLOAT);
527  TF_SetAttrType(op_desc, "Tperm", TF_INT32);
528  op = TF_FinishOperation(op_desc, tf_model->status);
529  if (TF_GetCode(tf_model->status) != TF_OK){
530  goto err;
531  }
532 
533  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
534  op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
535  input.oper = *cur_op;
536  TF_AddInput(op_desc, input);
537  input.oper = op;
538  TF_AddInput(op_desc, input);
539  TF_SetAttrType(op_desc, "T", TF_FLOAT);
540  TF_SetAttrIntList(op_desc, "strides", strides, 4);
541  TF_SetAttrString(op_desc, "padding", "VALID", 5);
542  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
543  if (TF_GetCode(tf_model->status) != TF_OK){
544  goto err;
545  }
546 
547  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
548  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
549  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
550  dims[0] = params->output_num;
551  dims_len = 1;
552  biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
553  memcpy(TF_TensorData(biases_tensor), params->biases, params->output_num * sizeof(float));
554  TF_SetAttrTensor(op_desc, "value", biases_tensor, tf_model->status);
555  if (TF_GetCode(tf_model->status) != TF_OK){
556  goto err;
557  }
558  op = TF_FinishOperation(op_desc, tf_model->status);
559  if (TF_GetCode(tf_model->status) != TF_OK){
560  goto err;
561  }
562 
563  snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
564  op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
565  input.oper = *cur_op;
566  TF_AddInput(op_desc, input);
567  input.oper = op;
568  TF_AddInput(op_desc, input);
569  TF_SetAttrType(op_desc, "T", TF_FLOAT);
570  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
571  if (TF_GetCode(tf_model->status) != TF_OK){
572  goto err;
573  }
574 
575  snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
576  switch (params->activation){
577  case RELU:
578  op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
579  break;
580  case TANH:
581  op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
582  break;
583  case SIGMOID:
584  op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
585  break;
586  default:
587  avpriv_report_missing_feature(ctx, "convolutional activation function %d", params->activation);
588  return AVERROR(ENOSYS);
589  }
590  input.oper = *cur_op;
591  TF_AddInput(op_desc, input);
592  TF_SetAttrType(op_desc, "T", TF_FLOAT);
593  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
594  if (TF_GetCode(tf_model->status) != TF_OK){
595  goto err;
596  }
597 
598  return 0;
599 err:
600  TF_DeleteTensor(kernel_tensor);
601  TF_DeleteTensor(biases_tensor);
602  av_log(ctx, AV_LOG_ERROR, "Failed to add conv layer %d\n", layer);
603  return DNN_GENERIC_ERROR;
604 }
605 
606 static int add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
607  DepthToSpaceParams *params, const int layer)
608 {
609  TFContext *ctx = &tf_model->ctx;
610  TF_OperationDescription *op_desc;
611  TF_Output input;
612  char name_buffer[NAME_BUFFER_SIZE];
613 
614  snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
615  op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
616  input.oper = *cur_op;
617  input.index = 0;
618  TF_AddInput(op_desc, input);
619  TF_SetAttrType(op_desc, "T", TF_FLOAT);
620  TF_SetAttrInt(op_desc, "block_size", params->block_size);
621  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
622  if (TF_GetCode(tf_model->status) != TF_OK){
623  av_log(ctx, AV_LOG_ERROR, "Failed to add depth_to_space to layer %d\n", layer);
624  return DNN_GENERIC_ERROR;
625  }
626 
627  return 0;
628 }
629 
630 static int add_pad_layer(TFModel *tf_model, TF_Operation **cur_op,
631  LayerPadParams *params, const int layer)
632 {
633  TFContext *ctx = &tf_model->ctx;
634  TF_Operation *op;
635  TF_Tensor *tensor;
636  TF_OperationDescription *op_desc;
637  TF_Output input;
638  int32_t *pads;
639  int64_t pads_shape[] = {4, 2};
640 
641  char name_buffer[NAME_BUFFER_SIZE];
642  snprintf(name_buffer, NAME_BUFFER_SIZE, "pad%d", layer);
643 
644  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
645  TF_SetAttrType(op_desc, "dtype", TF_INT32);
646  tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
647  pads = (int32_t *)TF_TensorData(tensor);
648  pads[0] = params->paddings[0][0];
649  pads[1] = params->paddings[0][1];
650  pads[2] = params->paddings[1][0];
651  pads[3] = params->paddings[1][1];
652  pads[4] = params->paddings[2][0];
653  pads[5] = params->paddings[2][1];
654  pads[6] = params->paddings[3][0];
655  pads[7] = params->paddings[3][1];
656  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
657  if (TF_GetCode(tf_model->status) != TF_OK){
658  TF_DeleteTensor(tensor);
659  av_log(ctx, AV_LOG_ERROR, "Failed to set value for pad of layer %d\n", layer);
660  return DNN_GENERIC_ERROR;
661  }
662  op = TF_FinishOperation(op_desc, tf_model->status);
663  if (TF_GetCode(tf_model->status) != TF_OK){
664  TF_DeleteTensor(tensor);
665  av_log(ctx, AV_LOG_ERROR, "Failed to add pad to layer %d\n", layer);
666  return DNN_GENERIC_ERROR;
667  }
668 
669  op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
670  input.oper = *cur_op;
671  input.index = 0;
672  TF_AddInput(op_desc, input);
673  input.oper = op;
674  TF_AddInput(op_desc, input);
675  TF_SetAttrType(op_desc, "T", TF_FLOAT);
676  TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
677  TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
678  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
679  if (TF_GetCode(tf_model->status) != TF_OK){
680  TF_DeleteTensor(tensor);
681  av_log(ctx, AV_LOG_ERROR, "Failed to add mirror_pad to layer %d\n", layer);
682  return DNN_GENERIC_ERROR;
683  }
684 
685  return 0;
686 }
687 
688 static int add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op,
689  DnnLayerMaximumParams *params, const int layer)
690 {
691  TFContext *ctx = &tf_model->ctx;
692  TF_Operation *op;
693  TF_Tensor *tensor;
694  TF_OperationDescription *op_desc;
695  TF_Output input;
696  float *y;
697 
698  char name_buffer[NAME_BUFFER_SIZE];
699  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum/y%d", layer);
700 
701  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
702  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
703  tensor = TF_AllocateTensor(TF_FLOAT, NULL, 0, TF_DataTypeSize(TF_FLOAT));
704  y = (float *)TF_TensorData(tensor);
705  *y = params->val.y;
706  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
707  if (TF_GetCode(tf_model->status) != TF_OK){
708  TF_DeleteTensor(tensor);
709  av_log(ctx, AV_LOG_ERROR, "Failed to set value for maximum/y of layer %d", layer);
710  return DNN_GENERIC_ERROR;
711  }
712  op = TF_FinishOperation(op_desc, tf_model->status);
713  if (TF_GetCode(tf_model->status) != TF_OK){
714  TF_DeleteTensor(tensor);
715  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum/y to layer %d\n", layer);
716  return DNN_GENERIC_ERROR;
717  }
718 
719  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum%d", layer);
720  op_desc = TF_NewOperation(tf_model->graph, "Maximum", name_buffer);
721  input.oper = *cur_op;
722  input.index = 0;
723  TF_AddInput(op_desc, input);
724  input.oper = op;
725  TF_AddInput(op_desc, input);
726  TF_SetAttrType(op_desc, "T", TF_FLOAT);
727  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
728  if (TF_GetCode(tf_model->status) != TF_OK){
729  TF_DeleteTensor(tensor);
730  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum to layer %d\n", layer);
731  return DNN_GENERIC_ERROR;
732  }
733 
734  return 0;
735 }
736 
737 static int load_native_model(TFModel *tf_model, const char *model_filename)
738 {
739  TFContext *ctx = &tf_model->ctx;
740  int32_t layer;
741  TF_OperationDescription *op_desc;
742  TF_Operation *op;
743  TF_Operation *transpose_op;
744  TF_Tensor *tensor = NULL;
745  TF_Output input;
747  int64_t transpose_perm_shape[] = {4};
748  int64_t input_shape[] = {1, -1, -1, -1};
749  int layer_add_res;
750  DNNModel *model = NULL;
751  NativeModel *native_model;
752 
753  model = ff_dnn_load_model_native(model_filename, DFT_PROCESS_FRAME, NULL, NULL);
754  if (!model){
755  av_log(ctx, AV_LOG_ERROR, "Failed to load native model\n");
756  return AVERROR(EINVAL);
757  }
758 
759  native_model = model->model;
760  tf_model->graph = TF_NewGraph();
761  tf_model->status = TF_NewStatus();
762 
763 #define CLEANUP_ON_ERROR(tf_model) \
764  { \
765  TF_DeleteTensor(tensor); \
766  TF_DeleteGraph(tf_model->graph); \
767  TF_DeleteStatus(tf_model->status); \
768  av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
769  return DNN_GENERIC_ERROR; \
770  }
771 
772  op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
773  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
774  TF_SetAttrShape(op_desc, "shape", input_shape, 4);
775  op = TF_FinishOperation(op_desc, tf_model->status);
776  if (TF_GetCode(tf_model->status) != TF_OK){
777  CLEANUP_ON_ERROR(tf_model);
778  }
779 
780  op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
781  TF_SetAttrType(op_desc, "dtype", TF_INT32);
782  tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
783  transpose_perm = (int32_t *)TF_TensorData(tensor);
784  transpose_perm[0] = 1;
785  transpose_perm[1] = 2;
786  transpose_perm[2] = 3;
787  transpose_perm[3] = 0;
788  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
789  if (TF_GetCode(tf_model->status) != TF_OK){
790  CLEANUP_ON_ERROR(tf_model);
791  }
792  transpose_op = TF_FinishOperation(op_desc, tf_model->status);
793  if (TF_GetCode(tf_model->status) != TF_OK){
794  CLEANUP_ON_ERROR(tf_model);
795  }
796 
797  for (layer = 0; layer < native_model->layers_num; ++layer){
798  switch (native_model->layers[layer].type){
799  case DLT_INPUT:
800  layer_add_res = 0;
801  break;
802  case DLT_CONV2D:
803  layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
804  (ConvolutionalParams *)native_model->layers[layer].params, layer);
805  break;
806  case DLT_DEPTH_TO_SPACE:
807  layer_add_res = add_depth_to_space_layer(tf_model, &op,
808  (DepthToSpaceParams *)native_model->layers[layer].params, layer);
809  break;
810  case DLT_MIRROR_PAD:
811  layer_add_res = add_pad_layer(tf_model, &op,
812  (LayerPadParams *)native_model->layers[layer].params, layer);
813  break;
814  case DLT_MAXIMUM:
815  layer_add_res = add_maximum_layer(tf_model, &op,
816  (DnnLayerMaximumParams *)native_model->layers[layer].params, layer);
817  break;
818  default:
819  CLEANUP_ON_ERROR(tf_model);
820  }
821 
822  if (layer_add_res != 0){
823  CLEANUP_ON_ERROR(tf_model);
824  }
825  }
826 
827  op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
828  input.oper = op;
829  input.index = 0;
830  TF_AddInput(op_desc, input);
831  TF_FinishOperation(op_desc, tf_model->status);
832  if (TF_GetCode(tf_model->status) != TF_OK){
833  CLEANUP_ON_ERROR(tf_model);
834  }
835 
836  ff_dnn_free_model_native(&model);
837 
838  return 0;
839 }
840 
841 DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
842 {
843  DNNModel *model = NULL;
844  TFModel *tf_model = NULL;
845  TFContext *ctx = NULL;
846 
847  model = av_mallocz(sizeof(DNNModel));
848  if (!model){
849  return NULL;
850  }
851 
852  tf_model = av_mallocz(sizeof(TFModel));
853  if (!tf_model){
854  av_freep(&model);
855  return NULL;
856  }
857  model->model = tf_model;
858  tf_model->model = model;
859  ctx = &tf_model->ctx;
860  ctx->class = &dnn_tensorflow_class;
861 
862  //parse options
864  if (av_opt_set_from_string(ctx, options, NULL, "=", "&") < 0) {
865  av_log(ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);
866  goto err;
867  }
868 
869  if (load_tf_model(tf_model, model_filename) != 0){
870  if (load_native_model(tf_model, model_filename) != 0){
871  goto err;
872  }
873  }
874 
875  if (ctx->options.nireq <= 0) {
876  ctx->options.nireq = av_cpu_count() / 2 + 1;
877  }
878 
879 #if !HAVE_PTHREAD_CANCEL
880  if (ctx->options.async) {
881  ctx->options.async = 0;
882  av_log(filter_ctx, AV_LOG_WARNING, "pthread is not supported, roll back to sync.\n");
883  }
884 #endif
885 
886  tf_model->request_queue = ff_safe_queue_create();
887  if (!tf_model->request_queue) {
888  goto err;
889  }
890 
891  for (int i = 0; i < ctx->options.nireq; i++) {
892  TFRequestItem *item = av_mallocz(sizeof(*item));
893  if (!item) {
894  goto err;
895  }
896  item->lltask = NULL;
898  if (!item->infer_request) {
899  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for TensorFlow inference request\n");
900  av_freep(&item);
901  goto err;
902  }
903  item->status = TF_NewStatus();
906  item->exec_module.args = item;
907 
908  if (ff_safe_queue_push_back(tf_model->request_queue, item) < 0) {
909  destroy_request_item(&item);
910  goto err;
911  }
912  }
913 
914  tf_model->lltask_queue = ff_queue_create();
915  if (!tf_model->lltask_queue) {
916  goto err;
917  }
918 
919  tf_model->task_queue = ff_queue_create();
920  if (!tf_model->task_queue) {
921  goto err;
922  }
923 
924  model->get_input = &get_input_tf;
925  model->get_output = &get_output_tf;
926  model->options = options;
927  model->filter_ctx = filter_ctx;
928  model->func_type = func_type;
929 
930  return model;
931 err:
932  ff_dnn_free_model_tf(&model);
933  return NULL;
934 }
935 
936 static int fill_model_input_tf(TFModel *tf_model, TFRequestItem *request) {
937  DNNData input;
938  LastLevelTaskItem *lltask;
939  TaskItem *task;
940  TFInferRequest *infer_request;
941  TFContext *ctx = &tf_model->ctx;
942  int ret = 0;
943 
944  lltask = ff_queue_pop_front(tf_model->lltask_queue);
945  av_assert0(lltask);
946  task = lltask->task;
947  request->lltask = lltask;
948 
949  ret = get_input_tf(tf_model, &input, task->input_name);
950  if (ret != 0) {
951  goto err;
952  }
953 
954  infer_request = request->infer_request;
955  input.height = task->in_frame->height;
956  input.width = task->in_frame->width;
957 
958  infer_request->tf_input = av_malloc(sizeof(TF_Output));
959  if (!infer_request->tf_input) {
960  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
961  ret = AVERROR(ENOMEM);
962  goto err;
963  }
964 
965  infer_request->tf_input->oper = TF_GraphOperationByName(tf_model->graph, task->input_name);
966  if (!infer_request->tf_input->oper){
967  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);
969  goto err;
970  }
971  infer_request->tf_input->index = 0;
972 
973  infer_request->input_tensor = allocate_input_tensor(&input);
974  if (!infer_request->input_tensor){
975  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
976  ret = AVERROR(ENOMEM);
977  goto err;
978  }
979  input.data = (float *)TF_TensorData(infer_request->input_tensor);
980 
981  switch (tf_model->model->func_type) {
982  case DFT_PROCESS_FRAME:
983  if (task->do_ioproc) {
984  if (tf_model->model->frame_pre_proc != NULL) {
985  tf_model->model->frame_pre_proc(task->in_frame, &input, tf_model->model->filter_ctx);
986  } else {
988  }
989  }
990  break;
993  break;
994  default:
995  avpriv_report_missing_feature(ctx, "model function type %d", tf_model->model->func_type);
996  break;
997  }
998 
999  infer_request->tf_outputs = av_malloc_array(task->nb_output, sizeof(TF_Output));
1000  if (infer_request->tf_outputs == NULL) {
1001  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n");
1002  ret = AVERROR(ENOMEM);
1003  goto err;
1004  }
1005 
1006  infer_request->output_tensors = av_calloc(task->nb_output, sizeof(*infer_request->output_tensors));
1007  if (!infer_request->output_tensors) {
1008  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output tensor\n");
1009  ret = AVERROR(ENOMEM);
1010  goto err;
1011  }
1012 
1013  for (int i = 0; i < task->nb_output; ++i) {
1014  infer_request->output_tensors[i] = NULL;
1015  infer_request->tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, task->output_names[i]);
1016  if (!infer_request->tf_outputs[i].oper) {
1017  av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", task->output_names[i]);
1019  goto err;
1020  }
1021  infer_request->tf_outputs[i].index = 0;
1022  }
1023 
1024  return 0;
1025 err:
1026  tf_free_request(infer_request);
1027  return ret;
1028 }
1029 
1030 static void infer_completion_callback(void *args) {
1031  TFRequestItem *request = args;
1032  LastLevelTaskItem *lltask = request->lltask;
1033  TaskItem *task = lltask->task;
1034  DNNData *outputs;
1035  TFInferRequest *infer_request = request->infer_request;
1036  TFModel *tf_model = task->model;
1037  TFContext *ctx = &tf_model->ctx;
1038 
1039  outputs = av_malloc_array(task->nb_output, sizeof(*outputs));
1040  if (!outputs) {
1041  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *outputs\n");
1042  goto err;
1043  }
1044 
1045  for (uint32_t i = 0; i < task->nb_output; ++i) {
1046  outputs[i].height = TF_Dim(infer_request->output_tensors[i], 1);
1047  outputs[i].width = TF_Dim(infer_request->output_tensors[i], 2);
1048  outputs[i].channels = TF_Dim(infer_request->output_tensors[i], 3);
1049  outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
1050  outputs[i].dt = TF_TensorType(infer_request->output_tensors[i]);
1051  }
1052  switch (tf_model->model->func_type) {
1053  case DFT_PROCESS_FRAME:
1054  //it only support 1 output if it's frame in & frame out
1055  if (task->do_ioproc) {
1056  if (tf_model->model->frame_post_proc != NULL) {
1057  tf_model->model->frame_post_proc(task->out_frame, outputs, tf_model->model->filter_ctx);
1058  } else {
1060  }
1061  } else {
1062  task->out_frame->width = outputs[0].width;
1063  task->out_frame->height = outputs[0].height;
1064  }
1065  break;
1066  case DFT_ANALYTICS_DETECT:
1067  if (!tf_model->model->detect_post_proc) {
1068  av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n");
1069  return;
1070  }
1071  tf_model->model->detect_post_proc(task->in_frame, outputs, task->nb_output, tf_model->model->filter_ctx);
1072  break;
1073  default:
1074  av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n");
1075  goto err;
1076  }
1077  task->inference_done++;
1078 err:
1079  tf_free_request(infer_request);
1080  av_freep(&outputs);
1081 
1082  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1083  destroy_request_item(&request);
1084  av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
1085  }
1086 }
1087 
1088 static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue)
1089 {
1090  TFModel *tf_model;
1091  TFContext *ctx;
1092  LastLevelTaskItem *lltask;
1093  TaskItem *task;
1094  int ret = 0;
1095 
1096  if (ff_queue_size(lltask_queue) == 0) {
1097  destroy_request_item(&request);
1098  return 0;
1099  }
1100 
1101  lltask = ff_queue_peek_front(lltask_queue);
1102  task = lltask->task;
1103  tf_model = task->model;
1104  ctx = &tf_model->ctx;
1105 
1106  ret = fill_model_input_tf(tf_model, request);
1107  if (ret != 0) {
1108  goto err;
1109  }
1110 
1111  if (task->async) {
1112  if (ff_dnn_start_inference_async(ctx, &request->exec_module) != 0) {
1113  goto err;
1114  }
1115  return 0;
1116  }
1117  else {
1118  ret = tf_start_inference(request);
1119  if (ret != 0) {
1120  goto err;
1121  }
1122  infer_completion_callback(request);
1123  return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;
1124  }
1125 err:
1126  tf_free_request(request->infer_request);
1127  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1128  destroy_request_item(&request);
1129  }
1130  ff_dnn_free_model_tf(&tf_model->model);
1131  return ret;
1132 }
1133 
1134 int ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
1135 {
1136  TFModel *tf_model = model->model;
1137  TFContext *ctx = &tf_model->ctx;
1138  TaskItem *task;
1139  TFRequestItem *request;
1140  int ret = 0;
1141 
1142  ret = ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params);
1143  if (ret != 0) {
1144  return ret;
1145  }
1146 
1147  task = av_malloc(sizeof(*task));
1148  if (!task) {
1149  av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
1150  return AVERROR(ENOMEM);
1151  }
1152 
1153  ret = ff_dnn_fill_task(task, exec_params, tf_model, ctx->options.async, 1);
1154  if (ret != 0) {
1155  av_log(ctx, AV_LOG_ERROR, "Fill task with invalid parameter(s).\n");
1156  av_freep(&task);
1157  return ret;
1158  }
1159 
1160  if (ff_queue_push_back(tf_model->task_queue, task) < 0) {
1161  av_freep(&task);
1162  av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
1163  return AVERROR(ENOMEM);
1164  }
1165 
1166  ret = extract_lltask_from_task(task, tf_model->lltask_queue);
1167  if (ret != 0) {
1168  av_freep(&task);
1169  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
1170  return ret;
1171  }
1172 
1173  request = ff_safe_queue_pop_front(tf_model->request_queue);
1174  if (!request) {
1175  av_freep(&task);
1176  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1177  return AVERROR(EINVAL);
1178  }
1179  return execute_model_tf(request, tf_model->lltask_queue);
1180 }
1181 
1183 {
1184  TFModel *tf_model = model->model;
1185  return ff_dnn_get_result_common(tf_model->task_queue, in, out);
1186 }
1187 
1188 int ff_dnn_flush_tf(const DNNModel *model)
1189 {
1190  TFModel *tf_model = model->model;
1191  TFContext *ctx = &tf_model->ctx;
1192  TFRequestItem *request;
1193  int ret;
1194 
1195  if (ff_queue_size(tf_model->lltask_queue) == 0) {
1196  // no pending task need to flush
1197  return 0;
1198  }
1199 
1200  request = ff_safe_queue_pop_front(tf_model->request_queue);
1201  if (!request) {
1202  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1203  return AVERROR(EINVAL);
1204  }
1205 
1206  ret = fill_model_input_tf(tf_model, request);
1207  if (ret != 0) {
1208  av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n");
1209  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1210  destroy_request_item(&request);
1211  }
1212  return ret;
1213  }
1214 
1215  return ff_dnn_start_inference_async(ctx, &request->exec_module);
1216 }
1217 
1219 {
1220  TFModel *tf_model;
1221 
1222  if (*model){
1223  tf_model = (*model)->model;
1224  while (ff_safe_queue_size(tf_model->request_queue) != 0) {
1226  destroy_request_item(&item);
1227  }
1229 
1230  while (ff_queue_size(tf_model->lltask_queue) != 0) {
1232  av_freep(&item);
1233  }
1234  ff_queue_destroy(tf_model->lltask_queue);
1235 
1236  while (ff_queue_size(tf_model->task_queue) != 0) {
1237  TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
1238  av_frame_free(&item->in_frame);
1239  av_frame_free(&item->out_frame);
1240  av_freep(&item);
1241  }
1242  ff_queue_destroy(tf_model->task_queue);
1243 
1244  if (tf_model->graph){
1245  TF_DeleteGraph(tf_model->graph);
1246  }
1247  if (tf_model->session){
1248  TF_CloseSession(tf_model->session, tf_model->status);
1249  TF_DeleteSession(tf_model->session, tf_model->status);
1250  }
1251  if (tf_model->status){
1252  TF_DeleteStatus(tf_model->status);
1253  }
1254  av_freep(&tf_model);
1255  av_freep(model);
1256  }
1257 }
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(dnn_tensorflow)
ff_dnn_get_result_tf
DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_backend_tf.c:1182
TFOptions::sess_config
char * sess_config
Definition: dnn_backend_tf.c:44
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
TFInferRequest
Stores execution parameters for single call to the TensorFlow C API.
Definition: dnn_backend_tf.c:69
TFInferRequest::tf_outputs
TF_Output * tf_outputs
Definition: dnn_backend_tf.c:70
execute_model_tf
static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue)
Definition: dnn_backend_tf.c:1088
FLAGS
#define FLAGS
Definition: dnn_backend_tf.c:84
status
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
TFModel::graph
TF_Graph * graph
Definition: dnn_backend_tf.c:57
av_opt_set_defaults
void av_opt_set_defaults(void *s)
Set the values of all AVOption fields to their default values.
Definition: opt.c:1459
ff_safe_queue_pop_front
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Definition: safe_queue.c:105
out
FILE * out
Definition: movenc.c:54
TFModel::ctx
TFContext ctx
Definition: dnn_backend_tf.c:55
DNNAsyncExecModule
Common Async Execution Mechanism for the DNN Backends.
Definition: dnn_backend_common.h:58
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
extract_lltask_from_task
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
Definition: dnn_backend_tf.c:202
add_conv_layer
static int add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)
Definition: dnn_backend_tf.c:484
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:29
ff_queue_size
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition: queue.c:88
DnnLayerMaximumParams::y
float y
Definition: dnn_backend_native_layer_maximum.h:36
DNN_GENERIC_ERROR
#define DNN_GENERIC_ERROR
Definition: dnn_interface.h:33
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:101
LastLevelTaskItem
Definition: dnn_backend_common.h:50
ConvolutionalParams::kernel
float * kernel
Definition: dnn_backend_native_layer_conv2d.h:33
test::height
int height
Definition: vc1dsp.c:39
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:330
AVFrame::width
int width
Definition: frame.h:402
ff_dnn_load_model_native
DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_native.c:139
SafeQueue
Double-ended queue with mutex locks ensuring data consistency while multithreading.
Definition: safe_queue.c:46
AVOption
AVOption.
Definition: opt.h:251
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:101
TFInferRequest::input_tensor
TF_Tensor * input_tensor
Definition: dnn_backend_tf.c:73
data
const char data[16]
Definition: mxf.c:146
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:68
load_tf_model
static int load_tf_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:387
dnn_io_proc.h
TFModel::request_queue
SafeQueue * request_queue
Definition: dnn_backend_tf.c:60
TaskItem
Definition: dnn_backend_common.h:36
DNNAsyncExecModule::callback
void(* callback)(void *args)
Completion Callback for the backend.
Definition: dnn_backend_common.h:70
DnnLayerMaximumParams
Definition: dnn_backend_native_layer_maximum.h:33
avio_size
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:344
tf_sess_config.config
config
Definition: tf_sess_config.py:33
OFFSET
#define OFFSET(x)
Definition: dnn_backend_tf.c:83
load_native_model
static int load_native_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:737
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:30
dnn_backend_native_layer_pad.h
destroy_request_item
static void destroy_request_item(TFRequestItem **arg)
Free the TFRequestItem completely.
Definition: dnn_backend_tf.c:188
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:90
ff_queue_create
Queue * ff_queue_create(void)
Create a Queue instance.
Definition: queue.c:47
DLT_INPUT
@ DLT_INPUT
Definition: dnn_backend_native.h:41
ff_dnn_execute_model_tf
int ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_tf.c:1134
TaskItem::model
void * model
Definition: dnn_backend_common.h:37
filter_ctx
static FilteringContext * filter_ctx
Definition: transcode.c:51
get_input_tf
static int get_input_tf(void *model, DNNData *input, const char *input_name)
Definition: dnn_backend_tf.c:277
SIGMOID
@ SIGMOID
Definition: dnn_backend_native.h:55
SPACE_CHARS
#define SPACE_CHARS
Definition: dnn_backend_tf.c:357
Queue
Linear double-ended data structure.
Definition: queue.c:33
add_depth_to_space_layer
static int add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)
Definition: dnn_backend_tf.c:606
ff_queue_push_back
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition: queue.c:130
avassert.h
DNN_BACKEND_COMMON_OPTIONS
#define DNN_BACKEND_COMMON_OPTIONS
Definition: dnn_backend_common.h:31
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
fill_model_input_tf
static int fill_model_input_tf(TFModel *tf_model, TFRequestItem *request)
Definition: dnn_backend_tf.c:936
TFRequestItem::exec_module
DNNAsyncExecModule exec_module
Definition: dnn_backend_tf.c:80
float
float
Definition: af_crystalizer.c:122
NativeModel::layers_num
int32_t layers_num
Definition: dnn_backend_native.h:128
ConvolutionalParams::input_num
int32_t input_num
Definition: dnn_backend_native_layer_conv2d.h:28
LastLevelTaskItem::task
TaskItem * task
Definition: dnn_backend_common.h:51
DLT_MAXIMUM
@ DLT_MAXIMUM
Definition: dnn_backend_native.h:45
TANH
@ TANH
Definition: dnn_backend_native.h:55
read_graph
static TF_Buffer * read_graph(const char *model_filename)
Definition: dnn_backend_tf.c:222
Layer::type
DNNLayerType type
Definition: dnn_backend_native.h:58
DLT_CONV2D
@ DLT_CONV2D
Definition: dnn_backend_native.h:42
op
static int op(uint8_t **dst, const uint8_t *dst_end, GetByteContext *gb, int pixel, int count, int *x, int width, int linesize)
Perform decode operation.
Definition: anm.c:76
ff_queue_destroy
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition: queue.c:72
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DNNData
Definition: dnn_interface.h:59
outputs
static const AVFilterPad outputs[]
Definition: af_acontrast.c:172
ConvolutionalParams::activation
DNNActivationFunc activation
Definition: dnn_backend_native_layer_conv2d.h:29
dnn_tensorflow_options
static const AVOption dnn_tensorflow_options[]
Definition: dnn_backend_tf.c:85
ff_dnn_fill_gettingoutput_task
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:162
LayerPadParams
Definition: dnn_backend_native_layer_pad.h:33
ctx
AVFormatContext * ctx
Definition: movenc.c:48
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:45
arg
const char * arg
Definition: jacosubdec.c:67
dnn_backend_native_layer_depth2space.h
ff_safe_queue_size
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
Definition: safe_queue.c:80
ff_proc_from_frame_to_dnn
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:100
ff_dnn_free_model_native
void ff_dnn_free_model_native(DNNModel **model)
Definition: dnn_backend_native.c:515
ff_dnn_flush_tf
int ff_dnn_flush_tf(const DNNModel *model)
Definition: dnn_backend_tf.c:1188
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
Layer::params
void * params
Definition: dnn_backend_native.h:66
ff_frame_to_dnn_detect
int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:254
NULL
#define NULL
Definition: coverity.c:32
ff_safe_queue_create
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
Definition: safe_queue.c:52
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:104
NativeModel
Definition: dnn_backend_native.h:124
av_opt_set_from_string
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition: opt.c:1667
tf_create_inference_request
static TFInferRequest * tf_create_inference_request(void)
Create a TensorFlow inference request.
Definition: dnn_backend_tf.c:137
ff_dnn_async_module_cleanup
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
Definition: dnn_backend_common.c:92
TFModel::task_queue
Queue * task_queue
Definition: dnn_backend_tf.c:62
infer_completion_callback
static void infer_completion_callback(void *args)
Definition: dnn_backend_tf.c:1030
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:38
TFModel::status
TF_Status * status
Definition: dnn_backend_tf.c:59
tf_free_request
static void tf_free_request(TFInferRequest *request)
Free the contents of TensorFlow inference request.
Definition: dnn_backend_tf.c:109
transpose_perm
static void transpose_perm(int16_t *out, int16_t *in, int num_vect, const uint8_t line_len[2], int length_div)
Interpret the input data as in the following table:
Definition: twinvq.c:623
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
ConvolutionalParams::kernel_size
int32_t kernel_size
Definition: dnn_backend_native_layer_conv2d.h:28
av_cpu_count
int av_cpu_count(void)
Definition: cpu.c:206
options
const OptionDef options[]
test::width
int width
Definition: vc1dsp.c:38
AVIOContext
Bytestream IO Context.
Definition: avio.h:166
DLT_MIRROR_PAD
@ DLT_MIRROR_PAD
Definition: dnn_backend_native.h:44
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:42
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:46
cpu.h
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:106
size
int size
Definition: twinvq_data.h:10344
NativeModel::layers
Layer * layers
Definition: dnn_backend_native.h:127
avio.h
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:92
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
TFOptions::nireq
uint32_t nireq
Definition: dnn_backend_tf.c:46
dnn_backend_native_layer_conv2d.h
ff_safe_queue_destroy
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
Definition: safe_queue.c:69
hex_to_data
static int hex_to_data(uint8_t *data, const char *p)
Definition: dnn_backend_tf.c:358
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
NAME_BUFFER_SIZE
#define NAME_BUFFER_SIZE
Definition: dnn_backend_tf.c:482
tf_start_inference
static int tf_start_inference(void *args)
Start synchronous inference for the TensorFlow model.
Definition: dnn_backend_tf.c:158
ff_dnn_fill_task
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:56
DepthToSpaceParams
Definition: dnn_backend_native_layer_depth2space.h:33
dnn_backend_native.h
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
ff_dnn_load_model_tf
DNNModel * ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_tf.c:841
RELU
@ RELU
Definition: dnn_backend_native.h:55
ff_safe_queue_push_back
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
Definition: safe_queue.c:95
avio_closep
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1280
ConvolutionalParams::output_num
int32_t output_num
Definition: dnn_backend_native_layer_conv2d.h:28
add_maximum_layer
static int add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)
Definition: dnn_backend_tf.c:688
TFContext::options
TFOptions options
Definition: dnn_backend_tf.c:51
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:269
LayerPadParams::paddings
int32_t paddings[4][2]
Definition: dnn_backend_native_layer_pad.h:34
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:55
TFRequestItem::status
TF_Status * status
Definition: dnn_backend_tf.c:79
TFInferRequest::output_tensors
TF_Tensor ** output_tensors
Definition: dnn_backend_tf.c:71
TFModel::session
TF_Session * session
Definition: dnn_backend_tf.c:58
TFRequestItem::infer_request
TFInferRequest * infer_request
Definition: dnn_backend_tf.c:77
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:31
DNNAsyncExecModule::start_inference
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
Definition: dnn_backend_common.h:63
DNNAsyncExecModule::args
void * args
Argument for the execution functions.
Definition: dnn_backend_common.h:76
av_toupper
static av_const int av_toupper(int c)
Locale-independent conversion of ASCII characters to uppercase.
Definition: avstring.h:228
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:254
TFOptions::async
uint8_t async
Definition: dnn_backend_tf.c:45
safe_queue.h
TaskItem::output_names
const char ** output_names
Definition: dnn_backend_common.h:41
len
int len
Definition: vorbis_enc_data.h:426
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:262
TFInferRequest::tf_input
TF_Output * tf_input
Definition: dnn_backend_tf.c:72
TFContext
Definition: dnn_backend_tf.c:49
DepthToSpaceParams::block_size
int block_size
Definition: dnn_backend_native_layer_depth2space.h:34
ret
ret
Definition: filter_design.txt:187
DNNModel::get_input
int(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:95
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
TFModel::model
DNNModel * model
Definition: dnn_backend_tf.c:56
TFModel
Definition: dnn_backend_tf.c:54
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: defs.h:40
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:39
AVFrame::height
int height
Definition: frame.h:402
allocate_input_tensor
static TF_Tensor * allocate_input_tensor(const DNNData *input)
Definition: dnn_backend_tf.c:255
dnn_backend_common.h
TFRequestItem::lltask
LastLevelTaskItem * lltask
Definition: dnn_backend_tf.c:78
defs.h
DnnLayerMaximumParams::val
union DnnLayerMaximumParams::@221 val
avio_read
int avio_read(AVIOContext *s, unsigned char *buf, int size)
Read size bytes from AVIOContext into buf.
Definition: aviobuf.c:633
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:142
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
avio_open
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1215
AVFilterContext
An instance of a filter.
Definition: avfilter.h:397
ff_dnn_free_model_tf
void ff_dnn_free_model_tf(DNNModel **model)
Definition: dnn_backend_tf.c:1218
ff_dnn_start_inference_async
int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
Definition: dnn_backend_common.c:111
DNNModel
Definition: dnn_interface.h:84
AVIO_FLAG_READ
#define AVIO_FLAG_READ
read-only
Definition: avio.h:623
TaskItem::input_name
const char * input_name
Definition: dnn_backend_common.h:40
DNNModel::options
const char * options
Definition: dnn_interface.h:88
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
TFOptions
Definition: dnn_backend_tf.c:43
free_buffer
static void free_buffer(void *data, size_t length)
Definition: dnn_backend_tf.c:97
int32_t
int32_t
Definition: audioconvert.c:56
get_output_tf
static int get_output_tf(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_tf.c:313
DNNExecBaseParams
Definition: dnn_interface.h:67
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:43
DNNModel::get_output
int(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:97
DLT_DEPTH_TO_SPACE
@ DLT_DEPTH_TO_SPACE
Definition: dnn_backend_native.h:43
avstring.h
dnn_backend_native_layer_maximum.h
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:229
TFRequestItem
Definition: dnn_backend_tf.c:76
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
dnn_backend_tf.h
snprintf
#define snprintf
Definition: snprintf.h:34
CLEANUP_ON_ERROR
#define CLEANUP_ON_ERROR(tf_model)
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
TFModel::lltask_queue
Queue * lltask_queue
Definition: dnn_backend_tf.c:61
TaskItem::nb_output
uint32_t nb_output
Definition: dnn_backend_common.h:44
ConvolutionalParams
Definition: dnn_backend_native_layer_conv2d.h:27
add_pad_layer
static int add_pad_layer(TFModel *tf_model, TF_Operation **cur_op, LayerPadParams *params, const int layer)
Definition: dnn_backend_tf.c:630
DNNModel::model
void * model
Definition: dnn_interface.h:86
ff_proc_from_dnn_to_frame
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:27
ConvolutionalParams::biases
float * biases
Definition: dnn_backend_native_layer_conv2d.h:34