FFmpeg
dnn_backend_tf.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN tensorflow backend implementation.
24  */
25 
26 #include "dnn_backend_tf.h"
27 #include "dnn_backend_native.h"
30 #include "libavformat/avio.h"
31 #include "libavutil/avassert.h"
34 
35 #include <tensorflow/c/c_api.h>
36 
37 typedef struct TFModel{
38  TF_Graph *graph;
39  TF_Session *session;
40  TF_Status *status;
41  TF_Output input;
42  TF_Tensor *input_tensor;
43  TF_Output *outputs;
44  TF_Tensor **output_tensors;
45  uint32_t nb_output;
46 } TFModel;
47 
48 static void free_buffer(void *data, size_t length)
49 {
50  av_freep(&data);
51 }
52 
53 static TF_Buffer *read_graph(const char *model_filename)
54 {
55  TF_Buffer *graph_buf;
56  unsigned char *graph_data = NULL;
57  AVIOContext *model_file_context;
58  long size, bytes_read;
59 
60  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
61  return NULL;
62  }
63 
64  size = avio_size(model_file_context);
65 
66  graph_data = av_malloc(size);
67  if (!graph_data){
68  avio_closep(&model_file_context);
69  return NULL;
70  }
71  bytes_read = avio_read(model_file_context, graph_data, size);
72  avio_closep(&model_file_context);
73  if (bytes_read != size){
74  av_freep(&graph_data);
75  return NULL;
76  }
77 
78  graph_buf = TF_NewBuffer();
79  graph_buf->data = (void *)graph_data;
80  graph_buf->length = size;
81  graph_buf->data_deallocator = free_buffer;
82 
83  return graph_buf;
84 }
85 
86 static TF_Tensor *allocate_input_tensor(const DNNData *input)
87 {
88  TF_DataType dt;
89  size_t size;
90  int64_t input_dims[] = {1, input->height, input->width, input->channels};
91  switch (input->dt) {
92  case DNN_FLOAT:
93  dt = TF_FLOAT;
94  size = sizeof(float);
95  break;
96  case DNN_UINT8:
97  dt = TF_UINT8;
98  size = sizeof(char);
99  break;
100  default:
101  av_assert0(!"should not reach here");
102  }
103 
104  return TF_AllocateTensor(dt, input_dims, 4,
105  input_dims[1] * input_dims[2] * input_dims[3] * size);
106 }
107 
108 static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
109 {
110  TFModel *tf_model = (TFModel *)model;
111  TF_Status *status;
112  int64_t dims[4];
113 
114  TF_Output tf_output;
115  tf_output.oper = TF_GraphOperationByName(tf_model->graph, input_name);
116  if (!tf_output.oper)
117  return DNN_ERROR;
118 
119  tf_output.index = 0;
120  input->dt = TF_OperationOutputType(tf_output);
121 
122  status = TF_NewStatus();
123  TF_GraphGetTensorShape(tf_model->graph, tf_output, dims, 4, status);
124  if (TF_GetCode(status) != TF_OK){
125  TF_DeleteStatus(status);
126  return DNN_ERROR;
127  }
128  TF_DeleteStatus(status);
129 
130  // currently only NHWC is supported
131  av_assert0(dims[0] == 1);
132  input->height = dims[1];
133  input->width = dims[2];
134  input->channels = dims[3];
135 
136  return DNN_SUCCESS;
137 }
138 
139 static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
140 {
141  TFModel *tf_model = (TFModel *)model;
142  TF_SessionOptions *sess_opts;
143  const TF_Operation *init_op = TF_GraphOperationByName(tf_model->graph, "init");
144 
145  // Input operation
146  tf_model->input.oper = TF_GraphOperationByName(tf_model->graph, input_name);
147  if (!tf_model->input.oper){
148  return DNN_ERROR;
149  }
150  tf_model->input.index = 0;
151  if (tf_model->input_tensor){
152  TF_DeleteTensor(tf_model->input_tensor);
153  }
154  tf_model->input_tensor = allocate_input_tensor(input);
155  if (!tf_model->input_tensor){
156  return DNN_ERROR;
157  }
158  input->data = (float *)TF_TensorData(tf_model->input_tensor);
159 
160  // Output operation
161  if (nb_output == 0)
162  return DNN_ERROR;
163 
164  av_freep(&tf_model->outputs);
165  tf_model->outputs = av_malloc_array(nb_output, sizeof(*tf_model->outputs));
166  if (!tf_model->outputs)
167  return DNN_ERROR;
168  for (int i = 0; i < nb_output; ++i) {
169  tf_model->outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]);
170  if (!tf_model->outputs[i].oper){
171  av_freep(&tf_model->outputs);
172  return DNN_ERROR;
173  }
174  tf_model->outputs[i].index = 0;
175  }
176 
177  if (tf_model->output_tensors) {
178  for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
179  if (tf_model->output_tensors[i]) {
180  TF_DeleteTensor(tf_model->output_tensors[i]);
181  tf_model->output_tensors[i] = NULL;
182  }
183  }
184  }
185  av_freep(&tf_model->output_tensors);
186  tf_model->output_tensors = av_mallocz_array(nb_output, sizeof(*tf_model->output_tensors));
187  if (!tf_model->output_tensors) {
188  av_freep(&tf_model->outputs);
189  return DNN_ERROR;
190  }
191 
192  tf_model->nb_output = nb_output;
193 
194  if (tf_model->session){
195  TF_CloseSession(tf_model->session, tf_model->status);
196  TF_DeleteSession(tf_model->session, tf_model->status);
197  }
198 
199  sess_opts = TF_NewSessionOptions();
200  tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
201  TF_DeleteSessionOptions(sess_opts);
202  if (TF_GetCode(tf_model->status) != TF_OK)
203  {
204  return DNN_ERROR;
205  }
206 
207  // Run initialization operation with name "init" if it is present in graph
208  if (init_op){
209  TF_SessionRun(tf_model->session, NULL,
210  NULL, NULL, 0,
211  NULL, NULL, 0,
212  &init_op, 1, NULL, tf_model->status);
213  if (TF_GetCode(tf_model->status) != TF_OK)
214  {
215  return DNN_ERROR;
216  }
217  }
218 
219  return DNN_SUCCESS;
220 }
221 
222 static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
223 {
224  TF_Buffer *graph_def;
225  TF_ImportGraphDefOptions *graph_opts;
226 
227  graph_def = read_graph(model_filename);
228  if (!graph_def){
229  return DNN_ERROR;
230  }
231  tf_model->graph = TF_NewGraph();
232  tf_model->status = TF_NewStatus();
233  graph_opts = TF_NewImportGraphDefOptions();
234  TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);
235  TF_DeleteImportGraphDefOptions(graph_opts);
236  TF_DeleteBuffer(graph_def);
237  if (TF_GetCode(tf_model->status) != TF_OK){
238  TF_DeleteGraph(tf_model->graph);
239  TF_DeleteStatus(tf_model->status);
240  return DNN_ERROR;
241  }
242 
243  return DNN_SUCCESS;
244 }
245 
246 #define NAME_BUFFER_SIZE 256
247 
248 static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
249  ConvolutionalParams* params, const int layer)
250 {
251  TF_Operation *op;
252  TF_OperationDescription *op_desc;
253  TF_Output input;
254  int64_t strides[] = {1, 1, 1, 1};
255  TF_Tensor *tensor;
256  int64_t dims[4];
257  int dims_len;
258  char name_buffer[NAME_BUFFER_SIZE];
259  int32_t size;
260 
261  size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
262  input.index = 0;
263 
264  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
265  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
266  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
267  dims[0] = params->output_num;
268  dims[1] = params->kernel_size;
269  dims[2] = params->kernel_size;
270  dims[3] = params->input_num;
271  dims_len = 4;
272  tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
273  memcpy(TF_TensorData(tensor), params->kernel, size * sizeof(float));
274  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
275  if (TF_GetCode(tf_model->status) != TF_OK){
276  return DNN_ERROR;
277  }
278  op = TF_FinishOperation(op_desc, tf_model->status);
279  if (TF_GetCode(tf_model->status) != TF_OK){
280  return DNN_ERROR;
281  }
282 
283  snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
284  op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
285  input.oper = op;
286  TF_AddInput(op_desc, input);
287  input.oper = transpose_op;
288  TF_AddInput(op_desc, input);
289  TF_SetAttrType(op_desc, "T", TF_FLOAT);
290  TF_SetAttrType(op_desc, "Tperm", TF_INT32);
291  op = TF_FinishOperation(op_desc, tf_model->status);
292  if (TF_GetCode(tf_model->status) != TF_OK){
293  return DNN_ERROR;
294  }
295 
296  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
297  op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
298  input.oper = *cur_op;
299  TF_AddInput(op_desc, input);
300  input.oper = op;
301  TF_AddInput(op_desc, input);
302  TF_SetAttrType(op_desc, "T", TF_FLOAT);
303  TF_SetAttrIntList(op_desc, "strides", strides, 4);
304  TF_SetAttrString(op_desc, "padding", "VALID", 5);
305  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
306  if (TF_GetCode(tf_model->status) != TF_OK){
307  return DNN_ERROR;
308  }
309 
310  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
311  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
312  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
313  dims[0] = params->output_num;
314  dims_len = 1;
315  tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
316  memcpy(TF_TensorData(tensor), params->biases, params->output_num * sizeof(float));
317  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
318  if (TF_GetCode(tf_model->status) != TF_OK){
319  return DNN_ERROR;
320  }
321  op = TF_FinishOperation(op_desc, tf_model->status);
322  if (TF_GetCode(tf_model->status) != TF_OK){
323  return DNN_ERROR;
324  }
325 
326  snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
327  op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
328  input.oper = *cur_op;
329  TF_AddInput(op_desc, input);
330  input.oper = op;
331  TF_AddInput(op_desc, input);
332  TF_SetAttrType(op_desc, "T", TF_FLOAT);
333  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
334  if (TF_GetCode(tf_model->status) != TF_OK){
335  return DNN_ERROR;
336  }
337 
338  snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
339  switch (params->activation){
340  case RELU:
341  op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
342  break;
343  case TANH:
344  op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
345  break;
346  case SIGMOID:
347  op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
348  break;
349  default:
350  return DNN_ERROR;
351  }
352  input.oper = *cur_op;
353  TF_AddInput(op_desc, input);
354  TF_SetAttrType(op_desc, "T", TF_FLOAT);
355  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
356  if (TF_GetCode(tf_model->status) != TF_OK){
357  return DNN_ERROR;
358  }
359 
360  return DNN_SUCCESS;
361 }
362 
363 static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
364  DepthToSpaceParams *params, const int layer)
365 {
366  TF_OperationDescription *op_desc;
367  TF_Output input;
368  char name_buffer[NAME_BUFFER_SIZE];
369 
370  snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
371  op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
372  input.oper = *cur_op;
373  input.index = 0;
374  TF_AddInput(op_desc, input);
375  TF_SetAttrType(op_desc, "T", TF_FLOAT);
376  TF_SetAttrInt(op_desc, "block_size", params->block_size);
377  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
378  if (TF_GetCode(tf_model->status) != TF_OK){
379  return DNN_ERROR;
380  }
381 
382  return DNN_SUCCESS;
383 }
384 
385 static DNNReturnType add_pad_layer(TFModel *tf_model, TF_Operation **cur_op,
386  LayerPadParams *params, const int layer)
387 {
388  TF_Operation *op;
389  TF_Tensor *tensor;
390  TF_OperationDescription *op_desc;
391  TF_Output input;
392  int32_t *pads;
393  int64_t pads_shape[] = {4, 2};
394 
395  char name_buffer[NAME_BUFFER_SIZE];
396  snprintf(name_buffer, NAME_BUFFER_SIZE, "pad%d", layer);
397 
398  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
399  TF_SetAttrType(op_desc, "dtype", TF_INT32);
400  tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
401  pads = (int32_t *)TF_TensorData(tensor);
402  pads[0] = params->paddings[0][0];
403  pads[1] = params->paddings[0][1];
404  pads[2] = params->paddings[1][0];
405  pads[3] = params->paddings[1][1];
406  pads[4] = params->paddings[2][0];
407  pads[5] = params->paddings[2][1];
408  pads[6] = params->paddings[3][0];
409  pads[7] = params->paddings[3][1];
410  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
411  if (TF_GetCode(tf_model->status) != TF_OK){
412  return DNN_ERROR;
413  }
414  op = TF_FinishOperation(op_desc, tf_model->status);
415  if (TF_GetCode(tf_model->status) != TF_OK){
416  return DNN_ERROR;
417  }
418 
419  op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
420  input.oper = *cur_op;
421  input.index = 0;
422  TF_AddInput(op_desc, input);
423  input.oper = op;
424  TF_AddInput(op_desc, input);
425  TF_SetAttrType(op_desc, "T", TF_FLOAT);
426  TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
427  TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
428  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
429  if (TF_GetCode(tf_model->status) != TF_OK){
430  return DNN_ERROR;
431  }
432 
433  return DNN_SUCCESS;
434 }
435 
436 static DNNReturnType add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op,
437  DnnLayerMaximumParams *params, const int layer)
438 {
439  TF_Operation *op;
440  TF_Tensor *tensor;
441  TF_OperationDescription *op_desc;
442  TF_Output input;
443  float *y;
444 
445  char name_buffer[NAME_BUFFER_SIZE];
446  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum/y%d", layer);
447 
448  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
449  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
450  tensor = TF_AllocateTensor(TF_FLOAT, NULL, 0, TF_DataTypeSize(TF_FLOAT));
451  y = (float *)TF_TensorData(tensor);
452  *y = params->val.y;
453  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
454  if (TF_GetCode(tf_model->status) != TF_OK){
455  return DNN_ERROR;
456  }
457  op = TF_FinishOperation(op_desc, tf_model->status);
458  if (TF_GetCode(tf_model->status) != TF_OK){
459  return DNN_ERROR;
460  }
461 
462  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum%d", layer);
463  op_desc = TF_NewOperation(tf_model->graph, "Maximum", name_buffer);
464  input.oper = *cur_op;
465  input.index = 0;
466  TF_AddInput(op_desc, input);
467  input.oper = op;
468  TF_AddInput(op_desc, input);
469  TF_SetAttrType(op_desc, "T", TF_FLOAT);
470  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
471  if (TF_GetCode(tf_model->status) != TF_OK){
472  return DNN_ERROR;
473  }
474 
475  return DNN_SUCCESS;
476 }
477 
478 static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
479 {
480  int32_t layer;
481  TF_OperationDescription *op_desc;
482  TF_Operation *op;
483  TF_Operation *transpose_op;
484  TF_Tensor *tensor;
485  TF_Output input;
487  int64_t transpose_perm_shape[] = {4};
488  int64_t input_shape[] = {1, -1, -1, -1};
489  DNNReturnType layer_add_res;
490  DNNModel *native_model = NULL;
491  ConvolutionalNetwork *conv_network;
492 
493  native_model = ff_dnn_load_model_native(model_filename);
494  if (!native_model){
495  return DNN_ERROR;
496  }
497 
498  conv_network = (ConvolutionalNetwork *)native_model->model;
499  tf_model->graph = TF_NewGraph();
500  tf_model->status = TF_NewStatus();
501 
502 #define CLEANUP_ON_ERROR(tf_model) \
503  { \
504  TF_DeleteGraph(tf_model->graph); \
505  TF_DeleteStatus(tf_model->status); \
506  return DNN_ERROR; \
507  }
508 
509  op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
510  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
511  TF_SetAttrShape(op_desc, "shape", input_shape, 4);
512  op = TF_FinishOperation(op_desc, tf_model->status);
513  if (TF_GetCode(tf_model->status) != TF_OK){
514  CLEANUP_ON_ERROR(tf_model);
515  }
516 
517  op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
518  TF_SetAttrType(op_desc, "dtype", TF_INT32);
519  tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
520  transpose_perm = (int32_t *)TF_TensorData(tensor);
521  transpose_perm[0] = 1;
522  transpose_perm[1] = 2;
523  transpose_perm[2] = 3;
524  transpose_perm[3] = 0;
525  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
526  if (TF_GetCode(tf_model->status) != TF_OK){
527  CLEANUP_ON_ERROR(tf_model);
528  }
529  transpose_op = TF_FinishOperation(op_desc, tf_model->status);
530 
531  for (layer = 0; layer < conv_network->layers_num; ++layer){
532  switch (conv_network->layers[layer].type){
533  case DLT_INPUT:
534  layer_add_res = DNN_SUCCESS;
535  break;
536  case DLT_CONV2D:
537  layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
538  (ConvolutionalParams *)conv_network->layers[layer].params, layer);
539  break;
540  case DLT_DEPTH_TO_SPACE:
541  layer_add_res = add_depth_to_space_layer(tf_model, &op,
542  (DepthToSpaceParams *)conv_network->layers[layer].params, layer);
543  break;
544  case DLT_MIRROR_PAD:
545  layer_add_res = add_pad_layer(tf_model, &op,
546  (LayerPadParams *)conv_network->layers[layer].params, layer);
547  break;
548  case DLT_MAXIMUM:
549  layer_add_res = add_maximum_layer(tf_model, &op,
550  (DnnLayerMaximumParams *)conv_network->layers[layer].params, layer);
551  break;
552  default:
553  CLEANUP_ON_ERROR(tf_model);
554  }
555 
556  if (layer_add_res != DNN_SUCCESS){
557  CLEANUP_ON_ERROR(tf_model);
558  }
559  }
560 
561  op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
562  input.oper = op;
563  input.index = 0;
564  TF_AddInput(op_desc, input);
565  TF_FinishOperation(op_desc, tf_model->status);
566  if (TF_GetCode(tf_model->status) != TF_OK){
567  CLEANUP_ON_ERROR(tf_model);
568  }
569 
570  ff_dnn_free_model_native(&native_model);
571 
572  return DNN_SUCCESS;
573 }
574 
575 DNNModel *ff_dnn_load_model_tf(const char *model_filename)
576 {
577  DNNModel *model = NULL;
578  TFModel *tf_model = NULL;
579 
580  model = av_malloc(sizeof(DNNModel));
581  if (!model){
582  return NULL;
583  }
584 
585  tf_model = av_mallocz(sizeof(TFModel));
586  if (!tf_model){
587  av_freep(&model);
588  return NULL;
589  }
590 
591  if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){
592  if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){
593  av_freep(&tf_model);
594  av_freep(&model);
595 
596  return NULL;
597  }
598  }
599 
600  model->model = (void *)tf_model;
602  model->get_input = &get_input_tf;
603 
604  return model;
605 }
606 
607 
608 
610 {
611  TFModel *tf_model = (TFModel *)model->model;
612  uint32_t nb = FFMIN(nb_output, tf_model->nb_output);
613  if (nb == 0)
614  return DNN_ERROR;
615 
616  av_assert0(tf_model->output_tensors);
617  for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
618  if (tf_model->output_tensors[i]) {
619  TF_DeleteTensor(tf_model->output_tensors[i]);
620  tf_model->output_tensors[i] = NULL;
621  }
622  }
623 
624  TF_SessionRun(tf_model->session, NULL,
625  &tf_model->input, &tf_model->input_tensor, 1,
626  tf_model->outputs, tf_model->output_tensors, nb,
627  NULL, 0, NULL, tf_model->status);
628 
629  if (TF_GetCode(tf_model->status) != TF_OK){
630  return DNN_ERROR;
631  }
632 
633  for (uint32_t i = 0; i < nb; ++i) {
634  outputs[i].height = TF_Dim(tf_model->output_tensors[i], 1);
635  outputs[i].width = TF_Dim(tf_model->output_tensors[i], 2);
636  outputs[i].channels = TF_Dim(tf_model->output_tensors[i], 3);
637  outputs[i].data = TF_TensorData(tf_model->output_tensors[i]);
638  outputs[i].dt = TF_TensorType(tf_model->output_tensors[i]);
639  }
640 
641  return DNN_SUCCESS;
642 }
643 
645 {
646  TFModel *tf_model;
647 
648  if (*model){
649  tf_model = (TFModel *)(*model)->model;
650  if (tf_model->graph){
651  TF_DeleteGraph(tf_model->graph);
652  }
653  if (tf_model->session){
654  TF_CloseSession(tf_model->session, tf_model->status);
655  TF_DeleteSession(tf_model->session, tf_model->status);
656  }
657  if (tf_model->status){
658  TF_DeleteStatus(tf_model->status);
659  }
660  if (tf_model->input_tensor){
661  TF_DeleteTensor(tf_model->input_tensor);
662  }
663  if (tf_model->output_tensors) {
664  for (uint32_t i = 0; i < tf_model->nb_output; ++i) {
665  if (tf_model->output_tensors[i]) {
666  TF_DeleteTensor(tf_model->output_tensors[i]);
667  tf_model->output_tensors[i] = NULL;
668  }
669  }
670  }
671  av_freep(&tf_model->outputs);
672  av_freep(&tf_model->output_tensors);
673  av_freep(&tf_model);
674  av_freep(model);
675  }
676 }
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1187
void * model
Definition: dnn_interface.h:45
#define NULL
Definition: coverity.c:32
Bytestream IO Context.
Definition: avio.h:161
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:339
Buffered I/O operations.
ptrdiff_t const GLvoid * data
Definition: opengl_enc.c:100
static TF_Buffer * read_graph(const char *model_filename)
int channels
Definition: dnn_interface.h:40
DNN inference functions interface for native backend.
#define AVIO_FLAG_READ
read-only
Definition: avio.h:674
DNNModel * ff_dnn_load_model_tf(const char *model_filename)
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:236
uint32_t nb_output
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
#define av_malloc(s)
TF_Status * status
DNN inference functions interface for TensorFlow backend.
ptrdiff_t size
Definition: opengl_enc.c:100
int avio_read(AVIOContext *s, unsigned char *buf, int size)
Read size bytes from AVIOContext into buf.
Definition: aviobuf.c:650
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:48
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
static DNNReturnType add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)
int height
Definition: dnn_interface.h:40
void * data
Definition: dnn_interface.h:38
static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
GLenum GLint * params
Definition: opengl_enc.c:113
simple assert() macros that are a bit more flexible than ISO C assert().
GLsizei GLsizei * length
Definition: opengl_enc.c:114
static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
TF_Tensor * input_tensor
TF_Output input
static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
#define FFMIN(a, b)
Definition: common.h:96
#define NAME_BUFFER_SIZE
int32_t
DNN inference functions interface for native backend.
DNN inference functions interface for native backend.
DNNReturnType
Definition: dnn_interface.h:31
void ff_dnn_free_model_native(DNNModel **model)
if(ret)
static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)
DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
static TF_Tensor * allocate_input_tensor(const DNNData *input)
static void free_buffer(void *data, size_t length)
static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)
layer pad (equivalent to tf.pad) for native backend.
TF_Tensor ** output_tensors
#define snprintf
Definition: snprintf.h:34
DNNLayerType type
TF_Session * session
static int op(uint8_t **dst, const uint8_t *dst_end, GetByteContext *gb, int pixel, int count, int *x, int width, int linesize)
Perform decode operation.
Definition: anm.c:78
TF_Output * outputs
TF_Graph * graph
void ff_dnn_free_model_tf(DNNModel **model)
void * params
static void transpose_perm(int16_t *out, int16_t *in, int num_vect, const uint8_t line_len[2], int length_div)
Interpret the input data as in the following table:
Definition: twinvq.c:630
DNNModel * ff_dnn_load_model_native(const char *model_filename)
#define av_freep(p)
DNNReturnType(* set_input_output)(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
Definition: dnn_interface.h:51
static DNNReturnType add_pad_layer(TFModel *tf_model, TF_Operation **cur_op, LayerPadParams *params, const int layer)
#define av_malloc_array(a, b)
static DNNReturnType set_input_output_tf(void *model, DNNData *input, const char *input_name, const char **output_names, uint32_t nb_output)
union DnnLayerMaximumParams::@219 val
DNNDataType dt
Definition: dnn_interface.h:39
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1242
#define CLEANUP_ON_ERROR(tf_model)
void * av_mallocz_array(size_t nmemb, size_t size)
Definition: mem.c:191