FFmpeg
dnn_backend_native.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN native backend implementation.
24  */
25 
26 #include "dnn_backend_native.h"
27 #include "libavutil/avassert.h"
29 
30 static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
31 {
32  ConvolutionalNetwork *network = (ConvolutionalNetwork *)model;
33  InputParams *input_params;
34  ConvolutionalParams *conv_params;
35  DepthToSpaceParams *depth_to_space_params;
36  LayerPadParams *pad_params;
37  int cur_width, cur_height, cur_channels;
38  int32_t layer;
39 
40  if (network->layers_num <= 0 || network->layers[0].type != INPUT){
41  return DNN_ERROR;
42  }
43  else{
44  input_params = (InputParams *)network->layers[0].params;
45  input_params->width = cur_width = input->width;
46  input_params->height = cur_height = input->height;
47  input_params->channels = cur_channels = input->channels;
48  if (input->data){
49  av_freep(&input->data);
50  }
51  av_assert0(input->dt == DNN_FLOAT);
52  network->layers[0].output = input->data = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
53  if (!network->layers[0].output){
54  return DNN_ERROR;
55  }
56  }
57 
58  for (layer = 1; layer < network->layers_num; ++layer){
59  switch (network->layers[layer].type){
60  case CONV:
61  conv_params = (ConvolutionalParams *)network->layers[layer].params;
62  if (conv_params->input_num != cur_channels){
63  return DNN_ERROR;
64  }
65  cur_channels = conv_params->output_num;
66 
67  if (conv_params->padding_method == VALID) {
68  int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation;
69  cur_height -= pad_size;
70  cur_width -= pad_size;
71  }
72  break;
73  case DEPTH_TO_SPACE:
74  depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
75  if (cur_channels % (depth_to_space_params->block_size * depth_to_space_params->block_size) != 0){
76  return DNN_ERROR;
77  }
78  cur_channels = cur_channels / (depth_to_space_params->block_size * depth_to_space_params->block_size);
79  cur_height *= depth_to_space_params->block_size;
80  cur_width *= depth_to_space_params->block_size;
81  break;
82  case MIRROR_PAD:
83  pad_params = (LayerPadParams *)network->layers[layer].params;
84  cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1];
85  cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1];
86  cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1];
87  break;
88  default:
89  return DNN_ERROR;
90  }
91  if (network->layers[layer].output){
92  av_freep(&network->layers[layer].output);
93  }
94 
95  if (cur_height <= 0 || cur_width <= 0)
96  return DNN_ERROR;
97 
98  network->layers[layer].output = av_malloc(cur_height * cur_width * cur_channels * sizeof(float));
99  if (!network->layers[layer].output){
100  return DNN_ERROR;
101  }
102  }
103 
104  return DNN_SUCCESS;
105 }
106 
107 // Loads model and its parameters that are stored in a binary file with following structure:
108 // layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
109 // For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
110 // For DEPTH_TO_SPACE layer: block_size
111 DNNModel *ff_dnn_load_model_native(const char *model_filename)
112 {
113  DNNModel *model = NULL;
114  ConvolutionalNetwork *network = NULL;
115  AVIOContext *model_file_context;
116  int file_size, dnn_size, kernel_size, i;
117  int32_t layer;
118  DNNLayerType layer_type;
119  ConvolutionalParams *conv_params;
120  DepthToSpaceParams *depth_to_space_params;
121  LayerPadParams *pad_params;
122 
123  model = av_malloc(sizeof(DNNModel));
124  if (!model){
125  return NULL;
126  }
127 
128  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
129  av_freep(&model);
130  return NULL;
131  }
132  file_size = avio_size(model_file_context);
133 
134  network = av_malloc(sizeof(ConvolutionalNetwork));
135  if (!network){
136  avio_closep(&model_file_context);
137  av_freep(&model);
138  return NULL;
139  }
140  model->model = (void *)network;
141 
142  network->layers_num = 1 + (int32_t)avio_rl32(model_file_context);
143  dnn_size = 4;
144 
145  network->layers = av_malloc(network->layers_num * sizeof(Layer));
146  if (!network->layers){
147  av_freep(&network);
148  avio_closep(&model_file_context);
149  av_freep(&model);
150  return NULL;
151  }
152 
153  for (layer = 0; layer < network->layers_num; ++layer){
154  network->layers[layer].output = NULL;
155  network->layers[layer].params = NULL;
156  }
157  network->layers[0].type = INPUT;
158  network->layers[0].params = av_malloc(sizeof(InputParams));
159  if (!network->layers[0].params){
160  avio_closep(&model_file_context);
161  ff_dnn_free_model_native(&model);
162  return NULL;
163  }
164 
165  for (layer = 1; layer < network->layers_num; ++layer){
166  layer_type = (int32_t)avio_rl32(model_file_context);
167  dnn_size += 4;
168  switch (layer_type){
169  case CONV:
170  conv_params = av_malloc(sizeof(ConvolutionalParams));
171  if (!conv_params){
172  avio_closep(&model_file_context);
173  ff_dnn_free_model_native(&model);
174  return NULL;
175  }
176  conv_params->dilation = (int32_t)avio_rl32(model_file_context);
177  conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
178  conv_params->activation = (int32_t)avio_rl32(model_file_context);
179  conv_params->input_num = (int32_t)avio_rl32(model_file_context);
180  conv_params->output_num = (int32_t)avio_rl32(model_file_context);
181  conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
182  kernel_size = conv_params->input_num * conv_params->output_num *
183  conv_params->kernel_size * conv_params->kernel_size;
184  dnn_size += 24 + (kernel_size + conv_params->output_num << 2);
185  if (dnn_size > file_size || conv_params->input_num <= 0 ||
186  conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
187  avio_closep(&model_file_context);
188  av_freep(&conv_params);
189  ff_dnn_free_model_native(&model);
190  return NULL;
191  }
192  conv_params->kernel = av_malloc(kernel_size * sizeof(float));
193  conv_params->biases = av_malloc(conv_params->output_num * sizeof(float));
194  if (!conv_params->kernel || !conv_params->biases){
195  avio_closep(&model_file_context);
196  av_freep(&conv_params->kernel);
197  av_freep(&conv_params->biases);
198  av_freep(&conv_params);
199  ff_dnn_free_model_native(&model);
200  return NULL;
201  }
202  for (i = 0; i < kernel_size; ++i){
203  conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context));
204  }
205  for (i = 0; i < conv_params->output_num; ++i){
206  conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
207  }
208  network->layers[layer].type = CONV;
209  network->layers[layer].params = conv_params;
210  break;
211  case DEPTH_TO_SPACE:
212  depth_to_space_params = av_malloc(sizeof(DepthToSpaceParams));
213  if (!depth_to_space_params){
214  avio_closep(&model_file_context);
215  ff_dnn_free_model_native(&model);
216  return NULL;
217  }
218  depth_to_space_params->block_size = (int32_t)avio_rl32(model_file_context);
219  dnn_size += 4;
220  network->layers[layer].type = DEPTH_TO_SPACE;
221  network->layers[layer].params = depth_to_space_params;
222  break;
223  case MIRROR_PAD:
224  pad_params = av_malloc(sizeof(LayerPadParams));
225  if (!pad_params){
226  avio_closep(&model_file_context);
227  ff_dnn_free_model_native(&model);
228  return NULL;
229  }
230  pad_params->mode = (int32_t)avio_rl32(model_file_context);
231  dnn_size += 4;
232  for (i = 0; i < 4; ++i) {
233  pad_params->paddings[i][0] = avio_rl32(model_file_context);
234  pad_params->paddings[i][1] = avio_rl32(model_file_context);
235  dnn_size += 8;
236  }
237  network->layers[layer].type = MIRROR_PAD;
238  network->layers[layer].params = pad_params;
239  break;
240  default:
241  avio_closep(&model_file_context);
242  ff_dnn_free_model_native(&model);
243  return NULL;
244  }
245  }
246 
247  avio_closep(&model_file_context);
248 
249  if (dnn_size != file_size){
250  ff_dnn_free_model_native(&model);
251  return NULL;
252  }
253 
255 
256  return model;
257 }
258 
259 #define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x)))
260 
261 static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height)
262 {
263  int radius = conv_params->kernel_size >> 1;
264  int src_linesize = width * conv_params->input_num;
265  int filter_linesize = conv_params->kernel_size * conv_params->input_num;
266  int filter_size = conv_params->kernel_size * filter_linesize;
267  int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0;
268 
269  for (int y = pad_size; y < height - pad_size; ++y) {
270  for (int x = pad_size; x < width - pad_size; ++x) {
271  for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) {
272  output[n_filter] = conv_params->biases[n_filter];
273 
274  for (int ch = 0; ch < conv_params->input_num; ++ch) {
275  for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) {
276  for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) {
277  float input_pel;
278  if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) {
279  int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height);
280  int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width);
281  input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
282  } else {
283  int y_pos = y + (kernel_y - radius) * conv_params->dilation;
284  int x_pos = x + (kernel_x - radius) * conv_params->dilation;
285  input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 :
286  input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
287  }
288 
289 
290  output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
291  kernel_x * conv_params->input_num + ch];
292  }
293  }
294  }
295  switch (conv_params->activation){
296  case RELU:
297  output[n_filter] = FFMAX(output[n_filter], 0.0);
298  break;
299  case TANH:
300  output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f;
301  break;
302  case SIGMOID:
303  output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter]));
304  break;
305  case NONE:
306  break;
307  case LEAKY_RELU:
308  output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0);
309  }
310  }
311  output += conv_params->output_num;
312  }
313  }
314 }
315 
316 static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels)
317 {
318  int y, x, by, bx, ch;
319  int new_channels = channels / (block_size * block_size);
320  int output_linesize = width * channels;
321  int by_linesize = output_linesize / block_size;
322  int x_linesize = new_channels * block_size;
323 
324  for (y = 0; y < height; ++y){
325  for (x = 0; x < width; ++x){
326  for (by = 0; by < block_size; ++by){
327  for (bx = 0; bx < block_size; ++bx){
328  for (ch = 0; ch < new_channels; ++ch){
329  output[by * by_linesize + x * x_linesize + bx * new_channels + ch] = input[ch];
330  }
331  input += new_channels;
332  }
333  }
334  }
335  output += output_linesize;
336  }
337 }
338 
340 {
341  ConvolutionalNetwork *network = (ConvolutionalNetwork *)model->model;
342  int cur_width, cur_height, cur_channels;
343  int32_t layer;
344  InputParams *input_params;
345  ConvolutionalParams *conv_params;
346  DepthToSpaceParams *depth_to_space_params;
347  LayerPadParams *pad_params;
348 
349  if (network->layers_num <= 0 || network->layers[0].type != INPUT || !network->layers[0].output){
350  return DNN_ERROR;
351  }
352  else{
353  input_params = (InputParams *)network->layers[0].params;
354  cur_width = input_params->width;
355  cur_height = input_params->height;
356  cur_channels = input_params->channels;
357  }
358 
359  for (layer = 1; layer < network->layers_num; ++layer){
360  if (!network->layers[layer].output){
361  return DNN_ERROR;
362  }
363  switch (network->layers[layer].type){
364  case CONV:
365  conv_params = (ConvolutionalParams *)network->layers[layer].params;
366  convolve(network->layers[layer - 1].output, network->layers[layer].output, conv_params, cur_width, cur_height);
367  cur_channels = conv_params->output_num;
368  if (conv_params->padding_method == VALID) {
369  int pad_size = (conv_params->kernel_size - 1) * conv_params->dilation;
370  cur_height -= pad_size;
371  cur_width -= pad_size;
372  }
373  break;
374  case DEPTH_TO_SPACE:
375  depth_to_space_params = (DepthToSpaceParams *)network->layers[layer].params;
376  depth_to_space(network->layers[layer - 1].output, network->layers[layer].output,
377  depth_to_space_params->block_size, cur_width, cur_height, cur_channels);
378  cur_height *= depth_to_space_params->block_size;
379  cur_width *= depth_to_space_params->block_size;
380  cur_channels /= depth_to_space_params->block_size * depth_to_space_params->block_size;
381  break;
382  case MIRROR_PAD:
383  pad_params = (LayerPadParams *)network->layers[layer].params;
384  dnn_execute_layer_pad(network->layers[layer - 1].output, network->layers[layer].output,
385  pad_params, 1, cur_height, cur_width, cur_channels);
386  cur_height = cur_height + pad_params->paddings[1][0] + pad_params->paddings[1][1];
387  cur_width = cur_width + pad_params->paddings[2][0] + pad_params->paddings[2][1];
388  cur_channels = cur_channels + pad_params->paddings[3][0] + pad_params->paddings[3][1];
389  break;
390  case INPUT:
391  return DNN_ERROR;
392  }
393  }
394 
395  // native mode does not support multiple outputs yet
396  if (nb_output > 1)
397  return DNN_ERROR;
398  outputs[0].data = network->layers[network->layers_num - 1].output;
399  outputs[0].height = cur_height;
400  outputs[0].width = cur_width;
401  outputs[0].channels = cur_channels;
402 
403  return DNN_SUCCESS;
404 }
405 
407 {
408  ConvolutionalNetwork *network;
409  ConvolutionalParams *conv_params;
410  int32_t layer;
411 
412  if (*model)
413  {
414  network = (ConvolutionalNetwork *)(*model)->model;
415  for (layer = 0; layer < network->layers_num; ++layer){
416  av_freep(&network->layers[layer].output);
417  if (network->layers[layer].type == CONV){
418  conv_params = (ConvolutionalParams *)network->layers[layer].params;
419  av_freep(&conv_params->kernel);
420  av_freep(&conv_params->biases);
421  }
422  av_freep(&network->layers[layer].params);
423  }
424  av_freep(&network->layers);
425  av_freep(&network);
426  av_freep(model);
427  }
428 }
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1187
void * model
Definition: dnn_interface.h:50
#define NULL
Definition: coverity.c:32
Bytestream IO Context.
Definition: avio.h:161
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:339
static av_always_inline float av_int2float(uint32_t i)
Reinterpret a 32-bit integer as a float.
Definition: intfloat.h:40
channels
Definition: aptx.c:30
int channels
Definition: dnn_interface.h:45
DNNActivationFunc activation
uint8_t pi<< 24) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_U8,(uint64_t)((*(const uint8_t *) pi-0x80U))<< 56) CONV_FUNC(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16,(*(const int16_t *) pi >>8)+0x80) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_S16,(uint64_t)(*(const int16_t *) pi)<< 48) CONV_FUNC(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32,(*(const int32_t *) pi >>24)+0x80) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_S32,(uint64_t)(*(const int32_t *) pi)<< 32) CONV_FUNC(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S64,(*(const int64_t *) pi >>56)+0x80) CONV_FUNC(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S64,*(const int64_t *) pi *(1.0f/(INT64_C(1)<< 63))) CONV_FUNC(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S64,*(const int64_t *) pi *(1.0/(INT64_C(1)<< 63))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_FLT, llrintf(*(const float *) pi *(INT64_C(1)<< 63))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31)))) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_DBL, llrint(*(const double *) pi *(INT64_C(1)<< 63)))#define FMT_PAIR_FUNC(out, in) static conv_func_type *const fmt_pair_to_conv_functions[AV_SAMPLE_FMT_NB *AV_SAMPLE_FMT_NB]={FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_S64),};static void cpy1(uint8_t **dst, const uint8_t **src, int len){memcpy(*dst,*src, len);}static void cpy2(uint8_t **dst, const uint8_t **src, int len){memcpy(*dst,*src, 2 *len);}static void cpy4(uint8_t **dst, const uint8_t **src, int len){memcpy(*dst,*src, 4 *len);}static void cpy8(uint8_t **dst, const uint8_t **src, int len){memcpy(*dst,*src, 8 *len);}AudioConvert *swri_audio_convert_alloc(enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, const int *ch_map, int flags){AudioConvert *ctx;conv_func_type *f=fmt_pair_to_conv_functions[av_get_packed_sample_fmt(out_fmt)+AV_SAMPLE_FMT_NB *av_get_packed_sample_fmt(in_fmt)];if(!f) return NULL;ctx=av_mallocz(sizeof(*ctx));if(!ctx) return NULL;if(channels==1){in_fmt=av_get_planar_sample_fmt(in_fmt);out_fmt=av_get_planar_sample_fmt(out_fmt);}ctx->channels=channels;ctx->conv_f=f;ctx->ch_map=ch_map;if(in_fmt==AV_SAMPLE_FMT_U8||in_fmt==AV_SAMPLE_FMT_U8P) memset(ctx->silence, 0x80, sizeof(ctx->silence));if(out_fmt==in_fmt &&!ch_map){switch(av_get_bytes_per_sample(in_fmt)){case 1:ctx->simd_f=cpy1;break;case 2:ctx->simd_f=cpy2;break;case 4:ctx->simd_f=cpy4;break;case 8:ctx->simd_f=cpy8;break;}}if(HAVE_X86ASM &&1) swri_audio_convert_init_x86(ctx, out_fmt, in_fmt, channels);if(ARCH_ARM) swri_audio_convert_init_arm(ctx, out_fmt, in_fmt, channels);if(ARCH_AARCH64) swri_audio_convert_init_aarch64(ctx, out_fmt, in_fmt, channels);return ctx;}void swri_audio_convert_free(AudioConvert **ctx){av_freep(ctx);}int swri_audio_convert(AudioConvert *ctx, AudioData *out, AudioData *in, int len){int ch;int off=0;const int os=(out->planar?1:out->ch_count)*out->bps;unsigned misaligned=0;av_assert0(ctx->channels==out->ch_count);if(ctx->in_simd_align_mask){int planes=in->planar?in->ch_count:1;unsigned m=0;for(ch=0;ch< planes;ch++) m|=(intptr_t) in->ch[ch];misaligned|=m &ctx->in_simd_align_mask;}if(ctx->out_simd_align_mask){int planes=out->planar?out->ch_count:1;unsigned m=0;for(ch=0;ch< planes;ch++) m|=(intptr_t) out->ch[ch];misaligned|=m &ctx->out_simd_align_mask;}if(ctx->simd_f &&!ctx->ch_map &&!misaligned){off=len &~15;av_assert1(off >=0);av_assert1(off<=len);av_assert2(ctx->channels==SWR_CH_MAX||!in->ch[ctx->channels]);if(off >0){if(out->planar==in->planar){int planes=out->planar?out->ch_count:1;for(ch=0;ch< planes;ch++){ctx->simd_f(out-> ch ch
Definition: audioconvert.c:56
DNN inference functions interface for native backend.
#define AVIO_FLAG_READ
read-only
Definition: avio.h:674
static void convolve(const float *input, float *output, const ConvolutionalParams *conv_params, int width, int height)
DNNDataType dt
Definition: dnn_interface.h:39
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
#define av_malloc(s)
#define f(width, name)
Definition: cbs_vp9.c:255
float * output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
#define height
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
int height
Definition: dnn_interface.h:45
unsigned int avio_rl32(AVIOContext *s)
Definition: aviobuf.c:772
simple assert() macros that are a bit more flexible than ISO C assert().
#define FFMAX(a, b)
Definition: common.h:94
int8_t exp
Definition: eval.c:72
static DNNReturnType set_input_output_native(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
#define FFMIN(a, b)
Definition: common.h:96
#define width
Definition: af_afade.c:54
int32_t
DNNReturnType ff_dnn_execute_model_native(const DNNModel *model, DNNData *outputs, uint32_t nb_output)
DNNReturnType
Definition: dnn_interface.h:31
void ff_dnn_free_model_native(DNNModel **model)
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
if(ret)
static void depth_to_space(const float *input, float *output, int block_size, int width, int height, int channels)
#define CLAMP_TO_EDGE(x, w)
layer pad (equivalent to tf.pad) for native backend.
DNNReturnType(* set_input_output)(void *model, DNNInputData *input, const char *input_name, const char **output_names, uint32_t nb_output)
Definition: dnn_interface.h:53
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
DNNConvPaddingParam padding_method
DNNLayerType type
DNNLayerType
float * data
Definition: dnn_interface.h:44
void * params
void dnn_execute_layer_pad(const float *input, float *output, const LayerPadParams *params, int number, int height, int width, int channel)
DNNModel * ff_dnn_load_model_native(const char *model_filename)
#define av_freep(p)
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1242
for(j=16;j >0;--j)