FFmpeg
dnn_backend_native_layer_conv2d.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "libavutil/avassert.h"
23 
24 #define CLAMP_TO_EDGE(x, w) ((x) < 0 ? 0 : ((x) >= (w) ? (w - 1) : (x)))
25 
26 int dnn_load_layer_conv2d(Layer *layer, AVIOContext *model_file_context, int file_size)
27 {
28  ConvolutionalParams *conv_params;
29  int kernel_size;
30  int dnn_size = 0;
31  conv_params = av_malloc(sizeof(*conv_params));
32  if (!conv_params)
33  return 0;
34 
35  conv_params->dilation = (int32_t)avio_rl32(model_file_context);
36  conv_params->padding_method = (int32_t)avio_rl32(model_file_context);
37  conv_params->activation = (int32_t)avio_rl32(model_file_context);
38  conv_params->input_num = (int32_t)avio_rl32(model_file_context);
39  conv_params->output_num = (int32_t)avio_rl32(model_file_context);
40  conv_params->kernel_size = (int32_t)avio_rl32(model_file_context);
41  conv_params->has_bias = (int32_t)avio_rl32(model_file_context);
42  dnn_size += 28;
43 
44  kernel_size = conv_params->input_num * conv_params->output_num *
45  conv_params->kernel_size * conv_params->kernel_size;
46  dnn_size += kernel_size * 4;
47  if (conv_params->has_bias)
48  dnn_size += conv_params->output_num * 4;
49 
50  if (dnn_size > file_size || conv_params->input_num <= 0 ||
51  conv_params->output_num <= 0 || conv_params->kernel_size <= 0){
52  av_freep(&conv_params);
53  return 0;
54  }
55 
56  conv_params->kernel = av_malloc(kernel_size * sizeof(float));
57  if (!conv_params->kernel) {
58  av_freep(&conv_params);
59  return 0;
60  }
61  for (int i = 0; i < kernel_size; ++i) {
62  conv_params->kernel[i] = av_int2float(avio_rl32(model_file_context));
63  }
64 
65  conv_params->biases = NULL;
66  if (conv_params->has_bias) {
67  conv_params->biases = av_malloc(conv_params->output_num * sizeof(float));
68  if (!conv_params->biases){
69  av_freep(&conv_params->kernel);
70  av_freep(&conv_params);
71  return 0;
72  }
73  for (int i = 0; i < conv_params->output_num; ++i){
74  conv_params->biases[i] = av_int2float(avio_rl32(model_file_context));
75  }
76  }
77 
78  layer->params = conv_params;
79 
80  layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
81  layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
82  dnn_size += 8;
83  return dnn_size;
84 }
85 
86 int dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_indexes,
87  int32_t output_operand_index, const void *parameters)
88 {
89  float *output;
90  int32_t input_operand_index = input_operand_indexes[0];
91  int number = operands[input_operand_index].dims[0];
92  int height = operands[input_operand_index].dims[1];
93  int width = operands[input_operand_index].dims[2];
94  int channel = operands[input_operand_index].dims[3];
95  const float *input = operands[input_operand_index].data;
96  const ConvolutionalParams *conv_params = (const ConvolutionalParams *)parameters;
97 
98  int radius = conv_params->kernel_size >> 1;
99  int src_linesize = width * conv_params->input_num;
100  int filter_linesize = conv_params->kernel_size * conv_params->input_num;
101  int filter_size = conv_params->kernel_size * filter_linesize;
102  int pad_size = (conv_params->padding_method == VALID) ? (conv_params->kernel_size - 1) / 2 * conv_params->dilation : 0;
103 
104  DnnOperand *output_operand = &operands[output_operand_index];
105  output_operand->dims[0] = number;
106  output_operand->dims[1] = height - pad_size * 2;
107  output_operand->dims[2] = width - pad_size * 2;
108  output_operand->dims[3] = conv_params->output_num;
109  output_operand->data_type = operands[input_operand_index].data_type;
110  output_operand->length = calculate_operand_data_length(output_operand);
111  output_operand->data = av_realloc(output_operand->data, output_operand->length);
112  if (!output_operand->data)
113  return -1;
114  output = output_operand->data;
115 
116  av_assert0(channel == conv_params->input_num);
117 
118  for (int y = pad_size; y < height - pad_size; ++y) {
119  for (int x = pad_size; x < width - pad_size; ++x) {
120  for (int n_filter = 0; n_filter < conv_params->output_num; ++n_filter) {
121  if (conv_params->has_bias)
122  output[n_filter] = conv_params->biases[n_filter];
123  else
124  output[n_filter] = 0.f;
125 
126  for (int ch = 0; ch < conv_params->input_num; ++ch) {
127  for (int kernel_y = 0; kernel_y < conv_params->kernel_size; ++kernel_y) {
128  for (int kernel_x = 0; kernel_x < conv_params->kernel_size; ++kernel_x) {
129  float input_pel;
130  if (conv_params->padding_method == SAME_CLAMP_TO_EDGE) {
131  int y_pos = CLAMP_TO_EDGE(y + (kernel_y - radius) * conv_params->dilation, height);
132  int x_pos = CLAMP_TO_EDGE(x + (kernel_x - radius) * conv_params->dilation, width);
133  input_pel = input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
134  } else {
135  int y_pos = y + (kernel_y - radius) * conv_params->dilation;
136  int x_pos = x + (kernel_x - radius) * conv_params->dilation;
137  input_pel = (x_pos < 0 || x_pos >= width || y_pos < 0 || y_pos >= height) ? 0.0 :
138  input[y_pos * src_linesize + x_pos * conv_params->input_num + ch];
139  }
140 
141 
142  output[n_filter] += input_pel * conv_params->kernel[n_filter * filter_size + kernel_y * filter_linesize +
143  kernel_x * conv_params->input_num + ch];
144  }
145  }
146  }
147  switch (conv_params->activation){
148  case RELU:
149  output[n_filter] = FFMAX(output[n_filter], 0.0);
150  break;
151  case TANH:
152  output[n_filter] = 2.0f / (1.0f + exp(-2.0f * output[n_filter])) - 1.0f;
153  break;
154  case SIGMOID:
155  output[n_filter] = 1.0f / (1.0f + exp(-output[n_filter]));
156  break;
157  case NONE:
158  break;
159  case LEAKY_RELU:
160  output[n_filter] = FFMAX(output[n_filter], 0.0) + 0.2 * FFMIN(output[n_filter], 0.0);
161  }
162  }
163  output += conv_params->output_num;
164  }
165  }
166  return 0;
167 }
#define NULL
Definition: coverity.c:32
Bytestream IO Context.
Definition: avio.h:161
void * av_realloc(void *ptr, size_t size)
Allocate, reallocate, or free a block of memory.
Definition: mem.c:135
static av_always_inline float av_int2float(uint32_t i)
Reinterpret a 32-bit integer as a float.
Definition: intfloat.h:40
int32_t input_operand_indexes[4]
a layer can have multiple inputs and one output.
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
#define av_malloc(s)
#define f(width, name)
Definition: cbs_vp9.c:255
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
#define height
DNNDataType data_type
support different kinds of data type such as float, half float, int8 etc, first support float now...
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
unsigned int avio_rl32(AVIOContext *s)
Definition: aviobuf.c:772
void * data
data pointer with data length in bytes.
simple assert() macros that are a bit more flexible than ISO C assert().
int32_t dims[4]
there are two memory layouts, NHWC or NCHW, so we use dims, dims[0] is Number.
#define FFMAX(a, b)
Definition: common.h:94
int8_t exp
Definition: eval.c:72
int dnn_load_layer_conv2d(Layer *layer, AVIOContext *model_file_context, int file_size)
int dnn_execute_layer_conv2d(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index, const void *parameters)
#define FFMIN(a, b)
Definition: common.h:96
#define width
Definition: af_afade.c:54
int32_t
#define CLAMP_TO_EDGE(x, w)
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
uint8_t pi<< 24) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_U8,(uint64_t)((*(const uint8_t *) pi-0x80U))<< 56) CONV_FUNC(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16,(*(const int16_t *) pi >>8)+0x80) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_S16,(uint64_t)(*(const int16_t *) pi)<< 48) CONV_FUNC(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32,(*(const int32_t *) pi >>24)+0x80) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_S32,(uint64_t)(*(const int32_t *) pi)<< 32) CONV_FUNC(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S64,(*(const int64_t *) pi >>56)+0x80) CONV_FUNC(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S64,*(const int64_t *) pi *(1.0f/(UINT64_C(1)<< 63))) CONV_FUNC(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S64,*(const int64_t *) pi *(1.0/(UINT64_C(1)<< 63))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_FLT, llrintf(*(const float *) pi *(UINT64_C(1)<< 63))) CONV_FUNC(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31)))) CONV_FUNC(AV_SAMPLE_FMT_S64, int64_t, AV_SAMPLE_FMT_DBL, llrint(*(const double *) pi *(UINT64_C(1)<< 63)))#define FMT_PAIR_FUNC(out, in) static conv_func_type *const fmt_pair_to_conv_functions[AV_SAMPLE_FMT_NB *AV_SAMPLE_FMT_NB]={FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_U8), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_S16), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_S32), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_FLT), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_DBL), FMT_PAIR_FUNC(AV_SAMPLE_FMT_U8, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S16, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S32, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_FLT, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_DBL, AV_SAMPLE_FMT_S64), FMT_PAIR_FUNC(AV_SAMPLE_FMT_S64, AV_SAMPLE_FMT_S64),};static void cpy1(uint8_t **dst, const uint8_t **src, int len){memcpy(*dst,*src, len);}static void cpy2(uint8_t **dst, const uint8_t **src, int len){memcpy(*dst,*src, 2 *len);}static void cpy4(uint8_t **dst, const uint8_t **src, int len){memcpy(*dst,*src, 4 *len);}static void cpy8(uint8_t **dst, const uint8_t **src, int len){memcpy(*dst,*src, 8 *len);}AudioConvert *swri_audio_convert_alloc(enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, const int *ch_map, int flags){AudioConvert *ctx;conv_func_type *f=fmt_pair_to_conv_functions[av_get_packed_sample_fmt(out_fmt)+AV_SAMPLE_FMT_NB *av_get_packed_sample_fmt(in_fmt)];if(!f) return NULL;ctx=av_mallocz(sizeof(*ctx));if(!ctx) return NULL;if(channels==1){in_fmt=av_get_planar_sample_fmt(in_fmt);out_fmt=av_get_planar_sample_fmt(out_fmt);}ctx->channels=channels;ctx->conv_f=f;ctx->ch_map=ch_map;if(in_fmt==AV_SAMPLE_FMT_U8||in_fmt==AV_SAMPLE_FMT_U8P) memset(ctx->silence, 0x80, sizeof(ctx->silence));if(out_fmt==in_fmt &&!ch_map){switch(av_get_bytes_per_sample(in_fmt)){case 1:ctx->simd_f=cpy1;break;case 2:ctx->simd_f=cpy2;break;case 4:ctx->simd_f=cpy4;break;case 8:ctx->simd_f=cpy8;break;}}if(HAVE_X86ASM &&1) swri_audio_convert_init_x86(ctx, out_fmt, in_fmt, channels);if(ARCH_ARM) swri_audio_convert_init_arm(ctx, out_fmt, in_fmt, channels);if(ARCH_AARCH64) swri_audio_convert_init_aarch64(ctx, out_fmt, in_fmt, channels);return ctx;}void swri_audio_convert_free(AudioConvert **ctx){av_freep(ctx);}int swri_audio_convert(AudioConvert *ctx, AudioData *out, AudioData *in, int len){int ch;int off=0;const int os=(out->planar?1:out->ch_count)*out->bps;unsigned misaligned=0;av_assert0(ctx->channels==out->ch_count);if(ctx->in_simd_align_mask){int planes=in->planar?in->ch_count:1;unsigned m=0;for(ch=0;ch< planes;ch++) m|=(intptr_t) in->ch[ch];misaligned|=m &ctx->in_simd_align_mask;}if(ctx->out_simd_align_mask){int planes=out->planar?out->ch_count:1;unsigned m=0;for(ch=0;ch< planes;ch++) m|=(intptr_t) out->ch[ch];misaligned|=m &ctx->out_simd_align_mask;}if(ctx->simd_f &&!ctx->ch_map &&!misaligned){off=len &~15;av_assert1(off >=0);av_assert1(off<=len);av_assert2(ctx->channels==SWR_CH_MAX||!in->ch[ctx->channels]);if(off >0){if(out->planar==in->planar){int planes=out->planar?out->ch_count:1;for(ch=0;ch< planes;ch++){ctx->simd_f(out-> ch ch
Definition: audioconvert.c:56
channel
Use these values when setting the channel map with ebur128_set_channel().
Definition: ebur128.h:39
int32_t calculate_operand_data_length(const DnnOperand *oprd)
void * params
#define av_freep(p)
int32_t output_operand_index