FFmpeg
Loading...
Searching...
No Matches
tx_float_init.c
Go to the documentation of this file.
1/*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#define TX_FLOAT
20#include "libavutil/tx_priv.h"
22#include "libavutil/mem.h"
23#include "libavutil/x86/cpu.h"
24
25#include "config.h"
26
27TX_DECL_FN(fft2, sse3)
28TX_DECL_FN(fft4_fwd, sse2)
29TX_DECL_FN(fft4_inv, sse2)
30TX_DECL_FN(fft8, sse3)
31TX_DECL_FN(fft8_ns, sse3)
32TX_DECL_FN(fft8, avx)
33TX_DECL_FN(fft8_ns, avx)
34TX_DECL_FN(fft15, avx2)
35TX_DECL_FN(fft15_ns, avx2)
36TX_DECL_FN(fft16, avx)
37TX_DECL_FN(fft16_ns, avx)
38TX_DECL_FN(fft16, fma3)
39TX_DECL_FN(fft16_ns, fma3)
40TX_DECL_FN(fft32, avx)
41TX_DECL_FN(fft32_ns, avx)
42TX_DECL_FN(fft32, fma3)
43TX_DECL_FN(fft32_ns, fma3)
44TX_DECL_FN(fft_sr, avx)
45TX_DECL_FN(fft_sr_ns, avx)
46TX_DECL_FN(fft_sr, fma3)
47TX_DECL_FN(fft_sr_ns, fma3)
48TX_DECL_FN(fft_sr, avx2)
49TX_DECL_FN(fft_sr_ns, avx2)
50
51TX_DECL_FN(fft_pfa_15xM, avx2)
52TX_DECL_FN(fft_pfa_15xM_ns, avx2)
53
54TX_DECL_FN(mdct_inv, avx2)
55
56TX_DECL_FN(fft2_asm, sse3)
57TX_DECL_FN(fft4_fwd_asm, sse2)
58TX_DECL_FN(fft4_inv_asm, sse2)
59TX_DECL_FN(fft8_asm, sse3)
60TX_DECL_FN(fft8_asm, avx)
61TX_DECL_FN(fft16_asm, avx)
62TX_DECL_FN(fft16_asm, fma3)
63TX_DECL_FN(fft32_asm, avx)
64TX_DECL_FN(fft32_asm, fma3)
65TX_DECL_FN(fft_sr_asm, avx)
66TX_DECL_FN(fft_sr_asm, fma3)
67TX_DECL_FN(fft_sr_asm, avx2)
68
69TX_DECL_FN(fft_pfa_15xM_asm, avx2)
70
71#define DECL_INIT_FN(basis, interleave) \
72static av_cold int b ##basis## _i ##interleave(AVTXContext *s, \
73 const FFTXCodelet *cd, \
74 uint64_t flags, \
75 FFTXCodeletOptions *opts, \
76 int len, int inv, \
77 const void *scale) \
78{ \
79 ff_tx_init_tabs_float(len); \
80 if (cd->max_len == 2) \
81 return ff_tx_gen_ptwo_revtab(s, opts); \
82 else \
83 return ff_tx_gen_split_radix_parity_revtab(s, len, inv, opts, \
84 basis, interleave); \
85}
86
87DECL_INIT_FN(8, 0)
88DECL_INIT_FN(8, 2)
89
90#if ARCH_X86_64
91static av_cold int factor_init(AVTXContext *s, const FFTXCodelet *cd,
92 uint64_t flags, FFTXCodeletOptions *opts,
93 int len, int inv, const void *scale)
94{
95 int ret;
96
97 /* The transformations below are performed in the gather domain,
98 * so override the option and let the infrastructure convert the map
99 * to SCATTER if needed. */
100 FFTXCodeletOptions sub_opts = { .map_dir = FF_TX_MAP_GATHER };
101
102 TX_TAB(ff_tx_init_tabs)(len);
103
104 if (len == 15)
105 ret = ff_tx_gen_pfa_input_map(s, &sub_opts, 3, 5);
106 else
107 ret = ff_tx_gen_default_map(s, &sub_opts);
108
109 if (ret < 0)
110 return ret;
111
112 if (len == 15) {
113 int cnt = 0, tmp[15];
114
115 /* Special permutation to simplify loads in the pre-permuted version */
116 memcpy(tmp, s->map, 15*sizeof(*tmp));
117 for (int i = 1; i < 15; i += 3) {
118 s->map[cnt] = tmp[i];
119 cnt++;
120 }
121 for (int i = 2; i < 15; i += 3) {
122 s->map[cnt] = tmp[i];
123 cnt++;
124 }
125 for (int i = 0; i < 15; i += 3) {
126 s->map[cnt] = tmp[i];
127 cnt++;
128 }
129 memmove(&s->map[7], &s->map[6], 4*sizeof(int));
130 memmove(&s->map[3], &s->map[1], 4*sizeof(int));
131 s->map[1] = tmp[2];
132 s->map[2] = tmp[0];
133 }
134
135 return 0;
136}
137
138static av_cold int m_inv_init(AVTXContext *s, const FFTXCodelet *cd,
139 uint64_t flags, FFTXCodeletOptions *opts,
140 int len, int inv, const void *scale)
141{
142 int ret;
143 FFTXCodeletOptions sub_opts = { .map_dir = FF_TX_MAP_GATHER };
144
145 s->scale_d = *((SCALE_TYPE *)scale);
146 s->scale_f = s->scale_d;
147
148 flags &= ~FF_TX_OUT_OF_PLACE; /* We want the subtransform to be */
149 flags |= AV_TX_INPLACE; /* in-place */
150 flags |= FF_TX_PRESHUFFLE; /* This function handles the permute step */
151 flags |= FF_TX_ASM_CALL; /* We want an assembly function, not C */
152
153 if ((ret = ff_tx_init_subtx(s, TX_TYPE(FFT), flags, &sub_opts, len >> 1,
154 inv, scale)))
155 return ret;
156
157 s->map = av_malloc(len*sizeof(*s->map));
158 if (!s->map)
159 return AVERROR(ENOMEM);
160
161 memcpy(s->map, s->sub->map, (len >> 1)*sizeof(*s->map));
162 /* Invert lookup table for unstrided path */
163 for (int i = 0; i < (len >> 1); i++)
164 s->map[(len >> 1) + s->map[i]] = i;
165
166 if ((ret = ff_tx_mdct_gen_exp_float(s, s->map)))
167 return ret;
168
169 return 0;
170}
171
173 const FFTXCodelet *cd,
174 uint64_t flags,
176 int len, int inv,
177 const void *scale)
178{
179 int ret;
180 int sub_len = len / cd->factors[0];
181 FFTXCodeletOptions sub_opts = { .map_dir = FF_TX_MAP_SCATTER };
182
183 flags &= ~FF_TX_OUT_OF_PLACE; /* We want the subtransform to be */
184 flags |= AV_TX_INPLACE; /* in-place */
185 flags |= FF_TX_PRESHUFFLE; /* This function handles the permute step */
186 flags |= FF_TX_ASM_CALL; /* We want an assembly function, not C */
187
188 if ((ret = ff_tx_init_subtx(s, TX_TYPE(FFT), flags, &sub_opts,
189 sub_len, inv, scale)))
190 return ret;
191
192 if ((ret = ff_tx_gen_compound_mapping(s, opts, s->inv, cd->factors[0], sub_len)))
193 return ret;
194
195 if (cd->factors[0] == 15) {
196 int tmp[15];
197
198 /* Our 15-point transform is also a compound one, so embed its input map */
199 TX_EMBED_INPUT_PFA_MAP(s->map, len, 3, 5);
200
201 /* Special permutation to simplify loads in the pre-permuted version */
202 for (int k = 0; k < s->sub[0].len; k++) {
203 int cnt = 0;
204 memcpy(tmp, &s->map[k*15], 15*sizeof(*tmp));
205 for (int i = 1; i < 15; i += 3) {
206 s->map[k*15 + cnt] = tmp[i];
207 cnt++;
208 }
209 for (int i = 2; i < 15; i += 3) {
210 s->map[k*15 + cnt] = tmp[i];
211 cnt++;
212 }
213 for (int i = 0; i < 15; i += 3) {
214 s->map[k*15 + cnt] = tmp[i];
215 cnt++;
216 }
217 memmove(&s->map[k*15 + 7], &s->map[k*15 + 6], 4*sizeof(int));
218 memmove(&s->map[k*15 + 3], &s->map[k*15 + 1], 4*sizeof(int));
219 s->map[k*15 + 1] = tmp[2];
220 s->map[k*15 + 2] = tmp[0];
221 }
222 }
223
224 if (!(s->tmp = av_malloc(len*sizeof(*s->tmp))))
225 return AVERROR(ENOMEM);
226
227 TX_TAB(ff_tx_init_tabs)(len / sub_len);
228
229 return 0;
230}
231#endif
232
234 TX_DEF(fft2, FFT, 2, 2, 2, 0, 128, NULL, sse3, SSE3, AV_TX_INPLACE, 0),
235 TX_DEF(fft2_asm, FFT, 2, 2, 2, 0, 192, b8_i0, sse3, SSE3,
237 TX_DEF(fft2, FFT, 2, 2, 2, 0, 192, b8_i0, sse3, SSE3, AV_TX_INPLACE | FF_TX_PRESHUFFLE, 0),
238 TX_DEF(fft4_fwd, FFT, 4, 4, 2, 0, 128, NULL, sse2, SSE2, AV_TX_INPLACE | FF_TX_FORWARD_ONLY, 0),
239 TX_DEF(fft4_fwd_asm, FFT, 4, 4, 2, 0, 192, b8_i0, sse2, SSE2,
241 TX_DEF(fft4_inv_asm, FFT, 4, 4, 2, 0, 128, NULL, sse2, SSE2,
243 TX_DEF(fft4_fwd, FFT, 4, 4, 2, 0, 192, b8_i0, sse2, SSE2, AV_TX_INPLACE | FF_TX_PRESHUFFLE, 0),
244 TX_DEF(fft4_inv, FFT, 4, 4, 2, 0, 128, NULL, sse2, SSE2, AV_TX_INPLACE | FF_TX_INVERSE_ONLY, 0),
245 TX_DEF(fft8, FFT, 8, 8, 2, 0, 128, b8_i0, sse3, SSE3, AV_TX_INPLACE, 0),
246 TX_DEF(fft8_asm, FFT, 8, 8, 2, 0, 192, b8_i0, sse3, SSE3,
248 TX_DEF(fft8_ns, FFT, 8, 8, 2, 0, 192, b8_i0, sse3, SSE3, AV_TX_INPLACE | FF_TX_PRESHUFFLE, 0),
249 TX_DEF(fft8, FFT, 8, 8, 2, 0, 256, b8_i0, avx, AVX, AV_TX_INPLACE, AV_CPU_FLAG_AVXSLOW),
250 TX_DEF(fft8_asm, FFT, 8, 8, 2, 0, 320, b8_i0, avx, AVX,
252 TX_DEF(fft8_ns, FFT, 8, 8, 2, 0, 320, b8_i0, avx, AVX, AV_TX_INPLACE | FF_TX_PRESHUFFLE,
254 TX_DEF(fft16, FFT, 16, 16, 2, 0, 256, b8_i2, avx, AVX, AV_TX_INPLACE, AV_CPU_FLAG_AVXSLOW),
255 TX_DEF(fft16_asm, FFT, 16, 16, 2, 0, 320, b8_i2, avx, AVX,
257 TX_DEF(fft16_ns, FFT, 16, 16, 2, 0, 320, b8_i2, avx, AVX, AV_TX_INPLACE | FF_TX_PRESHUFFLE,
259 TX_DEF(fft16, FFT, 16, 16, 2, 0, 288, b8_i2, fma3, FMA3, AV_TX_INPLACE, AV_CPU_FLAG_AVXSLOW),
260 TX_DEF(fft16_asm, FFT, 16, 16, 2, 0, 352, b8_i2, fma3, FMA3,
262 TX_DEF(fft16_ns, FFT, 16, 16, 2, 0, 352, b8_i2, fma3, FMA3, AV_TX_INPLACE | FF_TX_PRESHUFFLE,
264
265#if ARCH_X86_64
266 TX_DEF(fft32, FFT, 32, 32, 2, 0, 256, b8_i2, avx, AVX, AV_TX_INPLACE, AV_CPU_FLAG_AVXSLOW),
267 TX_DEF(fft32_asm, FFT, 32, 32, 2, 0, 320, b8_i2, avx, AVX,
269 TX_DEF(fft32_ns, FFT, 32, 32, 2, 0, 320, b8_i2, avx, AVX, AV_TX_INPLACE | FF_TX_PRESHUFFLE,
271 TX_DEF(fft32, FFT, 32, 32, 2, 0, 288, b8_i2, fma3, FMA3, AV_TX_INPLACE, AV_CPU_FLAG_AVXSLOW),
272 TX_DEF(fft32_asm, FFT, 32, 32, 2, 0, 352, b8_i2, fma3, FMA3,
274 TX_DEF(fft32_ns, FFT, 32, 32, 2, 0, 352, b8_i2, fma3, FMA3, AV_TX_INPLACE | FF_TX_PRESHUFFLE,
276 TX_DEF(fft_sr, FFT, 64, 131072, 2, 0, 256, b8_i2, avx, AVX, 0, AV_CPU_FLAG_AVXSLOW),
277 TX_DEF(fft_sr_asm, FFT, 64, 131072, 2, 0, 320, b8_i2, avx, AVX,
279 TX_DEF(fft_sr_ns, FFT, 64, 131072, 2, 0, 320, b8_i2, avx, AVX, AV_TX_INPLACE | FF_TX_PRESHUFFLE,
281 TX_DEF(fft_sr, FFT, 64, 131072, 2, 0, 288, b8_i2, fma3, FMA3, 0, AV_CPU_FLAG_AVXSLOW),
282 TX_DEF(fft_sr_asm, FFT, 64, 131072, 2, 0, 352, b8_i2, fma3, FMA3,
284 TX_DEF(fft_sr_ns, FFT, 64, 131072, 2, 0, 352, b8_i2, fma3, FMA3, AV_TX_INPLACE | FF_TX_PRESHUFFLE,
286
287 TX_DEF(fft15, FFT, 15, 15, 15, 0, 320, factor_init, avx2, AVX2,
289 TX_DEF(fft15_ns, FFT, 15, 15, 15, 0, 384, factor_init, avx2, AVX2,
291
292 TX_DEF(fft_sr, FFT, 64, 131072, 2, 0, 320, b8_i2, avx2, AVX2, 0,
294 TX_DEF(fft_sr_asm, FFT, 64, 131072, 2, 0, 384, b8_i2, avx2, AVX2,
296 TX_DEF(fft_sr_ns, FFT, 64, 131072, 2, 0, 384, b8_i2, avx2, AVX2, AV_TX_INPLACE | FF_TX_PRESHUFFLE,
298
299 TX_DEF(fft_pfa_15xM, FFT, 60, TX_LEN_UNLIMITED, 15, 2, 320, fft_pfa_init, avx2, AVX2,
301 TX_DEF(fft_pfa_15xM_asm, FFT, 60, TX_LEN_UNLIMITED, 15, 2, 384, fft_pfa_init, avx2, AVX2,
303 TX_DEF(fft_pfa_15xM_ns, FFT, 60, TX_LEN_UNLIMITED, 15, 2, 384, fft_pfa_init, avx2, AVX2,
305
306 TX_DEF(mdct_inv, MDCT, 16, TX_LEN_UNLIMITED, 2, TX_FACTOR_ANY, 384, m_inv_init, avx2, AVX2,
308#endif
309
310 NULL,
311};
static av_cold int fft_pfa_init(AVTXContext *s, const FFTXCodelet *cd, uint64_t flags, FFTXCodeletOptions *opts, int len, int inv, const void *scale)
#define TX_TYPE
static AVDictionary * opts
#define flags(name, subs,...)
Definition cbs_h264.c:74
#define i(width, name, range_min, range_max)
Definition cbs_h264.c:63
#define s(width, name)
Definition cbs_vp9.c:198
#define NULL
Definition coverity.c:32
int ret
Definition ffplay.c:391
#define AVERROR(e)
Definition error.h:45
static av_always_inline void scale(vec_s32 res[4], vec_s16 res_packed[2], const int shift)
Macro definitions for various function/variable attributes.
#define av_cold
Definition attributes.h:117
#define AV_CPU_FLAG_SLOW_GATHER
CPU has slow gathers.
Definition cpu.h:63
#define AV_CPU_FLAG_AVXSLOW
AVX supported, but slow when using YMM registers (e.g. Bulldozer)
Definition cpu.h:52
Memory handling functions.
#define av_malloc(s)
Definition ops_static.c:52
int factors[TX_MAX_FACTORS]
Definition tx_priv.h:208
static uint8_t tmp[40]
Definition aes_ctr.c:52
int ff_tx_gen_default_map(AVTXContext *s, FFTXCodeletOptions *opts)
Definition tx.c:525
av_cold int ff_tx_init_subtx(AVTXContext *s, enum AVTXType type, uint64_t flags, FFTXCodeletOptions *opts, int len, int inv, const void *scale)
Definition tx.c:712
int ff_tx_gen_compound_mapping(AVTXContext *s, FFTXCodeletOptions *opts, int inv, int n, int m)
Definition tx.c:75
int ff_tx_gen_pfa_input_map(AVTXContext *s, FFTXCodeletOptions *opts, int d1, int d2)
Definition tx.c:44
@ AV_TX_INPLACE
Allows for in-place transformations, where input == output.
Definition tx.h:161
#define TX_LEN_UNLIMITED
Definition tx_priv.h:216
#define FF_TX_OUT_OF_PLACE
Definition tx_priv.h:154
#define FF_TX_ASM_CALL
Definition tx_priv.h:159
#define FF_TX_INVERSE_ONLY
Definition tx_priv.h:157
#define FF_TX_FORWARD_ONLY
Definition tx_priv.h:158
const FFTXCodelet *const ff_tx_codelet_list_float_x86[]
#define TX_DECL_FN(fn, suffix)
Definition tx_priv.h:68
@ FF_TX_MAP_GATHER
Definition tx_priv.h:176
@ FF_TX_MAP_SCATTER
Definition tx_priv.h:179
#define TX_FACTOR_ANY
Definition tx_priv.h:209
#define TX_DEF(fn, tx_type, len_min, len_max, f1, f2, p, init_fn, suffix, cf, cd_flags, cf2)
Definition tx_priv.h:71
int ff_tx_mdct_gen_exp_float(AVTXContext *s, int *pre_tab)
#define FF_TX_PRESHUFFLE
Definition tx_priv.h:156
#define TX_EMBED_INPUT_PFA_MAP(map, tot_len, d1, d2)
Definition tx_priv.h:271
static av_always_inline void fft15(TXComplex *out, TXComplex *in, ptrdiff_t stride)
av_cold void TX_TAB ff_tx_init_tabs(int len)
int len
#define DECL_INIT_FN(basis, interleave)