FFmpeg
Loading...
Searching...
No Matches
dsp_init.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2013 Seppo Tomperi
3 * Copyright (c) 2013-2014 Pierre-Edouard Lepere
4 * Copyright (c) 2023-2024 Wu Jianhua
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23#include "config.h"
24
25#include "libavutil/cpu.h"
26#include "libavutil/mem_internal.h"
27#include "libavutil/x86/cpu.h"
28#include "libavcodec/hevc/dsp.h"
29#include "libavcodec/x86/hevc/dsp.h"
30#include "libavcodec/x86/h26x/h2656dsp.h"
31
32void ff_hevc_dequant_8_ssse3(int16_t *coeffs, int16_t log2_size);
33
34#define LFC_FUNC(DIR, DEPTH, OPT) \
35void ff_hevc_ ## DIR ## _loop_filter_chroma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, const int *tc, const uint8_t *no_p, const uint8_t *no_q);
36
37#define LFL_FUNC(DIR, DEPTH, OPT) \
38void ff_hevc_ ## DIR ## _loop_filter_luma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, int beta, const int *tc, const uint8_t *no_p, const uint8_t *no_q);
39
40#define LFC_FUNCS(type, depth, opt) \
41 LFC_FUNC(h, depth, opt) \
42 LFC_FUNC(v, depth, opt)
43
44#define LFL_FUNCS(type, depth, opt) \
45 LFL_FUNC(h, depth, opt) \
46 LFL_FUNC(v, depth, opt)
47
48LFC_FUNCS(uint8_t, 8, sse2)
49LFC_FUNCS(uint8_t, 10, sse2)
50LFC_FUNCS(uint8_t, 12, sse2)
51LFC_FUNCS(uint8_t, 8, avx)
52LFC_FUNCS(uint8_t, 10, avx)
53LFC_FUNCS(uint8_t, 12, avx)
54LFL_FUNCS(uint8_t, 8, sse2)
55LFL_FUNCS(uint8_t, 10, sse2)
56LFL_FUNCS(uint8_t, 12, sse2)
57LFL_FUNCS(uint8_t, 8, ssse3)
58LFL_FUNCS(uint8_t, 10, ssse3)
59LFL_FUNCS(uint8_t, 12, ssse3)
60LFL_FUNCS(uint8_t, 8, avx)
61LFL_FUNCS(uint8_t, 10, avx)
62LFL_FUNCS(uint8_t, 12, avx)
63
64#define IDCT_DC_FUNCS(W, opt) \
65void ff_hevc_idct_ ## W ## _dc_8_ ## opt(int16_t *coeffs); \
66void ff_hevc_idct_ ## W ## _dc_10_ ## opt(int16_t *coeffs); \
67void ff_hevc_idct_ ## W ## _dc_12_ ## opt(int16_t *coeffs)
68
69IDCT_DC_FUNCS(4x4, sse2);
70IDCT_DC_FUNCS(8x8, sse2);
71IDCT_DC_FUNCS(16x16, sse2);
72IDCT_DC_FUNCS(32x32, sse2);
73IDCT_DC_FUNCS(16x16, avx2);
74IDCT_DC_FUNCS(32x32, avx2);
75
76#define IDCT_FUNCS(opt) \
77void ff_hevc_idct_4x4_8_ ## opt(int16_t *coeffs, int col_limit); \
78void ff_hevc_idct_4x4_10_ ## opt(int16_t *coeffs, int col_limit); \
79void ff_hevc_idct_8x8_8_ ## opt(int16_t *coeffs, int col_limit); \
80void ff_hevc_idct_8x8_10_ ## opt(int16_t *coeffs, int col_limit); \
81void ff_hevc_idct_16x16_8_ ## opt(int16_t *coeffs, int col_limit); \
82void ff_hevc_idct_16x16_10_ ## opt(int16_t *coeffs, int col_limit); \
83void ff_hevc_idct_32x32_8_ ## opt(int16_t *coeffs, int col_limit); \
84void ff_hevc_idct_32x32_10_ ## opt(int16_t *coeffs, int col_limit);
85
86IDCT_FUNCS(sse2)
87IDCT_FUNCS(avx)
88
89#define TRANSFORM_LUMA_FUNCS(opt) \
90void ff_hevc_transform_4x4_luma_8_ ## opt(int16_t *coeffs); \
91void ff_hevc_transform_4x4_luma_10_ ## opt(int16_t *coeffs)
92
93TRANSFORM_LUMA_FUNCS(sse2);
94TRANSFORM_LUMA_FUNCS(avx);
95
96#define ff_hevc_pel_filters ff_hevc_qpel_filters
97#define DECL_HV_FILTER(f) \
98 const uint8_t *hf = ff_hevc_ ## f ## _filters[mx]; \
99 const uint8_t *vf = ff_hevc_ ## f ## _filters[my];
100
101#define FW_PUT(p, a, b, depth, opt) \
102static void hevc_put_ ## a ## _ ## depth ## _##opt(int16_t *dst, const uint8_t *src, ptrdiff_t srcstride, \
103 int height, intptr_t mx, intptr_t my,int width) \
104{ \
105 DECL_HV_FILTER(p) \
106 ff_h2656_put_ ## b ## _ ## depth ## _##opt(dst, 2 * MAX_PB_SIZE, src, srcstride, height, hf, vf, width); \
107}
108
109#define FW_PUT_UNI(p, a, b, depth, opt) \
110static void hevc_put_uni_ ## a ## _ ## depth ## _##opt(uint8_t *dst, ptrdiff_t dststride, \
111 const uint8_t *src, ptrdiff_t srcstride, \
112 int height, intptr_t mx, intptr_t my, int width) \
113{ \
114 DECL_HV_FILTER(p) \
115 ff_h2656_put_uni_ ## b ## _ ## depth ## _##opt(dst, dststride, src, srcstride, height, hf, vf, width); \
116}
117
118#if ARCH_X86_64 && HAVE_SSE4_EXTERNAL
119
120#define FW_PUT_FUNCS(p, a, b, depth, opt) \
121 FW_PUT(p, a, b, depth, opt) \
122 FW_PUT_UNI(p, a, b, depth, opt)
123
124#define FW_PEL(w, depth, opt) FW_PUT_FUNCS(pel, pel_pixels##w, pixels##w, depth, opt)
125
126#define FW_DIR(npel, n, w, depth, opt) \
127 FW_PUT_FUNCS(npel, npel ## _h##w, n ## tap_h##w, depth, opt) \
128 FW_PUT_FUNCS(npel, npel ## _v##w, n ## tap_v##w, depth, opt)
129
130#define FW_DIR_HV(npel, n, w, depth, opt) \
131 FW_PUT_FUNCS(npel, npel ## _hv##w, n ## tap_hv##w, depth, opt)
132
133FW_PEL(4, 8, sse4)
134FW_PEL(6, 8, sse4)
135FW_PEL(8, 8, sse4)
136FW_PEL(12, 8, sse4)
137FW_PEL(16, 8, sse4)
138FW_PEL(4, 10, sse4)
139FW_PEL(6, 10, sse4)
140FW_PEL(8, 10, sse4)
141FW_PEL(4, 12, sse4)
142FW_PEL(6, 12, sse4)
143FW_PEL(8, 12, sse4)
144
145#define FW_EPEL(w, depth, opt) FW_DIR(epel, 4, w, depth, opt)
146#define FW_EPEL_HV(w, depth, opt) FW_DIR_HV(epel, 4, w, depth, opt)
147#define FW_EPEL_FUNCS(w, depth, opt) \
148 FW_EPEL(w, depth, opt) \
149 FW_EPEL_HV(w, depth, opt)
150
151FW_EPEL(12, 8, sse4)
152
153FW_EPEL_FUNCS(4, 8, sse4)
154FW_EPEL_FUNCS(6, 8, sse4)
155FW_EPEL_FUNCS(8, 8, sse4)
156FW_EPEL_FUNCS(16, 8, sse4)
157FW_EPEL_FUNCS(4, 10, sse4)
158FW_EPEL_FUNCS(6, 10, sse4)
159FW_EPEL_FUNCS(8, 10, sse4)
160FW_EPEL_FUNCS(4, 12, sse4)
161FW_EPEL_FUNCS(6, 12, sse4)
162FW_EPEL_FUNCS(8, 12, sse4)
163
164#define FW_QPEL(w, depth, opt) FW_DIR(qpel, 8, w, depth, opt)
165#define FW_QPEL_HV(w, depth, opt) FW_DIR_HV(qpel, 8, w, depth, opt)
166#define FW_QPEL_FUNCS(w, depth, opt) \
167 FW_QPEL(w, depth, opt) \
168 FW_QPEL_HV(w, depth, opt)
169
170FW_QPEL(12, 8, sse4)
171FW_QPEL(16, 8, sse4)
172
173FW_QPEL_FUNCS(4, 8, sse4)
174FW_QPEL_FUNCS(8, 8, sse4)
175FW_QPEL_FUNCS(4, 10, sse4)
176FW_QPEL_FUNCS(8, 10, sse4)
177FW_QPEL_FUNCS(4, 12, sse4)
178FW_QPEL_FUNCS(8, 12, sse4)
179
180#if HAVE_AVX2_EXTERNAL
181
182FW_PEL(32, 8, avx2)
183FW_PUT(pel, pel_pixels16, pixels16, 10, avx2)
184
185FW_EPEL(32, 8, avx2)
186FW_EPEL(16, 10, avx2)
187
188FW_EPEL_HV(32, 8, avx2)
189FW_EPEL_HV(16, 10, avx2)
190
191FW_QPEL(32, 8, avx2)
192FW_QPEL(16, 10, avx2)
193
194FW_QPEL_HV(16, 10, avx2)
195
196#endif
197#endif
198
199#define mc_rep_func(name, bitd, step, W, opt) \
200static void hevc_put_##name##W##_##bitd##_##opt(int16_t *_dst, \
201 const uint8_t *_src, ptrdiff_t _srcstride, int height, \
202 intptr_t mx, intptr_t my, int width) \
203{ \
204 int i; \
205 int16_t *dst; \
206 for (i = 0; i < W; i += step) { \
207 const uint8_t *src = _src + (i * ((bitd + 7) / 8)); \
208 dst = _dst + i; \
209 hevc_put_##name##step##_##bitd##_##opt(dst, src, _srcstride, height, mx, my, width); \
210 } \
211}
212#define mc_rep_uni_func(name, bitd, step, W, opt) \
213static void hevc_put_uni_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, \
214 const uint8_t *_src, ptrdiff_t _srcstride, int height, \
215 intptr_t mx, intptr_t my, int width) \
216{ \
217 int i; \
218 uint8_t *dst; \
219 for (i = 0; i < W; i += step) { \
220 const uint8_t *src = _src + (i * ((bitd + 7) / 8)); \
221 dst = _dst + (i * ((bitd + 7) / 8)); \
222 hevc_put_uni_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, \
223 height, mx, my, width); \
224 } \
225}
226#define mc_rep_bi_func(name, bitd, step, W, opt) \
227static void ff_hevc_put_bi_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, const uint8_t *_src, \
228 ptrdiff_t _srcstride, const int16_t *_src2, \
229 int height, intptr_t mx, intptr_t my, int width) \
230{ \
231 int i; \
232 uint8_t *dst; \
233 for (i = 0; i < W ; i += step) { \
234 const uint8_t *src = _src + (i * ((bitd + 7) / 8)); \
235 const int16_t *src2 = _src2 + i; \
236 dst = _dst + (i * ((bitd + 7) / 8)); \
237 ff_hevc_put_bi_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, \
238 height, mx, my, width); \
239 } \
240}
241
242#define mc_rep_funcs(name, bitd, step, W, opt) \
243 mc_rep_func(name, bitd, step, W, opt) \
244 mc_rep_uni_func(name, bitd, step, W, opt) \
245 mc_rep_bi_func(name, bitd, step, W, opt)
246
247#define mc_rep_func2(name, bitd, step1, step2, W, opt) \
248static void hevc_put_##name##W##_##bitd##_##opt(int16_t *dst, \
249 const uint8_t *src, ptrdiff_t _srcstride, int height, \
250 intptr_t mx, intptr_t my, int width) \
251{ \
252 hevc_put_##name##step1##_##bitd##_##opt(dst, src, _srcstride, height, mx, my, width); \
253 hevc_put_##name##step2##_##bitd##_##opt(dst + step1, src + (step1 * ((bitd + 7) / 8)), \
254 _srcstride, height, mx, my, width); \
255}
256#define mc_rep_uni_func2(name, bitd, step1, step2, W, opt) \
257static void hevc_put_uni_##name##W##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, \
258 const uint8_t *src, ptrdiff_t _srcstride, int height, \
259 intptr_t mx, intptr_t my, int width) \
260{ \
261 hevc_put_uni_##name##step1##_##bitd##_##opt(dst, dststride, src, _srcstride, height, mx, my, width); \
262 hevc_put_uni_##name##step2##_##bitd##_##opt(dst + (step1 * ((bitd + 7) / 8)), dststride, \
263 src + (step1 * ((bitd + 7) / 8)), _srcstride, \
264 height, mx, my, width); \
265}
266#define mc_rep_bi_func2(name, bitd, step1, step2, W, opt) \
267static void ff_hevc_put_bi_##name##W##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, const uint8_t *src, \
268 ptrdiff_t _srcstride, const int16_t *src2, \
269 int height, intptr_t mx, intptr_t my, int width) \
270{ \
271 ff_hevc_put_bi_##name##step1##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, height, mx, my, width);\
272 ff_hevc_put_bi_##name##step2##_##bitd##_##opt(dst + (step1 * ((bitd + 7) / 8)), dststride, \
273 src + (step1 * ((bitd + 7) / 8)), _srcstride, \
274 src2 + step1, height, mx, my, width); \
275}
276
277#define mc_rep_funcs2(name, bitd, step1, step2, W, opt) \
278 mc_rep_func2(name, bitd, step1, step2, W, opt) \
279 mc_rep_uni_func2(name, bitd, step1, step2, W, opt) \
280 mc_rep_bi_func2(name, bitd, step1, step2, W, opt)
281
282#if ARCH_X86_64 && HAVE_SSE4_EXTERNAL
283
284#define mc_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
285static void hevc_put_##name##width1##_10_##opt1(int16_t *dst, const uint8_t *src, ptrdiff_t _srcstride, \
286 int height, intptr_t mx, intptr_t my, int width) \
287 \
288{ \
289 hevc_put_##name##width2##_10_##opt1(dst, src, _srcstride, height, mx, my, width); \
290 hevc_put_##name##width3##_10_##opt2(dst+ width2, src+ width4, _srcstride, height, mx, my, width); \
291}
292
293#define mc_bi_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
294static void ff_hevc_put_bi_##name##width1##_10_##opt1(uint8_t *dst, ptrdiff_t dststride, const uint8_t *src, \
295 ptrdiff_t _srcstride, const int16_t *src2, \
296 int height, intptr_t mx, intptr_t my, int width) \
297{ \
298 ff_hevc_put_bi_##name##width2##_10_##opt1(dst, dststride, src, _srcstride, src2, \
299 height, mx, my, width); \
300 ff_hevc_put_bi_##name##width3##_10_##opt2(dst+width4, dststride, src+width4, _srcstride, src2+width2, \
301 height, mx, my, width); \
302}
303
304#define mc_uni_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
305static void hevc_put_uni_##name##width1##_10_##opt1(uint8_t *dst, ptrdiff_t dststride, \
306 const uint8_t *src, ptrdiff_t _srcstride, int height, \
307 intptr_t mx, intptr_t my, int width) \
308{ \
309 hevc_put_uni_##name##width2##_10_##opt1(dst, dststride, src, _srcstride, \
310 height, mx, my, width); \
311 hevc_put_uni_##name##width3##_10_##opt2(dst+width4, dststride, src+width4, _srcstride, \
312 height, mx, my, width); \
313}
314
315#define mc_rep_mixs_10(name, width1, width2, width3, opt1, opt2, width4) \
316mc_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
317mc_bi_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
318mc_uni_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4)
319
320#define mc_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
321static void hevc_put_##name##width1##_8_##opt1(int16_t *dst, const uint8_t *src, ptrdiff_t _srcstride, \
322 int height, intptr_t mx, intptr_t my, int width) \
323 \
324{ \
325 hevc_put_##name##width2##_8_##opt1(dst, src, _srcstride, height, mx, my, width); \
326 hevc_put_##name##width3##_8_##opt2(dst+ width2, src+ width2, _srcstride, height, mx, my, width); \
327}
328
329#define mc_bi_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
330static void ff_hevc_put_bi_##name##width1##_8_##opt1(uint8_t *dst, ptrdiff_t dststride, const uint8_t *src, \
331 ptrdiff_t _srcstride, const int16_t *src2, \
332 int height, intptr_t mx, intptr_t my, int width) \
333{ \
334 ff_hevc_put_bi_##name##width2##_8_##opt1(dst, dststride, src, _srcstride, \
335 src2, height, mx, my, width); \
336 ff_hevc_put_bi_##name##width3##_8_##opt2(dst+width2, dststride, src+width2, _srcstride, \
337 src2+width2, height, mx, my, width); \
338}
339
340#define mc_uni_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
341static void hevc_put_uni_##name##width1##_8_##opt1(uint8_t *dst, ptrdiff_t dststride, \
342 const uint8_t *src, ptrdiff_t _srcstride, int height, \
343 intptr_t mx, intptr_t my, int width) \
344{ \
345 hevc_put_uni_##name##width2##_8_##opt1(dst, dststride, src, _srcstride, \
346 height, mx, my, width); \
347 hevc_put_uni_##name##width3##_8_##opt2(dst+width2, dststride, src+width2, _srcstride, \
348 height, mx, my, width); \
349}
350
351#define mc_rep_mixs_8(name, width1, width2, width3, opt1, opt2) \
352mc_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
353mc_bi_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
354mc_uni_rep_mix_8(name, width1, width2, width3, opt1, opt2)
355
356#if HAVE_AVX2_EXTERNAL
357
358mc_rep_mixs_8(pel_pixels, 48, 32, 16, avx2, sse4)
359mc_rep_mixs_8(epel_hv, 48, 32, 16, avx2, sse4)
360mc_rep_mixs_8(epel_h , 48, 32, 16, avx2, sse4)
361mc_rep_mixs_8(epel_v , 48, 32, 16, avx2, sse4)
362
363mc_rep_mix_10(pel_pixels, 24, 16, 8, avx2, sse4, 32)
364mc_bi_rep_mix_10(pel_pixels,24, 16, 8, avx2, sse4, 32)
365mc_rep_mixs_10(epel_hv, 24, 16, 8, avx2, sse4, 32)
366mc_rep_mixs_10(epel_h , 24, 16, 8, avx2, sse4, 32)
367mc_rep_mixs_10(epel_v , 24, 16, 8, avx2, sse4, 32)
368
369
370mc_rep_mixs_10(qpel_h , 24, 16, 8, avx2, sse4, 32)
371mc_rep_mixs_10(qpel_v , 24, 16, 8, avx2, sse4, 32)
372mc_rep_mixs_10(qpel_hv, 24, 16, 8, avx2, sse4, 32)
373
374
375mc_rep_funcs(pel_pixels, 8, 32, 64, avx2)
376
377mc_rep_uni_func(pel_pixels, 8, 64, 128, avx2)//used for 10bit
378mc_rep_uni_func(pel_pixels, 8, 32, 96, avx2) //used for 10bit
379
380mc_rep_func(pel_pixels, 10, 16, 32, avx2)
381mc_rep_func(pel_pixels, 10, 16, 48, avx2)
382mc_rep_func(pel_pixels, 10, 32, 64, avx2)
383
384mc_rep_bi_func(pel_pixels, 10, 16, 32, avx2)
385mc_rep_bi_func(pel_pixels, 10, 16, 48, avx2)
386mc_rep_bi_func(pel_pixels, 10, 32, 64, avx2)
387
388mc_rep_funcs(epel_h, 8, 32, 64, avx2)
389
390mc_rep_funcs(epel_v, 8, 32, 64, avx2)
391
392mc_rep_funcs(epel_h, 10, 16, 32, avx2)
393mc_rep_funcs(epel_h, 10, 16, 48, avx2)
394mc_rep_funcs(epel_h, 10, 32, 64, avx2)
395
396mc_rep_funcs(epel_v, 10, 16, 32, avx2)
397mc_rep_funcs(epel_v, 10, 16, 48, avx2)
398mc_rep_funcs(epel_v, 10, 32, 64, avx2)
399
400
401mc_rep_funcs(epel_hv, 8, 32, 64, avx2)
402
403mc_rep_funcs(epel_hv, 10, 16, 32, avx2)
404mc_rep_funcs(epel_hv, 10, 16, 48, avx2)
405mc_rep_funcs(epel_hv, 10, 32, 64, avx2)
406
407mc_rep_funcs(qpel_h, 8, 32, 64, avx2)
408mc_rep_mixs_8(qpel_h , 48, 32, 16, avx2, sse4)
409
410mc_rep_funcs(qpel_v, 8, 32, 64, avx2)
411mc_rep_mixs_8(qpel_v, 48, 32, 16, avx2, sse4)
412
413mc_rep_funcs(qpel_h, 10, 16, 32, avx2)
414mc_rep_funcs(qpel_h, 10, 16, 48, avx2)
415mc_rep_funcs(qpel_h, 10, 32, 64, avx2)
416
417mc_rep_funcs(qpel_v, 10, 16, 32, avx2)
418mc_rep_funcs(qpel_v, 10, 16, 48, avx2)
419mc_rep_funcs(qpel_v, 10, 32, 64, avx2)
420
421mc_rep_funcs(qpel_hv, 10, 16, 32, avx2)
422mc_rep_funcs(qpel_hv, 10, 16, 48, avx2)
423mc_rep_funcs(qpel_hv, 10, 32, 64, avx2)
424
425#endif //AVX2
426
427mc_rep_funcs(pel_pixels, 8, 16, 64, sse4)
428mc_rep_funcs(pel_pixels, 8, 16, 48, sse4)
429mc_rep_funcs(pel_pixels, 8, 16, 32, sse4)
430mc_rep_funcs(pel_pixels, 8, 8, 24, sse4)
431mc_rep_funcs(pel_pixels,10, 8, 64, sse4)
432mc_rep_funcs(pel_pixels,10, 8, 48, sse4)
433mc_rep_funcs(pel_pixels,10, 8, 32, sse4)
434mc_rep_funcs(pel_pixels,10, 8, 24, sse4)
435mc_rep_funcs(pel_pixels,10, 8, 16, sse4)
436mc_rep_funcs(pel_pixels,10, 4, 12, sse4)
437mc_rep_funcs(pel_pixels,12, 8, 64, sse4)
438mc_rep_funcs(pel_pixels,12, 8, 48, sse4)
439mc_rep_funcs(pel_pixels,12, 8, 32, sse4)
440mc_rep_funcs(pel_pixels,12, 8, 24, sse4)
441mc_rep_funcs(pel_pixels,12, 8, 16, sse4)
442mc_rep_funcs(pel_pixels,12, 4, 12, sse4)
443
444mc_rep_funcs(epel_h, 8, 16, 64, sse4)
445mc_rep_funcs(epel_h, 8, 16, 48, sse4)
446mc_rep_funcs(epel_h, 8, 16, 32, sse4)
447mc_rep_funcs(epel_h, 8, 8, 24, sse4)
448mc_rep_funcs(epel_h,10, 8, 64, sse4)
449mc_rep_funcs(epel_h,10, 8, 48, sse4)
450mc_rep_funcs(epel_h,10, 8, 32, sse4)
451mc_rep_funcs(epel_h,10, 8, 24, sse4)
452mc_rep_funcs(epel_h,10, 8, 16, sse4)
453mc_rep_funcs(epel_h,10, 4, 12, sse4)
454mc_rep_funcs(epel_h,12, 8, 64, sse4)
455mc_rep_funcs(epel_h,12, 8, 48, sse4)
456mc_rep_funcs(epel_h,12, 8, 32, sse4)
457mc_rep_funcs(epel_h,12, 8, 24, sse4)
458mc_rep_funcs(epel_h,12, 8, 16, sse4)
459mc_rep_funcs(epel_h,12, 4, 12, sse4)
460mc_rep_funcs(epel_v, 8, 16, 64, sse4)
461mc_rep_funcs(epel_v, 8, 16, 48, sse4)
462mc_rep_funcs(epel_v, 8, 16, 32, sse4)
463mc_rep_funcs(epel_v, 8, 8, 24, sse4)
464mc_rep_funcs(epel_v,10, 8, 64, sse4)
465mc_rep_funcs(epel_v,10, 8, 48, sse4)
466mc_rep_funcs(epel_v,10, 8, 32, sse4)
467mc_rep_funcs(epel_v,10, 8, 24, sse4)
468mc_rep_funcs(epel_v,10, 8, 16, sse4)
469mc_rep_funcs(epel_v,10, 4, 12, sse4)
470mc_rep_funcs(epel_v,12, 8, 64, sse4)
471mc_rep_funcs(epel_v,12, 8, 48, sse4)
472mc_rep_funcs(epel_v,12, 8, 32, sse4)
473mc_rep_funcs(epel_v,12, 8, 24, sse4)
474mc_rep_funcs(epel_v,12, 8, 16, sse4)
475mc_rep_funcs(epel_v,12, 4, 12, sse4)
476mc_rep_funcs(epel_hv, 8, 16, 64, sse4)
477mc_rep_funcs(epel_hv, 8, 16, 48, sse4)
478mc_rep_funcs(epel_hv, 8, 16, 32, sse4)
479mc_rep_funcs(epel_hv, 8, 8, 24, sse4)
480mc_rep_funcs2(epel_hv,8, 8, 4, 12, sse4)
481mc_rep_funcs(epel_hv,10, 8, 64, sse4)
482mc_rep_funcs(epel_hv,10, 8, 48, sse4)
483mc_rep_funcs(epel_hv,10, 8, 32, sse4)
484mc_rep_funcs(epel_hv,10, 8, 24, sse4)
485mc_rep_funcs(epel_hv,10, 8, 16, sse4)
486mc_rep_funcs(epel_hv,10, 4, 12, sse4)
487mc_rep_funcs(epel_hv,12, 8, 64, sse4)
488mc_rep_funcs(epel_hv,12, 8, 48, sse4)
489mc_rep_funcs(epel_hv,12, 8, 32, sse4)
490mc_rep_funcs(epel_hv,12, 8, 24, sse4)
491mc_rep_funcs(epel_hv,12, 8, 16, sse4)
492mc_rep_funcs(epel_hv,12, 4, 12, sse4)
493
494mc_rep_funcs(qpel_h, 8, 16, 64, sse4)
495mc_rep_funcs(qpel_h, 8, 16, 48, sse4)
496mc_rep_funcs(qpel_h, 8, 16, 32, sse4)
497mc_rep_funcs(qpel_h, 8, 8, 24, sse4)
498mc_rep_funcs(qpel_h,10, 8, 64, sse4)
499mc_rep_funcs(qpel_h,10, 8, 48, sse4)
500mc_rep_funcs(qpel_h,10, 8, 32, sse4)
501mc_rep_funcs(qpel_h,10, 8, 24, sse4)
502mc_rep_funcs(qpel_h,10, 8, 16, sse4)
503mc_rep_funcs(qpel_h,10, 4, 12, sse4)
504mc_rep_funcs(qpel_h,12, 8, 64, sse4)
505mc_rep_funcs(qpel_h,12, 8, 48, sse4)
506mc_rep_funcs(qpel_h,12, 8, 32, sse4)
507mc_rep_funcs(qpel_h,12, 8, 24, sse4)
508mc_rep_funcs(qpel_h,12, 8, 16, sse4)
509mc_rep_funcs(qpel_h,12, 4, 12, sse4)
510mc_rep_funcs(qpel_v, 8, 16, 64, sse4)
511mc_rep_funcs(qpel_v, 8, 16, 48, sse4)
512mc_rep_funcs(qpel_v, 8, 16, 32, sse4)
513mc_rep_funcs(qpel_v, 8, 8, 24, sse4)
514mc_rep_funcs(qpel_v,10, 8, 64, sse4)
515mc_rep_funcs(qpel_v,10, 8, 48, sse4)
516mc_rep_funcs(qpel_v,10, 8, 32, sse4)
517mc_rep_funcs(qpel_v,10, 8, 24, sse4)
518mc_rep_funcs(qpel_v,10, 8, 16, sse4)
519mc_rep_funcs(qpel_v,10, 4, 12, sse4)
520mc_rep_funcs(qpel_v,12, 8, 64, sse4)
521mc_rep_funcs(qpel_v,12, 8, 48, sse4)
522mc_rep_funcs(qpel_v,12, 8, 32, sse4)
523mc_rep_funcs(qpel_v,12, 8, 24, sse4)
524mc_rep_funcs(qpel_v,12, 8, 16, sse4)
525mc_rep_funcs(qpel_v,12, 4, 12, sse4)
526mc_rep_funcs(qpel_hv, 8, 8, 64, sse4)
527mc_rep_funcs(qpel_hv, 8, 8, 48, sse4)
528mc_rep_funcs(qpel_hv, 8, 8, 32, sse4)
529mc_rep_funcs(qpel_hv, 8, 8, 24, sse4)
530mc_rep_funcs(qpel_hv, 8, 8, 16, sse4)
531mc_rep_funcs2(qpel_hv,8, 8, 4, 12, sse4)
532mc_rep_funcs(qpel_hv,10, 8, 64, sse4)
533mc_rep_funcs(qpel_hv,10, 8, 48, sse4)
534mc_rep_funcs(qpel_hv,10, 8, 32, sse4)
535mc_rep_funcs(qpel_hv,10, 8, 24, sse4)
536mc_rep_funcs(qpel_hv,10, 8, 16, sse4)
537mc_rep_funcs(qpel_hv,10, 4, 12, sse4)
538mc_rep_funcs(qpel_hv,12, 8, 64, sse4)
539mc_rep_funcs(qpel_hv,12, 8, 48, sse4)
540mc_rep_funcs(qpel_hv,12, 8, 32, sse4)
541mc_rep_funcs(qpel_hv,12, 8, 24, sse4)
542mc_rep_funcs(qpel_hv,12, 8, 16, sse4)
543mc_rep_funcs(qpel_hv,12, 4, 12, sse4)
544
545#define mc_rep_uni_w(bitd, step, W, opt) \
546void ff_hevc_put_uni_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, const int16_t *_src, \
547 int height, int denom, int _wx, int _ox) \
548{ \
549 int i; \
550 uint8_t *dst; \
551 for (i = 0; i < W; i += step) { \
552 const int16_t *src = _src + i; \
553 dst= _dst + (i * ((bitd + 7) / 8)); \
554 ff_hevc_put_uni_w##step##_##bitd##_##opt(dst, dststride, src, \
555 height, denom, _wx, _ox); \
556 } \
557}
558
559mc_rep_uni_w(8, 6, 12, sse4)
560mc_rep_uni_w(8, 8, 16, sse4)
561mc_rep_uni_w(8, 8, 24, sse4)
562mc_rep_uni_w(8, 8, 32, sse4)
563mc_rep_uni_w(8, 8, 48, sse4)
564mc_rep_uni_w(8, 8, 64, sse4)
565
566mc_rep_uni_w(10, 6, 12, sse4)
567mc_rep_uni_w(10, 8, 16, sse4)
568mc_rep_uni_w(10, 8, 24, sse4)
569mc_rep_uni_w(10, 8, 32, sse4)
570mc_rep_uni_w(10, 8, 48, sse4)
571mc_rep_uni_w(10, 8, 64, sse4)
572
573mc_rep_uni_w(12, 6, 12, sse4)
574mc_rep_uni_w(12, 8, 16, sse4)
575mc_rep_uni_w(12, 8, 24, sse4)
576mc_rep_uni_w(12, 8, 32, sse4)
577mc_rep_uni_w(12, 8, 48, sse4)
578mc_rep_uni_w(12, 8, 64, sse4)
579
580#define mc_rep_bi_w(bitd, step, W, opt) \
581void ff_hevc_put_bi_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, const int16_t *_src, \
582 const int16_t *_src2, int height, \
583 int denom, int wx0, int wx1, int ox) \
584{ \
585 int i; \
586 uint8_t *dst; \
587 for (i = 0; i < W; i += step) { \
588 const int16_t *src = _src + i; \
589 const int16_t *src2 = _src2 + i; \
590 dst = _dst + (i * ((bitd + 7) / 8)); \
591 ff_hevc_put_bi_w##step##_##bitd##_##opt(dst, dststride, src, src2, \
592 height, denom, wx0, wx1, ox); \
593 } \
594}
595
596mc_rep_bi_w(8, 6, 12, sse4)
597mc_rep_bi_w(8, 8, 16, sse4)
598mc_rep_bi_w(8, 8, 24, sse4)
599mc_rep_bi_w(8, 8, 32, sse4)
600mc_rep_bi_w(8, 8, 48, sse4)
601mc_rep_bi_w(8, 8, 64, sse4)
602
603mc_rep_bi_w(10, 6, 12, sse4)
604mc_rep_bi_w(10, 8, 16, sse4)
605mc_rep_bi_w(10, 8, 24, sse4)
606mc_rep_bi_w(10, 8, 32, sse4)
607mc_rep_bi_w(10, 8, 48, sse4)
608mc_rep_bi_w(10, 8, 64, sse4)
609
610mc_rep_bi_w(12, 6, 12, sse4)
611mc_rep_bi_w(12, 8, 16, sse4)
612mc_rep_bi_w(12, 8, 24, sse4)
613mc_rep_bi_w(12, 8, 32, sse4)
614mc_rep_bi_w(12, 8, 48, sse4)
615mc_rep_bi_w(12, 8, 64, sse4)
616
617#define mc_uni_w_func(name, bitd, W, opt) \
618static void hevc_put_uni_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride, \
619 const uint8_t *_src, ptrdiff_t _srcstride, \
620 int height, int denom, \
621 int _wx, int _ox, \
622 intptr_t mx, intptr_t my, int width) \
623{ \
624 LOCAL_ALIGNED_16(int16_t, temp, [71 * MAX_PB_SIZE]); \
625 hevc_put_##name##W##_##bitd##_##opt(temp, _src, _srcstride, height, mx, my, width); \
626 ff_hevc_put_uni_w##W##_##bitd##_##opt(_dst, _dststride, temp, height, denom, _wx, _ox); \
627}
628
629#define mc_uni_w_funcs(name, bitd, opt) \
630 mc_uni_w_func(name, bitd, 4, opt) \
631 mc_uni_w_func(name, bitd, 8, opt) \
632 mc_uni_w_func(name, bitd, 12, opt) \
633 mc_uni_w_func(name, bitd, 16, opt) \
634 mc_uni_w_func(name, bitd, 24, opt) \
635 mc_uni_w_func(name, bitd, 32, opt) \
636 mc_uni_w_func(name, bitd, 48, opt) \
637 mc_uni_w_func(name, bitd, 64, opt)
638
639mc_uni_w_funcs(pel_pixels, 8, sse4)
640mc_uni_w_func(pel_pixels, 8, 6, sse4)
641mc_uni_w_funcs(epel_h, 8, sse4)
642mc_uni_w_func(epel_h, 8, 6, sse4)
643mc_uni_w_funcs(epel_v, 8, sse4)
644mc_uni_w_func(epel_v, 8, 6, sse4)
645mc_uni_w_funcs(epel_hv, 8, sse4)
646mc_uni_w_func(epel_hv, 8, 6, sse4)
647mc_uni_w_funcs(qpel_h, 8, sse4)
648mc_uni_w_funcs(qpel_v, 8, sse4)
649mc_uni_w_funcs(qpel_hv, 8, sse4)
650
651mc_uni_w_funcs(pel_pixels, 10, sse4)
652mc_uni_w_func(pel_pixels, 10, 6, sse4)
653mc_uni_w_funcs(epel_h, 10, sse4)
654mc_uni_w_func(epel_h, 10, 6, sse4)
655mc_uni_w_funcs(epel_v, 10, sse4)
656mc_uni_w_func(epel_v, 10, 6, sse4)
657mc_uni_w_funcs(epel_hv, 10, sse4)
658mc_uni_w_func(epel_hv, 10, 6, sse4)
659mc_uni_w_funcs(qpel_h, 10, sse4)
660mc_uni_w_funcs(qpel_v, 10, sse4)
661mc_uni_w_funcs(qpel_hv, 10, sse4)
662
663mc_uni_w_funcs(pel_pixels, 12, sse4)
664mc_uni_w_func(pel_pixels, 12, 6, sse4)
665mc_uni_w_funcs(epel_h, 12, sse4)
666mc_uni_w_func(epel_h, 12, 6, sse4)
667mc_uni_w_funcs(epel_v, 12, sse4)
668mc_uni_w_func(epel_v, 12, 6, sse4)
669mc_uni_w_funcs(epel_hv, 12, sse4)
670mc_uni_w_func(epel_hv, 12, 6, sse4)
671mc_uni_w_funcs(qpel_h, 12, sse4)
672mc_uni_w_funcs(qpel_v, 12, sse4)
673mc_uni_w_funcs(qpel_hv, 12, sse4)
674
675#define mc_bi_w_func(name, bitd, W, opt) \
676static void hevc_put_bi_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride, \
677 const uint8_t *_src, ptrdiff_t _srcstride, \
678 const int16_t *_src2, \
679 int height, int denom, \
680 int wx0, int wx1, int ox, \
681 intptr_t mx, intptr_t my, int width) \
682{ \
683 LOCAL_ALIGNED_16(int16_t, temp, [71 * MAX_PB_SIZE]); \
684 hevc_put_##name##W##_##bitd##_##opt(temp, _src, _srcstride, height, mx, my, width); \
685 ff_hevc_put_bi_w##W##_##bitd##_##opt(_dst, _dststride, temp, _src2, \
686 height, denom, wx0, wx1, ox); \
687}
688
689#define mc_bi_w_funcs(name, bitd, opt) \
690 mc_bi_w_func(name, bitd, 4, opt) \
691 mc_bi_w_func(name, bitd, 8, opt) \
692 mc_bi_w_func(name, bitd, 12, opt) \
693 mc_bi_w_func(name, bitd, 16, opt) \
694 mc_bi_w_func(name, bitd, 24, opt) \
695 mc_bi_w_func(name, bitd, 32, opt) \
696 mc_bi_w_func(name, bitd, 48, opt) \
697 mc_bi_w_func(name, bitd, 64, opt)
698
699mc_bi_w_funcs(pel_pixels, 8, sse4)
700mc_bi_w_func(pel_pixels, 8, 6, sse4)
701mc_bi_w_funcs(epel_h, 8, sse4)
702mc_bi_w_func(epel_h, 8, 6, sse4)
703mc_bi_w_funcs(epel_v, 8, sse4)
704mc_bi_w_func(epel_v, 8, 6, sse4)
705mc_bi_w_funcs(epel_hv, 8, sse4)
706mc_bi_w_func(epel_hv, 8, 6, sse4)
707mc_bi_w_funcs(qpel_h, 8, sse4)
708mc_bi_w_funcs(qpel_v, 8, sse4)
709mc_bi_w_funcs(qpel_hv, 8, sse4)
710
711mc_bi_w_funcs(pel_pixels, 10, sse4)
712mc_bi_w_func(pel_pixels, 10, 6, sse4)
713mc_bi_w_funcs(epel_h, 10, sse4)
714mc_bi_w_func(epel_h, 10, 6, sse4)
715mc_bi_w_funcs(epel_v, 10, sse4)
716mc_bi_w_func(epel_v, 10, 6, sse4)
717mc_bi_w_funcs(epel_hv, 10, sse4)
718mc_bi_w_func(epel_hv, 10, 6, sse4)
719mc_bi_w_funcs(qpel_h, 10, sse4)
720mc_bi_w_funcs(qpel_v, 10, sse4)
721mc_bi_w_funcs(qpel_hv, 10, sse4)
722
723mc_bi_w_funcs(pel_pixels, 12, sse4)
724mc_bi_w_func(pel_pixels, 12, 6, sse4)
725mc_bi_w_funcs(epel_h, 12, sse4)
726mc_bi_w_func(epel_h, 12, 6, sse4)
727mc_bi_w_funcs(epel_v, 12, sse4)
728mc_bi_w_func(epel_v, 12, 6, sse4)
729mc_bi_w_funcs(epel_hv, 12, sse4)
730mc_bi_w_func(epel_hv, 12, 6, sse4)
731mc_bi_w_funcs(qpel_h, 12, sse4)
732mc_bi_w_funcs(qpel_v, 12, sse4)
733mc_bi_w_funcs(qpel_hv, 12, sse4)
734#endif //ARCH_X86_64 && HAVE_SSE4_EXTERNAL
735
736#define SAO_BAND_FILTER_FUNCS(bitd, opt) \
737void ff_hevc_sao_band_filter_8_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
738 const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
739void ff_hevc_sao_band_filter_16_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
740 const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
741void ff_hevc_sao_band_filter_32_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
742 const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
743void ff_hevc_sao_band_filter_48_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
744 const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
745void ff_hevc_sao_band_filter_64_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
746 const int16_t *sao_offset_val, int sao_left_class, int width, int height);
747
748SAO_BAND_FILTER_FUNCS(8, sse2)
749SAO_BAND_FILTER_FUNCS(10, sse2)
750SAO_BAND_FILTER_FUNCS(12, sse2)
751SAO_BAND_FILTER_FUNCS(8, avx)
752SAO_BAND_FILTER_FUNCS(10, avx)
753SAO_BAND_FILTER_FUNCS(12, avx)
754SAO_BAND_FILTER_FUNCS(8, avx2)
755SAO_BAND_FILTER_FUNCS(10, avx2)
756SAO_BAND_FILTER_FUNCS(12, avx2)
757
758#define SAO_BAND_INIT(bitd, opt) do { \
759 c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_##bitd##_##opt; \
760 c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_##bitd##_##opt; \
761 c->sao_band_filter[2] = ff_hevc_sao_band_filter_32_##bitd##_##opt; \
762 c->sao_band_filter[3] = ff_hevc_sao_band_filter_48_##bitd##_##opt; \
763 c->sao_band_filter[4] = ff_hevc_sao_band_filter_64_##bitd##_##opt; \
764} while (0)
765
766#define SAO_EDGE_FILTER_FUNCS(bitd, opt) \
767void ff_hevc_sao_edge_filter_8_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
768 const int16_t *sao_offset_val, int eo, int width, int height); \
769void ff_hevc_sao_edge_filter_16_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
770 const int16_t *sao_offset_val, int eo, int width, int height); \
771void ff_hevc_sao_edge_filter_32_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
772 const int16_t *sao_offset_val, int eo, int width, int height); \
773void ff_hevc_sao_edge_filter_48_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
774 const int16_t *sao_offset_val, int eo, int width, int height); \
775void ff_hevc_sao_edge_filter_64_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
776 const int16_t *sao_offset_val, int eo, int width, int height); \
777
778SAO_EDGE_FILTER_FUNCS(8, ssse3)
779SAO_EDGE_FILTER_FUNCS(8, avx2)
780SAO_EDGE_FILTER_FUNCS(10, sse2)
781SAO_EDGE_FILTER_FUNCS(10, avx2)
782SAO_EDGE_FILTER_FUNCS(12, sse2)
783SAO_EDGE_FILTER_FUNCS(12, avx2)
784
785#define SAO_EDGE_INIT(bitd, opt) do { \
786 c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8_##bitd##_##opt; \
787 c->sao_edge_filter[1] = ff_hevc_sao_edge_filter_16_##bitd##_##opt; \
788 c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_##bitd##_##opt; \
789 c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_##bitd##_##opt; \
790 c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_##bitd##_##opt; \
791} while (0)
792
793#define PEL_LINK(dst, idx1, idx2, idx3, name, D, opt) \
794dst [idx1][idx2][idx3] = hevc_put_ ## name ## _ ## D ## _##opt; \
795dst ## _bi [idx1][idx2][idx3] = ff_hevc_put_bi_ ## name ## _ ## D ## _##opt; \
796dst ## _uni [idx1][idx2][idx3] = hevc_put_uni_ ## name ## _ ## D ## _##opt; \
797dst ## _uni_w[idx1][idx2][idx3] = hevc_put_uni_w_ ## name ## _ ## D ## _##opt; \
798dst ## _bi_w [idx1][idx2][idx3] = hevc_put_bi_w_ ## name ## _ ## D ## _##opt
799
800#define EPEL_LINKS(pointer, my, mx, fname, bitd, opt ) \
801 PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
802 PEL_LINK(pointer, 2, my , mx , fname##6 , bitd, opt ); \
803 PEL_LINK(pointer, 3, my , mx , fname##8 , bitd, opt ); \
804 PEL_LINK(pointer, 4, my , mx , fname##12, bitd, opt ); \
805 PEL_LINK(pointer, 5, my , mx , fname##16, bitd, opt ); \
806 PEL_LINK(pointer, 6, my , mx , fname##24, bitd, opt ); \
807 PEL_LINK(pointer, 7, my , mx , fname##32, bitd, opt ); \
808 PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
809 PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
810#define QPEL_LINKS(pointer, my, mx, fname, bitd, opt) \
811 PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
812 PEL_LINK(pointer, 3, my , mx , fname##8 , bitd, opt ); \
813 PEL_LINK(pointer, 4, my , mx , fname##12, bitd, opt ); \
814 PEL_LINK(pointer, 5, my , mx , fname##16, bitd, opt ); \
815 PEL_LINK(pointer, 6, my , mx , fname##24, bitd, opt ); \
816 PEL_LINK(pointer, 7, my , mx , fname##32, bitd, opt ); \
817 PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
818 PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
819
820void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
821{
822 int cpu_flags = av_get_cpu_flags();
823
824 if (bit_depth == 8) {
825 if (EXTERNAL_SSE2(cpu_flags)) {
826 c->add_residual[0] = ff_hevc_add_residual_4_8_sse2;
827
828 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_sse2;
829 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_sse2;
830#if ARCH_X86_64
831 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_sse2;
832 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_sse2;
833
834 c->idct[2] = ff_hevc_idct_16x16_8_sse2;
835 c->idct[3] = ff_hevc_idct_32x32_8_sse2;
836#endif
837 SAO_BAND_INIT(8, sse2);
838
839 c->idct_dc[0] = ff_hevc_idct_4x4_dc_8_sse2;
840 c->idct_dc[1] = ff_hevc_idct_8x8_dc_8_sse2;
841 c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_sse2;
842 c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_sse2;
843
844 c->idct[0] = ff_hevc_idct_4x4_8_sse2;
845 c->idct[1] = ff_hevc_idct_8x8_8_sse2;
846 c->transform_4x4_luma = ff_hevc_transform_4x4_luma_8_sse2;
847
848 c->add_residual[1] = ff_hevc_add_residual_8_8_sse2;
849 c->add_residual[2] = ff_hevc_add_residual_16_8_sse2;
850 c->add_residual[3] = ff_hevc_add_residual_32_8_sse2;
851 }
852 if (EXTERNAL_SSSE3(cpu_flags)) {
853#if ARCH_X86_64
854 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3;
855 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3;
856#endif
857 c->dequant = ff_hevc_dequant_8_ssse3;
858 SAO_EDGE_INIT(8, ssse3);
859 }
860#if HAVE_SSE4_EXTERNAL && ARCH_X86_64
861 if (EXTERNAL_SSE4(cpu_flags)) {
862
863 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 8, sse4);
864 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 8, sse4);
865 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 8, sse4);
866 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 8, sse4);
867
868 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 8, sse4);
869 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 8, sse4);
870 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 8, sse4);
871 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 8, sse4);
872 }
873#endif
874 if (EXTERNAL_AVX(cpu_flags)) {
875 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_avx;
876 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_avx;
877#if ARCH_X86_64
878 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_avx;
879 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_avx;
880
881 c->idct[2] = ff_hevc_idct_16x16_8_avx;
882 c->idct[3] = ff_hevc_idct_32x32_8_avx;
883#endif
884 SAO_BAND_INIT(8, avx);
885
886 c->idct[0] = ff_hevc_idct_4x4_8_avx;
887 c->idct[1] = ff_hevc_idct_8x8_8_avx;
888 c->transform_4x4_luma = ff_hevc_transform_4x4_luma_8_avx;
889 }
890 if (EXTERNAL_AVX2(cpu_flags)) {
891 c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_8_avx2;
892 c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_8_avx2;
893 }
894#if HAVE_AVX2_EXTERNAL
895 if (EXTERNAL_AVX2_FAST(cpu_flags)) {
896 c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_avx2;
897 c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_avx2;
898
899#if ARCH_X86_64
900 c->put_hevc_epel[7][0][0] = hevc_put_pel_pixels32_8_avx2;
901 c->put_hevc_epel[8][0][0] = hevc_put_pel_pixels48_8_avx2;
902 c->put_hevc_epel[9][0][0] = hevc_put_pel_pixels64_8_avx2;
903
904 c->put_hevc_qpel[7][0][0] = hevc_put_pel_pixels32_8_avx2;
905 c->put_hevc_qpel[8][0][0] = hevc_put_pel_pixels48_8_avx2;
906 c->put_hevc_qpel[9][0][0] = hevc_put_pel_pixels64_8_avx2;
907
908 c->put_hevc_epel_uni[7][0][0] = hevc_put_uni_pel_pixels32_8_avx2;
909 c->put_hevc_epel_uni[8][0][0] = hevc_put_uni_pel_pixels48_8_avx2;
910 c->put_hevc_epel_uni[9][0][0] = hevc_put_uni_pel_pixels64_8_avx2;
911
912 c->put_hevc_qpel_uni[7][0][0] = hevc_put_uni_pel_pixels32_8_avx2;
913 c->put_hevc_qpel_uni[8][0][0] = hevc_put_uni_pel_pixels48_8_avx2;
914 c->put_hevc_qpel_uni[9][0][0] = hevc_put_uni_pel_pixels64_8_avx2;
915
916 c->put_hevc_qpel_bi[7][0][0] = ff_hevc_put_bi_pel_pixels32_8_avx2;
917 c->put_hevc_qpel_bi[8][0][0] = ff_hevc_put_bi_pel_pixels48_8_avx2;
918 c->put_hevc_qpel_bi[9][0][0] = ff_hevc_put_bi_pel_pixels64_8_avx2;
919
920 c->put_hevc_epel_bi[7][0][0] = ff_hevc_put_bi_pel_pixels32_8_avx2;
921 c->put_hevc_epel_bi[8][0][0] = ff_hevc_put_bi_pel_pixels48_8_avx2;
922 c->put_hevc_epel_bi[9][0][0] = ff_hevc_put_bi_pel_pixels64_8_avx2;
923
924 c->put_hevc_epel[7][0][1] = hevc_put_epel_h32_8_avx2;
925 c->put_hevc_epel[8][0][1] = hevc_put_epel_h48_8_avx2;
926 c->put_hevc_epel[9][0][1] = hevc_put_epel_h64_8_avx2;
927
928 c->put_hevc_epel_uni[7][0][1] = hevc_put_uni_epel_h32_8_avx2;
929 c->put_hevc_epel_uni[8][0][1] = hevc_put_uni_epel_h48_8_avx2;
930 c->put_hevc_epel_uni[9][0][1] = hevc_put_uni_epel_h64_8_avx2;
931
932 c->put_hevc_epel_bi[7][0][1] = ff_hevc_put_bi_epel_h32_8_avx2;
933 c->put_hevc_epel_bi[8][0][1] = ff_hevc_put_bi_epel_h48_8_avx2;
934 c->put_hevc_epel_bi[9][0][1] = ff_hevc_put_bi_epel_h64_8_avx2;
935
936 c->put_hevc_epel[7][1][0] = hevc_put_epel_v32_8_avx2;
937 c->put_hevc_epel[8][1][0] = hevc_put_epel_v48_8_avx2;
938 c->put_hevc_epel[9][1][0] = hevc_put_epel_v64_8_avx2;
939
940 c->put_hevc_epel_uni[7][1][0] = hevc_put_uni_epel_v32_8_avx2;
941 c->put_hevc_epel_uni[8][1][0] = hevc_put_uni_epel_v48_8_avx2;
942 c->put_hevc_epel_uni[9][1][0] = hevc_put_uni_epel_v64_8_avx2;
943
944 c->put_hevc_epel_bi[7][1][0] = ff_hevc_put_bi_epel_v32_8_avx2;
945 c->put_hevc_epel_bi[8][1][0] = ff_hevc_put_bi_epel_v48_8_avx2;
946 c->put_hevc_epel_bi[9][1][0] = ff_hevc_put_bi_epel_v64_8_avx2;
947
948 c->put_hevc_epel[7][1][1] = hevc_put_epel_hv32_8_avx2;
949 c->put_hevc_epel[8][1][1] = hevc_put_epel_hv48_8_avx2;
950 c->put_hevc_epel[9][1][1] = hevc_put_epel_hv64_8_avx2;
951
952 c->put_hevc_epel_uni[7][1][1] = hevc_put_uni_epel_hv32_8_avx2;
953 c->put_hevc_epel_uni[8][1][1] = hevc_put_uni_epel_hv48_8_avx2;
954 c->put_hevc_epel_uni[9][1][1] = hevc_put_uni_epel_hv64_8_avx2;
955
956 c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_bi_epel_hv32_8_avx2;
957 c->put_hevc_epel_bi[8][1][1] = ff_hevc_put_bi_epel_hv48_8_avx2;
958 c->put_hevc_epel_bi[9][1][1] = ff_hevc_put_bi_epel_hv64_8_avx2;
959
960 c->put_hevc_qpel[7][0][1] = hevc_put_qpel_h32_8_avx2;
961 c->put_hevc_qpel[8][0][1] = hevc_put_qpel_h48_8_avx2;
962 c->put_hevc_qpel[9][0][1] = hevc_put_qpel_h64_8_avx2;
963
964 c->put_hevc_qpel[7][1][0] = hevc_put_qpel_v32_8_avx2;
965 c->put_hevc_qpel[8][1][0] = hevc_put_qpel_v48_8_avx2;
966 c->put_hevc_qpel[9][1][0] = hevc_put_qpel_v64_8_avx2;
967
968 c->put_hevc_qpel_uni[7][0][1] = hevc_put_uni_qpel_h32_8_avx2;
969 c->put_hevc_qpel_uni[8][0][1] = hevc_put_uni_qpel_h48_8_avx2;
970 c->put_hevc_qpel_uni[9][0][1] = hevc_put_uni_qpel_h64_8_avx2;
971
972 c->put_hevc_qpel_uni[7][1][0] = hevc_put_uni_qpel_v32_8_avx2;
973 c->put_hevc_qpel_uni[8][1][0] = hevc_put_uni_qpel_v48_8_avx2;
974 c->put_hevc_qpel_uni[9][1][0] = hevc_put_uni_qpel_v64_8_avx2;
975
976 c->put_hevc_qpel_bi[7][0][1] = ff_hevc_put_bi_qpel_h32_8_avx2;
977 c->put_hevc_qpel_bi[8][0][1] = ff_hevc_put_bi_qpel_h48_8_avx2;
978 c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_bi_qpel_h64_8_avx2;
979
980 c->put_hevc_qpel_bi[7][1][0] = ff_hevc_put_bi_qpel_v32_8_avx2;
981 c->put_hevc_qpel_bi[8][1][0] = ff_hevc_put_bi_qpel_v48_8_avx2;
982 c->put_hevc_qpel_bi[9][1][0] = ff_hevc_put_bi_qpel_v64_8_avx2;
983#endif /* ARCH_X86_64 */
984
985 SAO_BAND_INIT(8, avx2);
986
987 c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_8_avx2;
988 c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_8_avx2;
989 c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_8_avx2;
990
991 c->add_residual[3] = ff_hevc_add_residual_32_8_avx2;
992 }
993#endif /* HAVE_AVX2_EXTERNAL */
994#if ARCH_X86_64
995 if (EXTERNAL_AVX512ICL(cpu_flags)) {
996 c->put_hevc_qpel[1][0][1] = ff_hevc_put_qpel_h4_8_avx512icl;
997 c->put_hevc_qpel[3][0][1] = ff_hevc_put_qpel_h8_8_avx512icl;
998 c->put_hevc_qpel[5][0][1] = ff_hevc_put_qpel_h16_8_avx512icl;
999 c->put_hevc_qpel[7][0][1] = ff_hevc_put_qpel_h32_8_avx512icl;
1000 c->put_hevc_qpel[9][0][1] = ff_hevc_put_qpel_h64_8_avx512icl;
1001 c->put_hevc_qpel[3][1][1] = ff_hevc_put_qpel_hv8_8_avx512icl;
1002 }
1003#endif
1004 } else if (bit_depth == 10) {
1005 if (EXTERNAL_SSE2(cpu_flags)) {
1006 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_sse2;
1007 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_sse2;
1008#if ARCH_X86_64
1009 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_sse2;
1010 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_sse2;
1011
1012 c->idct[2] = ff_hevc_idct_16x16_10_sse2;
1013 c->idct[3] = ff_hevc_idct_32x32_10_sse2;
1014#endif
1015 SAO_BAND_INIT(10, sse2);
1016 SAO_EDGE_INIT(10, sse2);
1017
1018 c->idct_dc[0] = ff_hevc_idct_4x4_dc_10_sse2;
1019 c->idct_dc[1] = ff_hevc_idct_8x8_dc_10_sse2;
1020 c->idct_dc[2] = ff_hevc_idct_16x16_dc_10_sse2;
1021 c->idct_dc[3] = ff_hevc_idct_32x32_dc_10_sse2;
1022
1023 c->idct[0] = ff_hevc_idct_4x4_10_sse2;
1024 c->idct[1] = ff_hevc_idct_8x8_10_sse2;
1025 c->transform_4x4_luma = ff_hevc_transform_4x4_luma_10_sse2;
1026
1027 c->add_residual[0] = ff_hevc_add_residual_4_10_sse2;
1028 c->add_residual[1] = ff_hevc_add_residual_8_10_sse2;
1029 c->add_residual[2] = ff_hevc_add_residual_16_10_sse2;
1030 c->add_residual[3] = ff_hevc_add_residual_32_10_sse2;
1031 }
1032#if ARCH_X86_64
1033 if (EXTERNAL_SSSE3(cpu_flags)) {
1034 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_ssse3;
1035 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_ssse3;
1036 }
1037#endif
1038#if HAVE_SSE4_EXTERNAL && ARCH_X86_64
1039 if (EXTERNAL_SSE4(cpu_flags)) {
1040 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 10, sse4);
1041 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 10, sse4);
1042 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 10, sse4);
1043 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 10, sse4);
1044
1045 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 10, sse4);
1046 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 10, sse4);
1047 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 10, sse4);
1048 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 10, sse4);
1049 }
1050#endif
1051 if (EXTERNAL_AVX(cpu_flags)) {
1052 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_avx;
1053 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_avx;
1054#if ARCH_X86_64
1055 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_avx;
1056 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_avx;
1057
1058 c->idct[2] = ff_hevc_idct_16x16_10_avx;
1059 c->idct[3] = ff_hevc_idct_32x32_10_avx;
1060#endif
1061
1062 c->idct[0] = ff_hevc_idct_4x4_10_avx;
1063 c->idct[1] = ff_hevc_idct_8x8_10_avx;
1064 c->transform_4x4_luma = ff_hevc_transform_4x4_luma_10_avx;
1065
1066 SAO_BAND_INIT(10, avx);
1067 }
1068 if (EXTERNAL_AVX2(cpu_flags)) {
1069 c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_10_avx2;
1070 }
1071#if HAVE_AVX2_EXTERNAL
1072 if (EXTERNAL_AVX2_FAST(cpu_flags)) {
1073 c->idct_dc[2] = ff_hevc_idct_16x16_dc_10_avx2;
1074 c->idct_dc[3] = ff_hevc_idct_32x32_dc_10_avx2;
1075
1076#if ARCH_X86_64
1077 c->put_hevc_epel[5][0][0] = hevc_put_pel_pixels16_10_avx2;
1078 c->put_hevc_epel[6][0][0] = hevc_put_pel_pixels24_10_avx2;
1079 c->put_hevc_epel[7][0][0] = hevc_put_pel_pixels32_10_avx2;
1080 c->put_hevc_epel[8][0][0] = hevc_put_pel_pixels48_10_avx2;
1081 c->put_hevc_epel[9][0][0] = hevc_put_pel_pixels64_10_avx2;
1082
1083 c->put_hevc_qpel[5][0][0] = hevc_put_pel_pixels16_10_avx2;
1084 c->put_hevc_qpel[6][0][0] = hevc_put_pel_pixels24_10_avx2;
1085 c->put_hevc_qpel[7][0][0] = hevc_put_pel_pixels32_10_avx2;
1086 c->put_hevc_qpel[8][0][0] = hevc_put_pel_pixels48_10_avx2;
1087 c->put_hevc_qpel[9][0][0] = hevc_put_pel_pixels64_10_avx2;
1088
1089 c->put_hevc_epel_uni[5][0][0] = hevc_put_uni_pel_pixels32_8_avx2;
1090 c->put_hevc_epel_uni[6][0][0] = hevc_put_uni_pel_pixels48_8_avx2;
1091 c->put_hevc_epel_uni[7][0][0] = hevc_put_uni_pel_pixels64_8_avx2;
1092 c->put_hevc_epel_uni[8][0][0] = hevc_put_uni_pel_pixels96_8_avx2;
1093 c->put_hevc_epel_uni[9][0][0] = hevc_put_uni_pel_pixels128_8_avx2;
1094
1095 c->put_hevc_qpel_uni[5][0][0] = hevc_put_uni_pel_pixels32_8_avx2;
1096 c->put_hevc_qpel_uni[6][0][0] = hevc_put_uni_pel_pixels48_8_avx2;
1097 c->put_hevc_qpel_uni[7][0][0] = hevc_put_uni_pel_pixels64_8_avx2;
1098 c->put_hevc_qpel_uni[8][0][0] = hevc_put_uni_pel_pixels96_8_avx2;
1099 c->put_hevc_qpel_uni[9][0][0] = hevc_put_uni_pel_pixels128_8_avx2;
1100
1101 c->put_hevc_epel_bi[5][0][0] = ff_hevc_put_bi_pel_pixels16_10_avx2;
1102 c->put_hevc_epel_bi[6][0][0] = ff_hevc_put_bi_pel_pixels24_10_avx2;
1103 c->put_hevc_epel_bi[7][0][0] = ff_hevc_put_bi_pel_pixels32_10_avx2;
1104 c->put_hevc_epel_bi[8][0][0] = ff_hevc_put_bi_pel_pixels48_10_avx2;
1105 c->put_hevc_epel_bi[9][0][0] = ff_hevc_put_bi_pel_pixels64_10_avx2;
1106 c->put_hevc_qpel_bi[5][0][0] = ff_hevc_put_bi_pel_pixels16_10_avx2;
1107 c->put_hevc_qpel_bi[6][0][0] = ff_hevc_put_bi_pel_pixels24_10_avx2;
1108 c->put_hevc_qpel_bi[7][0][0] = ff_hevc_put_bi_pel_pixels32_10_avx2;
1109 c->put_hevc_qpel_bi[8][0][0] = ff_hevc_put_bi_pel_pixels48_10_avx2;
1110 c->put_hevc_qpel_bi[9][0][0] = ff_hevc_put_bi_pel_pixels64_10_avx2;
1111
1112 c->put_hevc_epel[5][0][1] = hevc_put_epel_h16_10_avx2;
1113 c->put_hevc_epel[6][0][1] = hevc_put_epel_h24_10_avx2;
1114 c->put_hevc_epel[7][0][1] = hevc_put_epel_h32_10_avx2;
1115 c->put_hevc_epel[8][0][1] = hevc_put_epel_h48_10_avx2;
1116 c->put_hevc_epel[9][0][1] = hevc_put_epel_h64_10_avx2;
1117
1118 c->put_hevc_epel_uni[5][0][1] = hevc_put_uni_epel_h16_10_avx2;
1119 c->put_hevc_epel_uni[6][0][1] = hevc_put_uni_epel_h24_10_avx2;
1120 c->put_hevc_epel_uni[7][0][1] = hevc_put_uni_epel_h32_10_avx2;
1121 c->put_hevc_epel_uni[8][0][1] = hevc_put_uni_epel_h48_10_avx2;
1122 c->put_hevc_epel_uni[9][0][1] = hevc_put_uni_epel_h64_10_avx2;
1123
1124 c->put_hevc_epel_bi[5][0][1] = ff_hevc_put_bi_epel_h16_10_avx2;
1125 c->put_hevc_epel_bi[6][0][1] = ff_hevc_put_bi_epel_h24_10_avx2;
1126 c->put_hevc_epel_bi[7][0][1] = ff_hevc_put_bi_epel_h32_10_avx2;
1127 c->put_hevc_epel_bi[8][0][1] = ff_hevc_put_bi_epel_h48_10_avx2;
1128 c->put_hevc_epel_bi[9][0][1] = ff_hevc_put_bi_epel_h64_10_avx2;
1129
1130 c->put_hevc_epel[5][1][0] = hevc_put_epel_v16_10_avx2;
1131 c->put_hevc_epel[6][1][0] = hevc_put_epel_v24_10_avx2;
1132 c->put_hevc_epel[7][1][0] = hevc_put_epel_v32_10_avx2;
1133 c->put_hevc_epel[8][1][0] = hevc_put_epel_v48_10_avx2;
1134 c->put_hevc_epel[9][1][0] = hevc_put_epel_v64_10_avx2;
1135
1136 c->put_hevc_epel_uni[5][1][0] = hevc_put_uni_epel_v16_10_avx2;
1137 c->put_hevc_epel_uni[6][1][0] = hevc_put_uni_epel_v24_10_avx2;
1138 c->put_hevc_epel_uni[7][1][0] = hevc_put_uni_epel_v32_10_avx2;
1139 c->put_hevc_epel_uni[8][1][0] = hevc_put_uni_epel_v48_10_avx2;
1140 c->put_hevc_epel_uni[9][1][0] = hevc_put_uni_epel_v64_10_avx2;
1141
1142 c->put_hevc_epel_bi[5][1][0] = ff_hevc_put_bi_epel_v16_10_avx2;
1143 c->put_hevc_epel_bi[6][1][0] = ff_hevc_put_bi_epel_v24_10_avx2;
1144 c->put_hevc_epel_bi[7][1][0] = ff_hevc_put_bi_epel_v32_10_avx2;
1145 c->put_hevc_epel_bi[8][1][0] = ff_hevc_put_bi_epel_v48_10_avx2;
1146 c->put_hevc_epel_bi[9][1][0] = ff_hevc_put_bi_epel_v64_10_avx2;
1147
1148 c->put_hevc_epel[5][1][1] = hevc_put_epel_hv16_10_avx2;
1149 c->put_hevc_epel[6][1][1] = hevc_put_epel_hv24_10_avx2;
1150 c->put_hevc_epel[7][1][1] = hevc_put_epel_hv32_10_avx2;
1151 c->put_hevc_epel[8][1][1] = hevc_put_epel_hv48_10_avx2;
1152 c->put_hevc_epel[9][1][1] = hevc_put_epel_hv64_10_avx2;
1153
1154 c->put_hevc_epel_uni[5][1][1] = hevc_put_uni_epel_hv16_10_avx2;
1155 c->put_hevc_epel_uni[6][1][1] = hevc_put_uni_epel_hv24_10_avx2;
1156 c->put_hevc_epel_uni[7][1][1] = hevc_put_uni_epel_hv32_10_avx2;
1157 c->put_hevc_epel_uni[8][1][1] = hevc_put_uni_epel_hv48_10_avx2;
1158 c->put_hevc_epel_uni[9][1][1] = hevc_put_uni_epel_hv64_10_avx2;
1159
1160 c->put_hevc_epel_bi[5][1][1] = ff_hevc_put_bi_epel_hv16_10_avx2;
1161 c->put_hevc_epel_bi[6][1][1] = ff_hevc_put_bi_epel_hv24_10_avx2;
1162 c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_bi_epel_hv32_10_avx2;
1163 c->put_hevc_epel_bi[8][1][1] = ff_hevc_put_bi_epel_hv48_10_avx2;
1164 c->put_hevc_epel_bi[9][1][1] = ff_hevc_put_bi_epel_hv64_10_avx2;
1165
1166 c->put_hevc_qpel[5][0][1] = hevc_put_qpel_h16_10_avx2;
1167 c->put_hevc_qpel[6][0][1] = hevc_put_qpel_h24_10_avx2;
1168 c->put_hevc_qpel[7][0][1] = hevc_put_qpel_h32_10_avx2;
1169 c->put_hevc_qpel[8][0][1] = hevc_put_qpel_h48_10_avx2;
1170 c->put_hevc_qpel[9][0][1] = hevc_put_qpel_h64_10_avx2;
1171
1172 c->put_hevc_qpel_uni[5][0][1] = hevc_put_uni_qpel_h16_10_avx2;
1173 c->put_hevc_qpel_uni[6][0][1] = hevc_put_uni_qpel_h24_10_avx2;
1174 c->put_hevc_qpel_uni[7][0][1] = hevc_put_uni_qpel_h32_10_avx2;
1175 c->put_hevc_qpel_uni[8][0][1] = hevc_put_uni_qpel_h48_10_avx2;
1176 c->put_hevc_qpel_uni[9][0][1] = hevc_put_uni_qpel_h64_10_avx2;
1177
1178 c->put_hevc_qpel_bi[5][0][1] = ff_hevc_put_bi_qpel_h16_10_avx2;
1179 c->put_hevc_qpel_bi[6][0][1] = ff_hevc_put_bi_qpel_h24_10_avx2;
1180 c->put_hevc_qpel_bi[7][0][1] = ff_hevc_put_bi_qpel_h32_10_avx2;
1181 c->put_hevc_qpel_bi[8][0][1] = ff_hevc_put_bi_qpel_h48_10_avx2;
1182 c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_bi_qpel_h64_10_avx2;
1183
1184 c->put_hevc_qpel[5][1][0] = hevc_put_qpel_v16_10_avx2;
1185 c->put_hevc_qpel[6][1][0] = hevc_put_qpel_v24_10_avx2;
1186 c->put_hevc_qpel[7][1][0] = hevc_put_qpel_v32_10_avx2;
1187 c->put_hevc_qpel[8][1][0] = hevc_put_qpel_v48_10_avx2;
1188 c->put_hevc_qpel[9][1][0] = hevc_put_qpel_v64_10_avx2;
1189
1190 c->put_hevc_qpel_uni[5][1][0] = hevc_put_uni_qpel_v16_10_avx2;
1191 c->put_hevc_qpel_uni[6][1][0] = hevc_put_uni_qpel_v24_10_avx2;
1192 c->put_hevc_qpel_uni[7][1][0] = hevc_put_uni_qpel_v32_10_avx2;
1193 c->put_hevc_qpel_uni[8][1][0] = hevc_put_uni_qpel_v48_10_avx2;
1194 c->put_hevc_qpel_uni[9][1][0] = hevc_put_uni_qpel_v64_10_avx2;
1195
1196 c->put_hevc_qpel_bi[5][1][0] = ff_hevc_put_bi_qpel_v16_10_avx2;
1197 c->put_hevc_qpel_bi[6][1][0] = ff_hevc_put_bi_qpel_v24_10_avx2;
1198 c->put_hevc_qpel_bi[7][1][0] = ff_hevc_put_bi_qpel_v32_10_avx2;
1199 c->put_hevc_qpel_bi[8][1][0] = ff_hevc_put_bi_qpel_v48_10_avx2;
1200 c->put_hevc_qpel_bi[9][1][0] = ff_hevc_put_bi_qpel_v64_10_avx2;
1201
1202 c->put_hevc_qpel[5][1][1] = hevc_put_qpel_hv16_10_avx2;
1203 c->put_hevc_qpel[6][1][1] = hevc_put_qpel_hv24_10_avx2;
1204 c->put_hevc_qpel[7][1][1] = hevc_put_qpel_hv32_10_avx2;
1205 c->put_hevc_qpel[8][1][1] = hevc_put_qpel_hv48_10_avx2;
1206 c->put_hevc_qpel[9][1][1] = hevc_put_qpel_hv64_10_avx2;
1207
1208 c->put_hevc_qpel_uni[5][1][1] = hevc_put_uni_qpel_hv16_10_avx2;
1209 c->put_hevc_qpel_uni[6][1][1] = hevc_put_uni_qpel_hv24_10_avx2;
1210 c->put_hevc_qpel_uni[7][1][1] = hevc_put_uni_qpel_hv32_10_avx2;
1211 c->put_hevc_qpel_uni[8][1][1] = hevc_put_uni_qpel_hv48_10_avx2;
1212 c->put_hevc_qpel_uni[9][1][1] = hevc_put_uni_qpel_hv64_10_avx2;
1213
1214 c->put_hevc_qpel_bi[5][1][1] = ff_hevc_put_bi_qpel_hv16_10_avx2;
1215 c->put_hevc_qpel_bi[6][1][1] = ff_hevc_put_bi_qpel_hv24_10_avx2;
1216 c->put_hevc_qpel_bi[7][1][1] = ff_hevc_put_bi_qpel_hv32_10_avx2;
1217 c->put_hevc_qpel_bi[8][1][1] = ff_hevc_put_bi_qpel_hv48_10_avx2;
1218 c->put_hevc_qpel_bi[9][1][1] = ff_hevc_put_bi_qpel_hv64_10_avx2;
1219#endif /* ARCH_X86_64 */
1220
1221 SAO_BAND_INIT(10, avx2);
1222 SAO_EDGE_INIT(10, avx2);
1223
1224 c->add_residual[2] = ff_hevc_add_residual_16_10_avx2;
1225 c->add_residual[3] = ff_hevc_add_residual_32_10_avx2;
1226 }
1227#endif /* HAVE_AVX2_EXTERNAL */
1228 } else if (bit_depth == 12) {
1229 if (EXTERNAL_SSE2(cpu_flags)) {
1230 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_sse2;
1231 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_sse2;
1232#if ARCH_X86_64
1233 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_sse2;
1234 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_sse2;
1235#endif
1236 SAO_BAND_INIT(12, sse2);
1237 SAO_EDGE_INIT(12, sse2);
1238
1239 c->idct_dc[0] = ff_hevc_idct_4x4_dc_12_sse2;
1240 c->idct_dc[1] = ff_hevc_idct_8x8_dc_12_sse2;
1241 c->idct_dc[2] = ff_hevc_idct_16x16_dc_12_sse2;
1242 c->idct_dc[3] = ff_hevc_idct_32x32_dc_12_sse2;
1243 }
1244#if ARCH_X86_64
1245 if (EXTERNAL_SSSE3(cpu_flags)) {
1246 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_ssse3;
1247 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_ssse3;
1248 }
1249#endif
1250#if HAVE_SSE4_EXTERNAL && ARCH_X86_64
1251 if (EXTERNAL_SSE4(cpu_flags)) {
1252 EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 12, sse4);
1253 EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 12, sse4);
1254 EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 12, sse4);
1255 EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 12, sse4);
1256
1257 QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 12, sse4);
1258 QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 12, sse4);
1259 QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 12, sse4);
1260 QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 12, sse4);
1261 }
1262#endif
1263 if (EXTERNAL_AVX(cpu_flags)) {
1264 c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_avx;
1265 c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_avx;
1266#if ARCH_X86_64
1267 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_avx;
1268 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_avx;
1269#endif
1270 SAO_BAND_INIT(12, avx);
1271 }
1272 if (EXTERNAL_AVX2(cpu_flags)) {
1273 c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_12_avx2;
1274 }
1275 if (EXTERNAL_AVX2_FAST(cpu_flags)) {
1276 c->idct_dc[2] = ff_hevc_idct_16x16_dc_12_avx2;
1277 c->idct_dc[3] = ff_hevc_idct_32x32_dc_12_avx2;
1278
1279 SAO_BAND_INIT(12, avx2);
1280 SAO_EDGE_INIT(12, avx2);
1281 }
1282 }
1283}
bit_depth
static void bit_depth(AudioStatsContext *s, const uint64_t *const mask, uint8_t *depth)
Definition af_astats.c:246
mc_rep_funcs
#define mc_rep_funcs(name, bitd, step, W, opt)
Definition h2656dsp.c:55
mc_rep_func
#define mc_rep_func(name, bitd, step, W, opt)
Definition h2656dsp.c:28
mc_rep_uni_func
#define mc_rep_uni_func(name, bitd, step, W, opt)
Definition h2656dsp.c:41
cpu_flags
static atomic_int cpu_flags
Definition cpu.c:56
av_get_cpu_flags
int av_get_cpu_flags(void)
Return the flags which specify extensions supported by the CPU.
Definition cpu.c:109
EXTERNAL_AVX
#define EXTERNAL_AVX(flags)
Definition cpu.h:64
EXTERNAL_AVX2_FAST
#define EXTERNAL_AVX2_FAST(flags)
Definition cpu.h:73
EXTERNAL_SSSE3
#define EXTERNAL_SSSE3(flags)
Definition cpu.h:59
EXTERNAL_AVX512ICL
#define EXTERNAL_AVX512ICL(flags)
Definition cpu.h:78
EXTERNAL_SSE4
#define EXTERNAL_SSE4(flags)
Definition cpu.h:62
EXTERNAL_AVX2
#define EXTERNAL_AVX2(flags)
Definition cpu.h:72
EXTERNAL_SSE2
#define EXTERNAL_SSE2(flags)
Definition cpu.h:53
mem_internal.h
c
static double c[64]
Definition vsrc_mptestsrc.c:89
ff_hevc_add_residual_32_10_sse2
void ff_hevc_add_residual_32_10_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_add_residual_16_8_sse2
void ff_hevc_add_residual_16_8_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_add_residual_16_10_sse2
void ff_hevc_add_residual_16_10_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_put_qpel_h64_8_avx512icl
void ff_hevc_put_qpel_h64_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
ff_hevc_add_residual_32_8_avx2
void ff_hevc_add_residual_32_8_avx2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_add_residual_4_10_sse2
void ff_hevc_add_residual_4_10_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_put_qpel_h32_8_avx512icl
void ff_hevc_put_qpel_h32_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
ff_hevc_add_residual_32_10_avx2
void ff_hevc_add_residual_32_10_avx2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_add_residual_16_10_avx2
void ff_hevc_add_residual_16_10_avx2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_put_qpel_h8_8_avx512icl
void ff_hevc_put_qpel_h8_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
ff_hevc_put_qpel_h4_8_avx512icl
void ff_hevc_put_qpel_h4_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
ff_hevc_put_qpel_h16_8_avx512icl
void ff_hevc_put_qpel_h16_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
ff_hevc_add_residual_8_8_sse2
void ff_hevc_add_residual_8_8_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_add_residual_8_10_sse2
void ff_hevc_add_residual_8_10_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_add_residual_32_8_sse2
void ff_hevc_add_residual_32_8_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_put_qpel_hv8_8_avx512icl
void ff_hevc_put_qpel_hv8_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
ff_hevc_add_residual_4_8_sse2
void ff_hevc_add_residual_4_8_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
FW_PUT
#define FW_PUT(p, a, b, depth, opt)
Definition dsp_init.c:101
LFL_FUNCS
#define LFL_FUNCS(type, depth, opt)
Definition dsp_init.c:44
LFC_FUNCS
#define LFC_FUNCS(type, depth, opt)
Definition dsp_init.c:40
mc_rep_funcs2
#define mc_rep_funcs2(name, bitd, step1, step2, W, opt)
Definition dsp_init.c:277
IDCT_FUNCS
#define IDCT_FUNCS(opt)
Definition dsp_init.c:76
ff_hevc_dsp_init_x86
void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
Definition dsp_init.c:820
mc_rep_funcs
#define mc_rep_funcs(name, bitd, step, W, opt)
Definition dsp_init.c:242
SAO_BAND_INIT
#define SAO_BAND_INIT(bitd, opt)
Definition dsp_init.c:758
ff_hevc_dequant_8_ssse3
void ff_hevc_dequant_8_ssse3(int16_t *coeffs, int16_t log2_size)
IDCT_DC_FUNCS
#define IDCT_DC_FUNCS(W, opt)
Definition dsp_init.c:64
TRANSFORM_LUMA_FUNCS
#define TRANSFORM_LUMA_FUNCS(opt)
Definition dsp_init.c:89
mc_rep_bi_func
#define mc_rep_bi_func(name, bitd, step, W, opt)
Definition dsp_init.c:226
QPEL_LINKS
#define QPEL_LINKS(pointer, my, mx, fname, bitd, opt)
Definition dsp_init.c:810
SAO_BAND_FILTER_FUNCS
#define SAO_BAND_FILTER_FUNCS(bitd, opt)
Definition dsp_init.c:736
EPEL_LINKS
#define EPEL_LINKS(pointer, my, mx, fname, bitd, opt)
Definition dsp_init.c:800
SAO_EDGE_INIT
#define SAO_EDGE_INIT(bitd, opt)
Definition dsp_init.c:785
SAO_EDGE_FILTER_FUNCS
#define SAO_EDGE_FILTER_FUNCS(bitd, opt)
Definition dsp_init.c:766
Generated on Sun Aug 16 2026 20:20:44 for FFmpeg by doxygen 1.13.2