FFmpeg
dsp_init.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2013 Seppo Tomperi
3  * Copyright (c) 2013-2014 Pierre-Edouard Lepere
4  * Copyright (c) 2023-2024 Wu Jianhua
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22 
23 #include "config.h"
24 
25 #include "libavutil/cpu.h"
26 #include "libavutil/mem_internal.h"
27 #include "libavutil/x86/cpu.h"
28 #include "libavcodec/hevc/dsp.h"
29 #include "libavcodec/x86/hevc/dsp.h"
30 #include "libavcodec/x86/h26x/h2656dsp.h"
31 
32 void ff_hevc_dequant_8_ssse3(int16_t *coeffs, int16_t log2_size);
33 
34 #define LFC_FUNC(DIR, DEPTH, OPT) \
35 void ff_hevc_ ## DIR ## _loop_filter_chroma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, const int *tc, const uint8_t *no_p, const uint8_t *no_q);
36 
37 #define LFL_FUNC(DIR, DEPTH, OPT) \
38 void ff_hevc_ ## DIR ## _loop_filter_luma_ ## DEPTH ## _ ## OPT(uint8_t *pix, ptrdiff_t stride, int beta, const int *tc, const uint8_t *no_p, const uint8_t *no_q);
39 
40 #define LFC_FUNCS(type, depth, opt) \
41  LFC_FUNC(h, depth, opt) \
42  LFC_FUNC(v, depth, opt)
43 
44 #define LFL_FUNCS(type, depth, opt) \
45  LFL_FUNC(h, depth, opt) \
46  LFL_FUNC(v, depth, opt)
47 
48 LFC_FUNCS(uint8_t, 8, sse2)
49 LFC_FUNCS(uint8_t, 10, sse2)
50 LFC_FUNCS(uint8_t, 12, sse2)
51 LFC_FUNCS(uint8_t, 8, avx)
52 LFC_FUNCS(uint8_t, 10, avx)
53 LFC_FUNCS(uint8_t, 12, avx)
54 LFL_FUNCS(uint8_t, 8, sse2)
55 LFL_FUNCS(uint8_t, 10, sse2)
56 LFL_FUNCS(uint8_t, 12, sse2)
57 LFL_FUNCS(uint8_t, 8, ssse3)
58 LFL_FUNCS(uint8_t, 10, ssse3)
59 LFL_FUNCS(uint8_t, 12, ssse3)
60 LFL_FUNCS(uint8_t, 8, avx)
61 LFL_FUNCS(uint8_t, 10, avx)
62 LFL_FUNCS(uint8_t, 12, avx)
63 
64 #define IDCT_DC_FUNCS(W, opt) \
65 void ff_hevc_idct_ ## W ## _dc_8_ ## opt(int16_t *coeffs); \
66 void ff_hevc_idct_ ## W ## _dc_10_ ## opt(int16_t *coeffs); \
67 void ff_hevc_idct_ ## W ## _dc_12_ ## opt(int16_t *coeffs)
68 
69 IDCT_DC_FUNCS(4x4, sse2);
70 IDCT_DC_FUNCS(8x8, sse2);
71 IDCT_DC_FUNCS(16x16, sse2);
72 IDCT_DC_FUNCS(32x32, sse2);
73 IDCT_DC_FUNCS(16x16, avx2);
74 IDCT_DC_FUNCS(32x32, avx2);
75 
76 #define IDCT_FUNCS(opt) \
77 void ff_hevc_idct_4x4_8_ ## opt(int16_t *coeffs, int col_limit); \
78 void ff_hevc_idct_4x4_10_ ## opt(int16_t *coeffs, int col_limit); \
79 void ff_hevc_idct_8x8_8_ ## opt(int16_t *coeffs, int col_limit); \
80 void ff_hevc_idct_8x8_10_ ## opt(int16_t *coeffs, int col_limit); \
81 void ff_hevc_idct_16x16_8_ ## opt(int16_t *coeffs, int col_limit); \
82 void ff_hevc_idct_16x16_10_ ## opt(int16_t *coeffs, int col_limit); \
83 void ff_hevc_idct_32x32_8_ ## opt(int16_t *coeffs, int col_limit); \
84 void ff_hevc_idct_32x32_10_ ## opt(int16_t *coeffs, int col_limit);
85 
86 IDCT_FUNCS(sse2)
87 IDCT_FUNCS(avx)
88 
89 #define TRANSFORM_LUMA_FUNCS(opt) \
90 void ff_hevc_transform_4x4_luma_8_ ## opt(int16_t *coeffs); \
91 void ff_hevc_transform_4x4_luma_10_ ## opt(int16_t *coeffs)
92 
93 TRANSFORM_LUMA_FUNCS(sse2);
94 TRANSFORM_LUMA_FUNCS(avx);
95 
96 #define ff_hevc_pel_filters ff_hevc_qpel_filters
97 #define DECL_HV_FILTER(f) \
98  const uint8_t *hf = ff_hevc_ ## f ## _filters[mx]; \
99  const uint8_t *vf = ff_hevc_ ## f ## _filters[my];
100 
101 #define FW_PUT(p, a, b, depth, opt) \
102 static void hevc_put_ ## a ## _ ## depth ## _##opt(int16_t *dst, const uint8_t *src, ptrdiff_t srcstride, \
103  int height, intptr_t mx, intptr_t my,int width) \
104 { \
105  DECL_HV_FILTER(p) \
106  ff_h2656_put_ ## b ## _ ## depth ## _##opt(dst, 2 * MAX_PB_SIZE, src, srcstride, height, hf, vf, width); \
107 }
108 
109 #define FW_PUT_UNI(p, a, b, depth, opt) \
110 static void hevc_put_uni_ ## a ## _ ## depth ## _##opt(uint8_t *dst, ptrdiff_t dststride, \
111  const uint8_t *src, ptrdiff_t srcstride, \
112  int height, intptr_t mx, intptr_t my, int width) \
113 { \
114  DECL_HV_FILTER(p) \
115  ff_h2656_put_uni_ ## b ## _ ## depth ## _##opt(dst, dststride, src, srcstride, height, hf, vf, width); \
116 }
117 
118 #if ARCH_X86_64 && HAVE_SSE4_EXTERNAL
119 
120 #define FW_PUT_FUNCS(p, a, b, depth, opt) \
121  FW_PUT(p, a, b, depth, opt) \
122  FW_PUT_UNI(p, a, b, depth, opt)
123 
124 #define FW_PEL(w, depth, opt) FW_PUT_FUNCS(pel, pel_pixels##w, pixels##w, depth, opt)
125 
126 #define FW_DIR(npel, n, w, depth, opt) \
127  FW_PUT_FUNCS(npel, npel ## _h##w, n ## tap_h##w, depth, opt) \
128  FW_PUT_FUNCS(npel, npel ## _v##w, n ## tap_v##w, depth, opt)
129 
130 #define FW_DIR_HV(npel, n, w, depth, opt) \
131  FW_PUT_FUNCS(npel, npel ## _hv##w, n ## tap_hv##w, depth, opt)
132 
133 FW_PEL(4, 8, sse4)
134 FW_PEL(6, 8, sse4)
135 FW_PEL(8, 8, sse4)
136 FW_PEL(12, 8, sse4)
137 FW_PEL(16, 8, sse4)
138 FW_PEL(4, 10, sse4)
139 FW_PEL(6, 10, sse4)
140 FW_PEL(8, 10, sse4)
141 FW_PEL(4, 12, sse4)
142 FW_PEL(6, 12, sse4)
143 FW_PEL(8, 12, sse4)
144 
145 #define FW_EPEL(w, depth, opt) FW_DIR(epel, 4, w, depth, opt)
146 #define FW_EPEL_HV(w, depth, opt) FW_DIR_HV(epel, 4, w, depth, opt)
147 #define FW_EPEL_FUNCS(w, depth, opt) \
148  FW_EPEL(w, depth, opt) \
149  FW_EPEL_HV(w, depth, opt)
150 
151 FW_EPEL(12, 8, sse4)
152 
153 FW_EPEL_FUNCS(4, 8, sse4)
154 FW_EPEL_FUNCS(6, 8, sse4)
155 FW_EPEL_FUNCS(8, 8, sse4)
156 FW_EPEL_FUNCS(16, 8, sse4)
157 FW_EPEL_FUNCS(4, 10, sse4)
158 FW_EPEL_FUNCS(6, 10, sse4)
159 FW_EPEL_FUNCS(8, 10, sse4)
160 FW_EPEL_FUNCS(4, 12, sse4)
161 FW_EPEL_FUNCS(6, 12, sse4)
162 FW_EPEL_FUNCS(8, 12, sse4)
163 
164 #define FW_QPEL(w, depth, opt) FW_DIR(qpel, 8, w, depth, opt)
165 #define FW_QPEL_HV(w, depth, opt) FW_DIR_HV(qpel, 8, w, depth, opt)
166 #define FW_QPEL_FUNCS(w, depth, opt) \
167  FW_QPEL(w, depth, opt) \
168  FW_QPEL_HV(w, depth, opt)
169 
170 FW_QPEL(12, 8, sse4)
171 FW_QPEL(16, 8, sse4)
172 
173 FW_QPEL_FUNCS(4, 8, sse4)
174 FW_QPEL_FUNCS(8, 8, sse4)
175 FW_QPEL_FUNCS(4, 10, sse4)
176 FW_QPEL_FUNCS(8, 10, sse4)
177 FW_QPEL_FUNCS(4, 12, sse4)
178 FW_QPEL_FUNCS(8, 12, sse4)
179 
180 #if HAVE_AVX2_EXTERNAL
181 
182 FW_PEL(32, 8, avx2)
183 FW_PUT(pel, pel_pixels16, pixels16, 10, avx2)
184 
185 FW_EPEL(32, 8, avx2)
186 FW_EPEL(16, 10, avx2)
187 
188 FW_EPEL_HV(32, 8, avx2)
189 FW_EPEL_HV(16, 10, avx2)
190 
191 FW_QPEL(32, 8, avx2)
192 FW_QPEL(16, 10, avx2)
193 
194 FW_QPEL_HV(16, 10, avx2)
195 
196 #endif
197 #endif
198 
199 #define mc_rep_func(name, bitd, step, W, opt) \
200 static void hevc_put_##name##W##_##bitd##_##opt(int16_t *_dst, \
201  const uint8_t *_src, ptrdiff_t _srcstride, int height, \
202  intptr_t mx, intptr_t my, int width) \
203 { \
204  int i; \
205  int16_t *dst; \
206  for (i = 0; i < W; i += step) { \
207  const uint8_t *src = _src + (i * ((bitd + 7) / 8)); \
208  dst = _dst + i; \
209  hevc_put_##name##step##_##bitd##_##opt(dst, src, _srcstride, height, mx, my, width); \
210  } \
211 }
212 #define mc_rep_uni_func(name, bitd, step, W, opt) \
213 static void hevc_put_uni_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, \
214  const uint8_t *_src, ptrdiff_t _srcstride, int height, \
215  intptr_t mx, intptr_t my, int width) \
216 { \
217  int i; \
218  uint8_t *dst; \
219  for (i = 0; i < W; i += step) { \
220  const uint8_t *src = _src + (i * ((bitd + 7) / 8)); \
221  dst = _dst + (i * ((bitd + 7) / 8)); \
222  hevc_put_uni_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, \
223  height, mx, my, width); \
224  } \
225 }
226 #define mc_rep_bi_func(name, bitd, step, W, opt) \
227 static void ff_hevc_put_bi_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, const uint8_t *_src, \
228  ptrdiff_t _srcstride, const int16_t *_src2, \
229  int height, intptr_t mx, intptr_t my, int width) \
230 { \
231  int i; \
232  uint8_t *dst; \
233  for (i = 0; i < W ; i += step) { \
234  const uint8_t *src = _src + (i * ((bitd + 7) / 8)); \
235  const int16_t *src2 = _src2 + i; \
236  dst = _dst + (i * ((bitd + 7) / 8)); \
237  ff_hevc_put_bi_##name##step##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, \
238  height, mx, my, width); \
239  } \
240 }
241 
242 #define mc_rep_funcs(name, bitd, step, W, opt) \
243  mc_rep_func(name, bitd, step, W, opt) \
244  mc_rep_uni_func(name, bitd, step, W, opt) \
245  mc_rep_bi_func(name, bitd, step, W, opt)
246 
247 #define mc_rep_func2(name, bitd, step1, step2, W, opt) \
248 static void hevc_put_##name##W##_##bitd##_##opt(int16_t *dst, \
249  const uint8_t *src, ptrdiff_t _srcstride, int height, \
250  intptr_t mx, intptr_t my, int width) \
251 { \
252  hevc_put_##name##step1##_##bitd##_##opt(dst, src, _srcstride, height, mx, my, width); \
253  hevc_put_##name##step2##_##bitd##_##opt(dst + step1, src + (step1 * ((bitd + 7) / 8)), \
254  _srcstride, height, mx, my, width); \
255 }
256 #define mc_rep_uni_func2(name, bitd, step1, step2, W, opt) \
257 static void hevc_put_uni_##name##W##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, \
258  const uint8_t *src, ptrdiff_t _srcstride, int height, \
259  intptr_t mx, intptr_t my, int width) \
260 { \
261  hevc_put_uni_##name##step1##_##bitd##_##opt(dst, dststride, src, _srcstride, height, mx, my, width); \
262  hevc_put_uni_##name##step2##_##bitd##_##opt(dst + (step1 * ((bitd + 7) / 8)), dststride, \
263  src + (step1 * ((bitd + 7) / 8)), _srcstride, \
264  height, mx, my, width); \
265 }
266 #define mc_rep_bi_func2(name, bitd, step1, step2, W, opt) \
267 static void ff_hevc_put_bi_##name##W##_##bitd##_##opt(uint8_t *dst, ptrdiff_t dststride, const uint8_t *src, \
268  ptrdiff_t _srcstride, const int16_t *src2, \
269  int height, intptr_t mx, intptr_t my, int width) \
270 { \
271  ff_hevc_put_bi_##name##step1##_##bitd##_##opt(dst, dststride, src, _srcstride, src2, height, mx, my, width);\
272  ff_hevc_put_bi_##name##step2##_##bitd##_##opt(dst + (step1 * ((bitd + 7) / 8)), dststride, \
273  src + (step1 * ((bitd + 7) / 8)), _srcstride, \
274  src2 + step1, height, mx, my, width); \
275 }
276 
277 #define mc_rep_funcs2(name, bitd, step1, step2, W, opt) \
278  mc_rep_func2(name, bitd, step1, step2, W, opt) \
279  mc_rep_uni_func2(name, bitd, step1, step2, W, opt) \
280  mc_rep_bi_func2(name, bitd, step1, step2, W, opt)
281 
282 #if ARCH_X86_64 && HAVE_SSE4_EXTERNAL
283 
284 #define mc_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
285 static void hevc_put_##name##width1##_10_##opt1(int16_t *dst, const uint8_t *src, ptrdiff_t _srcstride, \
286  int height, intptr_t mx, intptr_t my, int width) \
287  \
288 { \
289  hevc_put_##name##width2##_10_##opt1(dst, src, _srcstride, height, mx, my, width); \
290  hevc_put_##name##width3##_10_##opt2(dst+ width2, src+ width4, _srcstride, height, mx, my, width); \
291 }
292 
293 #define mc_bi_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
294 static void ff_hevc_put_bi_##name##width1##_10_##opt1(uint8_t *dst, ptrdiff_t dststride, const uint8_t *src, \
295  ptrdiff_t _srcstride, const int16_t *src2, \
296  int height, intptr_t mx, intptr_t my, int width) \
297 { \
298  ff_hevc_put_bi_##name##width2##_10_##opt1(dst, dststride, src, _srcstride, src2, \
299  height, mx, my, width); \
300  ff_hevc_put_bi_##name##width3##_10_##opt2(dst+width4, dststride, src+width4, _srcstride, src2+width2, \
301  height, mx, my, width); \
302 }
303 
304 #define mc_uni_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
305 static void hevc_put_uni_##name##width1##_10_##opt1(uint8_t *dst, ptrdiff_t dststride, \
306  const uint8_t *src, ptrdiff_t _srcstride, int height, \
307  intptr_t mx, intptr_t my, int width) \
308 { \
309  hevc_put_uni_##name##width2##_10_##opt1(dst, dststride, src, _srcstride, \
310  height, mx, my, width); \
311  hevc_put_uni_##name##width3##_10_##opt2(dst+width4, dststride, src+width4, _srcstride, \
312  height, mx, my, width); \
313 }
314 
315 #define mc_rep_mixs_10(name, width1, width2, width3, opt1, opt2, width4) \
316 mc_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
317 mc_bi_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4) \
318 mc_uni_rep_mix_10(name, width1, width2, width3, opt1, opt2, width4)
319 
320 #define mc_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
321 static void hevc_put_##name##width1##_8_##opt1(int16_t *dst, const uint8_t *src, ptrdiff_t _srcstride, \
322  int height, intptr_t mx, intptr_t my, int width) \
323  \
324 { \
325  hevc_put_##name##width2##_8_##opt1(dst, src, _srcstride, height, mx, my, width); \
326  hevc_put_##name##width3##_8_##opt2(dst+ width2, src+ width2, _srcstride, height, mx, my, width); \
327 }
328 
329 #define mc_bi_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
330 static void ff_hevc_put_bi_##name##width1##_8_##opt1(uint8_t *dst, ptrdiff_t dststride, const uint8_t *src, \
331  ptrdiff_t _srcstride, const int16_t *src2, \
332  int height, intptr_t mx, intptr_t my, int width) \
333 { \
334  ff_hevc_put_bi_##name##width2##_8_##opt1(dst, dststride, src, _srcstride, \
335  src2, height, mx, my, width); \
336  ff_hevc_put_bi_##name##width3##_8_##opt2(dst+width2, dststride, src+width2, _srcstride, \
337  src2+width2, height, mx, my, width); \
338 }
339 
340 #define mc_uni_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
341 static void hevc_put_uni_##name##width1##_8_##opt1(uint8_t *dst, ptrdiff_t dststride, \
342  const uint8_t *src, ptrdiff_t _srcstride, int height, \
343  intptr_t mx, intptr_t my, int width) \
344 { \
345  hevc_put_uni_##name##width2##_8_##opt1(dst, dststride, src, _srcstride, \
346  height, mx, my, width); \
347  hevc_put_uni_##name##width3##_8_##opt2(dst+width2, dststride, src+width2, _srcstride, \
348  height, mx, my, width); \
349 }
350 
351 #define mc_rep_mixs_8(name, width1, width2, width3, opt1, opt2) \
352 mc_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
353 mc_bi_rep_mix_8(name, width1, width2, width3, opt1, opt2) \
354 mc_uni_rep_mix_8(name, width1, width2, width3, opt1, opt2)
355 
356 #if HAVE_AVX2_EXTERNAL
357 
358 mc_rep_mixs_8(pel_pixels, 48, 32, 16, avx2, sse4)
359 mc_rep_mixs_8(epel_hv, 48, 32, 16, avx2, sse4)
360 mc_rep_mixs_8(epel_h , 48, 32, 16, avx2, sse4)
361 mc_rep_mixs_8(epel_v , 48, 32, 16, avx2, sse4)
362 
363 mc_rep_mix_10(pel_pixels, 24, 16, 8, avx2, sse4, 32)
364 mc_bi_rep_mix_10(pel_pixels,24, 16, 8, avx2, sse4, 32)
365 mc_rep_mixs_10(epel_hv, 24, 16, 8, avx2, sse4, 32)
366 mc_rep_mixs_10(epel_h , 24, 16, 8, avx2, sse4, 32)
367 mc_rep_mixs_10(epel_v , 24, 16, 8, avx2, sse4, 32)
368 
369 
370 mc_rep_mixs_10(qpel_h , 24, 16, 8, avx2, sse4, 32)
371 mc_rep_mixs_10(qpel_v , 24, 16, 8, avx2, sse4, 32)
372 mc_rep_mixs_10(qpel_hv, 24, 16, 8, avx2, sse4, 32)
373 
374 
375 mc_rep_funcs(pel_pixels, 8, 32, 64, avx2)
376 
377 mc_rep_uni_func(pel_pixels, 8, 64, 128, avx2)//used for 10bit
378 mc_rep_uni_func(pel_pixels, 8, 32, 96, avx2) //used for 10bit
379 
380 mc_rep_func(pel_pixels, 10, 16, 32, avx2)
381 mc_rep_func(pel_pixels, 10, 16, 48, avx2)
382 mc_rep_func(pel_pixels, 10, 32, 64, avx2)
383 
384 mc_rep_bi_func(pel_pixels, 10, 16, 32, avx2)
385 mc_rep_bi_func(pel_pixels, 10, 16, 48, avx2)
386 mc_rep_bi_func(pel_pixels, 10, 32, 64, avx2)
387 
388 mc_rep_funcs(epel_h, 8, 32, 64, avx2)
389 
390 mc_rep_funcs(epel_v, 8, 32, 64, avx2)
391 
392 mc_rep_funcs(epel_h, 10, 16, 32, avx2)
393 mc_rep_funcs(epel_h, 10, 16, 48, avx2)
394 mc_rep_funcs(epel_h, 10, 32, 64, avx2)
395 
396 mc_rep_funcs(epel_v, 10, 16, 32, avx2)
397 mc_rep_funcs(epel_v, 10, 16, 48, avx2)
398 mc_rep_funcs(epel_v, 10, 32, 64, avx2)
399 
400 
401 mc_rep_funcs(epel_hv, 8, 32, 64, avx2)
402 
403 mc_rep_funcs(epel_hv, 10, 16, 32, avx2)
404 mc_rep_funcs(epel_hv, 10, 16, 48, avx2)
405 mc_rep_funcs(epel_hv, 10, 32, 64, avx2)
406 
407 mc_rep_funcs(qpel_h, 8, 32, 64, avx2)
408 mc_rep_mixs_8(qpel_h , 48, 32, 16, avx2, sse4)
409 
410 mc_rep_funcs(qpel_v, 8, 32, 64, avx2)
411 mc_rep_mixs_8(qpel_v, 48, 32, 16, avx2, sse4)
412 
413 mc_rep_funcs(qpel_h, 10, 16, 32, avx2)
414 mc_rep_funcs(qpel_h, 10, 16, 48, avx2)
415 mc_rep_funcs(qpel_h, 10, 32, 64, avx2)
416 
417 mc_rep_funcs(qpel_v, 10, 16, 32, avx2)
418 mc_rep_funcs(qpel_v, 10, 16, 48, avx2)
419 mc_rep_funcs(qpel_v, 10, 32, 64, avx2)
420 
421 mc_rep_funcs(qpel_hv, 10, 16, 32, avx2)
422 mc_rep_funcs(qpel_hv, 10, 16, 48, avx2)
423 mc_rep_funcs(qpel_hv, 10, 32, 64, avx2)
424 
425 #endif //AVX2
426 
427 mc_rep_funcs(pel_pixels, 8, 16, 64, sse4)
428 mc_rep_funcs(pel_pixels, 8, 16, 48, sse4)
429 mc_rep_funcs(pel_pixels, 8, 16, 32, sse4)
430 mc_rep_funcs(pel_pixels, 8, 8, 24, sse4)
431 mc_rep_funcs(pel_pixels,10, 8, 64, sse4)
432 mc_rep_funcs(pel_pixels,10, 8, 48, sse4)
433 mc_rep_funcs(pel_pixels,10, 8, 32, sse4)
434 mc_rep_funcs(pel_pixels,10, 8, 24, sse4)
435 mc_rep_funcs(pel_pixels,10, 8, 16, sse4)
436 mc_rep_funcs(pel_pixels,10, 4, 12, sse4)
437 mc_rep_funcs(pel_pixels,12, 8, 64, sse4)
438 mc_rep_funcs(pel_pixels,12, 8, 48, sse4)
439 mc_rep_funcs(pel_pixels,12, 8, 32, sse4)
440 mc_rep_funcs(pel_pixels,12, 8, 24, sse4)
441 mc_rep_funcs(pel_pixels,12, 8, 16, sse4)
442 mc_rep_funcs(pel_pixels,12, 4, 12, sse4)
443 
444 mc_rep_funcs(epel_h, 8, 16, 64, sse4)
445 mc_rep_funcs(epel_h, 8, 16, 48, sse4)
446 mc_rep_funcs(epel_h, 8, 16, 32, sse4)
447 mc_rep_funcs(epel_h, 8, 8, 24, sse4)
448 mc_rep_funcs(epel_h,10, 8, 64, sse4)
449 mc_rep_funcs(epel_h,10, 8, 48, sse4)
450 mc_rep_funcs(epel_h,10, 8, 32, sse4)
451 mc_rep_funcs(epel_h,10, 8, 24, sse4)
452 mc_rep_funcs(epel_h,10, 8, 16, sse4)
453 mc_rep_funcs(epel_h,10, 4, 12, sse4)
454 mc_rep_funcs(epel_h,12, 8, 64, sse4)
455 mc_rep_funcs(epel_h,12, 8, 48, sse4)
456 mc_rep_funcs(epel_h,12, 8, 32, sse4)
457 mc_rep_funcs(epel_h,12, 8, 24, sse4)
458 mc_rep_funcs(epel_h,12, 8, 16, sse4)
459 mc_rep_funcs(epel_h,12, 4, 12, sse4)
460 mc_rep_funcs(epel_v, 8, 16, 64, sse4)
461 mc_rep_funcs(epel_v, 8, 16, 48, sse4)
462 mc_rep_funcs(epel_v, 8, 16, 32, sse4)
463 mc_rep_funcs(epel_v, 8, 8, 24, sse4)
464 mc_rep_funcs(epel_v,10, 8, 64, sse4)
465 mc_rep_funcs(epel_v,10, 8, 48, sse4)
466 mc_rep_funcs(epel_v,10, 8, 32, sse4)
467 mc_rep_funcs(epel_v,10, 8, 24, sse4)
468 mc_rep_funcs(epel_v,10, 8, 16, sse4)
469 mc_rep_funcs(epel_v,10, 4, 12, sse4)
470 mc_rep_funcs(epel_v,12, 8, 64, sse4)
471 mc_rep_funcs(epel_v,12, 8, 48, sse4)
472 mc_rep_funcs(epel_v,12, 8, 32, sse4)
473 mc_rep_funcs(epel_v,12, 8, 24, sse4)
474 mc_rep_funcs(epel_v,12, 8, 16, sse4)
475 mc_rep_funcs(epel_v,12, 4, 12, sse4)
476 mc_rep_funcs(epel_hv, 8, 16, 64, sse4)
477 mc_rep_funcs(epel_hv, 8, 16, 48, sse4)
478 mc_rep_funcs(epel_hv, 8, 16, 32, sse4)
479 mc_rep_funcs(epel_hv, 8, 8, 24, sse4)
480 mc_rep_funcs2(epel_hv,8, 8, 4, 12, sse4)
481 mc_rep_funcs(epel_hv,10, 8, 64, sse4)
482 mc_rep_funcs(epel_hv,10, 8, 48, sse4)
483 mc_rep_funcs(epel_hv,10, 8, 32, sse4)
484 mc_rep_funcs(epel_hv,10, 8, 24, sse4)
485 mc_rep_funcs(epel_hv,10, 8, 16, sse4)
486 mc_rep_funcs(epel_hv,10, 4, 12, sse4)
487 mc_rep_funcs(epel_hv,12, 8, 64, sse4)
488 mc_rep_funcs(epel_hv,12, 8, 48, sse4)
489 mc_rep_funcs(epel_hv,12, 8, 32, sse4)
490 mc_rep_funcs(epel_hv,12, 8, 24, sse4)
491 mc_rep_funcs(epel_hv,12, 8, 16, sse4)
492 mc_rep_funcs(epel_hv,12, 4, 12, sse4)
493 
494 mc_rep_funcs(qpel_h, 8, 16, 64, sse4)
495 mc_rep_funcs(qpel_h, 8, 16, 48, sse4)
496 mc_rep_funcs(qpel_h, 8, 16, 32, sse4)
497 mc_rep_funcs(qpel_h, 8, 8, 24, sse4)
498 mc_rep_funcs(qpel_h,10, 8, 64, sse4)
499 mc_rep_funcs(qpel_h,10, 8, 48, sse4)
500 mc_rep_funcs(qpel_h,10, 8, 32, sse4)
501 mc_rep_funcs(qpel_h,10, 8, 24, sse4)
502 mc_rep_funcs(qpel_h,10, 8, 16, sse4)
503 mc_rep_funcs(qpel_h,10, 4, 12, sse4)
504 mc_rep_funcs(qpel_h,12, 8, 64, sse4)
505 mc_rep_funcs(qpel_h,12, 8, 48, sse4)
506 mc_rep_funcs(qpel_h,12, 8, 32, sse4)
507 mc_rep_funcs(qpel_h,12, 8, 24, sse4)
508 mc_rep_funcs(qpel_h,12, 8, 16, sse4)
509 mc_rep_funcs(qpel_h,12, 4, 12, sse4)
510 mc_rep_funcs(qpel_v, 8, 16, 64, sse4)
511 mc_rep_funcs(qpel_v, 8, 16, 48, sse4)
512 mc_rep_funcs(qpel_v, 8, 16, 32, sse4)
513 mc_rep_funcs(qpel_v, 8, 8, 24, sse4)
514 mc_rep_funcs(qpel_v,10, 8, 64, sse4)
515 mc_rep_funcs(qpel_v,10, 8, 48, sse4)
516 mc_rep_funcs(qpel_v,10, 8, 32, sse4)
517 mc_rep_funcs(qpel_v,10, 8, 24, sse4)
518 mc_rep_funcs(qpel_v,10, 8, 16, sse4)
519 mc_rep_funcs(qpel_v,10, 4, 12, sse4)
520 mc_rep_funcs(qpel_v,12, 8, 64, sse4)
521 mc_rep_funcs(qpel_v,12, 8, 48, sse4)
522 mc_rep_funcs(qpel_v,12, 8, 32, sse4)
523 mc_rep_funcs(qpel_v,12, 8, 24, sse4)
524 mc_rep_funcs(qpel_v,12, 8, 16, sse4)
525 mc_rep_funcs(qpel_v,12, 4, 12, sse4)
526 mc_rep_funcs(qpel_hv, 8, 8, 64, sse4)
527 mc_rep_funcs(qpel_hv, 8, 8, 48, sse4)
528 mc_rep_funcs(qpel_hv, 8, 8, 32, sse4)
529 mc_rep_funcs(qpel_hv, 8, 8, 24, sse4)
530 mc_rep_funcs(qpel_hv, 8, 8, 16, sse4)
531 mc_rep_funcs2(qpel_hv,8, 8, 4, 12, sse4)
532 mc_rep_funcs(qpel_hv,10, 8, 64, sse4)
533 mc_rep_funcs(qpel_hv,10, 8, 48, sse4)
534 mc_rep_funcs(qpel_hv,10, 8, 32, sse4)
535 mc_rep_funcs(qpel_hv,10, 8, 24, sse4)
536 mc_rep_funcs(qpel_hv,10, 8, 16, sse4)
537 mc_rep_funcs(qpel_hv,10, 4, 12, sse4)
538 mc_rep_funcs(qpel_hv,12, 8, 64, sse4)
539 mc_rep_funcs(qpel_hv,12, 8, 48, sse4)
540 mc_rep_funcs(qpel_hv,12, 8, 32, sse4)
541 mc_rep_funcs(qpel_hv,12, 8, 24, sse4)
542 mc_rep_funcs(qpel_hv,12, 8, 16, sse4)
543 mc_rep_funcs(qpel_hv,12, 4, 12, sse4)
544 
545 #define mc_rep_uni_w(bitd, step, W, opt) \
546 void ff_hevc_put_uni_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, const int16_t *_src, \
547  int height, int denom, int _wx, int _ox) \
548 { \
549  int i; \
550  uint8_t *dst; \
551  for (i = 0; i < W; i += step) { \
552  const int16_t *src = _src + i; \
553  dst= _dst + (i * ((bitd + 7) / 8)); \
554  ff_hevc_put_uni_w##step##_##bitd##_##opt(dst, dststride, src, \
555  height, denom, _wx, _ox); \
556  } \
557 }
558 
559 mc_rep_uni_w(8, 6, 12, sse4)
560 mc_rep_uni_w(8, 8, 16, sse4)
561 mc_rep_uni_w(8, 8, 24, sse4)
562 mc_rep_uni_w(8, 8, 32, sse4)
563 mc_rep_uni_w(8, 8, 48, sse4)
564 mc_rep_uni_w(8, 8, 64, sse4)
565 
566 mc_rep_uni_w(10, 6, 12, sse4)
567 mc_rep_uni_w(10, 8, 16, sse4)
568 mc_rep_uni_w(10, 8, 24, sse4)
569 mc_rep_uni_w(10, 8, 32, sse4)
570 mc_rep_uni_w(10, 8, 48, sse4)
571 mc_rep_uni_w(10, 8, 64, sse4)
572 
573 mc_rep_uni_w(12, 6, 12, sse4)
574 mc_rep_uni_w(12, 8, 16, sse4)
575 mc_rep_uni_w(12, 8, 24, sse4)
576 mc_rep_uni_w(12, 8, 32, sse4)
577 mc_rep_uni_w(12, 8, 48, sse4)
578 mc_rep_uni_w(12, 8, 64, sse4)
579 
580 #define mc_rep_bi_w(bitd, step, W, opt) \
581 void ff_hevc_put_bi_w##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t dststride, const int16_t *_src, \
582  const int16_t *_src2, int height, \
583  int denom, int wx0, int wx1, int ox) \
584 { \
585  int i; \
586  uint8_t *dst; \
587  for (i = 0; i < W; i += step) { \
588  const int16_t *src = _src + i; \
589  const int16_t *src2 = _src2 + i; \
590  dst = _dst + (i * ((bitd + 7) / 8)); \
591  ff_hevc_put_bi_w##step##_##bitd##_##opt(dst, dststride, src, src2, \
592  height, denom, wx0, wx1, ox); \
593  } \
594 }
595 
596 mc_rep_bi_w(8, 6, 12, sse4)
597 mc_rep_bi_w(8, 8, 16, sse4)
598 mc_rep_bi_w(8, 8, 24, sse4)
599 mc_rep_bi_w(8, 8, 32, sse4)
600 mc_rep_bi_w(8, 8, 48, sse4)
601 mc_rep_bi_w(8, 8, 64, sse4)
602 
603 mc_rep_bi_w(10, 6, 12, sse4)
604 mc_rep_bi_w(10, 8, 16, sse4)
605 mc_rep_bi_w(10, 8, 24, sse4)
606 mc_rep_bi_w(10, 8, 32, sse4)
607 mc_rep_bi_w(10, 8, 48, sse4)
608 mc_rep_bi_w(10, 8, 64, sse4)
609 
610 mc_rep_bi_w(12, 6, 12, sse4)
611 mc_rep_bi_w(12, 8, 16, sse4)
612 mc_rep_bi_w(12, 8, 24, sse4)
613 mc_rep_bi_w(12, 8, 32, sse4)
614 mc_rep_bi_w(12, 8, 48, sse4)
615 mc_rep_bi_w(12, 8, 64, sse4)
616 
617 #define mc_uni_w_func(name, bitd, W, opt) \
618 static void hevc_put_uni_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride, \
619  const uint8_t *_src, ptrdiff_t _srcstride, \
620  int height, int denom, \
621  int _wx, int _ox, \
622  intptr_t mx, intptr_t my, int width) \
623 { \
624  LOCAL_ALIGNED_16(int16_t, temp, [71 * MAX_PB_SIZE]); \
625  hevc_put_##name##W##_##bitd##_##opt(temp, _src, _srcstride, height, mx, my, width); \
626  ff_hevc_put_uni_w##W##_##bitd##_##opt(_dst, _dststride, temp, height, denom, _wx, _ox); \
627 }
628 
629 #define mc_uni_w_funcs(name, bitd, opt) \
630  mc_uni_w_func(name, bitd, 4, opt) \
631  mc_uni_w_func(name, bitd, 8, opt) \
632  mc_uni_w_func(name, bitd, 12, opt) \
633  mc_uni_w_func(name, bitd, 16, opt) \
634  mc_uni_w_func(name, bitd, 24, opt) \
635  mc_uni_w_func(name, bitd, 32, opt) \
636  mc_uni_w_func(name, bitd, 48, opt) \
637  mc_uni_w_func(name, bitd, 64, opt)
638 
639 mc_uni_w_funcs(pel_pixels, 8, sse4)
640 mc_uni_w_func(pel_pixels, 8, 6, sse4)
641 mc_uni_w_funcs(epel_h, 8, sse4)
642 mc_uni_w_func(epel_h, 8, 6, sse4)
643 mc_uni_w_funcs(epel_v, 8, sse4)
644 mc_uni_w_func(epel_v, 8, 6, sse4)
645 mc_uni_w_funcs(epel_hv, 8, sse4)
646 mc_uni_w_func(epel_hv, 8, 6, sse4)
647 mc_uni_w_funcs(qpel_h, 8, sse4)
648 mc_uni_w_funcs(qpel_v, 8, sse4)
649 mc_uni_w_funcs(qpel_hv, 8, sse4)
650 
651 mc_uni_w_funcs(pel_pixels, 10, sse4)
652 mc_uni_w_func(pel_pixels, 10, 6, sse4)
653 mc_uni_w_funcs(epel_h, 10, sse4)
654 mc_uni_w_func(epel_h, 10, 6, sse4)
655 mc_uni_w_funcs(epel_v, 10, sse4)
656 mc_uni_w_func(epel_v, 10, 6, sse4)
657 mc_uni_w_funcs(epel_hv, 10, sse4)
658 mc_uni_w_func(epel_hv, 10, 6, sse4)
659 mc_uni_w_funcs(qpel_h, 10, sse4)
660 mc_uni_w_funcs(qpel_v, 10, sse4)
661 mc_uni_w_funcs(qpel_hv, 10, sse4)
662 
663 mc_uni_w_funcs(pel_pixels, 12, sse4)
664 mc_uni_w_func(pel_pixels, 12, 6, sse4)
665 mc_uni_w_funcs(epel_h, 12, sse4)
666 mc_uni_w_func(epel_h, 12, 6, sse4)
667 mc_uni_w_funcs(epel_v, 12, sse4)
668 mc_uni_w_func(epel_v, 12, 6, sse4)
669 mc_uni_w_funcs(epel_hv, 12, sse4)
670 mc_uni_w_func(epel_hv, 12, 6, sse4)
671 mc_uni_w_funcs(qpel_h, 12, sse4)
672 mc_uni_w_funcs(qpel_v, 12, sse4)
673 mc_uni_w_funcs(qpel_hv, 12, sse4)
674 
675 #define mc_bi_w_func(name, bitd, W, opt) \
676 static void hevc_put_bi_w_##name##W##_##bitd##_##opt(uint8_t *_dst, ptrdiff_t _dststride, \
677  const uint8_t *_src, ptrdiff_t _srcstride, \
678  const int16_t *_src2, \
679  int height, int denom, \
680  int wx0, int wx1, int ox, \
681  intptr_t mx, intptr_t my, int width) \
682 { \
683  LOCAL_ALIGNED_16(int16_t, temp, [71 * MAX_PB_SIZE]); \
684  hevc_put_##name##W##_##bitd##_##opt(temp, _src, _srcstride, height, mx, my, width); \
685  ff_hevc_put_bi_w##W##_##bitd##_##opt(_dst, _dststride, temp, _src2, \
686  height, denom, wx0, wx1, ox); \
687 }
688 
689 #define mc_bi_w_funcs(name, bitd, opt) \
690  mc_bi_w_func(name, bitd, 4, opt) \
691  mc_bi_w_func(name, bitd, 8, opt) \
692  mc_bi_w_func(name, bitd, 12, opt) \
693  mc_bi_w_func(name, bitd, 16, opt) \
694  mc_bi_w_func(name, bitd, 24, opt) \
695  mc_bi_w_func(name, bitd, 32, opt) \
696  mc_bi_w_func(name, bitd, 48, opt) \
697  mc_bi_w_func(name, bitd, 64, opt)
698 
699 mc_bi_w_funcs(pel_pixels, 8, sse4)
700 mc_bi_w_func(pel_pixels, 8, 6, sse4)
701 mc_bi_w_funcs(epel_h, 8, sse4)
702 mc_bi_w_func(epel_h, 8, 6, sse4)
703 mc_bi_w_funcs(epel_v, 8, sse4)
704 mc_bi_w_func(epel_v, 8, 6, sse4)
705 mc_bi_w_funcs(epel_hv, 8, sse4)
706 mc_bi_w_func(epel_hv, 8, 6, sse4)
707 mc_bi_w_funcs(qpel_h, 8, sse4)
708 mc_bi_w_funcs(qpel_v, 8, sse4)
709 mc_bi_w_funcs(qpel_hv, 8, sse4)
710 
711 mc_bi_w_funcs(pel_pixels, 10, sse4)
712 mc_bi_w_func(pel_pixels, 10, 6, sse4)
713 mc_bi_w_funcs(epel_h, 10, sse4)
714 mc_bi_w_func(epel_h, 10, 6, sse4)
715 mc_bi_w_funcs(epel_v, 10, sse4)
716 mc_bi_w_func(epel_v, 10, 6, sse4)
717 mc_bi_w_funcs(epel_hv, 10, sse4)
718 mc_bi_w_func(epel_hv, 10, 6, sse4)
719 mc_bi_w_funcs(qpel_h, 10, sse4)
720 mc_bi_w_funcs(qpel_v, 10, sse4)
721 mc_bi_w_funcs(qpel_hv, 10, sse4)
722 
723 mc_bi_w_funcs(pel_pixels, 12, sse4)
724 mc_bi_w_func(pel_pixels, 12, 6, sse4)
725 mc_bi_w_funcs(epel_h, 12, sse4)
726 mc_bi_w_func(epel_h, 12, 6, sse4)
727 mc_bi_w_funcs(epel_v, 12, sse4)
728 mc_bi_w_func(epel_v, 12, 6, sse4)
729 mc_bi_w_funcs(epel_hv, 12, sse4)
730 mc_bi_w_func(epel_hv, 12, 6, sse4)
731 mc_bi_w_funcs(qpel_h, 12, sse4)
732 mc_bi_w_funcs(qpel_v, 12, sse4)
733 mc_bi_w_funcs(qpel_hv, 12, sse4)
734 #endif //ARCH_X86_64 && HAVE_SSE4_EXTERNAL
735 
736 #define SAO_BAND_FILTER_FUNCS(bitd, opt) \
737 void ff_hevc_sao_band_filter_8_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
738  const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
739 void ff_hevc_sao_band_filter_16_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
740  const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
741 void ff_hevc_sao_band_filter_32_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
742  const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
743 void ff_hevc_sao_band_filter_48_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
744  const int16_t *sao_offset_val, int sao_left_class, int width, int height); \
745 void ff_hevc_sao_band_filter_64_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
746  const int16_t *sao_offset_val, int sao_left_class, int width, int height);
747 
748 SAO_BAND_FILTER_FUNCS(8, sse2)
749 SAO_BAND_FILTER_FUNCS(10, sse2)
750 SAO_BAND_FILTER_FUNCS(12, sse2)
751 SAO_BAND_FILTER_FUNCS(8, avx)
752 SAO_BAND_FILTER_FUNCS(10, avx)
753 SAO_BAND_FILTER_FUNCS(12, avx)
754 SAO_BAND_FILTER_FUNCS(8, avx2)
755 SAO_BAND_FILTER_FUNCS(10, avx2)
756 SAO_BAND_FILTER_FUNCS(12, avx2)
757 
758 #define SAO_BAND_INIT(bitd, opt) do { \
759  c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_##bitd##_##opt; \
760  c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_##bitd##_##opt; \
761  c->sao_band_filter[2] = ff_hevc_sao_band_filter_32_##bitd##_##opt; \
762  c->sao_band_filter[3] = ff_hevc_sao_band_filter_48_##bitd##_##opt; \
763  c->sao_band_filter[4] = ff_hevc_sao_band_filter_64_##bitd##_##opt; \
764 } while (0)
765 
766 #define SAO_EDGE_FILTER_FUNCS(bitd, opt) \
767 void ff_hevc_sao_edge_filter_8_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
768  const int16_t *sao_offset_val, int eo, int width, int height); \
769 void ff_hevc_sao_edge_filter_16_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
770  const int16_t *sao_offset_val, int eo, int width, int height); \
771 void ff_hevc_sao_edge_filter_32_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
772  const int16_t *sao_offset_val, int eo, int width, int height); \
773 void ff_hevc_sao_edge_filter_48_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
774  const int16_t *sao_offset_val, int eo, int width, int height); \
775 void ff_hevc_sao_edge_filter_64_##bitd##_##opt(uint8_t *_dst, const uint8_t *_src, ptrdiff_t stride_dst, \
776  const int16_t *sao_offset_val, int eo, int width, int height); \
777 
778 SAO_EDGE_FILTER_FUNCS(8, ssse3)
779 SAO_EDGE_FILTER_FUNCS(8, avx2)
780 SAO_EDGE_FILTER_FUNCS(10, sse2)
781 SAO_EDGE_FILTER_FUNCS(10, avx2)
782 SAO_EDGE_FILTER_FUNCS(12, sse2)
783 SAO_EDGE_FILTER_FUNCS(12, avx2)
784 
785 #define SAO_EDGE_INIT(bitd, opt) do { \
786  c->sao_edge_filter[0] = ff_hevc_sao_edge_filter_8_##bitd##_##opt; \
787  c->sao_edge_filter[1] = ff_hevc_sao_edge_filter_16_##bitd##_##opt; \
788  c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_##bitd##_##opt; \
789  c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_##bitd##_##opt; \
790  c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_##bitd##_##opt; \
791 } while (0)
792 
793 #define PEL_LINK(dst, idx1, idx2, idx3, name, D, opt) \
794 dst [idx1][idx2][idx3] = hevc_put_ ## name ## _ ## D ## _##opt; \
795 dst ## _bi [idx1][idx2][idx3] = ff_hevc_put_bi_ ## name ## _ ## D ## _##opt; \
796 dst ## _uni [idx1][idx2][idx3] = hevc_put_uni_ ## name ## _ ## D ## _##opt; \
797 dst ## _uni_w[idx1][idx2][idx3] = hevc_put_uni_w_ ## name ## _ ## D ## _##opt; \
798 dst ## _bi_w [idx1][idx2][idx3] = hevc_put_bi_w_ ## name ## _ ## D ## _##opt
799 
800 #define EPEL_LINKS(pointer, my, mx, fname, bitd, opt ) \
801  PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
802  PEL_LINK(pointer, 2, my , mx , fname##6 , bitd, opt ); \
803  PEL_LINK(pointer, 3, my , mx , fname##8 , bitd, opt ); \
804  PEL_LINK(pointer, 4, my , mx , fname##12, bitd, opt ); \
805  PEL_LINK(pointer, 5, my , mx , fname##16, bitd, opt ); \
806  PEL_LINK(pointer, 6, my , mx , fname##24, bitd, opt ); \
807  PEL_LINK(pointer, 7, my , mx , fname##32, bitd, opt ); \
808  PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
809  PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
810 #define QPEL_LINKS(pointer, my, mx, fname, bitd, opt) \
811  PEL_LINK(pointer, 1, my , mx , fname##4 , bitd, opt ); \
812  PEL_LINK(pointer, 3, my , mx , fname##8 , bitd, opt ); \
813  PEL_LINK(pointer, 4, my , mx , fname##12, bitd, opt ); \
814  PEL_LINK(pointer, 5, my , mx , fname##16, bitd, opt ); \
815  PEL_LINK(pointer, 6, my , mx , fname##24, bitd, opt ); \
816  PEL_LINK(pointer, 7, my , mx , fname##32, bitd, opt ); \
817  PEL_LINK(pointer, 8, my , mx , fname##48, bitd, opt ); \
818  PEL_LINK(pointer, 9, my , mx , fname##64, bitd, opt )
819 
820 void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
821 {
822  int cpu_flags = av_get_cpu_flags();
823 
824  if (bit_depth == 8) {
825  if (EXTERNAL_SSE2(cpu_flags)) {
826  c->add_residual[0] = ff_hevc_add_residual_4_8_sse2;
827 
828  c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_sse2;
829  c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_sse2;
830 #if ARCH_X86_64
831  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_sse2;
832  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_sse2;
833 
834  c->idct[2] = ff_hevc_idct_16x16_8_sse2;
835  c->idct[3] = ff_hevc_idct_32x32_8_sse2;
836 #endif
837  SAO_BAND_INIT(8, sse2);
838 
839  c->idct_dc[0] = ff_hevc_idct_4x4_dc_8_sse2;
840  c->idct_dc[1] = ff_hevc_idct_8x8_dc_8_sse2;
841  c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_sse2;
842  c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_sse2;
843 
844  c->idct[0] = ff_hevc_idct_4x4_8_sse2;
845  c->idct[1] = ff_hevc_idct_8x8_8_sse2;
846  c->transform_4x4_luma = ff_hevc_transform_4x4_luma_8_sse2;
847 
848  c->add_residual[1] = ff_hevc_add_residual_8_8_sse2;
849  c->add_residual[2] = ff_hevc_add_residual_16_8_sse2;
850  c->add_residual[3] = ff_hevc_add_residual_32_8_sse2;
851  }
852  if (EXTERNAL_SSSE3(cpu_flags)) {
853 #if ARCH_X86_64
854  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_ssse3;
855  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_ssse3;
856 #endif
857  c->dequant = ff_hevc_dequant_8_ssse3;
858  SAO_EDGE_INIT(8, ssse3);
859  }
860 #if HAVE_SSE4_EXTERNAL && ARCH_X86_64
861  if (EXTERNAL_SSE4(cpu_flags)) {
862 
863  EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 8, sse4);
864  EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 8, sse4);
865  EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 8, sse4);
866  EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 8, sse4);
867 
868  QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 8, sse4);
869  QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 8, sse4);
870  QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 8, sse4);
871  QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 8, sse4);
872  }
873 #endif
874  if (EXTERNAL_AVX(cpu_flags)) {
875  c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_8_avx;
876  c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_8_avx;
877 #if ARCH_X86_64
878  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_avx;
879  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_avx;
880 
881  c->idct[2] = ff_hevc_idct_16x16_8_avx;
882  c->idct[3] = ff_hevc_idct_32x32_8_avx;
883 #endif
884  SAO_BAND_INIT(8, avx);
885 
886  c->idct[0] = ff_hevc_idct_4x4_8_avx;
887  c->idct[1] = ff_hevc_idct_8x8_8_avx;
888  c->transform_4x4_luma = ff_hevc_transform_4x4_luma_8_avx;
889  }
890  if (EXTERNAL_AVX2(cpu_flags)) {
891  c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_8_avx2;
892  c->sao_band_filter[1] = ff_hevc_sao_band_filter_16_8_avx2;
893  }
894 #if HAVE_AVX2_EXTERNAL
895  if (EXTERNAL_AVX2_FAST(cpu_flags)) {
896  c->idct_dc[2] = ff_hevc_idct_16x16_dc_8_avx2;
897  c->idct_dc[3] = ff_hevc_idct_32x32_dc_8_avx2;
898 
899 #if ARCH_X86_64
900  c->put_hevc_epel[7][0][0] = hevc_put_pel_pixels32_8_avx2;
901  c->put_hevc_epel[8][0][0] = hevc_put_pel_pixels48_8_avx2;
902  c->put_hevc_epel[9][0][0] = hevc_put_pel_pixels64_8_avx2;
903 
904  c->put_hevc_qpel[7][0][0] = hevc_put_pel_pixels32_8_avx2;
905  c->put_hevc_qpel[8][0][0] = hevc_put_pel_pixels48_8_avx2;
906  c->put_hevc_qpel[9][0][0] = hevc_put_pel_pixels64_8_avx2;
907 
908  c->put_hevc_epel_uni[7][0][0] = hevc_put_uni_pel_pixels32_8_avx2;
909  c->put_hevc_epel_uni[8][0][0] = hevc_put_uni_pel_pixels48_8_avx2;
910  c->put_hevc_epel_uni[9][0][0] = hevc_put_uni_pel_pixels64_8_avx2;
911 
912  c->put_hevc_qpel_uni[7][0][0] = hevc_put_uni_pel_pixels32_8_avx2;
913  c->put_hevc_qpel_uni[8][0][0] = hevc_put_uni_pel_pixels48_8_avx2;
914  c->put_hevc_qpel_uni[9][0][0] = hevc_put_uni_pel_pixels64_8_avx2;
915 
916  c->put_hevc_qpel_bi[7][0][0] = ff_hevc_put_bi_pel_pixels32_8_avx2;
917  c->put_hevc_qpel_bi[8][0][0] = ff_hevc_put_bi_pel_pixels48_8_avx2;
918  c->put_hevc_qpel_bi[9][0][0] = ff_hevc_put_bi_pel_pixels64_8_avx2;
919 
920  c->put_hevc_epel_bi[7][0][0] = ff_hevc_put_bi_pel_pixels32_8_avx2;
921  c->put_hevc_epel_bi[8][0][0] = ff_hevc_put_bi_pel_pixels48_8_avx2;
922  c->put_hevc_epel_bi[9][0][0] = ff_hevc_put_bi_pel_pixels64_8_avx2;
923 
924  c->put_hevc_epel[7][0][1] = hevc_put_epel_h32_8_avx2;
925  c->put_hevc_epel[8][0][1] = hevc_put_epel_h48_8_avx2;
926  c->put_hevc_epel[9][0][1] = hevc_put_epel_h64_8_avx2;
927 
928  c->put_hevc_epel_uni[7][0][1] = hevc_put_uni_epel_h32_8_avx2;
929  c->put_hevc_epel_uni[8][0][1] = hevc_put_uni_epel_h48_8_avx2;
930  c->put_hevc_epel_uni[9][0][1] = hevc_put_uni_epel_h64_8_avx2;
931 
932  c->put_hevc_epel_bi[7][0][1] = ff_hevc_put_bi_epel_h32_8_avx2;
933  c->put_hevc_epel_bi[8][0][1] = ff_hevc_put_bi_epel_h48_8_avx2;
934  c->put_hevc_epel_bi[9][0][1] = ff_hevc_put_bi_epel_h64_8_avx2;
935 
936  c->put_hevc_epel[7][1][0] = hevc_put_epel_v32_8_avx2;
937  c->put_hevc_epel[8][1][0] = hevc_put_epel_v48_8_avx2;
938  c->put_hevc_epel[9][1][0] = hevc_put_epel_v64_8_avx2;
939 
940  c->put_hevc_epel_uni[7][1][0] = hevc_put_uni_epel_v32_8_avx2;
941  c->put_hevc_epel_uni[8][1][0] = hevc_put_uni_epel_v48_8_avx2;
942  c->put_hevc_epel_uni[9][1][0] = hevc_put_uni_epel_v64_8_avx2;
943 
944  c->put_hevc_epel_bi[7][1][0] = ff_hevc_put_bi_epel_v32_8_avx2;
945  c->put_hevc_epel_bi[8][1][0] = ff_hevc_put_bi_epel_v48_8_avx2;
946  c->put_hevc_epel_bi[9][1][0] = ff_hevc_put_bi_epel_v64_8_avx2;
947 
948  c->put_hevc_epel[7][1][1] = hevc_put_epel_hv32_8_avx2;
949  c->put_hevc_epel[8][1][1] = hevc_put_epel_hv48_8_avx2;
950  c->put_hevc_epel[9][1][1] = hevc_put_epel_hv64_8_avx2;
951 
952  c->put_hevc_epel_uni[7][1][1] = hevc_put_uni_epel_hv32_8_avx2;
953  c->put_hevc_epel_uni[8][1][1] = hevc_put_uni_epel_hv48_8_avx2;
954  c->put_hevc_epel_uni[9][1][1] = hevc_put_uni_epel_hv64_8_avx2;
955 
956  c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_bi_epel_hv32_8_avx2;
957  c->put_hevc_epel_bi[8][1][1] = ff_hevc_put_bi_epel_hv48_8_avx2;
958  c->put_hevc_epel_bi[9][1][1] = ff_hevc_put_bi_epel_hv64_8_avx2;
959 
960  c->put_hevc_qpel[7][0][1] = hevc_put_qpel_h32_8_avx2;
961  c->put_hevc_qpel[8][0][1] = hevc_put_qpel_h48_8_avx2;
962  c->put_hevc_qpel[9][0][1] = hevc_put_qpel_h64_8_avx2;
963 
964  c->put_hevc_qpel[7][1][0] = hevc_put_qpel_v32_8_avx2;
965  c->put_hevc_qpel[8][1][0] = hevc_put_qpel_v48_8_avx2;
966  c->put_hevc_qpel[9][1][0] = hevc_put_qpel_v64_8_avx2;
967 
968  c->put_hevc_qpel_uni[7][0][1] = hevc_put_uni_qpel_h32_8_avx2;
969  c->put_hevc_qpel_uni[8][0][1] = hevc_put_uni_qpel_h48_8_avx2;
970  c->put_hevc_qpel_uni[9][0][1] = hevc_put_uni_qpel_h64_8_avx2;
971 
972  c->put_hevc_qpel_uni[7][1][0] = hevc_put_uni_qpel_v32_8_avx2;
973  c->put_hevc_qpel_uni[8][1][0] = hevc_put_uni_qpel_v48_8_avx2;
974  c->put_hevc_qpel_uni[9][1][0] = hevc_put_uni_qpel_v64_8_avx2;
975 
976  c->put_hevc_qpel_bi[7][0][1] = ff_hevc_put_bi_qpel_h32_8_avx2;
977  c->put_hevc_qpel_bi[8][0][1] = ff_hevc_put_bi_qpel_h48_8_avx2;
978  c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_bi_qpel_h64_8_avx2;
979 
980  c->put_hevc_qpel_bi[7][1][0] = ff_hevc_put_bi_qpel_v32_8_avx2;
981  c->put_hevc_qpel_bi[8][1][0] = ff_hevc_put_bi_qpel_v48_8_avx2;
982  c->put_hevc_qpel_bi[9][1][0] = ff_hevc_put_bi_qpel_v64_8_avx2;
983 #endif /* ARCH_X86_64 */
984 
985  SAO_BAND_INIT(8, avx2);
986 
987  c->sao_edge_filter[2] = ff_hevc_sao_edge_filter_32_8_avx2;
988  c->sao_edge_filter[3] = ff_hevc_sao_edge_filter_48_8_avx2;
989  c->sao_edge_filter[4] = ff_hevc_sao_edge_filter_64_8_avx2;
990 
991  c->add_residual[3] = ff_hevc_add_residual_32_8_avx2;
992  }
993 #endif /* HAVE_AVX2_EXTERNAL */
994 #if ARCH_X86_64
995  if (EXTERNAL_AVX512ICL(cpu_flags)) {
996  c->put_hevc_qpel[1][0][1] = ff_hevc_put_qpel_h4_8_avx512icl;
997  c->put_hevc_qpel[3][0][1] = ff_hevc_put_qpel_h8_8_avx512icl;
998  c->put_hevc_qpel[5][0][1] = ff_hevc_put_qpel_h16_8_avx512icl;
999  c->put_hevc_qpel[7][0][1] = ff_hevc_put_qpel_h32_8_avx512icl;
1000  c->put_hevc_qpel[9][0][1] = ff_hevc_put_qpel_h64_8_avx512icl;
1001  c->put_hevc_qpel[3][1][1] = ff_hevc_put_qpel_hv8_8_avx512icl;
1002  }
1003 #endif
1004  } else if (bit_depth == 10) {
1005  if (EXTERNAL_MMXEXT(cpu_flags)) {
1006  c->add_residual[0] = ff_hevc_add_residual_4_10_mmxext;
1007  }
1008  if (EXTERNAL_SSE2(cpu_flags)) {
1009  c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_sse2;
1010  c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_sse2;
1011 #if ARCH_X86_64
1012  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_sse2;
1013  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_sse2;
1014 
1015  c->idct[2] = ff_hevc_idct_16x16_10_sse2;
1016  c->idct[3] = ff_hevc_idct_32x32_10_sse2;
1017 #endif
1018  SAO_BAND_INIT(10, sse2);
1019  SAO_EDGE_INIT(10, sse2);
1020 
1021  c->idct_dc[0] = ff_hevc_idct_4x4_dc_10_sse2;
1022  c->idct_dc[1] = ff_hevc_idct_8x8_dc_10_sse2;
1023  c->idct_dc[2] = ff_hevc_idct_16x16_dc_10_sse2;
1024  c->idct_dc[3] = ff_hevc_idct_32x32_dc_10_sse2;
1025 
1026  c->idct[0] = ff_hevc_idct_4x4_10_sse2;
1027  c->idct[1] = ff_hevc_idct_8x8_10_sse2;
1028  c->transform_4x4_luma = ff_hevc_transform_4x4_luma_10_sse2;
1029 
1030  c->add_residual[1] = ff_hevc_add_residual_8_10_sse2;
1031  c->add_residual[2] = ff_hevc_add_residual_16_10_sse2;
1032  c->add_residual[3] = ff_hevc_add_residual_32_10_sse2;
1033  }
1034 #if ARCH_X86_64
1035  if (EXTERNAL_SSSE3(cpu_flags)) {
1036  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_ssse3;
1037  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_ssse3;
1038  }
1039 #endif
1040 #if HAVE_SSE4_EXTERNAL && ARCH_X86_64
1041  if (EXTERNAL_SSE4(cpu_flags)) {
1042  EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 10, sse4);
1043  EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 10, sse4);
1044  EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 10, sse4);
1045  EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 10, sse4);
1046 
1047  QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 10, sse4);
1048  QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 10, sse4);
1049  QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 10, sse4);
1050  QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 10, sse4);
1051  }
1052 #endif
1053  if (EXTERNAL_AVX(cpu_flags)) {
1054  c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_10_avx;
1055  c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_10_avx;
1056 #if ARCH_X86_64
1057  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_avx;
1058  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_avx;
1059 
1060  c->idct[2] = ff_hevc_idct_16x16_10_avx;
1061  c->idct[3] = ff_hevc_idct_32x32_10_avx;
1062 #endif
1063 
1064  c->idct[0] = ff_hevc_idct_4x4_10_avx;
1065  c->idct[1] = ff_hevc_idct_8x8_10_avx;
1066  c->transform_4x4_luma = ff_hevc_transform_4x4_luma_10_avx;
1067 
1068  SAO_BAND_INIT(10, avx);
1069  }
1070  if (EXTERNAL_AVX2(cpu_flags)) {
1071  c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_10_avx2;
1072  }
1073 #if HAVE_AVX2_EXTERNAL
1074  if (EXTERNAL_AVX2_FAST(cpu_flags)) {
1075  c->idct_dc[2] = ff_hevc_idct_16x16_dc_10_avx2;
1076  c->idct_dc[3] = ff_hevc_idct_32x32_dc_10_avx2;
1077 
1078 #if ARCH_X86_64
1079  c->put_hevc_epel[5][0][0] = hevc_put_pel_pixels16_10_avx2;
1080  c->put_hevc_epel[6][0][0] = hevc_put_pel_pixels24_10_avx2;
1081  c->put_hevc_epel[7][0][0] = hevc_put_pel_pixels32_10_avx2;
1082  c->put_hevc_epel[8][0][0] = hevc_put_pel_pixels48_10_avx2;
1083  c->put_hevc_epel[9][0][0] = hevc_put_pel_pixels64_10_avx2;
1084 
1085  c->put_hevc_qpel[5][0][0] = hevc_put_pel_pixels16_10_avx2;
1086  c->put_hevc_qpel[6][0][0] = hevc_put_pel_pixels24_10_avx2;
1087  c->put_hevc_qpel[7][0][0] = hevc_put_pel_pixels32_10_avx2;
1088  c->put_hevc_qpel[8][0][0] = hevc_put_pel_pixels48_10_avx2;
1089  c->put_hevc_qpel[9][0][0] = hevc_put_pel_pixels64_10_avx2;
1090 
1091  c->put_hevc_epel_uni[5][0][0] = hevc_put_uni_pel_pixels32_8_avx2;
1092  c->put_hevc_epel_uni[6][0][0] = hevc_put_uni_pel_pixels48_8_avx2;
1093  c->put_hevc_epel_uni[7][0][0] = hevc_put_uni_pel_pixels64_8_avx2;
1094  c->put_hevc_epel_uni[8][0][0] = hevc_put_uni_pel_pixels96_8_avx2;
1095  c->put_hevc_epel_uni[9][0][0] = hevc_put_uni_pel_pixels128_8_avx2;
1096 
1097  c->put_hevc_qpel_uni[5][0][0] = hevc_put_uni_pel_pixels32_8_avx2;
1098  c->put_hevc_qpel_uni[6][0][0] = hevc_put_uni_pel_pixels48_8_avx2;
1099  c->put_hevc_qpel_uni[7][0][0] = hevc_put_uni_pel_pixels64_8_avx2;
1100  c->put_hevc_qpel_uni[8][0][0] = hevc_put_uni_pel_pixels96_8_avx2;
1101  c->put_hevc_qpel_uni[9][0][0] = hevc_put_uni_pel_pixels128_8_avx2;
1102 
1103  c->put_hevc_epel_bi[5][0][0] = ff_hevc_put_bi_pel_pixels16_10_avx2;
1104  c->put_hevc_epel_bi[6][0][0] = ff_hevc_put_bi_pel_pixels24_10_avx2;
1105  c->put_hevc_epel_bi[7][0][0] = ff_hevc_put_bi_pel_pixels32_10_avx2;
1106  c->put_hevc_epel_bi[8][0][0] = ff_hevc_put_bi_pel_pixels48_10_avx2;
1107  c->put_hevc_epel_bi[9][0][0] = ff_hevc_put_bi_pel_pixels64_10_avx2;
1108  c->put_hevc_qpel_bi[5][0][0] = ff_hevc_put_bi_pel_pixels16_10_avx2;
1109  c->put_hevc_qpel_bi[6][0][0] = ff_hevc_put_bi_pel_pixels24_10_avx2;
1110  c->put_hevc_qpel_bi[7][0][0] = ff_hevc_put_bi_pel_pixels32_10_avx2;
1111  c->put_hevc_qpel_bi[8][0][0] = ff_hevc_put_bi_pel_pixels48_10_avx2;
1112  c->put_hevc_qpel_bi[9][0][0] = ff_hevc_put_bi_pel_pixels64_10_avx2;
1113 
1114  c->put_hevc_epel[5][0][1] = hevc_put_epel_h16_10_avx2;
1115  c->put_hevc_epel[6][0][1] = hevc_put_epel_h24_10_avx2;
1116  c->put_hevc_epel[7][0][1] = hevc_put_epel_h32_10_avx2;
1117  c->put_hevc_epel[8][0][1] = hevc_put_epel_h48_10_avx2;
1118  c->put_hevc_epel[9][0][1] = hevc_put_epel_h64_10_avx2;
1119 
1120  c->put_hevc_epel_uni[5][0][1] = hevc_put_uni_epel_h16_10_avx2;
1121  c->put_hevc_epel_uni[6][0][1] = hevc_put_uni_epel_h24_10_avx2;
1122  c->put_hevc_epel_uni[7][0][1] = hevc_put_uni_epel_h32_10_avx2;
1123  c->put_hevc_epel_uni[8][0][1] = hevc_put_uni_epel_h48_10_avx2;
1124  c->put_hevc_epel_uni[9][0][1] = hevc_put_uni_epel_h64_10_avx2;
1125 
1126  c->put_hevc_epel_bi[5][0][1] = ff_hevc_put_bi_epel_h16_10_avx2;
1127  c->put_hevc_epel_bi[6][0][1] = ff_hevc_put_bi_epel_h24_10_avx2;
1128  c->put_hevc_epel_bi[7][0][1] = ff_hevc_put_bi_epel_h32_10_avx2;
1129  c->put_hevc_epel_bi[8][0][1] = ff_hevc_put_bi_epel_h48_10_avx2;
1130  c->put_hevc_epel_bi[9][0][1] = ff_hevc_put_bi_epel_h64_10_avx2;
1131 
1132  c->put_hevc_epel[5][1][0] = hevc_put_epel_v16_10_avx2;
1133  c->put_hevc_epel[6][1][0] = hevc_put_epel_v24_10_avx2;
1134  c->put_hevc_epel[7][1][0] = hevc_put_epel_v32_10_avx2;
1135  c->put_hevc_epel[8][1][0] = hevc_put_epel_v48_10_avx2;
1136  c->put_hevc_epel[9][1][0] = hevc_put_epel_v64_10_avx2;
1137 
1138  c->put_hevc_epel_uni[5][1][0] = hevc_put_uni_epel_v16_10_avx2;
1139  c->put_hevc_epel_uni[6][1][0] = hevc_put_uni_epel_v24_10_avx2;
1140  c->put_hevc_epel_uni[7][1][0] = hevc_put_uni_epel_v32_10_avx2;
1141  c->put_hevc_epel_uni[8][1][0] = hevc_put_uni_epel_v48_10_avx2;
1142  c->put_hevc_epel_uni[9][1][0] = hevc_put_uni_epel_v64_10_avx2;
1143 
1144  c->put_hevc_epel_bi[5][1][0] = ff_hevc_put_bi_epel_v16_10_avx2;
1145  c->put_hevc_epel_bi[6][1][0] = ff_hevc_put_bi_epel_v24_10_avx2;
1146  c->put_hevc_epel_bi[7][1][0] = ff_hevc_put_bi_epel_v32_10_avx2;
1147  c->put_hevc_epel_bi[8][1][0] = ff_hevc_put_bi_epel_v48_10_avx2;
1148  c->put_hevc_epel_bi[9][1][0] = ff_hevc_put_bi_epel_v64_10_avx2;
1149 
1150  c->put_hevc_epel[5][1][1] = hevc_put_epel_hv16_10_avx2;
1151  c->put_hevc_epel[6][1][1] = hevc_put_epel_hv24_10_avx2;
1152  c->put_hevc_epel[7][1][1] = hevc_put_epel_hv32_10_avx2;
1153  c->put_hevc_epel[8][1][1] = hevc_put_epel_hv48_10_avx2;
1154  c->put_hevc_epel[9][1][1] = hevc_put_epel_hv64_10_avx2;
1155 
1156  c->put_hevc_epel_uni[5][1][1] = hevc_put_uni_epel_hv16_10_avx2;
1157  c->put_hevc_epel_uni[6][1][1] = hevc_put_uni_epel_hv24_10_avx2;
1158  c->put_hevc_epel_uni[7][1][1] = hevc_put_uni_epel_hv32_10_avx2;
1159  c->put_hevc_epel_uni[8][1][1] = hevc_put_uni_epel_hv48_10_avx2;
1160  c->put_hevc_epel_uni[9][1][1] = hevc_put_uni_epel_hv64_10_avx2;
1161 
1162  c->put_hevc_epel_bi[5][1][1] = ff_hevc_put_bi_epel_hv16_10_avx2;
1163  c->put_hevc_epel_bi[6][1][1] = ff_hevc_put_bi_epel_hv24_10_avx2;
1164  c->put_hevc_epel_bi[7][1][1] = ff_hevc_put_bi_epel_hv32_10_avx2;
1165  c->put_hevc_epel_bi[8][1][1] = ff_hevc_put_bi_epel_hv48_10_avx2;
1166  c->put_hevc_epel_bi[9][1][1] = ff_hevc_put_bi_epel_hv64_10_avx2;
1167 
1168  c->put_hevc_qpel[5][0][1] = hevc_put_qpel_h16_10_avx2;
1169  c->put_hevc_qpel[6][0][1] = hevc_put_qpel_h24_10_avx2;
1170  c->put_hevc_qpel[7][0][1] = hevc_put_qpel_h32_10_avx2;
1171  c->put_hevc_qpel[8][0][1] = hevc_put_qpel_h48_10_avx2;
1172  c->put_hevc_qpel[9][0][1] = hevc_put_qpel_h64_10_avx2;
1173 
1174  c->put_hevc_qpel_uni[5][0][1] = hevc_put_uni_qpel_h16_10_avx2;
1175  c->put_hevc_qpel_uni[6][0][1] = hevc_put_uni_qpel_h24_10_avx2;
1176  c->put_hevc_qpel_uni[7][0][1] = hevc_put_uni_qpel_h32_10_avx2;
1177  c->put_hevc_qpel_uni[8][0][1] = hevc_put_uni_qpel_h48_10_avx2;
1178  c->put_hevc_qpel_uni[9][0][1] = hevc_put_uni_qpel_h64_10_avx2;
1179 
1180  c->put_hevc_qpel_bi[5][0][1] = ff_hevc_put_bi_qpel_h16_10_avx2;
1181  c->put_hevc_qpel_bi[6][0][1] = ff_hevc_put_bi_qpel_h24_10_avx2;
1182  c->put_hevc_qpel_bi[7][0][1] = ff_hevc_put_bi_qpel_h32_10_avx2;
1183  c->put_hevc_qpel_bi[8][0][1] = ff_hevc_put_bi_qpel_h48_10_avx2;
1184  c->put_hevc_qpel_bi[9][0][1] = ff_hevc_put_bi_qpel_h64_10_avx2;
1185 
1186  c->put_hevc_qpel[5][1][0] = hevc_put_qpel_v16_10_avx2;
1187  c->put_hevc_qpel[6][1][0] = hevc_put_qpel_v24_10_avx2;
1188  c->put_hevc_qpel[7][1][0] = hevc_put_qpel_v32_10_avx2;
1189  c->put_hevc_qpel[8][1][0] = hevc_put_qpel_v48_10_avx2;
1190  c->put_hevc_qpel[9][1][0] = hevc_put_qpel_v64_10_avx2;
1191 
1192  c->put_hevc_qpel_uni[5][1][0] = hevc_put_uni_qpel_v16_10_avx2;
1193  c->put_hevc_qpel_uni[6][1][0] = hevc_put_uni_qpel_v24_10_avx2;
1194  c->put_hevc_qpel_uni[7][1][0] = hevc_put_uni_qpel_v32_10_avx2;
1195  c->put_hevc_qpel_uni[8][1][0] = hevc_put_uni_qpel_v48_10_avx2;
1196  c->put_hevc_qpel_uni[9][1][0] = hevc_put_uni_qpel_v64_10_avx2;
1197 
1198  c->put_hevc_qpel_bi[5][1][0] = ff_hevc_put_bi_qpel_v16_10_avx2;
1199  c->put_hevc_qpel_bi[6][1][0] = ff_hevc_put_bi_qpel_v24_10_avx2;
1200  c->put_hevc_qpel_bi[7][1][0] = ff_hevc_put_bi_qpel_v32_10_avx2;
1201  c->put_hevc_qpel_bi[8][1][0] = ff_hevc_put_bi_qpel_v48_10_avx2;
1202  c->put_hevc_qpel_bi[9][1][0] = ff_hevc_put_bi_qpel_v64_10_avx2;
1203 
1204  c->put_hevc_qpel[5][1][1] = hevc_put_qpel_hv16_10_avx2;
1205  c->put_hevc_qpel[6][1][1] = hevc_put_qpel_hv24_10_avx2;
1206  c->put_hevc_qpel[7][1][1] = hevc_put_qpel_hv32_10_avx2;
1207  c->put_hevc_qpel[8][1][1] = hevc_put_qpel_hv48_10_avx2;
1208  c->put_hevc_qpel[9][1][1] = hevc_put_qpel_hv64_10_avx2;
1209 
1210  c->put_hevc_qpel_uni[5][1][1] = hevc_put_uni_qpel_hv16_10_avx2;
1211  c->put_hevc_qpel_uni[6][1][1] = hevc_put_uni_qpel_hv24_10_avx2;
1212  c->put_hevc_qpel_uni[7][1][1] = hevc_put_uni_qpel_hv32_10_avx2;
1213  c->put_hevc_qpel_uni[8][1][1] = hevc_put_uni_qpel_hv48_10_avx2;
1214  c->put_hevc_qpel_uni[9][1][1] = hevc_put_uni_qpel_hv64_10_avx2;
1215 
1216  c->put_hevc_qpel_bi[5][1][1] = ff_hevc_put_bi_qpel_hv16_10_avx2;
1217  c->put_hevc_qpel_bi[6][1][1] = ff_hevc_put_bi_qpel_hv24_10_avx2;
1218  c->put_hevc_qpel_bi[7][1][1] = ff_hevc_put_bi_qpel_hv32_10_avx2;
1219  c->put_hevc_qpel_bi[8][1][1] = ff_hevc_put_bi_qpel_hv48_10_avx2;
1220  c->put_hevc_qpel_bi[9][1][1] = ff_hevc_put_bi_qpel_hv64_10_avx2;
1221 #endif /* ARCH_X86_64 */
1222 
1223  SAO_BAND_INIT(10, avx2);
1224  SAO_EDGE_INIT(10, avx2);
1225 
1226  c->add_residual[2] = ff_hevc_add_residual_16_10_avx2;
1227  c->add_residual[3] = ff_hevc_add_residual_32_10_avx2;
1228  }
1229 #endif /* HAVE_AVX2_EXTERNAL */
1230  } else if (bit_depth == 12) {
1231  if (EXTERNAL_SSE2(cpu_flags)) {
1232  c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_sse2;
1233  c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_sse2;
1234 #if ARCH_X86_64
1235  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_sse2;
1236  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_sse2;
1237 #endif
1238  SAO_BAND_INIT(12, sse2);
1239  SAO_EDGE_INIT(12, sse2);
1240 
1241  c->idct_dc[0] = ff_hevc_idct_4x4_dc_12_sse2;
1242  c->idct_dc[1] = ff_hevc_idct_8x8_dc_12_sse2;
1243  c->idct_dc[2] = ff_hevc_idct_16x16_dc_12_sse2;
1244  c->idct_dc[3] = ff_hevc_idct_32x32_dc_12_sse2;
1245  }
1246 #if ARCH_X86_64
1247  if (EXTERNAL_SSSE3(cpu_flags)) {
1248  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_ssse3;
1249  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_ssse3;
1250  }
1251 #endif
1252 #if HAVE_SSE4_EXTERNAL && ARCH_X86_64
1253  if (EXTERNAL_SSE4(cpu_flags)) {
1254  EPEL_LINKS(c->put_hevc_epel, 0, 0, pel_pixels, 12, sse4);
1255  EPEL_LINKS(c->put_hevc_epel, 0, 1, epel_h, 12, sse4);
1256  EPEL_LINKS(c->put_hevc_epel, 1, 0, epel_v, 12, sse4);
1257  EPEL_LINKS(c->put_hevc_epel, 1, 1, epel_hv, 12, sse4);
1258 
1259  QPEL_LINKS(c->put_hevc_qpel, 0, 0, pel_pixels, 12, sse4);
1260  QPEL_LINKS(c->put_hevc_qpel, 0, 1, qpel_h, 12, sse4);
1261  QPEL_LINKS(c->put_hevc_qpel, 1, 0, qpel_v, 12, sse4);
1262  QPEL_LINKS(c->put_hevc_qpel, 1, 1, qpel_hv, 12, sse4);
1263  }
1264 #endif
1265  if (EXTERNAL_AVX(cpu_flags)) {
1266  c->hevc_v_loop_filter_chroma = ff_hevc_v_loop_filter_chroma_12_avx;
1267  c->hevc_h_loop_filter_chroma = ff_hevc_h_loop_filter_chroma_12_avx;
1268 #if ARCH_X86_64
1269  c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_avx;
1270  c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_avx;
1271 #endif
1272  SAO_BAND_INIT(12, avx);
1273  }
1274  if (EXTERNAL_AVX2(cpu_flags)) {
1275  c->sao_band_filter[0] = ff_hevc_sao_band_filter_8_12_avx2;
1276  }
1277  if (EXTERNAL_AVX2_FAST(cpu_flags)) {
1278  c->idct_dc[2] = ff_hevc_idct_16x16_dc_12_avx2;
1279  c->idct_dc[3] = ff_hevc_idct_32x32_dc_12_avx2;
1280 
1281  SAO_BAND_INIT(12, avx2);
1282  SAO_EDGE_INIT(12, avx2);
1283  }
1284  }
1285 }
SAO_BAND_FILTER_FUNCS
#define SAO_BAND_FILTER_FUNCS(bitd, opt)
Definition: dsp_init.c:736
mc_rep_bi_func
#define mc_rep_bi_func(name, bitd, step, W, opt)
Definition: dsp_init.c:226
mem_internal.h
ff_hevc_add_residual_4_10_mmxext
void ff_hevc_add_residual_4_10_mmxext(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_put_qpel_h8_8_avx512icl
void ff_hevc_put_qpel_h8_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
ff_hevc_dsp_init_x86
void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
Definition: dsp_init.c:820
mc_rep_funcs2
#define mc_rep_funcs2(name, bitd, step1, step2, W, opt)
Definition: dsp_init.c:277
SAO_EDGE_INIT
#define SAO_EDGE_INIT(bitd, opt)
Definition: dsp_init.c:785
bit_depth
static void bit_depth(AudioStatsContext *s, const uint64_t *const mask, uint8_t *depth)
Definition: af_astats.c:246
cpu.h
FW_PUT
#define FW_PUT(p, a, b, depth, opt)
Definition: dsp_init.c:101
ff_hevc_add_residual_4_8_sse2
void ff_hevc_add_residual_4_8_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
SAO_EDGE_FILTER_FUNCS
#define SAO_EDGE_FILTER_FUNCS(bitd, opt)
Definition: dsp_init.c:766
EXTERNAL_SSSE3
#define EXTERNAL_SSSE3(flags)
Definition: cpu.h:59
ff_hevc_put_qpel_h32_8_avx512icl
void ff_hevc_put_qpel_h32_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
ff_hevc_dequant_8_ssse3
void ff_hevc_dequant_8_ssse3(int16_t *coeffs, int16_t log2_size)
ff_hevc_add_residual_16_10_sse2
void ff_hevc_add_residual_16_10_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
av_get_cpu_flags
int av_get_cpu_flags(void)
Return the flags which specify extensions supported by the CPU.
Definition: cpu.c:109
ff_hevc_add_residual_32_10_sse2
void ff_hevc_add_residual_32_10_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
EXTERNAL_AVX2
#define EXTERNAL_AVX2(flags)
Definition: cpu.h:72
mc_rep_uni_func
#define mc_rep_uni_func(name, bitd, step, W, opt)
Definition: dsp_init.c:212
ff_hevc_add_residual_8_8_sse2
void ff_hevc_add_residual_8_8_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_add_residual_8_10_sse2
void ff_hevc_add_residual_8_10_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
EXTERNAL_AVX2_FAST
#define EXTERNAL_AVX2_FAST(flags)
Definition: cpu.h:73
ff_hevc_add_residual_16_10_avx2
void ff_hevc_add_residual_16_10_avx2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
EXTERNAL_AVX
#define EXTERNAL_AVX(flags)
Definition: cpu.h:64
mc_rep_func
#define mc_rep_func(name, bitd, step, W, opt)
Definition: dsp_init.c:199
ff_hevc_put_qpel_h64_8_avx512icl
void ff_hevc_put_qpel_h64_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
IDCT_DC_FUNCS
#define IDCT_DC_FUNCS(W, opt)
Definition: dsp_init.c:64
ff_hevc_add_residual_32_10_avx2
void ff_hevc_add_residual_32_10_avx2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
SAO_BAND_INIT
#define SAO_BAND_INIT(bitd, opt)
Definition: dsp_init.c:758
QPEL_LINKS
#define QPEL_LINKS(pointer, my, mx, fname, bitd, opt)
Definition: dsp_init.c:810
dsp.h
dsp.h
TRANSFORM_LUMA_FUNCS
#define TRANSFORM_LUMA_FUNCS(opt)
Definition: dsp_init.c:89
ff_hevc_add_residual_32_8_sse2
void ff_hevc_add_residual_32_8_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
LFL_FUNCS
#define LFL_FUNCS(type, depth, opt)
Definition: dsp_init.c:44
cpu_flags
CheckasmCpu cpu_flags
Definition: checkasm.c:84
EXTERNAL_SSE2
#define EXTERNAL_SSE2(flags)
Definition: cpu.h:53
HEVCDSPContext
Definition: dsp.h:47
EXTERNAL_SSE4
#define EXTERNAL_SSE4(flags)
Definition: cpu.h:62
h2656dsp.h
ff_hevc_add_residual_16_8_sse2
void ff_hevc_add_residual_16_8_sse2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_add_residual_32_8_avx2
void ff_hevc_add_residual_32_8_avx2(uint8_t *dst, const int16_t *res, ptrdiff_t stride)
ff_hevc_put_qpel_h4_8_avx512icl
void ff_hevc_put_qpel_h4_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
mc_rep_funcs
#define mc_rep_funcs(name, bitd, step, W, opt)
Definition: dsp_init.c:242
EPEL_LINKS
#define EPEL_LINKS(pointer, my, mx, fname, bitd, opt)
Definition: dsp_init.c:800
ff_hevc_put_qpel_h16_8_avx512icl
void ff_hevc_put_qpel_h16_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
LFC_FUNCS
#define LFC_FUNCS(type, depth, opt)
Definition: dsp_init.c:40
EXTERNAL_MMXEXT
#define EXTERNAL_MMXEXT(flags)
Definition: cpu.h:51
cpu.h
IDCT_FUNCS
#define IDCT_FUNCS(opt)
Definition: dsp_init.c:76
ff_hevc_put_qpel_hv8_8_avx512icl
void ff_hevc_put_qpel_hv8_8_avx512icl(int16_t *dst, const uint8_t *_src, ptrdiff_t _srcstride, int height, intptr_t mx, intptr_t my, int width)
EXTERNAL_AVX512ICL
#define EXTERNAL_AVX512ICL(flags)
Definition: cpu.h:78
Generated on Wed Jul 29 2026 19:21:49 for FFmpeg by   doxygen 1.8.17