FFmpeg
Loading...
Searching...
No Matches
hevcdsp_lsx.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2022 Loongson Technology Corporation Limited
3 * Contributed by Lu Wang <wanglu@loongson.cn>
4 * Hao Chen <chenhao@loongson.cn>
5 *
6 * This file is part of FFmpeg.
7 *
8 * FFmpeg is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Lesser General Public
10 * License as published by the Free Software Foundation; either
11 * version 2.1 of the License, or (at your option) any later version.
12 *
13 * FFmpeg is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Lesser General Public License for more details.
17 *
18 * You should have received a copy of the GNU Lesser General Public
19 * License along with FFmpeg; if not, write to the Free Software
20 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 */
22
23#include "libavutil/loongarch/loongson_intrinsics.h"
24#include "hevcdsp_lsx.h"
25
26static const uint8_t ff_hevc_mask_arr[16 * 2] __attribute__((aligned(0x40))) = {
27 /* 8 width cases */
28 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
29 /* 4 width cases */
30 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20
31};
32
33/* hevc_copy: dst = src << 6 */
34static void hevc_copy_4w_lsx(const uint8_t *src, int32_t src_stride,
35 int16_t *dst, int32_t dst_stride,
36 int32_t height)
37{
38 int32_t src_stride_2x = (src_stride << 1);
39 int32_t dst_stride_2x = (dst_stride << 1);
40 int32_t src_stride_4x = (src_stride << 2);
41 int32_t dst_stride_4x = (dst_stride << 2);
42 int32_t src_stride_3x = src_stride_2x + src_stride;
43 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
44 int32_t loop_cnt = height >> 3;
45 int32_t res = height & 0x07;
46
47 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
48 __m128i in0, in1, in2, in3;
49 for (; loop_cnt--;) {
50 src0 = __lsx_vld(src, 0);
51 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x,
52 src1, src2);
53 src3 = __lsx_vldx(src, src_stride_3x);
54 src += src_stride_4x;
55 src4 = __lsx_vld(src, 0);
56 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x,
57 src5, src6);
58 src7 = __lsx_vldx(src, src_stride_3x);
59 src += src_stride_4x;
60
61 DUP4_ARG2(__lsx_vilvl_w, src1, src0, src3, src2, src5, src4, src7, src6,
62 src0, src1, src2, src3);
63 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
64 in0, in1, in2, in3);
65
66 __lsx_vstelm_d(in0, dst, 0, 0);
67 __lsx_vstelm_d(in0, dst + dst_stride, 0, 1);
68 __lsx_vstelm_d(in1, dst + dst_stride_2x, 0, 0);
69 __lsx_vstelm_d(in1, dst + dst_stride_3x, 0, 1);
70 dst += dst_stride_4x;
71 __lsx_vstelm_d(in2, dst, 0, 0);
72 __lsx_vstelm_d(in2, dst + dst_stride, 0, 1);
73 __lsx_vstelm_d(in3, dst + dst_stride_2x, 0, 0);
74 __lsx_vstelm_d(in3, dst + dst_stride_3x, 0, 1);
75 dst += dst_stride_4x;
76 }
77 for (;res--;) {
78 src0 = __lsx_vld(src, 0);
79 in0 = __lsx_vsllwil_hu_bu(src0, 6);
80 __lsx_vstelm_d(in0, dst, 0, 0);
81 src += src_stride;
82 dst += dst_stride;
83 }
84}
85
86static void hevc_copy_6w_lsx(const uint8_t *src, int32_t src_stride,
87 int16_t *dst, int32_t dst_stride,
88 int32_t height)
89{
90 int32_t loop_cnt = (height >> 3);
91 int32_t res = height & 0x07;
92 int32_t src_stride_2x = (src_stride << 1);
93 int32_t src_stride_4x = (src_stride << 2);
94 int32_t src_stride_3x = src_stride_2x + src_stride;
95
96 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
97 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
98
99 for (loop_cnt = (height >> 3); loop_cnt--;) {
100 src0 = __lsx_vld(src, 0);
101 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
102 src3 = __lsx_vldx(src, src_stride_3x);
103 src += src_stride_4x;
104 src4 = __lsx_vld(src, 0);
105 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
106 src7 = __lsx_vldx(src, src_stride_3x);
107 src += src_stride_4x;
108
109 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
110 in0, in1, in2, in3);
111 DUP4_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, src6, 6, src7, 6,
112 in4, in5, in6, in7);
113
114 __lsx_vstelm_d(in0, dst, 0, 0);
115 __lsx_vstelm_w(in0, dst, 8, 2);
116 dst += dst_stride;
117 __lsx_vstelm_d(in1, dst, 0, 0);
118 __lsx_vstelm_w(in1, dst, 8, 2);
119 dst += dst_stride;
120 __lsx_vstelm_d(in2, dst, 0, 0);
121 __lsx_vstelm_w(in2, dst, 8, 2);
122 dst += dst_stride;
123 __lsx_vstelm_d(in3, dst, 0, 0);
124 __lsx_vstelm_w(in3, dst, 8, 2);
125 dst += dst_stride;
126 __lsx_vstelm_d(in4, dst, 0, 0);
127 __lsx_vstelm_w(in4, dst, 8, 2);
128 dst += dst_stride;
129 __lsx_vstelm_d(in5, dst, 0, 0);
130 __lsx_vstelm_w(in5, dst, 8, 2);
131 dst += dst_stride;
132 __lsx_vstelm_d(in6, dst, 0, 0);
133 __lsx_vstelm_w(in6, dst, 8, 2);
134 dst += dst_stride;
135 __lsx_vstelm_d(in7, dst, 0, 0);
136 __lsx_vstelm_w(in7, dst, 8, 2);
137 dst += dst_stride;
138 }
139 for (;res--;) {
140 src0 = __lsx_vld(src, 0);
141 in0 = __lsx_vsllwil_hu_bu(src0, 6);
142 src += src_stride;
143 __lsx_vstelm_d(in0, dst, 0, 0);
144 __lsx_vstelm_w(in0, dst, 8, 2);
145 dst += dst_stride;
146 }
147}
148
149static void hevc_copy_8w_lsx(const uint8_t *src, int32_t src_stride,
150 int16_t *dst, int32_t dst_stride,
151 int32_t height)
152{
153 int32_t src_stride_2x = (src_stride << 1);
154 int32_t dst_stride_x = (dst_stride << 1);
155 int32_t src_stride_4x = (src_stride << 2);
156 int32_t dst_stride_2x = (dst_stride_x << 1);
157 int32_t src_stride_3x = src_stride_2x + src_stride;
158 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
159 int32_t loop_cnt = height >> 3;
160 int32_t res = height & 0x07;
161 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
162 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
163
164 for (loop_cnt = (height >> 3); loop_cnt--;) {
165 src0 = __lsx_vld(src, 0);
166 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x,
167 src1, src2);
168 src3 = __lsx_vldx(src, src_stride_3x);
169 src += src_stride_4x;
170 src4 = __lsx_vld(src, 0);
171 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x,
172 src5, src6);
173 src7 = __lsx_vldx(src, src_stride_3x);
174 src += src_stride_4x;
175
176 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
177 in0, in1, in2, in3);
178 DUP4_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, src6, 6, src7, 6,
179 in4, in5, in6, in7);
180 __lsx_vst(in0, dst, 0);
181 __lsx_vstx(in1, dst, dst_stride_x);
182 __lsx_vstx(in2, dst, dst_stride_2x);
183 __lsx_vstx(in3, dst, dst_stride_3x);
184 dst += dst_stride_2x;
185 __lsx_vst(in4, dst, 0);
186 __lsx_vstx(in5, dst, dst_stride_x);
187 __lsx_vstx(in6, dst, dst_stride_2x);
188 __lsx_vstx(in7, dst, dst_stride_3x);
189 dst += dst_stride_2x;
190 }
191 for (;res--;) {
192 src0 = __lsx_vld(src, 0);
193 in0 = __lsx_vsllwil_hu_bu(src0, 6);
194 __lsx_vst(in0, dst, 0);
195 src += src_stride;
196 dst += dst_stride;
197 }
198}
199
200static void hevc_copy_12w_lsx(const uint8_t *src, int32_t src_stride,
201 int16_t *dst, int32_t dst_stride,
202 int32_t height)
203{
204 uint32_t loop_cnt;
205 uint32_t res = height & 0x07;
206 int32_t src_stride_2x = (src_stride << 1);
207 int32_t dst_stride_x = (dst_stride << 1);
208 int32_t src_stride_4x = (src_stride << 2);
209 int32_t dst_stride_2x = (dst_stride_x << 1);
210 int32_t src_stride_3x = src_stride_2x + src_stride;
211 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
212 __m128i zero = __lsx_vldi(0);
213 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
214 __m128i in0, in1, in0_r, in1_r, in2_r, in3_r;
215
216 for (loop_cnt = (height >> 3); loop_cnt--;) {
217 src0 = __lsx_vld(src, 0);
218 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
219 src3 = __lsx_vldx(src, src_stride_3x);
220 src += src_stride_4x;
221 src4 = __lsx_vld(src, 0);
222 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
223 src7 = __lsx_vldx(src, src_stride_3x);
224 src += src_stride_4x;
225
226 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
227 in0_r, in1_r, in2_r, in3_r);
228 DUP2_ARG2(__lsx_vilvh_w, src1, src0, src3, src2, src0, src1);
229 DUP2_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, in0, in1);
230 __lsx_vst(in0_r, dst, 0);
231 __lsx_vstx(in1_r, dst, dst_stride_x);
232 __lsx_vstx(in2_r, dst, dst_stride_2x);
233 __lsx_vstx(in3_r, dst, dst_stride_3x);
234 __lsx_vstelm_d(in0, dst, 16, 0);
235 dst += dst_stride;
236 __lsx_vstelm_d(in0, dst, 16, 1);
237 dst += dst_stride;
238 __lsx_vstelm_d(in1, dst, 16, 0);
239 dst += dst_stride;
240 __lsx_vstelm_d(in1, dst, 16, 1);
241 dst += dst_stride;
242
243 DUP4_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, src6, 6, src7, 6,
244 in0_r, in1_r, in2_r, in3_r);
245 DUP2_ARG2(__lsx_vilvh_w, src5, src4, src7, src6, src0, src1);
246 DUP2_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, in0, in1);
247 __lsx_vst(in0_r, dst, 0);
248 __lsx_vstx(in1_r, dst, dst_stride_x);
249 __lsx_vstx(in2_r, dst, dst_stride_2x);
250 __lsx_vstx(in3_r, dst, dst_stride_3x);
251 __lsx_vstelm_d(in0, dst, 16, 0);
252 dst += dst_stride;
253 __lsx_vstelm_d(in0, dst, 16, 1);
254 dst += dst_stride;
255 __lsx_vstelm_d(in1, dst, 16, 0);
256 dst += dst_stride;
257 __lsx_vstelm_d(in1, dst, 16, 1);
258 dst += dst_stride;
259 }
260 for (;res--;) {
261 src0 = __lsx_vld(src, 0);
262 in0 = __lsx_vsllwil_hu_bu(src0, 6);
263 src1 = __lsx_vilvh_b(zero, src0);
264 in1 = __lsx_vslli_h(src1, 6);
265 __lsx_vst(in0, dst, 0);
266 __lsx_vstelm_d(in1, dst, 16, 0);
267 src += src_stride;
268 dst += dst_stride;
269 }
270}
271
272static void hevc_copy_16w_lsx(const uint8_t *src, int32_t src_stride,
273 int16_t *dst, int32_t dst_stride,
274 int32_t height)
275{
276 __m128i zero = __lsx_vldi(0);
277 int32_t src_stride_2x = (src_stride << 1);
278 int32_t dst_stride_x = (dst_stride << 1);
279 int32_t src_stride_4x = (src_stride << 2);
280 int32_t dst_stride_2x = (dst_stride << 2);
281 int32_t src_stride_3x = src_stride_2x + src_stride;
282 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
283 int32_t loop_cnt = height >> 3;
284 int32_t res = height & 0x07;
285 int16_t* dst1 = dst + 8;
286 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
287 __m128i in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l;
288
289 for (loop_cnt = (height >> 3); loop_cnt--;) {
290 src0 = __lsx_vld(src, 0);
291 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x,
292 src1, src2);
293 src3 = __lsx_vldx(src, src_stride_3x);
294 src += src_stride_4x;
295 src4 = __lsx_vld(src, 0);
296 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x,
297 src5, src6);
298 src7 = __lsx_vldx(src, src_stride_3x);
299 src += src_stride_4x;
300 DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3,
301 in0_l, in1_l, in2_l, in3_l);
302 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
303 in0_r, in1_r, in2_r, in3_r);
304 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l,
305 in1_l, in2_l, in3_l);
306 __lsx_vst(in0_r, dst, 0);
307 __lsx_vstx(in1_r, dst, dst_stride_x);
308 __lsx_vstx(in2_r, dst, dst_stride_2x);
309 __lsx_vstx(in3_r, dst, dst_stride_3x);
310 __lsx_vst(in0_l, dst1, 0);
311 __lsx_vstx(in1_l, dst1, dst_stride_x);
312 __lsx_vstx(in2_l, dst1, dst_stride_2x);
313 __lsx_vstx(in3_l, dst1, dst_stride_3x);
314 dst += dst_stride_2x;
315 dst1 += dst_stride_2x;
316
317 DUP4_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, zero, src6, zero, src7,
318 in0_l, in1_l, in2_l, in3_l);
319 DUP4_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, src6, 6, src7, 6,
320 in0_r, in1_r, in2_r, in3_r);
321 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l,
322 in1_l, in2_l, in3_l);
323 __lsx_vst(in0_r, dst, 0);
324 __lsx_vstx(in1_r, dst, dst_stride_x);
325 __lsx_vstx(in2_r, dst, dst_stride_2x);
326 __lsx_vstx(in3_r, dst, dst_stride_3x);
327 __lsx_vst(in0_l, dst1, 0);
328 __lsx_vstx(in1_l, dst1, dst_stride_x);
329 __lsx_vstx(in2_l, dst1, dst_stride_2x);
330 __lsx_vstx(in3_l, dst1, dst_stride_3x);
331 dst += dst_stride_2x;
332 dst1 += dst_stride_2x;
333 }
334 if (res) {
335 src0 = __lsx_vld(src, 0);
336 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
337 src3 = __lsx_vldx(src, src_stride_3x);
338
339 DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero, src3,
340 in0_l, in1_l, in2_l, in3_l);
341 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
342 in0_r, in1_r, in2_r, in3_r);
343 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l,
344 in1_l, in2_l, in3_l);
345 __lsx_vst(in0_r, dst, 0);
346 __lsx_vstx(in1_r, dst, dst_stride_x);
347 __lsx_vstx(in2_r, dst, dst_stride_2x);
348 __lsx_vstx(in3_r, dst, dst_stride_3x);
349 dst += 8;
350 __lsx_vst(in0_l, dst, 0);
351 __lsx_vstx(in1_l, dst, dst_stride_x);
352 __lsx_vstx(in2_l, dst, dst_stride_2x);
353 __lsx_vstx(in3_l, dst, dst_stride_3x);
354 }
355}
356
357static void hevc_copy_24w_lsx(const uint8_t *src, int32_t src_stride,
358 int16_t *dst, int32_t dst_stride,
359 int32_t height)
360{
361 uint32_t loop_cnt;
362 int32_t src_stride_2x = (src_stride << 1);
363 int32_t dst_stride_x = (dst_stride << 1);
364 int32_t src_stride_4x = (src_stride << 2);
365 int32_t dst_stride_2x = (dst_stride << 2);
366 int32_t src_stride_3x = src_stride_2x + src_stride;
367 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
368 const uint8_t *_src = src + 16;
369 int16_t *dst1 = dst;
370 __m128i zero = __lsx_vldi(0);
371 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
372 __m128i in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l;
373
374 for (loop_cnt = (height >> 2); loop_cnt--;) {
375 src0 = __lsx_vld(src, 0);
376 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
377 src3 = __lsx_vldx(src, src_stride_3x);
378 src += src_stride_4x;
379 src4 = __lsx_vld(_src, 0);
380 DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride_2x,
381 src5, src6);
382 src7 = __lsx_vldx(_src, src_stride_3x);
383 _src += src_stride_4x;
384
385 DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero,
386 src3, in0_l, in1_l, in2_l, in3_l);
387 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
388 in0_r, in1_r, in2_r, in3_r);
389 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6,
390 in0_l, in1_l, in2_l, in3_l);
391 __lsx_vst(in0_r, dst, 0);
392 __lsx_vstx(in1_r, dst, dst_stride_x);
393 __lsx_vstx(in2_r, dst, dst_stride_2x);
394 __lsx_vstx(in3_r, dst, dst_stride_3x);
395 dst1 = dst + 8;
396 __lsx_vst(in0_l, dst1, 0);
397 __lsx_vstx(in1_l, dst1, dst_stride_x);
398 __lsx_vstx(in2_l, dst1, dst_stride_2x);
399 __lsx_vstx(in3_l, dst1, dst_stride_3x);
400 DUP4_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, src6, 6, src7, 6,
401 in0_r, in1_r, in2_r, in3_r);
402 dst1 = dst1 + 8;
403 __lsx_vst(in0_r, dst1, 0);
404 __lsx_vstx(in1_r, dst1, dst_stride_x);
405 __lsx_vstx(in2_r, dst1, dst_stride_2x);
406 __lsx_vstx(in3_r, dst1, dst_stride_3x);
407 dst += dst_stride_2x;
408 }
409}
410
411static void hevc_copy_32w_lsx(const uint8_t *src, int32_t src_stride,
412 int16_t *dst, int32_t dst_stride,
413 int32_t height)
414{
415 uint32_t loop_cnt;
416 int32_t src_stride_2x = (src_stride << 1);
417 int32_t src_stride_4x = (src_stride << 2);
418 int32_t src_stride_3x = src_stride_2x + src_stride;
419 const uint8_t *_src = src + 16;
420 __m128i zero = {0};
421 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
422 __m128i in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l;
423
424 for (loop_cnt = (height >> 2); loop_cnt--;) {
425 src0 = __lsx_vld(src, 0);
426 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src2, src4);
427 src6 = __lsx_vldx(src, src_stride_3x);
428 src += src_stride_4x;
429 src1 = __lsx_vld(_src, 0);
430 DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride_2x,
431 src3, src5);
432 src7 = __lsx_vldx(_src, src_stride_3x);
433 _src += src_stride_4x;
434
435 DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero,
436 src3, in0_l, in1_l, in2_l, in3_l);
437 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
438 in0_r, in1_r, in2_r, in3_r);
439 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6,
440 in0_l, in1_l, in2_l, in3_l);
441 __lsx_vst(in0_r, dst, 0);
442 __lsx_vst(in0_l, dst, 16);
443 __lsx_vst(in1_r, dst, 32);
444 __lsx_vst(in1_l, dst, 48);
445 dst += dst_stride;
446 __lsx_vst(in2_r, dst, 0);
447 __lsx_vst(in2_l, dst, 16);
448 __lsx_vst(in3_r, dst, 32);
449 __lsx_vst(in3_l, dst, 48);
450 dst += dst_stride;
451
452 DUP4_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, zero, src6, zero, src7,
453 in0_l, in1_l, in2_l, in3_l);
454 DUP4_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, src6, 6, src7, 6,
455 in0_r, in1_r, in2_r, in3_r);
456 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l,
457 in1_l, in2_l, in3_l);
458 __lsx_vst(in0_r, dst, 0);
459 __lsx_vst(in0_l, dst, 16);
460 __lsx_vst(in1_r, dst, 32);
461 __lsx_vst(in1_l, dst, 48);
462 dst += dst_stride;
463 __lsx_vst(in2_r, dst, 0);
464 __lsx_vst(in2_l, dst, 16);
465 __lsx_vst(in3_r, dst, 32);
466 __lsx_vst(in3_l, dst, 48);
467 dst += dst_stride;
468 }
469}
470
471static void hevc_copy_48w_lsx(const uint8_t *src, int32_t src_stride,
472 int16_t *dst, int32_t dst_stride,
473 int32_t height)
474{
475 uint32_t loop_cnt;
476 __m128i zero = {0};
477 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
478 __m128i src8, src9, src10, src11;
479 __m128i in0_r, in1_r, in2_r, in3_r, in4_r, in5_r;
480 __m128i in0_l, in1_l, in2_l, in3_l, in4_l, in5_l;
481
482 for (loop_cnt = (height >> 2); loop_cnt--;) {
483 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
484 src2 = __lsx_vld(src, 32);
485 src += src_stride;
486 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src3, src4);
487 src5 = __lsx_vld(src, 32);
488 src += src_stride;
489 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src6, src7);
490 src8 = __lsx_vld(src, 32);
491 src += src_stride;
492 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src9, src10);
493 src11 = __lsx_vld(src, 32);
494 src += src_stride;
495
496 DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero,
497 src3, in0_l, in1_l, in2_l, in3_l);
498 DUP2_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, in4_l, in5_l);
499 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
500 in0_r, in1_r, in2_r, in3_r);
501 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l,
502 in1_l, in2_l, in3_l);
503 DUP2_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, in4_r, in5_r);
504 DUP2_ARG2(__lsx_vslli_h, in4_l, 6, in5_l, 6, in4_l, in5_l);
505 __lsx_vst(in0_r, dst, 0);
506 __lsx_vst(in0_l, dst, 16);
507 __lsx_vst(in1_r, dst, 32);
508 __lsx_vst(in1_l, dst, 48);
509 __lsx_vst(in2_r, dst, 64);
510 __lsx_vst(in2_l, dst, 80);
511 dst += dst_stride;
512 __lsx_vst(in3_r, dst, 0);
513 __lsx_vst(in3_l, dst, 16);
514 __lsx_vst(in4_r, dst, 32);
515 __lsx_vst(in4_l, dst, 48);
516 __lsx_vst(in5_r, dst, 64);
517 __lsx_vst(in5_l, dst, 80);
518 dst += dst_stride;
519
520 DUP4_ARG2(__lsx_vilvh_b, zero, src6, zero, src7, zero, src8, zero, src9,
521 in0_l, in1_l, in2_l, in3_l);
522 DUP2_ARG2(__lsx_vilvh_b, zero, src10, zero, src11, in4_l, in5_l);
523 DUP4_ARG2(__lsx_vsllwil_hu_bu, src6, 6, src7, 6, src8, 6, src9, 6,
524 in0_r, in1_r, in2_r, in3_r);
525 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l,
526 in1_l, in2_l, in3_l);
527 DUP2_ARG2(__lsx_vsllwil_hu_bu, src10, 6, src11, 6, in4_r, in5_r);
528 DUP2_ARG2(__lsx_vslli_h, in4_l, 6, in5_l, 6, in4_l, in5_l);
529 __lsx_vst(in0_r, dst, 0);
530 __lsx_vst(in0_l, dst, 16);
531 __lsx_vst(in1_r, dst, 32);
532 __lsx_vst(in1_l, dst, 48);
533 __lsx_vst(in2_r, dst, 64);
534 __lsx_vst(in2_l, dst, 80);
535 dst += dst_stride;
536 __lsx_vst(in3_r, dst, 0);
537 __lsx_vst(in3_l, dst, 16);
538 __lsx_vst(in4_r, dst, 32);
539 __lsx_vst(in4_l, dst, 48);
540 __lsx_vst(in5_r, dst, 64);
541 __lsx_vst(in5_l, dst, 80);
542 dst += dst_stride;
543 }
544}
545
546static void hevc_copy_64w_lsx(const uint8_t *src, int32_t src_stride,
547 int16_t *dst, int32_t dst_stride,
548 int32_t height)
549{
550 uint32_t loop_cnt;
551 __m128i zero = {0};
552 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
553 __m128i in0_r, in1_r, in2_r, in3_r, in0_l, in1_l, in2_l, in3_l;
554
555
556 for (loop_cnt = (height >> 1); loop_cnt--;) {
557 DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48,
558 src0, src1, src2, src3);
559 src += src_stride;
560 DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48,
561 src4, src5, src6, src7);
562 src += src_stride;
563
564 DUP4_ARG2(__lsx_vilvh_b, zero, src0, zero, src1, zero, src2, zero,
565 src3, in0_l, in1_l, in2_l, in3_l);
566 DUP4_ARG2(__lsx_vsllwil_hu_bu, src0, 6, src1, 6, src2, 6, src3, 6,
567 in0_r, in1_r, in2_r, in3_r);
568 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6,
569 in0_l, in1_l, in2_l, in3_l);
570 __lsx_vst(in0_r, dst, 0);
571 __lsx_vst(in0_l, dst, 16);
572 __lsx_vst(in1_r, dst, 32);
573 __lsx_vst(in1_l, dst, 48);
574 __lsx_vst(in2_r, dst, 64);
575 __lsx_vst(in2_l, dst, 80);
576 __lsx_vst(in3_r, dst, 96);
577 __lsx_vst(in3_l, dst, 112);
578 dst += dst_stride;
579
580 DUP4_ARG2(__lsx_vilvh_b, zero, src4, zero, src5, zero, src6, zero,
581 src7, in0_l, in1_l, in2_l, in3_l);
582 DUP4_ARG2(__lsx_vsllwil_hu_bu, src4, 6, src5, 6, src6, 6, src7, 6,
583 in0_r, in1_r, in2_r, in3_r);
584 DUP4_ARG2(__lsx_vslli_h, in0_l, 6, in1_l, 6, in2_l, 6, in3_l, 6, in0_l,
585 in1_l, in2_l, in3_l);
586 __lsx_vst(in0_r, dst, 0);
587 __lsx_vst(in0_l, dst, 16);
588 __lsx_vst(in1_r, dst, 32);
589 __lsx_vst(in1_l, dst, 48);
590 __lsx_vst(in2_r, dst, 64);
591 __lsx_vst(in2_l, dst, 80);
592 __lsx_vst(in3_r, dst, 96);
593 __lsx_vst(in3_l, dst, 112);
594 dst += dst_stride;
595 }
596}
597
598static void hevc_hz_8t_4w_lsx(const uint8_t *src, int32_t src_stride,
599 int16_t *dst, int32_t dst_stride,
600 const int8_t *filter, int32_t height)
601{
602 uint32_t loop_cnt = height >> 3;
603 uint32_t res = (height & 0x7) >> 1;
604 int32_t src_stride_2x = (src_stride << 1);
605 int32_t dst_stride_2x = (dst_stride << 1);
606 int32_t src_stride_4x = (src_stride << 2);
607 int32_t dst_stride_4x = (dst_stride << 2);
608 int32_t src_stride_3x = src_stride_2x + src_stride;
609 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
610 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
611 __m128i filt0, filt1, filt2, filt3;
612 __m128i mask1, mask2, mask3;
613 __m128i vec0, vec1, vec2, vec3;
614 __m128i dst0, dst1, dst2, dst3;
615 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 16);
616
617 src -= 3;
618 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
619 filt0, filt1, filt2, filt3);
620
621 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
622 mask3 = __lsx_vaddi_bu(mask0, 6);
623
624 for (;loop_cnt--;) {
625 src0 = __lsx_vld(src, 0);
626 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
627 src3 = __lsx_vldx(src, src_stride_3x);
628 src += src_stride_4x;
629 src4 = __lsx_vld(src, 0);
630 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
631 src7 = __lsx_vldx(src, src_stride_3x);
632 src += src_stride_4x;
633 DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask0, src1, src0, mask1, src1,
634 src0, mask2, src1, src0, mask3, vec0, vec1, vec2, vec3);
635 dst0 = __lsx_vdp2_h_bu_b(vec0, filt0);
636 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst0, vec2, filt2,
637 dst0, dst0);
638 dst0 = __lsx_vdp2add_h_bu_b(dst0, vec3, filt3);
639 DUP4_ARG3(__lsx_vshuf_b, src3, src2, mask0, src3, src2, mask1, src3,
640 src2, mask2, src3, src2, mask3, vec0, vec1, vec2, vec3);
641 dst1 = __lsx_vdp2_h_bu_b(vec0, filt0);
642 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1, vec1, filt1, dst1, vec2, filt2,
643 dst1, dst1);
644 dst1 = __lsx_vdp2add_h_bu_b(dst1, vec3, filt3);
645 DUP4_ARG3(__lsx_vshuf_b, src5, src4, mask0, src5, src4, mask1, src5,
646 src4, mask2, src5, src4, mask3, vec0, vec1, vec2, vec3);
647 dst2 = __lsx_vdp2_h_bu_b(vec0, filt0);
648 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2, vec1, filt1, dst2, vec2, filt2,
649 dst2, dst2);
650 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec3, filt3);
651 DUP4_ARG3(__lsx_vshuf_b, src7, src6, mask0, src7, src6, mask1, src7,
652 src6, mask2, src7, src6, mask3, vec0, vec1, vec2, vec3);
653 dst3 = __lsx_vdp2_h_bu_b(vec0, filt0);
654 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst3, vec2, filt2,
655 dst3, dst3);
656 dst3 = __lsx_vdp2add_h_bu_b(dst3, vec3, filt3);
657
658 __lsx_vstelm_d(dst0, dst, 0, 0);
659 __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1);
660 __lsx_vstelm_d(dst1, dst + dst_stride_2x, 0, 0);
661 __lsx_vstelm_d(dst1, dst + dst_stride_3x, 0, 1);
662 dst += dst_stride_4x;
663 __lsx_vstelm_d(dst2, dst, 0, 0);
664 __lsx_vstelm_d(dst2, dst + dst_stride, 0, 1);
665 __lsx_vstelm_d(dst3, dst + dst_stride_2x, 0, 0);
666 __lsx_vstelm_d(dst3, dst + dst_stride_3x, 0, 1);
667 dst += dst_stride_4x;
668 }
669 for (;res--;) {
670 src0 = __lsx_vld(src, 0);
671 src1 = __lsx_vldx(src, src_stride);
672 DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask0, src1, src0, mask1, src1,
673 src0, mask2, src1, src0, mask3, vec0, vec1, vec2, vec3);
674 dst0 = __lsx_vdp2_h_bu_b(vec0, filt0);
675 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst0, vec2, filt2,
676 dst0, dst0);
677 dst0 = __lsx_vdp2add_h_bu_b(dst0, vec3, filt3);
678 __lsx_vstelm_d(dst0, dst, 0, 0);
679 __lsx_vstelm_d(dst0, dst + dst_stride, 0, 1);
680 src += src_stride_2x;
681 dst += dst_stride_2x;
682 }
683}
684
685static void hevc_hz_8t_8w_lsx(const uint8_t *src, int32_t src_stride,
686 int16_t *dst, int32_t dst_stride,
687 const int8_t *filter, int32_t height)
688{
689 uint32_t loop_cnt;
690 int32_t src_stride_2x = (src_stride << 1);
691 int32_t dst_stride_x = (dst_stride << 1);
692 int32_t src_stride_4x = (src_stride << 2);
693 int32_t dst_stride_2x = (dst_stride << 2);
694 int32_t src_stride_3x = src_stride_2x + src_stride;
695 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
696 __m128i src0, src1, src2, src3;
697 __m128i filt0, filt1, filt2, filt3;
698 __m128i mask1, mask2, mask3;
699 __m128i vec0, vec1, vec2, vec3;
700 __m128i dst0, dst1, dst2, dst3;
701 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
702
703 src -= 3;
704 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
705 filt0, filt1, filt2, filt3);
706
707 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
708 mask3 = __lsx_vaddi_bu(mask0, 6);
709
710 for (loop_cnt = (height >> 2); loop_cnt--;) {
711 src0 = __lsx_vld(src, 0);
712 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
713 src3 = __lsx_vldx(src, src_stride_3x);
714 src += src_stride_4x;
715
716 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0,
717 src0, mask2, src0, src0, mask3, vec0, vec1, vec2, vec3);
718 dst0 = __lsx_vdp2_h_bu_b(vec0, filt0);
719 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst0, vec2, filt2,
720 dst0, dst0);
721 dst0 = __lsx_vdp2add_h_bu_b(dst0, vec3, filt3);
722 DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1,
723 src1, mask2, src1, src1, mask3, vec0, vec1, vec2, vec3);
724 dst1 = __lsx_vdp2_h_bu_b(vec0, filt0);
725 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1, vec1, filt1, dst1, vec2, filt2,
726 dst1, dst1);
727 dst1 = __lsx_vdp2add_h_bu_b(dst1, vec3, filt3);
728 DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2,
729 src2, mask2, src2, src2, mask3, vec0, vec1, vec2, vec3);
730 dst2 = __lsx_vdp2_h_bu_b(vec0, filt0);
731 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2, vec1, filt1, dst2, vec2, filt2,
732 dst2, dst2);
733 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec3, filt3);
734 DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3,
735 src3, mask2, src3, src3, mask3, vec0, vec1, vec2, vec3);
736 dst3 = __lsx_vdp2_h_bu_b(vec0, filt0);
737 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst3, vec2, filt2,
738 dst3, dst3);
739 dst3 = __lsx_vdp2add_h_bu_b(dst3, vec3, filt3);
740
741 __lsx_vst(dst0, dst, 0);
742 __lsx_vstx(dst1, dst, dst_stride_x);
743 __lsx_vstx(dst2, dst, dst_stride_2x);
744 __lsx_vstx(dst3, dst, dst_stride_3x);
745 dst += dst_stride_2x;
746 }
747}
748
749static void hevc_hz_8t_12w_lsx(const uint8_t *src, int32_t src_stride,
750 int16_t *dst, int32_t dst_stride,
751 const int8_t *filter, int32_t height)
752{
753 uint32_t loop_cnt;
754 int32_t src_stride_2x = (src_stride << 1);
755 int32_t src_stride_4x = (src_stride << 2);
756 int32_t src_stride_3x = src_stride_2x + src_stride;
757 const uint8_t *_src;
758 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
759 __m128i mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
760 __m128i vec0, vec1, vec2, vec3, vec4, vec5;
761 __m128i filt0, filt1, filt2, filt3, dst0, dst1, dst2, dst3, dst4, dst5;
762
763 src -= 3;
764 _src = src + 8;
765 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
766 filt0, filt1, filt2, filt3);
767
768 mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
769 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
770 mask3 = __lsx_vaddi_bu(mask0, 6);
771 mask4 = __lsx_vld(ff_hevc_mask_arr, 16);
772 DUP2_ARG2(__lsx_vaddi_bu, mask4, 2, mask4, 4, mask5, mask6);
773 mask7 = __lsx_vaddi_bu(mask4, 6);
774
775 for (loop_cnt = height >> 2; loop_cnt--;) {
776 src0 = __lsx_vld(src, 0);
777 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
778 src3 = __lsx_vldx(src, src_stride_3x);
779 src4 = __lsx_vld(_src, 0);
780 DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride_2x, src5, src6);
781 src7 = __lsx_vldx(_src, src_stride_3x);
782 src += src_stride_4x;
783 _src += src_stride_4x;
784
785 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0, vec0, vec1);
786 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0, vec2, vec3);
787 DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask4, src7, src6, mask4, vec4, vec5);
788 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
789 vec3, filt0, dst0, dst1, dst2, dst3);
790 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec4, filt0, vec5, filt0, dst4, dst5);
791 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1, vec0, vec1);
792 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1, vec2, vec3);
793 DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask5, src7, src6, mask5, vec4, vec5);
794 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
795 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
796 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt1, dst5, vec5, filt1, dst4, dst5);
797 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2, vec0, vec1);
798 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2, vec2, vec3);
799 DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask6, src7, src6, mask6, vec4, vec5);
800 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt2, dst1, vec1, filt2,
801 dst2, vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3);
802 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt2, dst5, vec5, filt2, dst4, dst5);
803 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3, vec0, vec1);
804 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask3, src3, src3, mask3, vec2, vec3);
805 DUP2_ARG3(__lsx_vshuf_b, src5, src4, mask7, src7, src6, mask7, vec4, vec5);
806 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt3, dst1, vec1, filt3,
807 dst2, vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3);
808 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt3, dst5, vec5, filt3, dst4, dst5);
809
810 __lsx_vst(dst0, dst, 0);
811 __lsx_vstelm_d(dst4, dst, 16, 0);
812 dst += dst_stride;
813 __lsx_vst(dst1, dst, 0);
814 __lsx_vstelm_d(dst4, dst, 16, 1);
815 dst += dst_stride;
816 __lsx_vst(dst2, dst, 0);
817 __lsx_vstelm_d(dst5, dst, 16, 0);
818 dst += dst_stride;
819 __lsx_vst(dst3, dst, 0);
820 __lsx_vstelm_d(dst5, dst, 16, 1);
821 dst += dst_stride;
822 }
823}
824
825static void hevc_hz_8t_16w_lsx(const uint8_t *src, int32_t src_stride,
826 int16_t *dst, int32_t dst_stride,
827 const int8_t *filter, int32_t height)
828{
829 uint32_t loop_cnt;
830 __m128i src0, src1, src2, src3;
831 __m128i filt0, filt1, filt2, filt3;
832 __m128i mask1, mask2, mask3;
833 __m128i vec0, vec1, vec2, vec3;
834 __m128i dst0, dst1, dst2, dst3;
835 __m128i mask0;
836
837 src -= 3;
838 mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
839 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
840 filt0, filt1, filt2, filt3);
841
842 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
843 mask3 = __lsx_vaddi_bu(mask0, 6);
844
845 for (loop_cnt = (height >> 1); loop_cnt--;) {
846 DUP2_ARG2(__lsx_vld, src, 0, src, 8, src0, src1);
847 src += src_stride;
848 DUP2_ARG2(__lsx_vld, src, 0, src, 8, src2, src3);
849 src += src_stride;
850
851 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src1, mask0,
852 vec0, vec1);
853 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0,
854 vec2, vec3);
855 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
856 vec3, filt0, dst0, dst1, dst2, dst3);
857 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src1, mask1,
858 vec0, vec1);
859 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1,
860 vec2, vec3);
861 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
862 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
863 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src1, mask2,
864 vec0, vec1);
865 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2,
866 vec2, vec3);
867 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt2, dst1, vec1, filt2,
868 dst2, vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3);
869 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src1, mask3,
870 vec0, vec1);
871 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask3, src3, src3, mask3,
872 vec2, vec3);
873 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt3, dst1, vec1, filt3,
874 dst2, vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3);
875
876 __lsx_vst(dst0, dst, 0);
877 __lsx_vst(dst1, dst, 16);
878 dst += dst_stride;
879 __lsx_vst(dst2, dst, 0);
880 __lsx_vst(dst3, dst, 16);
881 dst += dst_stride;
882 }
883}
884
885static void hevc_hz_8t_24w_lsx(const uint8_t *src, int32_t src_stride,
886 int16_t *dst, int32_t dst_stride,
887 const int8_t *filter, int32_t height)
888{
889 uint32_t loop_cnt;
890 __m128i src0, src1, src2, src3;
891 __m128i filt0, filt1, filt2, filt3;
892 __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7;
893 __m128i vec0, vec1, vec2, vec3, vec4, vec5;
894 __m128i dst0, dst1, dst2, dst3, dst4, dst5;
895 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
896
897 src -= 3;
898 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
899 filt0, filt1, filt2, filt3);
900
901 DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1,
902 mask2, mask3, mask4);
903 DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6);
904 mask7 = __lsx_vaddi_bu(mask0, 14);
905
906 for (loop_cnt = (height >> 1); loop_cnt--;) {
907 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
908 src += src_stride;
909 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src2, src3);
910 src += src_stride;
911
912 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, src1,
913 src1, mask0, src2, src2, mask0, vec0, vec1, vec2, vec3);
914 DUP2_ARG3(__lsx_vshuf_b, src3, src2, mask4, src3, src3, mask0,
915 vec4, vec5);
916 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
917 vec3, filt0, dst0, dst1, dst2, dst3);
918 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec4, filt0, vec5, filt0, dst4, dst5);
919 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask5, src1,
920 src1, mask1, src2, src2, mask1, vec0, vec1, vec2, vec3);
921 DUP2_ARG3(__lsx_vshuf_b, src3, src2, mask5, src3, src3, mask1,
922 vec4, vec5);
923 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
924 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
925 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt1, dst5, vec5, filt1,
926 dst4, dst5);
927 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src0, mask6, src1,
928 src1, mask2, src2, src2, mask2, vec0, vec1, vec2, vec3);
929 DUP2_ARG3(__lsx_vshuf_b, src3, src2, mask6, src3, src3, mask2,
930 vec4, vec5);
931 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt2, dst1, vec1, filt2,
932 dst2, vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3);
933 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt2, dst5, vec5, filt2,
934 dst4, dst5);
935 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src0, mask7, src1,
936 src1, mask3, src2, src2, mask3, vec0, vec1, vec2, vec3);
937 DUP2_ARG3(__lsx_vshuf_b, src3, src2, mask7, src3, src3, mask3,
938 vec4, vec5);
939 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt3, dst1, vec1, filt3,
940 dst2, vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3);
941 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt3, dst5, vec5, filt3,
942 dst4, dst5);
943
944 __lsx_vst(dst0, dst, 0);
945 __lsx_vst(dst1, dst, 16);
946 __lsx_vst(dst2, dst, 32);
947 dst += dst_stride;
948 __lsx_vst(dst3, dst, 0);
949 __lsx_vst(dst4, dst, 16);
950 __lsx_vst(dst5, dst, 32);
951 dst += dst_stride;
952 }
953}
954
955static void hevc_hz_8t_32w_lsx(const uint8_t *src, int32_t src_stride,
956 int16_t *dst, int32_t dst_stride,
957 const int8_t *filter, int32_t height)
958{
959 uint32_t loop_cnt;
960 __m128i src0, src1, src2;
961 __m128i filt0, filt1, filt2, filt3;
962 __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7;
963 __m128i vec0, vec1, vec2, vec3;
964 __m128i dst0, dst1, dst2, dst3;
965 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
966
967 src -= 3;
968 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
969 filt0, filt1, filt2, filt3);
970
971 DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8,
972 mask1, mask2, mask3, mask4);
973 DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6);
974 mask7 = __lsx_vaddi_bu(mask0, 14);
975
976 for (loop_cnt = height; loop_cnt--;) {
977 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
978 src2 = __lsx_vld(src, 24);
979 src += src_stride;
980
981 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0,
982 src0, mask2, src0, src0, mask3, vec0, vec1, vec2, vec3);
983 dst0 = __lsx_vdp2_h_bu_b(vec0, filt0);
984 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst0, vec2, filt2,
985 dst0, dst0);
986 dst0 = __lsx_vdp2add_h_bu_b(dst0, vec3, filt3);
987
988 DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask4, src1, src0, mask5, src1,
989 src0, mask6, src1, src0, mask7, vec0, vec1, vec2, vec3);
990 dst1 = __lsx_vdp2_h_bu_b(vec0, filt0);
991 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1, vec1, filt1, dst1, vec2, filt2,
992 dst1, dst1);
993 dst1 = __lsx_vdp2add_h_bu_b(dst1, vec3, filt3);
994 DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1,
995 mask2, src1, src1, mask3, vec0, vec1, vec2, vec3);
996 dst2 = __lsx_vdp2_h_bu_b(vec0, filt0);
997 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2, vec1, filt1, dst2, vec2, filt2,
998 dst2, dst2);
999 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec3, filt3);
1000 DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2,
1001 mask2, src2, src2, mask3, vec0, vec1, vec2, vec3);
1002 dst3 = __lsx_vdp2_h_bu_b(vec0, filt0);
1003 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst3, vec2, filt2,
1004 dst3, dst3);
1005 dst3 = __lsx_vdp2add_h_bu_b(dst3, vec3, filt3);
1006
1007 __lsx_vst(dst0, dst, 0);
1008 __lsx_vst(dst1, dst, 16);
1009 __lsx_vst(dst2, dst, 32);
1010 __lsx_vst(dst3, dst, 48);
1011 dst += dst_stride;
1012 }
1013}
1014
1015static void hevc_hz_8t_48w_lsx(const uint8_t *src, int32_t src_stride,
1016 int16_t *dst, int32_t dst_stride,
1017 const int8_t *filter, int32_t height)
1018{
1019 uint32_t loop_cnt;
1020 __m128i src0, src1, src2, src3;
1021 __m128i filt0, filt1, filt2, filt3;
1022 __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1023 __m128i vec0, vec1, vec2, vec3, vec4, vec5;
1024 __m128i dst0, dst1, dst2, dst3, dst4, dst5;
1025 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
1026
1027 src -= 3;
1028 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
1029 filt0, filt1, filt2, filt3);
1030
1031 DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1,
1032 mask2, mask3, mask4);
1033 DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6);
1034 mask7 = __lsx_vaddi_bu(mask0, 14);
1035
1036 for (loop_cnt = height; loop_cnt--;) {
1037 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
1038 src2 = __lsx_vld(src, 32);
1039 src3 = __lsx_vld(src, 40);
1040 src += src_stride;
1041
1042 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask4, src1, src1,
1043 mask0, src2, src1, mask4, vec0, vec1, vec2, vec3);
1044 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
1045 vec3, filt0, dst0, dst1, dst2, dst3);
1046 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask5, src1,
1047 src1, mask1, src2, src1, mask5, vec0, vec1, vec2, vec3);
1048 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
1049 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
1050 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask2, src1, src0, mask6, src1,
1051 src1, mask2, src2, src1, mask6, vec0, vec1, vec2, vec3);
1052 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt2, dst1, vec1, filt2,
1053 dst2, vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3);
1054 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask3, src1, src0, mask7, src1,
1055 src1, mask3, src2, src1, mask7, vec0, vec1, vec2, vec3);
1056 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt3, dst1, vec1, filt3,
1057 dst2, vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3);
1058 __lsx_vst(dst0, dst, 0);
1059 __lsx_vst(dst1, dst, 16);
1060 __lsx_vst(dst2, dst, 32);
1061 __lsx_vst(dst3, dst, 48);
1062
1063 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src3, src3, mask0,
1064 vec4, vec5);
1065 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec4, filt0, vec5, filt0, dst4, dst5);
1066 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask1, src3, src3, mask1,
1067 vec4, vec5);
1068 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt1, dst5, vec5, filt1,
1069 dst4, dst5);
1070 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask2, src3, src3, mask2,
1071 vec4, vec5);
1072 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt2, dst5, vec5, filt2,
1073 dst4, dst5);
1074 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask3, src3, src3, mask3,
1075 vec4, vec5);
1076 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec4, filt3, dst5, vec5, filt3,
1077 dst4, dst5);
1078 __lsx_vst(dst4, dst, 64);
1079 __lsx_vst(dst5, dst, 80);
1080 dst += dst_stride;
1081 }
1082}
1083
1084static void hevc_hz_8t_64w_lsx(const uint8_t *src, int32_t src_stride,
1085 int16_t *dst, int32_t dst_stride,
1086 const int8_t *filter, int32_t height)
1087{
1088 uint32_t loop_cnt;
1089 __m128i src0, src1, src2, src3, src4;
1090 __m128i filt0, filt1, filt2, filt3;
1091 __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1092 __m128i vec0, vec1, vec2, vec3;
1093 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1094 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
1095
1096 src -= 3;
1097 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
1098 filt0, filt1, filt2, filt3);
1099
1100 DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1,
1101 mask2, mask3, mask4);
1102 DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6)
1103 mask7 = __lsx_vaddi_bu(mask0, 14);
1104
1105 for (loop_cnt = height; loop_cnt--;) {
1106 DUP4_ARG2(__lsx_vld, src, 0, src, 16, src, 32, src, 48,
1107 src0, src1, src2, src3);
1108 src4 = __lsx_vld(src, 56);
1109 src += src_stride;
1110
1111 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0,
1112 src0, mask2, src0, src0, mask3, vec0, vec1, vec2, vec3);
1113 dst0 = __lsx_vdp2_h_bu_b(vec0, filt0);
1114 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst0, vec2, filt2,
1115 dst0, dst0);
1116 dst0 = __lsx_vdp2add_h_bu_b(dst0, vec3, filt3);
1117 __lsx_vst(dst0, dst, 0);
1118
1119 DUP4_ARG3(__lsx_vshuf_b, src1, src0, mask4, src1, src0, mask5, src1,
1120 src0, mask6, src1, src0, mask7, vec0, vec1, vec2, vec3);
1121 dst1 = __lsx_vdp2_h_bu_b(vec0, filt0);
1122 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1, vec1, filt1, dst1, vec2, filt2,
1123 dst1, dst1);
1124 dst1 = __lsx_vdp2add_h_bu_b(dst1, vec3, filt3);
1125 __lsx_vst(dst1, dst, 16);
1126
1127 DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1,
1128 src1, mask2, src1, src1, mask3, vec0, vec1, vec2, vec3);
1129 dst2 = __lsx_vdp2_h_bu_b(vec0, filt0);
1130 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2, vec1, filt1, dst2, vec2, filt2,
1131 dst2, dst2);
1132 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec3, filt3);
1133 __lsx_vst(dst2, dst, 32);
1134
1135 DUP4_ARG3(__lsx_vshuf_b, src2, src1, mask4, src2, src1, mask5, src2,
1136 src1, mask6, src2, src1, mask7, vec0, vec1, vec2, vec3);
1137 dst3 = __lsx_vdp2_h_bu_b(vec0, filt0);
1138 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst3, vec2, filt2,
1139 dst3, dst3);
1140 dst3 = __lsx_vdp2add_h_bu_b(dst3, vec3, filt3);
1141 __lsx_vst(dst3, dst, 48);
1142
1143 DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2,
1144 src2, mask2, src2, src2, mask3, vec0, vec1, vec2, vec3);
1145 dst4 = __lsx_vdp2_h_bu_b(vec0, filt0);
1146 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec1, filt1, dst4, vec2, filt2,
1147 dst4, dst4);
1148 dst4 = __lsx_vdp2add_h_bu_b(dst4, vec3, filt3);
1149 __lsx_vst(dst4, dst, 64);
1150
1151 DUP4_ARG3(__lsx_vshuf_b, src3, src2, mask4, src3, src2, mask5, src3,
1152 src2, mask6, src3, src2, mask7, vec0, vec1, vec2, vec3);
1153 dst5 = __lsx_vdp2_h_bu_b(vec0, filt0);
1154 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst5, vec1, filt1, dst5, vec2, filt2,
1155 dst5, dst5);
1156 dst5 = __lsx_vdp2add_h_bu_b(dst5, vec3, filt3);
1157 __lsx_vst(dst5, dst, 80);
1158
1159 DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3,
1160 src3, mask2, src3, src3, mask3, vec0, vec1, vec2, vec3);
1161 dst6 = __lsx_vdp2_h_bu_b(vec0, filt0);
1162 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst6, vec1, filt1, dst6, vec2, filt2,
1163 dst6, dst6);
1164 dst6 = __lsx_vdp2add_h_bu_b(dst6, vec3, filt3);
1165 __lsx_vst(dst6, dst, 96);
1166
1167 DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4,
1168 src4, mask2, src4, src4, mask3, vec0, vec1, vec2, vec3);
1169 dst7 = __lsx_vdp2_h_bu_b(vec0, filt0);
1170 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst7, vec1, filt1, dst7, vec2, filt2,
1171 dst7, dst7);
1172 dst7 = __lsx_vdp2add_h_bu_b(dst7, vec3, filt3);
1173 __lsx_vst(dst7, dst, 112);
1174 dst += dst_stride;
1175 }
1176}
1177
1178static void hevc_vt_8t_4w_lsx(const uint8_t *src, int32_t src_stride,
1179 int16_t *dst, int32_t dst_stride,
1180 const int8_t *filter, int32_t height)
1181{
1182 int32_t loop_cnt;
1183 int32_t res = (height & 0x07) >> 1;
1184 int32_t src_stride_2x = (src_stride << 1);
1185 int32_t src_stride_4x = (src_stride << 2);
1186 int32_t src_stride_3x = src_stride_2x + src_stride;
1187 __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8;
1188 __m128i src9, src10, src11, src12, src13, src14;
1189 __m128i src10_r, src32_r, src54_r, src76_r, src98_r;
1190 __m128i src21_r, src43_r, src65_r, src87_r, src109_r;
1191 __m128i src1110_r, src1211_r, src1312_r, src1413_r;
1192 __m128i src2110, src4332, src6554, src8776, src10998;
1193 __m128i src12111110, src14131312;
1194 __m128i dst10, dst32, dst54, dst76;
1195 __m128i filt0, filt1, filt2, filt3;
1196
1197 src -= src_stride_3x;
1198
1199 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
1200 filt0, filt1, filt2, filt3);
1201
1202 src0 = __lsx_vld(src, 0);
1203 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
1204 src3 = __lsx_vldx(src, src_stride_3x);
1205 src += src_stride_4x;
1206 src4 = __lsx_vld(src, 0);
1207 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
1208 src += src_stride_3x;
1209 DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1,
1210 src10_r, src32_r, src54_r, src21_r);
1211 DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r);
1212 DUP2_ARG2(__lsx_vilvl_d, src21_r, src10_r, src43_r, src32_r,
1213 src2110, src4332);
1214 src6554 = __lsx_vilvl_d(src65_r, src54_r);
1215
1216 for (loop_cnt = (height >> 3); loop_cnt--;) {
1217 src7 = __lsx_vld(src, 0);
1218 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src8, src9);
1219 src10 = __lsx_vldx(src, src_stride_3x);
1220 src += src_stride_4x;
1221 src11 = __lsx_vld(src, 0);
1222 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x,
1223 src12, src13);
1224 src14 = __lsx_vldx(src, src_stride_3x);
1225 src += src_stride_4x;
1226
1227 DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
1228 src76_r, src87_r, src98_r, src109_r);
1229 DUP4_ARG2(__lsx_vilvl_b, src11, src10, src12, src11, src13, src12, src14,
1230 src13, src1110_r, src1211_r, src1312_r, src1413_r);
1231 DUP4_ARG2(__lsx_vilvl_d, src87_r, src76_r, src109_r, src98_r, src1211_r,
1232 src1110_r, src1413_r, src1312_r, src8776, src10998,
1233 src12111110, src14131312);
1234
1235 dst10 = __lsx_vdp2_h_bu_b(src2110, filt0);
1236 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst10, src4332, filt1, dst10, src6554,
1237 filt2, dst10, dst10);
1238 dst10 = __lsx_vdp2add_h_bu_b(dst10, src8776, filt3);
1239 dst32 = __lsx_vdp2_h_bu_b(src4332, filt0);
1240 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst32, src6554, filt1, dst32, src8776,
1241 filt2, dst32, dst32);
1242 dst32 = __lsx_vdp2add_h_bu_b(dst32, src10998, filt3);
1243 dst54 = __lsx_vdp2_h_bu_b(src6554, filt0);
1244 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst54, src8776, filt1,
1245 dst54, src10998, filt2, dst54, dst54);
1246 dst54 = __lsx_vdp2add_h_bu_b(dst54, src12111110, filt3);
1247 dst76 = __lsx_vdp2_h_bu_b(src8776, filt0);
1248 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst76, src10998, filt1, dst76,
1249 src12111110, filt2, dst76, dst76);
1250 dst76 = __lsx_vdp2add_h_bu_b(dst76, src14131312, filt3);
1251
1252 __lsx_vstelm_d(dst10, dst, 0, 0);
1253 dst += dst_stride;
1254 __lsx_vstelm_d(dst10, dst, 0, 1);
1255 dst += dst_stride;
1256 __lsx_vstelm_d(dst32, dst, 0, 0);
1257 dst += dst_stride;
1258 __lsx_vstelm_d(dst32, dst, 0, 1);
1259 dst += dst_stride;
1260 __lsx_vstelm_d(dst54, dst, 0, 0);
1261 dst += dst_stride;
1262 __lsx_vstelm_d(dst54, dst, 0, 1);
1263 dst += dst_stride;
1264 __lsx_vstelm_d(dst76, dst, 0, 0);
1265 dst += dst_stride;
1266 __lsx_vstelm_d(dst76, dst, 0, 1);
1267 dst += dst_stride;
1268
1269 src2110 = src10998;
1270 src4332 = src12111110;
1271 src6554 = src14131312;
1272 src6 = src14;
1273 }
1274 for (;res--;) {
1275 src7 = __lsx_vld(src, 0);
1276 src8 = __lsx_vldx(src, src_stride);
1277 DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r);
1278 src += src_stride_2x;
1279 src8776 = __lsx_vilvl_d(src87_r, src76_r);
1280
1281 dst10 = __lsx_vdp2_h_bu_b(src2110, filt0);
1282 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst10, src4332, filt1, dst10, src6554,
1283 filt2, dst10, dst10);
1284 dst10 = __lsx_vdp2add_h_bu_b(dst10, src8776, filt3);
1285
1286 __lsx_vstelm_d(dst10, dst, 0, 0);
1287 dst += dst_stride;
1288 __lsx_vstelm_d(dst10, dst, 0, 1);
1289 dst += dst_stride;
1290
1291 src2110 = src4332;
1292 src4332 = src6554;
1293 src6554 = src8776;
1294 src6 = src8;
1295 }
1296}
1297
1298static void hevc_vt_8t_8w_lsx(const uint8_t *src, int32_t src_stride,
1299 int16_t *dst, int32_t dst_stride,
1300 const int8_t *filter, int32_t height)
1301{
1302 int32_t loop_cnt;
1303 int32_t src_stride_2x = (src_stride << 1);
1304 int32_t dst_stride_x = (dst_stride << 1);
1305 int32_t src_stride_4x = (src_stride << 2);
1306 int32_t dst_stride_2x = (dst_stride << 2);
1307 int32_t src_stride_3x = src_stride_2x + src_stride;
1308 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
1309 __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1310 __m128i src10_r, src32_r, src54_r, src76_r, src98_r;
1311 __m128i src21_r, src43_r, src65_r, src87_r, src109_r;
1312 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
1313 __m128i filt0, filt1, filt2, filt3;
1314
1315 src -= src_stride_3x;
1316
1317 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
1318 filt0, filt1, filt2, filt3);
1319
1320 src0 = __lsx_vld(src, 0);
1321 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
1322 src3 = __lsx_vldx(src, src_stride_3x);
1323 src += src_stride_4x;
1324 src4 = __lsx_vld(src, 0);
1325 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
1326 src += src_stride_3x;
1327 DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1,
1328 src10_r, src32_r, src54_r, src21_r);
1329 DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r);
1330
1331 for (loop_cnt = (height >> 2); loop_cnt--;) {
1332 src7 = __lsx_vld(src, 0);
1333 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src8, src9);
1334 src10 = __lsx_vldx(src, src_stride_3x);
1335 src += src_stride_4x;
1336 DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10,
1337 src9, src76_r, src87_r, src98_r, src109_r);
1338
1339 dst0_r = __lsx_vdp2_h_bu_b(src10_r, filt0);
1340 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst0_r,
1341 src54_r, filt2, dst0_r, dst0_r);
1342 dst0_r = __lsx_vdp2add_h_bu_b(dst0_r, src76_r, filt3);
1343 dst1_r = __lsx_vdp2_h_bu_b(src21_r, filt0);
1344 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1_r, src43_r, filt1, dst1_r,
1345 src65_r, filt2, dst1_r, dst1_r);
1346 dst1_r = __lsx_vdp2add_h_bu_b(dst1_r, src87_r, filt3);
1347 dst2_r = __lsx_vdp2_h_bu_b(src32_r, filt0);
1348 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src54_r, filt1, dst2_r,
1349 src76_r, filt2, dst2_r, dst2_r);
1350 dst2_r = __lsx_vdp2add_h_bu_b(dst2_r, src98_r, filt3);
1351 dst3_r = __lsx_vdp2_h_bu_b(src43_r, filt0);
1352 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3_r, src65_r, filt1, dst3_r,
1353 src87_r, filt2, dst3_r, dst3_r);
1354 dst3_r = __lsx_vdp2add_h_bu_b(dst3_r, src109_r, filt3);
1355
1356 __lsx_vst(dst0_r, dst, 0);
1357 __lsx_vstx(dst1_r, dst, dst_stride_x);
1358 __lsx_vstx(dst2_r, dst, dst_stride_2x);
1359 __lsx_vstx(dst3_r, dst, dst_stride_3x);
1360 dst += dst_stride_2x;
1361
1362 src10_r = src54_r;
1363 src32_r = src76_r;
1364 src54_r = src98_r;
1365 src21_r = src65_r;
1366 src43_r = src87_r;
1367 src65_r = src109_r;
1368 src6 = src10;
1369 }
1370}
1371
1372static void hevc_vt_8t_12w_lsx(const uint8_t *src, int32_t src_stride,
1373 int16_t *dst, int32_t dst_stride,
1374 const int8_t *filter, int32_t height)
1375{
1376 int32_t loop_cnt;
1377 int32_t src_stride_2x = (src_stride << 1);
1378 int32_t src_stride_4x = (src_stride << 2);
1379 int32_t src_stride_3x = src_stride_2x + src_stride;
1380 __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1381 __m128i src10_r, src32_r, src54_r, src76_r, src98_r;
1382 __m128i src21_r, src43_r, src65_r, src87_r, src109_r;
1383 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
1384 __m128i src10_l, src32_l, src54_l, src76_l, src98_l;
1385 __m128i src21_l, src43_l, src65_l, src87_l, src109_l;
1386 __m128i src2110, src4332, src6554, src8776, src10998;
1387 __m128i dst0_l, dst1_l;
1388 __m128i filt0, filt1, filt2, filt3;
1389
1390 src -= src_stride_3x;
1391
1392 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
1393 filt0, filt1, filt2, filt3);
1394 src0 = __lsx_vld(src, 0);
1395 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
1396 src3 = __lsx_vldx(src, src_stride_3x);
1397 src += src_stride_4x;
1398 src4 = __lsx_vld(src, 0);
1399 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
1400 src += src_stride_3x;
1401 DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1,
1402 src10_r, src32_r, src54_r, src21_r);
1403 DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r);
1404 DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1,
1405 src10_l, src32_l, src54_l, src21_l);
1406 DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, src43_l, src65_l);
1407 DUP2_ARG2(__lsx_vilvl_d, src21_l, src10_l, src43_l, src32_l,
1408 src2110, src4332);
1409 src6554 = __lsx_vilvl_d(src65_l, src54_l);
1410
1411 for (loop_cnt = (height >> 2); loop_cnt--;) {
1412 src7 = __lsx_vld(src, 0);
1413 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src8, src9);
1414 src10 = __lsx_vldx(src, src_stride_3x);
1415 src += src_stride_4x;
1416 DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10,
1417 src9, src76_r, src87_r, src98_r, src109_r);
1418 DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8, src10,
1419 src9, src76_l, src87_l, src98_l, src109_l);
1420 DUP2_ARG2(__lsx_vilvl_d, src87_l, src76_l, src109_l, src98_l,
1421 src8776, src10998);
1422
1423 dst0_r = __lsx_vdp2_h_bu_b(src10_r, filt0);
1424 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst0_r,
1425 src54_r, filt2, dst0_r, dst0_r);
1426 dst0_r = __lsx_vdp2add_h_bu_b(dst0_r, src76_r, filt3);
1427 dst1_r = __lsx_vdp2_h_bu_b(src21_r, filt0);
1428 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1_r, src43_r, filt1, dst1_r,
1429 src65_r, filt2, dst1_r, dst1_r);
1430 dst1_r = __lsx_vdp2add_h_bu_b(dst1_r, src87_r, filt3);
1431 dst2_r = __lsx_vdp2_h_bu_b(src32_r, filt0);
1432 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src54_r, filt1, dst2_r,
1433 src76_r, filt2, dst2_r, dst2_r);
1434 dst2_r = __lsx_vdp2add_h_bu_b(dst2_r, src98_r, filt3);
1435 dst3_r = __lsx_vdp2_h_bu_b(src43_r, filt0);
1436 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3_r, src65_r, filt1, dst3_r,
1437 src87_r, filt2, dst3_r, dst3_r);
1438 dst3_r = __lsx_vdp2add_h_bu_b(dst3_r, src109_r, filt3);
1439 dst0_l = __lsx_vdp2_h_bu_b(src2110, filt0);
1440 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0_l, src4332, filt1, dst0_l,
1441 src6554, filt2, dst0_l, dst0_l);
1442 dst0_l = __lsx_vdp2add_h_bu_b(dst0_l, src8776, filt3);
1443 dst1_l = __lsx_vdp2_h_bu_b(src4332, filt0);
1444 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1_l, src6554, filt1, dst1_l,
1445 src8776, filt2, dst1_l, dst1_l);
1446 dst1_l = __lsx_vdp2add_h_bu_b(dst1_l, src10998, filt3);
1447
1448 __lsx_vst(dst0_r, dst, 0);
1449 __lsx_vstelm_d(dst0_l, dst, 16, 0);
1450 dst += dst_stride;
1451 __lsx_vst(dst1_r, dst, 0);
1452 __lsx_vstelm_d(dst0_l, dst, 16, 1);
1453 dst += dst_stride;
1454 __lsx_vst(dst2_r, dst, 0);
1455 __lsx_vstelm_d(dst1_l, dst, 16, 0);
1456 dst += dst_stride;
1457 __lsx_vst(dst3_r, dst, 0);
1458 __lsx_vstelm_d(dst1_l, dst, 16, 1);
1459 dst += dst_stride;
1460
1461 src10_r = src54_r;
1462 src32_r = src76_r;
1463 src54_r = src98_r;
1464 src21_r = src65_r;
1465 src43_r = src87_r;
1466 src65_r = src109_r;
1467 src2110 = src6554;
1468 src4332 = src8776;
1469 src6554 = src10998;
1470 src6 = src10;
1471 }
1472}
1473
1474static void hevc_vt_8t_16multx4mult_lsx(const uint8_t *src,
1475 int32_t src_stride,
1476 int16_t *dst,
1477 int32_t dst_stride,
1478 const int8_t *filter,
1479 int32_t height,
1480 int32_t width)
1481{
1482 const uint8_t *src_tmp;
1483 int16_t *dst_tmp;
1484 int32_t loop_cnt, cnt;
1485 int32_t src_stride_2x = (src_stride << 1);
1486 int32_t src_stride_4x = (src_stride << 2);
1487 int32_t src_stride_3x = src_stride_2x + src_stride;
1488 __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1489 __m128i src10_r, src32_r, src54_r, src76_r, src98_r;
1490 __m128i src21_r, src43_r, src65_r, src87_r, src109_r;
1491 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
1492 __m128i src10_l, src32_l, src54_l, src76_l, src98_l;
1493 __m128i src21_l, src43_l, src65_l, src87_l, src109_l;
1494 __m128i dst0_l, dst1_l, dst2_l, dst3_l;
1495 __m128i filt0, filt1, filt2, filt3;
1496
1497 src -= src_stride_3x;
1498
1499 DUP4_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filter, 4, filter, 6,
1500 filt0, filt1, filt2, filt3);
1501
1502 for (cnt = width >> 4; cnt--;) {
1503 src_tmp = src;
1504 dst_tmp = dst;
1505
1506 src0 = __lsx_vld(src_tmp, 0);
1507 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
1508 src1, src2);
1509 src3 = __lsx_vldx(src_tmp, src_stride_3x);
1510 src_tmp += src_stride_4x;
1511 src4 = __lsx_vld(src_tmp, 0);
1512 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
1513 src5, src6);
1514 src_tmp += src_stride_3x;
1515 DUP4_ARG2(__lsx_vilvl_b, src1, src0, src3, src2, src5, src4, src2, src1,
1516 src10_r, src32_r, src54_r, src21_r);
1517 DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r);
1518 DUP4_ARG2(__lsx_vilvh_b, src1, src0, src3, src2, src5, src4, src2, src1,
1519 src10_l, src32_l, src54_l, src21_l);
1520 DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, src43_l, src65_l);
1521
1522 for (loop_cnt = (height >> 2); loop_cnt--;) {
1523 src7 = __lsx_vld(src_tmp, 0);
1524 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
1525 src8, src9);
1526 src10 = __lsx_vldx(src_tmp, src_stride_3x);
1527 src_tmp += src_stride_4x;
1528 DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8,
1529 src10, src9, src76_r, src87_r, src98_r, src109_r);
1530 DUP4_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src9, src8,
1531 src10, src9, src76_l, src87_l, src98_l, src109_l);
1532
1533 dst0_r = __lsx_vdp2_h_bu_b(src10_r, filt0);
1534 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst0_r,
1535 src54_r, filt2, dst0_r, dst0_r);
1536 dst0_r = __lsx_vdp2add_h_bu_b(dst0_r, src76_r, filt3);
1537 dst1_r = __lsx_vdp2_h_bu_b(src21_r, filt0);
1538 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1_r, src43_r, filt1, dst1_r,
1539 src65_r, filt2, dst1_r, dst1_r);
1540 dst1_r = __lsx_vdp2add_h_bu_b(dst1_r, src87_r, filt3);
1541 dst2_r = __lsx_vdp2_h_bu_b(src32_r, filt0);
1542 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src54_r, filt1, dst2_r,
1543 src76_r, filt2, dst2_r, dst2_r);
1544 dst2_r = __lsx_vdp2add_h_bu_b(dst2_r, src98_r, filt3);
1545 dst3_r = __lsx_vdp2_h_bu_b(src43_r, filt0);
1546 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3_r, src65_r, filt1, dst3_r,
1547 src87_r, filt2, dst3_r, dst3_r);
1548 dst3_r = __lsx_vdp2add_h_bu_b(dst3_r, src109_r, filt3);
1549 dst0_l = __lsx_vdp2_h_bu_b(src10_l, filt0);
1550 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0_l, src32_l, filt1, dst0_l,
1551 src54_l, filt2, dst0_l, dst0_l);
1552 dst0_l = __lsx_vdp2add_h_bu_b(dst0_l, src76_l, filt3);
1553 dst1_l = __lsx_vdp2_h_bu_b(src21_l, filt0);
1554 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1_l, src43_l, filt1, dst1_l,
1555 src65_l, filt2, dst1_l, dst1_l);
1556 dst1_l = __lsx_vdp2add_h_bu_b(dst1_l, src87_l, filt3);
1557 dst2_l = __lsx_vdp2_h_bu_b(src32_l, filt0);
1558 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2_l, src54_l, filt1, dst2_l,
1559 src76_l, filt2, dst2_l, dst2_l);
1560 dst2_l = __lsx_vdp2add_h_bu_b(dst2_l, src98_l, filt3);
1561 dst3_l = __lsx_vdp2_h_bu_b(src43_l, filt0);
1562 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3_l, src65_l, filt1, dst3_l,
1563 src87_l, filt2, dst3_l, dst3_l);
1564 dst3_l = __lsx_vdp2add_h_bu_b(dst3_l, src109_l, filt3);
1565
1566 __lsx_vst(dst0_r, dst_tmp, 0);
1567 __lsx_vst(dst0_l, dst_tmp, 16);
1568 dst_tmp += dst_stride;
1569 __lsx_vst(dst1_r, dst_tmp, 0);
1570 __lsx_vst(dst1_l, dst_tmp, 16);
1571 dst_tmp += dst_stride;
1572 __lsx_vst(dst2_r, dst_tmp, 0);
1573 __lsx_vst(dst2_l, dst_tmp, 16);
1574 dst_tmp += dst_stride;
1575 __lsx_vst(dst3_r, dst_tmp, 0);
1576 __lsx_vst(dst3_l, dst_tmp, 16);
1577 dst_tmp += dst_stride;
1578
1579 src10_r = src54_r;
1580 src32_r = src76_r;
1581 src54_r = src98_r;
1582 src21_r = src65_r;
1583 src43_r = src87_r;
1584 src65_r = src109_r;
1585 src10_l = src54_l;
1586 src32_l = src76_l;
1587 src54_l = src98_l;
1588 src21_l = src65_l;
1589 src43_l = src87_l;
1590 src65_l = src109_l;
1591 src6 = src10;
1592 }
1593 src += 16;
1594 dst += 16;
1595 }
1596}
1597
1598static void hevc_vt_8t_16w_lsx(const uint8_t *src, int32_t src_stride,
1599 int16_t *dst, int32_t dst_stride,
1600 const int8_t *filter, int32_t height)
1601{
1602 hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride,
1603 filter, height, 16);
1604}
1605
1606static void hevc_vt_8t_24w_lsx(const uint8_t *src, int32_t src_stride,
1607 int16_t *dst, int32_t dst_stride,
1608 const int8_t *filter, int32_t height)
1609{
1610 hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride,
1611 filter, height, 16);
1612 hevc_vt_8t_8w_lsx(src + 16, src_stride, dst + 16, dst_stride,
1613 filter, height);
1614}
1615
1616static void hevc_vt_8t_32w_lsx(const uint8_t *src, int32_t src_stride,
1617 int16_t *dst, int32_t dst_stride,
1618 const int8_t *filter, int32_t height)
1619{
1620 hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride,
1621 filter, height, 32);
1622}
1623
1624static void hevc_vt_8t_48w_lsx(const uint8_t *src, int32_t src_stride,
1625 int16_t *dst, int32_t dst_stride,
1626 const int8_t *filter, int32_t height)
1627{
1628 hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride,
1629 filter, height, 48);
1630}
1631
1632static void hevc_vt_8t_64w_lsx(const uint8_t *src, int32_t src_stride,
1633 int16_t *dst, int32_t dst_stride,
1634 const int8_t *filter, int32_t height)
1635{
1636 hevc_vt_8t_16multx4mult_lsx(src, src_stride, dst, dst_stride,
1637 filter, height, 64);
1638}
1639
1640static void hevc_hv_8t_4w_lsx(const uint8_t *src, int32_t src_stride,
1641 int16_t *dst, int32_t dst_stride,
1642 const int8_t *filter_x, const int8_t *filter_y,
1643 int32_t height)
1644{
1645 uint32_t loop_cnt;
1646 int32_t src_stride_2x = (src_stride << 1);
1647 int32_t src_stride_4x = (src_stride << 2);
1648 int32_t src_stride_3x = src_stride_2x + src_stride;
1649 __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1650 __m128i filt0, filt1, filt2, filt3;
1651 __m128i filt_h0, filt_h1, filt_h2, filt_h3;
1652 __m128i mask1, mask2, mask3;
1653 __m128i filter_vec;
1654 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1655 __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1656 __m128i dst30, dst41, dst52, dst63, dst66, dst97, dst108;
1657 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
1658 __m128i dst10_r, dst32_r, dst54_r, dst76_r, dst98_r;
1659 __m128i dst21_r, dst43_r, dst65_r, dst87_r, dst109_r;
1660 __m128i mask0;
1661
1662 mask0 = __lsx_vld(ff_hevc_mask_arr, 16);
1663
1664 src -= src_stride_3x + 3;
1665 DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4,
1666 filter_x, 6, filt0, filt1, filt2, filt3);
1667 filter_vec = __lsx_vld(filter_y, 0);
1668 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
1669
1670 DUP4_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filter_vec, 2,
1671 filter_vec, 3, filt_h0, filt_h1, filt_h2, filt_h3);
1672 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
1673 mask3 = __lsx_vaddi_bu(mask0, 6);
1674
1675 src0 = __lsx_vld(src, 0);
1676 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
1677 src3 = __lsx_vldx(src, src_stride_3x);
1678 src += src_stride_4x;
1679 src4 = __lsx_vld(src, 0);
1680 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
1681 src += src_stride_3x;
1682
1683 DUP4_ARG3(__lsx_vshuf_b, src3, src0, mask0, src3, src0, mask1, src3, src0,
1684 mask2, src3, src0, mask3, vec0, vec1, vec2, vec3);
1685 DUP4_ARG3(__lsx_vshuf_b, src4, src1, mask0, src4, src1, mask1, src4, src1,
1686 mask2, src4, src1, mask3, vec4, vec5, vec6, vec7);
1687 DUP4_ARG3(__lsx_vshuf_b, src5, src2, mask0, src5, src2, mask1, src5, src2,
1688 mask2, src5, src2, mask3, vec8, vec9, vec10, vec11);
1689 DUP4_ARG3(__lsx_vshuf_b, src6, src3, mask0, src6, src3, mask1, src6, src3,
1690 mask2, src6, src3, mask3, vec12, vec13, vec14, vec15);
1691 dst30 = __lsx_vdp2_h_bu_b(vec0, filt0);
1692 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst30, vec1, filt1, dst30, vec2, filt2,
1693 dst30, dst30);
1694 dst30 = __lsx_vdp2add_h_bu_b(dst30, vec3, filt3);
1695 dst41 = __lsx_vdp2_h_bu_b(vec4, filt0);
1696 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst41, vec5, filt1, dst41, vec6, filt2,
1697 dst41, dst41);
1698 dst41 = __lsx_vdp2add_h_bu_b(dst41, vec7, filt3);
1699 dst52 = __lsx_vdp2_h_bu_b(vec8, filt0);
1700 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst52, vec9, filt1, dst52, vec10, filt2,
1701 dst52, dst52);
1702 dst52 = __lsx_vdp2add_h_bu_b(dst52, vec11, filt3);
1703 dst63 = __lsx_vdp2_h_bu_b(vec12, filt0);
1704 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst63, vec13, filt1, dst63, vec14, filt2,
1705 dst63, dst63);
1706 dst63 = __lsx_vdp2add_h_bu_b(dst63, vec15, filt3);
1707
1708 DUP2_ARG2(__lsx_vilvl_h, dst41, dst30, dst52, dst41, dst10_r, dst21_r);
1709 DUP2_ARG2(__lsx_vilvh_h, dst41, dst30, dst52, dst41, dst43_r, dst54_r);
1710 dst32_r = __lsx_vilvl_h(dst63, dst52);
1711 dst65_r = __lsx_vilvh_h(dst63, dst52);
1712 dst66 = __lsx_vreplvei_d(dst63, 1);
1713
1714 for (loop_cnt = height >> 2; loop_cnt--;) {
1715 src7 = __lsx_vld(src, 0);
1716 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src8, src9);
1717 src10 = __lsx_vldx(src, src_stride_3x);
1718 src += src_stride_4x;
1719
1720 DUP4_ARG3(__lsx_vshuf_b, src9, src7, mask0, src9, src7, mask1, src9, src7,
1721 mask2, src9, src7, mask3, vec0, vec1, vec2, vec3);
1722 DUP4_ARG3(__lsx_vshuf_b, src10, src8, mask0, src10, src8, mask1, src10, src8,
1723 mask2, src10, src8, mask3, vec4, vec5, vec6, vec7);
1724
1725 dst97 = __lsx_vdp2_h_bu_b(vec0, filt0);
1726 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst97, vec1, filt1, dst97, vec2, filt2,
1727 dst97, dst97);
1728 dst97 = __lsx_vdp2add_h_bu_b(dst97, vec3, filt3);
1729 dst108 = __lsx_vdp2_h_bu_b(vec4, filt0);
1730 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst108, vec5, filt1, dst108, vec6,
1731 filt2, dst108, dst108);
1732 dst108 = __lsx_vdp2add_h_bu_b(dst108, vec7, filt3);
1733
1734 DUP2_ARG2(__lsx_vilvl_h, dst97, dst66, dst108, dst97, dst76_r, dst87_r);
1735 dst109_r = __lsx_vilvh_h(dst108, dst97);
1736 dst66 = __lsx_vreplvei_d(dst97, 1);
1737 dst98_r = __lsx_vilvl_h(dst66, dst108);
1738
1739 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst21_r, filt_h0, dst32_r,
1740 filt_h0, dst43_r, filt_h0, dst0_r, dst1_r, dst2_r, dst3_r);
1741 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst1_r, dst43_r,
1742 filt_h1, dst2_r, dst54_r, filt_h1, dst3_r, dst65_r, filt_h1,
1743 dst0_r, dst1_r, dst2_r, dst3_r);
1744 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst54_r, filt_h2, dst1_r, dst65_r,
1745 filt_h2, dst2_r, dst76_r, filt_h2, dst3_r, dst87_r, filt_h2,
1746 dst0_r, dst1_r, dst2_r, dst3_r);
1747 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst76_r, filt_h3, dst1_r, dst87_r,
1748 filt_h3, dst2_r, dst98_r, filt_h3, dst3_r, dst109_r, filt_h3,
1749 dst0_r, dst1_r, dst2_r, dst3_r);
1750 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst1_r, 6, dst2_r, 6, dst3_r, 6,
1751 dst0_r, dst1_r, dst2_r, dst3_r);
1752 DUP2_ARG2(__lsx_vpickev_h, dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst2_r);
1753 __lsx_vstelm_d(dst0_r, dst, 0, 0);
1754 dst += dst_stride;
1755 __lsx_vstelm_d(dst0_r, dst, 0, 1);
1756 dst += dst_stride;
1757 __lsx_vstelm_d(dst2_r, dst, 0, 0);
1758 dst += dst_stride;
1759 __lsx_vstelm_d(dst2_r, dst, 0, 1);
1760 dst += dst_stride;
1761
1762 dst10_r = dst54_r;
1763 dst32_r = dst76_r;
1764 dst54_r = dst98_r;
1765 dst21_r = dst65_r;
1766 dst43_r = dst87_r;
1767 dst65_r = dst109_r;
1768 dst66 = __lsx_vreplvei_d(dst108, 1);
1769 }
1770}
1771
1772static void hevc_hv_8t_8multx1mult_lsx(const uint8_t *src,
1773 int32_t src_stride,
1774 int16_t *dst,
1775 int32_t dst_stride,
1776 const int8_t *filter_x,
1777 const int8_t *filter_y,
1778 int32_t height,
1779 int32_t width)
1780{
1781 uint32_t loop_cnt, cnt;
1782 const uint8_t *src_tmp;
1783 int16_t *dst_tmp;
1784 int32_t src_stride_2x = (src_stride << 1);
1785 int32_t src_stride_4x = (src_stride << 2);
1786 int32_t src_stride_3x = src_stride_2x + src_stride;
1787 __m128i src0, src1, src2, src3, src4, src5, src6, src7;
1788 __m128i filt0, filt1, filt2, filt3;
1789 __m128i filt_h0, filt_h1, filt_h2, filt_h3;
1790 __m128i mask1, mask2, mask3;
1791 __m128i filter_vec;
1792 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1793 __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1794 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1795 __m128i dst0_r, dst0_l;
1796 __m128i dst10_r, dst32_r, dst54_r, dst76_r;
1797 __m128i dst10_l, dst32_l, dst54_l, dst76_l;
1798 __m128i mask0 = {0x403030202010100, 0x807070606050504};
1799
1800 src -= src_stride_3x + 3;
1801 DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4,
1802 filter_x, 6, filt0, filt1, filt2, filt3);
1803
1804 filter_vec = __lsx_vld(filter_y, 0);
1805 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
1806
1807 DUP4_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filter_vec, 2,
1808 filter_vec, 3, filt_h0, filt_h1, filt_h2, filt_h3);
1809
1810 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
1811 mask3 = __lsx_vaddi_bu(mask0, 6);
1812
1813 for (cnt = width >> 3; cnt--;) {
1814 src_tmp = src;
1815 dst_tmp = dst;
1816 src0 = __lsx_vld(src_tmp, 0);
1817 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
1818 src1, src2);
1819 src3 = __lsx_vldx(src_tmp, src_stride_3x);
1820 src_tmp += src_stride_4x;
1821 src4 = __lsx_vld(src_tmp, 0);
1822 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
1823 src5, src6);
1824 src_tmp += src_stride_3x;
1825
1826 /* row 0 row 1 row 2 row 3 */
1827 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0,
1828 src0, mask2, src0, src0, mask3, vec0, vec1, vec2, vec3);
1829 DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1,
1830 src1, mask2, src1, src1, mask3, vec4, vec5, vec6, vec7);
1831 DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2,
1832 src2, mask2, src2, src2, mask3, vec8, vec9, vec10, vec11);
1833 DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3,
1834 src3, mask2, src3, src3, mask3, vec12, vec13, vec14, vec15);
1835 dst0 = __lsx_vdp2_h_bu_b(vec0, filt0);
1836 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst0, vec2, filt2,
1837 dst0, dst0);
1838 dst0 = __lsx_vdp2add_h_bu_b(dst0, vec3, filt3);
1839 dst1 = __lsx_vdp2_h_bu_b(vec4, filt0);
1840 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1, vec5, filt1, dst1, vec6, filt2,
1841 dst1, dst1);
1842 dst1 = __lsx_vdp2add_h_bu_b(dst1, vec7, filt3);
1843 dst2 = __lsx_vdp2_h_bu_b(vec8, filt0);
1844 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2, vec9, filt1, dst2, vec10, filt2,
1845 dst2, dst2);
1846 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec11, filt3);
1847 dst3 = __lsx_vdp2_h_bu_b(vec12, filt0);
1848 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec13, filt1, dst3, vec14, filt2,
1849 dst3, dst3);
1850 dst3 = __lsx_vdp2add_h_bu_b(dst3, vec15, filt3);
1851
1852 /* row 4 row 5 row 6 */
1853 DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4,
1854 src4, mask2, src4, src4, mask3, vec0, vec1, vec2, vec3);
1855 DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src5,
1856 src5, mask2, src5, src5, mask3, vec4, vec5, vec6, vec7);
1857 DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src6,
1858 src6, mask2, src6, src6, mask3, vec8, vec9, vec10, vec11);
1859 dst4 = __lsx_vdp2_h_bu_b(vec0, filt0);
1860 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec1, filt1, dst4, vec2, filt2,
1861 dst4, dst4);
1862 dst4 = __lsx_vdp2add_h_bu_b(dst4, vec3, filt3);
1863 dst5 = __lsx_vdp2_h_bu_b(vec4, filt0);
1864 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst5, vec5, filt1, dst5, vec6, filt2,
1865 dst5, dst5);
1866 dst5 = __lsx_vdp2add_h_bu_b(dst5, vec7, filt3);
1867 dst6 = __lsx_vdp2_h_bu_b(vec8, filt0);
1868 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst6, vec9, filt1, dst6, vec10, filt2,
1869 dst6, dst6);
1870 dst6 = __lsx_vdp2add_h_bu_b(dst6, vec11, filt3);
1871
1872 for (loop_cnt = height; loop_cnt--;) {
1873 src7 = __lsx_vld(src_tmp, 0);
1874 src_tmp += src_stride;
1875
1876 DUP4_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, src7,
1877 src7, mask2, src7, src7, mask3, vec0, vec1, vec2, vec3);
1878 dst7 = __lsx_vdp2_h_bu_b(vec0, filt0);
1879 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst7, vec1, filt1, dst7, vec2,
1880 filt2, dst7, dst7);
1881 dst7 = __lsx_vdp2add_h_bu_b(dst7, vec3, filt3);
1882
1883 DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7,
1884 dst6, dst10_r, dst32_r, dst54_r, dst76_r);
1885 DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7,
1886 dst6, dst10_l, dst32_l, dst54_l, dst76_l);
1887
1888 DUP2_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0,
1889 dst0_r, dst0_l);
1890 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l,
1891 dst32_l, filt_h1, dst0_r, dst54_r, filt_h2, dst0_l,
1892 dst54_l, filt_h2, dst0_r, dst0_l, dst0_r, dst0_l);
1893 DUP2_ARG3(__lsx_vdp2add_w_h, dst0_r, dst76_r, filt_h3, dst0_l,
1894 dst76_l, filt_h3, dst0_r, dst0_l);
1895 dst0_r = __lsx_vsrai_w(dst0_r, 6);
1896 dst0_l = __lsx_vsrai_w(dst0_l, 6);
1897
1898 dst0_r = __lsx_vpickev_h(dst0_l, dst0_r);
1899 __lsx_vst(dst0_r, dst_tmp, 0);
1900 dst_tmp += dst_stride;
1901
1902 dst0 = dst1;
1903 dst1 = dst2;
1904 dst2 = dst3;
1905 dst3 = dst4;
1906 dst4 = dst5;
1907 dst5 = dst6;
1908 dst6 = dst7;
1909 }
1910 src += 8;
1911 dst += 8;
1912 }
1913}
1914
1915static void hevc_hv_8t_8w_lsx(const uint8_t *src, int32_t src_stride,
1916 int16_t *dst, int32_t dst_stride,
1917 const int8_t *filter_x, const int8_t *filter_y,
1918 int32_t height)
1919{
1920 hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride,
1921 filter_x, filter_y, height, 8);
1922}
1923
1924static void hevc_hv_8t_12w_lsx(const uint8_t *src, int32_t src_stride,
1925 int16_t *dst, int32_t dst_stride,
1926 const int8_t *filter_x, const int8_t *filter_y,
1927 int32_t height)
1928{
1929 uint32_t loop_cnt;
1930 const uint8_t *src_tmp;
1931 int16_t *dst_tmp;
1932 int32_t src_stride_2x = (src_stride << 1);
1933 int32_t src_stride_4x = (src_stride << 2);
1934 int32_t src_stride_3x = src_stride_2x + src_stride;
1935 __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1936 __m128i mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1937 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1938 __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1939 __m128i filt0, filt1, filt2, filt3, filt_h0, filt_h1, filt_h2, filt_h3;
1940 __m128i filter_vec;
1941 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1942 __m128i dst30, dst41, dst52, dst63, dst66, dst97, dst108;
1943 __m128i dst10_r, dst32_r, dst54_r, dst76_r, dst98_r, dst21_r, dst43_r;
1944 __m128i dst65_r, dst87_r, dst109_r, dst10_l, dst32_l, dst54_l, dst76_l;
1945 __m128i dst0_r, dst0_l, dst1_r, dst2_r, dst3_r;
1946
1947 src -= src_stride_3x + 3;
1948 DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4,
1949 filter_x, 6, filt0, filt1, filt2, filt3);
1950
1951 filter_vec = __lsx_vld(filter_y, 0);
1952 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
1953
1954 DUP4_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filter_vec, 2,
1955 filter_vec, 3, filt_h0, filt_h1, filt_h2, filt_h3);
1956
1957 mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
1958 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
1959 mask3 = __lsx_vaddi_bu(mask0, 6);
1960
1961 src_tmp = src;
1962 dst_tmp = dst;
1963
1964 src0 = __lsx_vld(src_tmp, 0);
1965 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
1966 src1, src2);
1967 src3 = __lsx_vldx(src_tmp, src_stride_3x);
1968 src_tmp += src_stride_4x;
1969 src4 = __lsx_vld(src_tmp, 0);
1970 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
1971 src5, src6);
1972 src_tmp += src_stride_3x;
1973
1974 /* row 0 row 1 row 2 row 3 */
1975 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src0, src0,
1976 mask2, src0, src0, mask3, vec0, vec1, vec2, vec3);
1977 DUP4_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, src1, src1,
1978 mask2, src1, src1, mask3, vec4, vec5, vec6, vec7);
1979 DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, src2, src2,
1980 mask2, src2, src2, mask3, vec8, vec9, vec10, vec11);
1981 DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3, src3,
1982 mask2, src3, src3, mask3, vec12, vec13, vec14, vec15);
1983 dst0 = __lsx_vdp2_h_bu_b(vec0, filt0);
1984 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst0, vec2, filt2,
1985 dst0, dst0);
1986 dst0 = __lsx_vdp2add_h_bu_b(dst0, vec3, filt3);
1987 dst1 = __lsx_vdp2_h_bu_b(vec4, filt0);
1988 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst1, vec5, filt1, dst1, vec6, filt2,
1989 dst1, dst1);
1990 dst1 = __lsx_vdp2add_h_bu_b(dst1, vec7, filt3);
1991 dst2 = __lsx_vdp2_h_bu_b(vec8, filt0);
1992 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2, vec9, filt1, dst2, vec10, filt2,
1993 dst2, dst2);
1994 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec11, filt3);
1995 dst3 = __lsx_vdp2_h_bu_b(vec12, filt0);
1996 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec13, filt1, dst3, vec14, filt2,
1997 dst3, dst3);
1998 dst3 = __lsx_vdp2add_h_bu_b(dst3, vec15, filt3);
1999
2000 /* row 4 row 5 row 6 */
2001 DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4, src4,
2002 mask2, src4, src4, mask3, vec0, vec1, vec2, vec3);
2003 DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src5, src5,
2004 mask2, src5, src5, mask3, vec4, vec5, vec6, vec7);
2005 DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src6, src6,
2006 mask2, src6, src6, mask3, vec8, vec9, vec10, vec11);
2007 dst4 = __lsx_vdp2_h_bu_b(vec0, filt0);
2008 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec1, filt1, dst4, vec2, filt2,
2009 dst4, dst4);
2010 dst4 = __lsx_vdp2add_h_bu_b(dst4, vec3, filt3);
2011 dst5 = __lsx_vdp2_h_bu_b(vec4, filt0);
2012 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst5, vec5, filt1, dst5, vec6, filt2,
2013 dst5, dst5);
2014 dst5 = __lsx_vdp2add_h_bu_b(dst5, vec7, filt3);
2015 dst6 = __lsx_vdp2_h_bu_b(vec8, filt0);
2016 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst6, vec9, filt1, dst6, vec10, filt2,
2017 dst6, dst6);
2018 dst6 = __lsx_vdp2add_h_bu_b(dst6, vec11, filt3);
2019
2020 for (loop_cnt = height; loop_cnt--;) {
2021 src7 = __lsx_vld(src_tmp, 0);
2022 src_tmp += src_stride;
2023
2024 DUP4_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, src7,
2025 src7, mask2, src7, src7, mask3, vec0, vec1, vec2, vec3);
2026 dst7 = __lsx_vdp2_h_bu_b(vec0, filt0);
2027 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst7, vec1, filt1, dst7, vec2, filt2,
2028 dst7, dst7);
2029 dst7 = __lsx_vdp2add_h_bu_b(dst7, vec3, filt3);
2030 DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
2031 dst10_r, dst32_r, dst54_r, dst76_r);
2032 DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
2033 dst10_l, dst32_l, dst54_l, dst76_l);
2034 DUP2_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0,
2035 dst0_r, dst0_l);
2036 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
2037 filt_h1, dst0_r, dst54_r, filt_h2, dst0_l, dst54_l, filt_h2,
2038 dst0_r, dst0_l, dst0_r, dst0_l);
2039 DUP2_ARG3(__lsx_vdp2add_w_h, dst0_r, dst76_r, filt_h3, dst0_l, dst76_l,
2040 filt_h3, dst0_r, dst0_l)
2041 dst0_r = __lsx_vsrai_w(dst0_r, 6);
2042 dst0_l = __lsx_vsrai_w(dst0_l, 6);
2043
2044 dst0_r = __lsx_vpickev_h(dst0_l, dst0_r);
2045 __lsx_vst(dst0_r, dst_tmp, 0);
2046 dst_tmp += dst_stride;
2047
2048 dst0 = dst1;
2049 dst1 = dst2;
2050 dst2 = dst3;
2051 dst3 = dst4;
2052 dst4 = dst5;
2053 dst5 = dst6;
2054 dst6 = dst7;
2055 }
2056 src += 8;
2057 dst += 8;
2058
2059 mask4 = __lsx_vld(ff_hevc_mask_arr, 16);
2060 DUP2_ARG2(__lsx_vaddi_bu, mask4, 2, mask4, 4, mask5, mask6);
2061 mask7 = __lsx_vaddi_bu(mask4, 6);
2062
2063 src0 = __lsx_vld(src, 0);
2064 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
2065 src3 = __lsx_vldx(src, src_stride_3x);
2066 src += src_stride_4x;
2067 src4 = __lsx_vld(src, 0);
2068 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
2069 src += src_stride_3x;
2070
2071 DUP4_ARG3(__lsx_vshuf_b, src3, src0, mask4, src3, src0, mask5, src3, src0,
2072 mask6, src3, src0, mask7, vec0, vec1, vec2, vec3);
2073 DUP4_ARG3(__lsx_vshuf_b, src4, src1, mask4, src4, src1, mask5, src4, src1,
2074 mask6, src4, src1, mask7, vec4, vec5, vec6, vec7);
2075 DUP4_ARG3(__lsx_vshuf_b, src5, src2, mask4, src5, src2, mask5, src5, src2,
2076 mask6, src5, src2, mask7, vec8, vec9, vec10, vec11);
2077 DUP4_ARG3(__lsx_vshuf_b, src6, src3, mask4, src6, src3, mask5, src6, src3,
2078 mask6, src6, src3, mask7, vec12, vec13, vec14, vec15);
2079 dst30 = __lsx_vdp2_h_bu_b(vec0, filt0);
2080 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst30, vec1, filt1, dst30, vec2, filt2,
2081 dst30, dst30);
2082 dst30 = __lsx_vdp2add_h_bu_b(dst30, vec3, filt3);
2083 dst41 = __lsx_vdp2_h_bu_b(vec4, filt0);
2084 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst41, vec5, filt1, dst41, vec6, filt2,
2085 dst41, dst41);
2086 dst41 = __lsx_vdp2add_h_bu_b(dst41, vec7, filt3);
2087 dst52 = __lsx_vdp2_h_bu_b(vec8, filt0);
2088 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst52, vec9, filt1, dst52, vec10, filt2,
2089 dst52, dst52);
2090 dst52 = __lsx_vdp2add_h_bu_b(dst52, vec11, filt3);
2091 dst63 = __lsx_vdp2_h_bu_b(vec12, filt0);
2092 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst63, vec13, filt1, dst63, vec14, filt2,
2093 dst63, dst63);
2094 dst63 = __lsx_vdp2add_h_bu_b(dst63, vec15, filt3);
2095
2096 DUP2_ARG2(__lsx_vilvl_h, dst41, dst30, dst52, dst41, dst10_r, dst21_r);
2097 DUP2_ARG2(__lsx_vilvh_h, dst41, dst30, dst52, dst41, dst43_r, dst54_r);
2098 dst32_r = __lsx_vilvl_h(dst63, dst52);
2099 dst65_r = __lsx_vilvh_h(dst63, dst52);
2100
2101 dst66 = __lsx_vreplvei_d(dst63, 1);
2102
2103 for (loop_cnt = height >> 2; loop_cnt--;) {
2104 src7 = __lsx_vld(src, 0);
2105 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src8, src9);
2106 src10 = __lsx_vldx(src, src_stride_3x);
2107 src += src_stride_4x;
2108
2109 DUP4_ARG3(__lsx_vshuf_b, src9, src7, mask4, src9, src7, mask5, src9,
2110 src7, mask6, src9, src7, mask7, vec0, vec1, vec2, vec3);
2111 DUP4_ARG3(__lsx_vshuf_b, src10, src8, mask4, src10, src8, mask5, src10,
2112 src8, mask6, src10, src8, mask7, vec4, vec5, vec6, vec7);
2113 dst97 = __lsx_vdp2_h_bu_b(vec0, filt0);
2114 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst97, vec1, filt1, dst97, vec2, filt2,
2115 dst97, dst97);
2116 dst97 = __lsx_vdp2add_h_bu_b(dst97, vec3, filt3);
2117 dst108 = __lsx_vdp2_h_bu_b(vec4, filt0);
2118 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst108, vec5, filt1, dst108, vec6,
2119 filt2, dst108, dst108);
2120 dst108 = __lsx_vdp2add_h_bu_b(dst108, vec7, filt3);
2121
2122 DUP2_ARG2(__lsx_vilvl_h, dst97, dst66, dst108, dst97, dst76_r, dst87_r);
2123 dst109_r = __lsx_vilvh_h(dst108, dst97);
2124 dst66 = __lsx_vreplvei_d(dst97, 1);
2125 dst98_r = __lsx_vilvl_h(dst66, dst108);
2126
2127 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst21_r, filt_h0, dst32_r,
2128 filt_h0, dst43_r, filt_h0, dst0_r, dst1_r, dst2_r, dst3_r);
2129 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst1_r, dst43_r,
2130 filt_h1, dst2_r, dst54_r, filt_h1, dst3_r, dst65_r, filt_h1,
2131 dst0_r, dst1_r, dst2_r, dst3_r);
2132 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst54_r, filt_h2, dst1_r, dst65_r,
2133 filt_h2, dst2_r, dst76_r, filt_h2, dst3_r, dst87_r, filt_h2,
2134 dst0_r, dst1_r, dst2_r, dst3_r);
2135 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst76_r, filt_h3, dst1_r, dst87_r,
2136 filt_h3, dst2_r, dst98_r, filt_h3, dst3_r, dst109_r, filt_h3,
2137 dst0_r, dst1_r, dst2_r, dst3_r);
2138 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst1_r, 6, dst2_r, 6, dst3_r, 6,
2139 dst0_r, dst1_r, dst2_r, dst3_r);
2140 DUP2_ARG2(__lsx_vpickev_h, dst1_r, dst0_r, dst3_r, dst2_r,
2141 dst0_r, dst2_r);
2142 __lsx_vstelm_d(dst0_r, dst, 0, 0);
2143 dst += dst_stride;
2144 __lsx_vstelm_d(dst0_r, dst, 0, 1);
2145 dst += dst_stride;
2146 __lsx_vstelm_d(dst2_r, dst, 0, 0);
2147 dst += dst_stride;
2148 __lsx_vstelm_d(dst2_r, dst, 0, 1);
2149 dst += dst_stride;
2150
2151 dst10_r = dst54_r;
2152 dst32_r = dst76_r;
2153 dst54_r = dst98_r;
2154 dst21_r = dst65_r;
2155 dst43_r = dst87_r;
2156 dst65_r = dst109_r;
2157 dst66 = __lsx_vreplvei_d(dst108, 1);
2158 }
2159}
2160
2161static void hevc_hv_8t_16w_lsx(const uint8_t *src, int32_t src_stride,
2162 int16_t *dst, int32_t dst_stride,
2163 const int8_t *filter_x, const int8_t *filter_y,
2164 int32_t height)
2165{
2166 hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride,
2167 filter_x, filter_y, height, 16);
2168}
2169
2170static void hevc_hv_8t_24w_lsx(const uint8_t *src, int32_t src_stride,
2171 int16_t *dst, int32_t dst_stride,
2172 const int8_t *filter_x, const int8_t *filter_y,
2173 int32_t height)
2174{
2175 hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride,
2176 filter_x, filter_y, height, 24);
2177}
2178
2179static void hevc_hv_8t_32w_lsx(const uint8_t *src, int32_t src_stride,
2180 int16_t *dst, int32_t dst_stride,
2181 const int8_t *filter_x, const int8_t *filter_y,
2182 int32_t height)
2183{
2184 hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride,
2185 filter_x, filter_y, height, 32);
2186}
2187
2188static void hevc_hv_8t_48w_lsx(const uint8_t *src, int32_t src_stride,
2189 int16_t *dst, int32_t dst_stride,
2190 const int8_t *filter_x, const int8_t *filter_y,
2191 int32_t height)
2192{
2193 hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride,
2194 filter_x, filter_y, height, 48);
2195}
2196
2197static void hevc_hv_8t_64w_lsx(const uint8_t *src, int32_t src_stride,
2198 int16_t *dst, int32_t dst_stride,
2199 const int8_t *filter_x, const int8_t *filter_y,
2200 int32_t height)
2201{
2202 hevc_hv_8t_8multx1mult_lsx(src, src_stride, dst, dst_stride,
2203 filter_x, filter_y, height, 64);
2204}
2205
2206static void hevc_hz_4t_32w_lsx(const uint8_t *src,
2207 int32_t src_stride,
2208 int16_t *dst,
2209 int32_t dst_stride,
2210 const int8_t *filter,
2211 int32_t height)
2212{
2213 uint32_t loop_cnt;
2214 __m128i src0, src1, src2;
2215 __m128i filt0, filt1;
2216 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
2217 __m128i mask1, mask2, mask3;
2218 __m128i dst0, dst1, dst2, dst3;
2219 __m128i vec0, vec1, vec2, vec3;
2220
2221 src -= 1;
2222 DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1);
2223
2224 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 8, mask1, mask2);
2225 mask3 = __lsx_vaddi_bu(mask0, 10);
2226
2227 for (loop_cnt = height; loop_cnt--;) {
2228 DUP2_ARG2(__lsx_vld, src, 0, src, 16, src0, src1);
2229 src2 = __lsx_vld(src, 24);
2230 src += src_stride;
2231
2232 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src1, src0, mask2,
2233 vec0, vec1);
2234 DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src2, src2, mask0,
2235 vec2, vec3);
2236 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
2237 vec3, filt0, dst0, dst1, dst2, dst3);
2238 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask1, src1, src0, mask3,
2239 vec0, vec1);
2240 DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask1, src2, src2, mask1,
2241 vec2, vec3);
2242 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
2243 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
2244 __lsx_vst(dst0, dst, 0);
2245 __lsx_vst(dst1, dst, 16);
2246 __lsx_vst(dst2, dst, 32);
2247 __lsx_vst(dst3, dst, 48);
2248 dst += dst_stride;
2249 }
2250}
2251
2252static void hevc_vt_4t_16w_lsx(const uint8_t *src,
2253 int32_t src_stride,
2254 int16_t *dst,
2255 int32_t dst_stride,
2256 const int8_t *filter,
2257 int32_t height)
2258{
2259 int32_t loop_cnt;
2260 int32_t src_stride_2x = (src_stride << 1);
2261 int32_t src_stride_3x = src_stride_2x + src_stride;
2262 __m128i src0, src1, src2, src3, src4, src5;
2263 __m128i src10_r, src32_r, src21_r, src43_r;
2264 __m128i src10_l, src32_l, src21_l, src43_l;
2265 __m128i dst0_r, dst1_r, dst0_l, dst1_l;
2266 __m128i filt0, filt1;
2267
2268 src -= src_stride;
2269 DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1);
2270
2271 src0 = __lsx_vld(src, 0);
2272 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
2273 src += src_stride_3x;
2274 DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r);
2275 DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l);
2276
2277 for (loop_cnt = (height >> 2); loop_cnt--;) {
2278 src3 = __lsx_vld(src, 0);
2279 src4 = __lsx_vldx(src, src_stride);
2280 src += src_stride_2x;
2281 DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r);
2282 DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l);
2283 DUP4_ARG2(__lsx_vdp2_h_bu_b, src10_r, filt0, src10_l, filt0, src21_r,
2284 filt0, src21_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
2285 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst0_l,
2286 src32_l, filt1, dst1_r, src43_r, filt1, dst1_l, src43_l,
2287 filt1, dst0_r, dst0_l, dst1_r, dst1_l);
2288 __lsx_vst(dst0_r, dst, 0);
2289 __lsx_vst(dst0_l, dst, 16);
2290 dst += dst_stride;
2291 __lsx_vst(dst1_r, dst, 0);
2292 __lsx_vst(dst1_l, dst, 16);
2293 dst += dst_stride;
2294
2295 src5 = __lsx_vld(src, 0);
2296 src2 = __lsx_vldx(src, src_stride);
2297 src += src_stride_2x;
2298 DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r);
2299 DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l);
2300 DUP4_ARG2(__lsx_vdp2_h_bu_b, src32_r, filt0, src32_l, filt0, src43_r,
2301 filt0, src43_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
2302 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src10_r, filt1, dst0_l,
2303 src10_l, filt1, dst1_r, src21_r, filt1, dst1_l, src21_l,
2304 filt1, dst0_r, dst0_l, dst1_r, dst1_l);
2305 __lsx_vst(dst0_r, dst, 0);
2306 __lsx_vst(dst0_l, dst, 16);
2307 dst += dst_stride;
2308 __lsx_vst(dst1_r, dst, 0);
2309 __lsx_vst(dst1_l, dst, 16);
2310 dst += dst_stride;
2311 }
2312}
2313
2314static void hevc_vt_4t_24w_lsx(const uint8_t *src,
2315 int32_t src_stride,
2316 int16_t *dst,
2317 int32_t dst_stride,
2318 const int8_t *filter,
2319 int32_t height)
2320{
2321 int32_t loop_cnt;
2322 int32_t src_stride_2x = (src_stride << 1);
2323 int32_t src_stride_3x = src_stride_2x + src_stride;
2324 const uint8_t *_src;
2325
2326 __m128i src0, src1, src2, src3, src4, src5;
2327 __m128i src6, src7, src8, src9, src10, src11;
2328 __m128i src10_r, src32_r, src76_r, src98_r;
2329 __m128i src21_r, src43_r, src87_r, src109_r;
2330 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
2331 __m128i src10_l, src32_l, src21_l, src43_l;
2332 __m128i dst0_l, dst1_l;
2333 __m128i filt0, filt1;
2334
2335 src -= src_stride;
2336 _src = src + 16;
2337 DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1);
2338
2339 src0 = __lsx_vld(src, 0);
2340 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
2341 DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r);
2342 DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l);
2343
2344 src6 = __lsx_vld(_src, 0);
2345 DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride_2x, src7, src8);
2346 src += src_stride_3x;
2347 _src += src_stride_3x;
2348 DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r);
2349
2350 for (loop_cnt = (height >> 2); loop_cnt--;) {
2351 DUP2_ARG2(__lsx_vld, src, 0, _src, 0, src3, src9);
2352 DUP2_ARG2(__lsx_vldx, src, src_stride, _src, src_stride, src4, src10);
2353 src += src_stride_2x;
2354 _src += src_stride_2x;
2355 DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r);
2356 DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l);
2357
2358 DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r);
2359 DUP4_ARG2(__lsx_vdp2_h_bu_b, src10_r, filt0, src10_l, filt0, src21_r,
2360 filt0, src21_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
2361 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst0_l,
2362 src32_l, filt1, dst1_r, src43_r, filt1, dst1_l, src43_l,
2363 filt1, dst0_r, dst0_l, dst1_r, dst1_l);
2364 DUP2_ARG2(__lsx_vdp2_h_bu_b, src76_r, filt0, src87_r, filt0,
2365 dst2_r, dst3_r);
2366 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src98_r, filt1, dst3_r,
2367 src109_r, filt1, dst2_r, dst3_r);
2368
2369 __lsx_vst(dst0_r, dst, 0);
2370 __lsx_vst(dst0_l, dst, 16);
2371 __lsx_vst(dst2_r, dst, 32);
2372 dst += dst_stride;
2373 __lsx_vst(dst1_r, dst, 0);
2374 __lsx_vst(dst1_l, dst, 16);
2375 __lsx_vst(dst3_r, dst, 32);
2376 dst += dst_stride;
2377
2378 DUP2_ARG2(__lsx_vld, src, 0, _src, 0, src5, src11);
2379 DUP2_ARG2(__lsx_vldx, src, src_stride, _src, src_stride, src2, src8);
2380 src += src_stride_2x;
2381 _src += src_stride_2x;
2382 DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r);
2383 DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l);
2384
2385 DUP2_ARG2(__lsx_vilvl_b, src11, src10, src8, src11, src76_r, src87_r);
2386
2387 DUP4_ARG2(__lsx_vdp2_h_bu_b, src32_r, filt0, src32_l, filt0, src43_r,
2388 filt0, src43_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
2389 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src10_r, filt1, dst0_l, src10_l,
2390 filt1, dst1_r, src21_r, filt1, dst1_l, src21_l, filt1,
2391 dst0_r, dst0_l, dst1_r, dst1_l);
2392 DUP2_ARG2(__lsx_vdp2_h_bu_b, src98_r, filt0, src109_r, filt0,
2393 dst2_r, dst3_r);
2394 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src76_r, filt1, dst3_r, src87_r,
2395 filt1, dst2_r, dst3_r);
2396
2397 __lsx_vst(dst0_r, dst, 0);
2398 __lsx_vst(dst0_l, dst, 16);
2399 __lsx_vst(dst2_r, dst, 32);
2400 dst += dst_stride;
2401 __lsx_vst(dst1_r, dst, 0);
2402 __lsx_vst(dst1_l, dst, 16);
2403 __lsx_vst(dst3_r, dst, 32);
2404 dst += dst_stride;
2405 }
2406}
2407
2408static void hevc_vt_4t_32w_lsx(const uint8_t *src,
2409 int32_t src_stride,
2410 int16_t *dst,
2411 int32_t dst_stride,
2412 const int8_t *filter,
2413 int32_t height)
2414{
2415 int32_t loop_cnt;
2416 int32_t src_stride_2x = (src_stride << 1);
2417 int32_t src_stride_3x = src_stride_2x + src_stride;
2418 const uint8_t *_src;
2419
2420 __m128i src0, src1, src2, src3, src4, src5;
2421 __m128i src6, src7, src8, src9, src10, src11;
2422 __m128i src10_r, src32_r, src76_r, src98_r;
2423 __m128i src21_r, src43_r, src87_r, src109_r;
2424 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
2425 __m128i src10_l, src32_l, src76_l, src98_l;
2426 __m128i src21_l, src43_l, src87_l, src109_l;
2427 __m128i dst0_l, dst1_l, dst2_l, dst3_l;
2428 __m128i filt0, filt1;
2429
2430 src -= src_stride;
2431 _src = src + 16;
2432 DUP2_ARG2(__lsx_vldrepl_h, filter, 0, filter, 2, filt0, filt1);
2433
2434 src0 = __lsx_vld(src, 0);
2435 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
2436 DUP2_ARG2(__lsx_vilvl_b, src1, src0, src2, src1, src10_r, src21_r);
2437 DUP2_ARG2(__lsx_vilvh_b, src1, src0, src2, src1, src10_l, src21_l);
2438
2439 src6 = __lsx_vld(_src, 0);
2440 DUP2_ARG2(__lsx_vldx, _src, src_stride, _src, src_stride_2x, src7, src8);
2441 src += src_stride_3x;
2442 _src += src_stride_3x;
2443 DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r);
2444 DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src76_l, src87_l);
2445
2446 for (loop_cnt = (height >> 2); loop_cnt--;) {
2447 DUP2_ARG2(__lsx_vld, src, 0, _src, 0, src3, src9);
2448 DUP2_ARG2(__lsx_vldx, src, src_stride, _src, src_stride, src4, src10);
2449 src += src_stride_2x;
2450 _src += src_stride_2x;
2451 DUP2_ARG2(__lsx_vilvl_b, src3, src2, src4, src3, src32_r, src43_r);
2452 DUP2_ARG2(__lsx_vilvh_b, src3, src2, src4, src3, src32_l, src43_l);
2453
2454 DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r);
2455 DUP2_ARG2(__lsx_vilvh_b, src9, src8, src10, src9, src98_l, src109_l);
2456
2457 DUP4_ARG2(__lsx_vdp2_h_bu_b, src10_r, filt0, src10_l, filt0, src21_r,
2458 filt0, src21_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
2459 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst0_l,
2460 src32_l, filt1, dst1_r, src43_r, filt1, dst1_l,src43_l,
2461 filt1, dst0_r, dst0_l, dst1_r, dst1_l);
2462 DUP4_ARG2(__lsx_vdp2_h_bu_b, src76_r, filt0, src76_l, filt0, src87_r,
2463 filt0, src87_l, filt0, dst2_r, dst2_l, dst3_r, dst3_l);
2464 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src98_r, filt1, dst2_l, src98_l,
2465 filt1, dst3_r, src109_r, filt1, dst3_l, src109_l, filt1,
2466 dst2_r, dst2_l, dst3_r, dst3_l);
2467
2468 __lsx_vst(dst0_r, dst, 0);
2469 __lsx_vst(dst0_l, dst, 16);
2470 __lsx_vst(dst2_r, dst, 32);
2471 __lsx_vst(dst2_l, dst, 48);
2472 dst += dst_stride;
2473 __lsx_vst(dst1_r, dst, 0);
2474 __lsx_vst(dst1_l, dst, 16);
2475 __lsx_vst(dst3_r, dst, 32);
2476 __lsx_vst(dst3_l, dst, 48);
2477 dst += dst_stride;
2478
2479 DUP2_ARG2(__lsx_vld, src, 0, _src, 0, src5, src11);
2480 DUP2_ARG2(__lsx_vldx, src, src_stride, _src, src_stride, src2, src8);
2481 src += src_stride_2x;
2482 _src += src_stride_2x;
2483 DUP2_ARG2(__lsx_vilvl_b, src5, src4, src2, src5, src10_r, src21_r);
2484 DUP2_ARG2(__lsx_vilvh_b, src5, src4, src2, src5, src10_l, src21_l);
2485
2486 DUP2_ARG2(__lsx_vilvl_b, src11, src10, src8, src11, src76_r, src87_r);
2487 DUP2_ARG2(__lsx_vilvh_b, src11, src10, src8, src11, src76_l, src87_l);
2488
2489 DUP4_ARG2(__lsx_vdp2_h_bu_b, src32_r, filt0, src32_l, filt0, src43_r,
2490 filt0, src43_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
2491 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src10_r, filt1, dst0_l,
2492 src10_l, filt1, dst1_r, src21_r, filt1, dst1_l, src21_l,
2493 filt1, dst0_r, dst0_l, dst1_r, dst1_l);
2494
2495 DUP4_ARG2(__lsx_vdp2_h_bu_b, src98_r, filt0, src98_l, filt0, src109_r,
2496 filt0, src109_l, filt0, dst2_r, dst2_l, dst3_r, dst3_l);
2497 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src76_r, filt1, dst2_l, src76_l,
2498 filt1, dst3_r, src87_r, filt1, dst3_l, src87_l, filt1,
2499 dst2_r, dst2_l, dst3_r, dst3_l);
2500
2501 __lsx_vst(dst0_r, dst, 0);
2502 __lsx_vst(dst0_l, dst, 16);
2503 __lsx_vst(dst2_r, dst, 32);
2504 __lsx_vst(dst2_l, dst, 48);
2505 dst += dst_stride;
2506 __lsx_vst(dst1_r, dst, 0);
2507 __lsx_vst(dst1_l, dst, 16);
2508 __lsx_vst(dst3_r, dst, 32);
2509 __lsx_vst(dst3_l, dst, 48);
2510 dst += dst_stride;
2511 }
2512}
2513
2514static void hevc_hv_4t_8x2_lsx(const uint8_t *src,
2515 int32_t src_stride,
2516 int16_t *dst,
2517 int32_t dst_stride,
2518 const int8_t *filter_x,
2519 const int8_t *filter_y)
2520{
2521 int32_t src_stride_2x = (src_stride << 1);
2522 int32_t src_stride_4x = (src_stride << 2);
2523 int32_t src_stride_3x = src_stride_2x + src_stride;
2524
2525 __m128i src0, src1, src2, src3, src4;
2526 __m128i filt0, filt1;
2527 __m128i filt_h0, filt_h1;
2528 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
2529 __m128i mask1;
2530 __m128i filter_vec;
2531 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
2532 __m128i dst0, dst1, dst2, dst3, dst4;
2533 __m128i dst0_r, dst0_l, dst1_r, dst1_l;
2534 __m128i dst10_r, dst32_r, dst21_r, dst43_r;
2535 __m128i dst10_l, dst32_l, dst21_l, dst43_l;
2536
2537 src -= (src_stride + 1);
2538 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
2539
2540 filter_vec = __lsx_vld(filter_y, 0);
2541 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
2542 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
2543
2544 mask1 = __lsx_vaddi_bu(mask0, 2);
2545
2546 src0 = __lsx_vld(src, 0);
2547 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src1, src2);
2548 src3 = __lsx_vldx(src, src_stride_3x);
2549 src4 = __lsx_vldx(src, src_stride_4x);
2550
2551 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1);
2552 DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3);
2553 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5);
2554 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec6, vec7);
2555 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec8, vec9);
2556
2557 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0,
2558 vec6, filt0, dst0, dst1, dst2, dst3);
2559 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1,
2560 dst2, vec5, filt1, dst3, vec7, filt1, dst0, dst1, dst2, dst3);
2561 dst4 = __lsx_vdp2_h_bu_b(vec8, filt0);
2562 dst4 = __lsx_vdp2add_h_bu_b(dst4, vec9, filt1);
2563
2564 DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r);
2565 DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l);
2566 DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r);
2567 DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l);
2568
2569 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
2570 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
2571 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
2572 filt_h1, dst1_r, dst43_r, filt_h1, dst1_l, dst43_l, filt_h1,
2573 dst0_r, dst0_l, dst1_r, dst1_l);
2574 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6,
2575 dst0_r, dst0_l, dst1_r, dst1_l);
2576 DUP2_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
2577 __lsx_vst(dst0_r, dst, 0);
2578 __lsx_vst(dst1_r, dst + dst_stride, 0);
2579}
2580
2581static void hevc_hv_4t_8multx4_lsx(const uint8_t *src, int32_t src_stride,
2582 int16_t *dst, int32_t dst_stride,
2583 const int8_t *filter_x,
2584 const int8_t *filter_y, int32_t width8mult)
2585{
2586 int32_t cnt;
2587 int32_t src_stride_2x = (src_stride << 1);
2588 int32_t dst_stride_x = (dst_stride << 1);
2589 int32_t src_stride_4x = (src_stride << 2);
2590 int32_t dst_stride_2x = (dst_stride << 2);
2591 int32_t src_stride_3x = src_stride_2x + src_stride;
2592 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
2593
2594 __m128i src0, src1, src2, src3, src4, src5, src6, mask0, mask1;
2595 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2596 __m128i filt0, filt1, filt_h0, filt_h1, filter_vec;
2597 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6;
2598 __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
2599 __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
2600 __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
2601
2602 src -= (src_stride + 1);
2603 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
2604
2605 filter_vec = __lsx_vld(filter_y, 0);
2606 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
2607 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
2608
2609 mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
2610 mask1 = __lsx_vaddi_bu(mask0, 2);
2611
2612 for (cnt = width8mult; cnt--;) {
2613 src0 = __lsx_vld(src, 0);
2614 DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src,
2615 src_stride_3x, src, src_stride_4x, src1, src2, src3, src4);
2616 src += src_stride_4x;
2617 DUP2_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src5, src6);
2618 src += (8 - src_stride_4x);
2619
2620 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1,
2621 vec0, vec1);
2622 DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1,
2623 vec2, vec3);
2624 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1,
2625 vec4, vec5);
2626
2627 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, dst0, dst1);
2628 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1,
2629 dst0, dst1);
2630 dst2 = __lsx_vdp2_h_bu_b(vec4, filt0);
2631 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec5, filt1);
2632
2633 DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r);
2634 DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l);
2635
2636 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1,
2637 vec0, vec1);
2638 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1,
2639 vec2, vec3);
2640 DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1,
2641 vec4, vec5);
2642 DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1,
2643 vec6, vec7);
2644 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0,
2645 vec6, filt0, dst3, dst4, dst5, dst6);
2646 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst4, vec3, filt1,
2647 dst5, vec5, filt1, dst6, vec7, filt1, dst3, dst4, dst5, dst6);
2648 DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r);
2649 DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l);
2650 DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r);
2651 DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l);
2652
2653 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
2654 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
2655 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
2656 filt_h1, dst1_r, dst43_r, filt_h1, dst1_l, dst43_l, filt_h1,
2657 dst0_r, dst0_l, dst1_r, dst1_l);
2658
2659 DUP4_ARG2(__lsx_vdp2_w_h, dst32_r, filt_h0, dst32_l, filt_h0, dst43_r,
2660 filt_h0, dst43_l, filt_h0, dst2_r, dst2_l, dst3_r, dst3_l);
2661 DUP4_ARG3(__lsx_vdp2add_w_h, dst2_r, dst54_r, filt_h1, dst2_l, dst54_l,
2662 filt_h1, dst3_r, dst65_r, filt_h1, dst3_l, dst65_l, filt_h1,
2663 dst2_r, dst2_l, dst3_r, dst3_l);
2664 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6,
2665 dst0_r, dst0_l, dst1_r, dst1_l);
2666 DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6,
2667 dst2_r, dst2_l, dst3_r, dst3_l);
2668 DUP2_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r,
2669 dst0_r, dst1_r);
2670 DUP2_ARG2(__lsx_vpickev_h, dst2_l, dst2_r, dst3_l, dst3_r,
2671 dst2_r, dst3_r);
2672
2673 __lsx_vst(dst0_r, dst, 0);
2674 __lsx_vstx(dst1_r, dst, dst_stride_x);
2675 __lsx_vstx(dst2_r, dst, dst_stride_2x);
2676 __lsx_vstx(dst3_r, dst, dst_stride_3x);
2677 dst += 8;
2678 }
2679}
2680
2681static void hevc_hv_4t_8x6_lsx(const uint8_t *src,
2682 int32_t src_stride,
2683 int16_t *dst,
2684 int32_t dst_stride,
2685 const int8_t *filter_x,
2686 const int8_t *filter_y)
2687{
2688 int32_t src_stride_2x = (src_stride << 1);
2689 int32_t dst_stride_2x = (dst_stride << 1);
2690 int32_t src_stride_4x = (src_stride << 2);
2691 int32_t src_stride_3x = src_stride_2x + src_stride;
2692 __m128i src0, src1, src2, src3, src4, src5, src6, src7, src8;
2693 __m128i filt0, filt1;
2694 __m128i filt_h0, filt_h1;
2695 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
2696 __m128i mask1, filter_vec;
2697 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
2698 __m128i vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17;
2699 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
2700 __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
2701 __m128i dst4_r, dst4_l, dst5_r, dst5_l;
2702 __m128i dst10_r, dst32_r, dst10_l, dst32_l;
2703 __m128i dst21_r, dst43_r, dst21_l, dst43_l;
2704 __m128i dst54_r, dst54_l, dst65_r, dst65_l;
2705 __m128i dst76_r, dst76_l, dst87_r, dst87_l;
2706
2707 src -= (src_stride + 1);
2708 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
2709
2710 filter_vec = __lsx_vld(filter_y, 0);
2711 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
2712 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
2713
2714 mask1 = __lsx_vaddi_bu(mask0, 2);
2715
2716 src0 = __lsx_vld(src, 0);
2717 DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src,
2718 src_stride_3x, src, src_stride_4x, src1, src2, src3, src4);
2719 src += src_stride_4x;
2720 DUP4_ARG2(__lsx_vldx, src, src_stride, src, src_stride_2x, src,
2721 src_stride_3x, src, src_stride_4x, src5, src6, src7, src8);
2722
2723 DUP4_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, src1, src1,
2724 mask0, src1, src1, mask1, vec0, vec1, vec2, vec3);
2725 DUP4_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1,src3, src3,
2726 mask0, src3, src3, mask1, vec4, vec5, vec6, vec7);
2727 DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src5, src5,
2728 mask0, src5, src5, mask1, vec8, vec9, vec10, vec11);
2729 DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src7, src7,
2730 mask0, src7, src7, mask1, vec12, vec13, vec14, vec15);
2731 DUP2_ARG3(__lsx_vshuf_b, src8, src8, mask0, src8, src8, mask1,
2732 vec16, vec17);
2733
2734 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0, vec6,
2735 filt0, dst0, dst1, dst2, dst3);
2736 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1,
2737 dst2, vec5, filt1, dst3, vec7, filt1, dst0, dst1, dst2, dst3);
2738 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec8, filt0, vec10, filt0, vec12, filt0,
2739 vec14, filt0, dst4, dst5, dst6, dst7);
2740 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec9, filt1, dst5, vec11, filt1, dst6,
2741 vec13, filt1, dst7, vec15, filt1, dst4, dst5, dst6, dst7);
2742 dst8 = __lsx_vdp2_h_bu_b(vec16, filt0);
2743 dst8 = __lsx_vdp2add_h_bu_b(dst8, vec17, filt1);
2744
2745 DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3,
2746 dst10_r, dst21_r, dst32_r, dst43_r);
2747 DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3,
2748 dst10_l, dst21_l, dst32_l, dst43_l);
2749 DUP4_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7,
2750 dst54_r, dst65_r, dst76_r, dst87_r);
2751 DUP4_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7,
2752 dst54_l, dst65_l, dst76_l, dst87_l);
2753
2754 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
2755 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
2756 DUP4_ARG2(__lsx_vdp2_w_h, dst32_r, filt_h0, dst32_l, filt_h0, dst43_r,
2757 filt_h0, dst43_l, filt_h0, dst2_r, dst2_l, dst3_r, dst3_l);
2758 DUP4_ARG2(__lsx_vdp2_w_h, dst54_r, filt_h0, dst54_l, filt_h0, dst65_r,
2759 filt_h0, dst65_l, filt_h0, dst4_r, dst4_l, dst5_r, dst5_l);
2760 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
2761 filt_h1, dst1_r, dst43_r, filt_h1, dst1_l, dst43_l, filt_h1,
2762 dst0_r, dst0_l, dst1_r, dst1_l);
2763 DUP4_ARG3(__lsx_vdp2add_w_h, dst2_r, dst54_r, filt_h1, dst2_l, dst54_l,
2764 filt_h1, dst3_r, dst65_r, filt_h1, dst3_l, dst65_l, filt_h1,
2765 dst2_r, dst2_l, dst3_r, dst3_l);
2766 DUP4_ARG3(__lsx_vdp2add_w_h, dst4_r, dst76_r, filt_h1, dst4_l, dst76_l,
2767 filt_h1, dst5_r, dst87_r, filt_h1, dst5_l, dst87_l, filt_h1,
2768 dst4_r, dst4_l, dst5_r, dst5_l);
2769 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6, dst0_r,
2770 dst0_l, dst1_r, dst1_l);
2771 DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6, dst2_r,
2772 dst2_l, dst3_r, dst3_l);
2773 DUP4_ARG2(__lsx_vsrai_w, dst4_r, 6, dst4_l, 6, dst5_r, 6, dst5_l, 6, dst4_r,
2774 dst4_l, dst5_r, dst5_l);
2775
2776 DUP4_ARG2(__lsx_vpickev_h,dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r,
2777 dst3_l, dst3_r, dst0_r, dst1_r, dst2_r, dst3_r);
2778 DUP2_ARG2(__lsx_vpickev_h, dst4_l, dst4_r, dst5_l, dst5_r, dst4_r, dst5_r);
2779
2780 __lsx_vst(dst0_r, dst, 0);
2781 __lsx_vstx(dst1_r, dst, dst_stride_2x);
2782 dst += dst_stride_2x;
2783 __lsx_vst(dst2_r, dst, 0);
2784 __lsx_vstx(dst3_r, dst, dst_stride_2x);
2785 dst += dst_stride_2x;
2786 __lsx_vst(dst4_r, dst, 0);
2787 __lsx_vstx(dst5_r, dst, dst_stride_2x);
2788}
2789
2790static void hevc_hv_4t_8multx4mult_lsx(const uint8_t *src,
2791 int32_t src_stride,
2792 int16_t *dst,
2793 int32_t dst_stride,
2794 const int8_t *filter_x,
2795 const int8_t *filter_y,
2796 int32_t height,
2797 int32_t width8mult)
2798{
2799 uint32_t loop_cnt, cnt;
2800 const uint8_t *src_tmp;
2801 int16_t *dst_tmp;
2802 int32_t src_stride_2x = (src_stride << 1);
2803 int32_t dst_stride_x = (dst_stride << 1);
2804 int32_t src_stride_4x = (src_stride << 2);
2805 int32_t dst_stride_2x = (dst_stride << 2);
2806 int32_t src_stride_3x = src_stride_2x + src_stride;
2807 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
2808
2809 __m128i src0, src1, src2, src3, src4, src5, src6;
2810 __m128i filt0, filt1;
2811 __m128i filt_h0, filt_h1;
2812 __m128i mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
2813 __m128i mask1, filter_vec;
2814 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2815 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6;
2816 __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
2817 __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
2818 __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
2819
2820 src -= (src_stride + 1);
2821 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
2822
2823 filter_vec = __lsx_vld(filter_y, 0);
2824 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
2825 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
2826
2827 mask1 = __lsx_vaddi_bu(mask0, 2);
2828
2829 for (cnt = width8mult; cnt--;) {
2830 src_tmp = src;
2831 dst_tmp = dst;
2832
2833 src0 = __lsx_vld(src_tmp, 0);
2834 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
2835 src1, src2);
2836 src_tmp += src_stride_3x;
2837
2838 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1,
2839 vec0, vec1);
2840 DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1,
2841 vec2, vec3);
2842 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1,
2843 vec4, vec5);
2844
2845 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, dst0, dst1);
2846 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1,
2847 dst0, dst1);
2848 dst2 = __lsx_vdp2_h_bu_b(vec4, filt0);
2849 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec5, filt1);
2850
2851 DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r);
2852 DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l);
2853
2854 for (loop_cnt = height >> 2; loop_cnt--;) {
2855 src3 = __lsx_vld(src_tmp, 0);
2856 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x,
2857 src4, src5);
2858 src6 = __lsx_vldx(src_tmp, src_stride_3x);
2859 src_tmp += src_stride_4x;
2860
2861 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1,
2862 vec0, vec1);
2863 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1,
2864 vec2, vec3);
2865 DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1,
2866 vec4, vec5);
2867 DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1,
2868 vec6, vec7);
2869
2870 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0,
2871 vec6, filt0, dst3, dst4, dst5, dst6);
2872 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst4, vec3,
2873 filt1, dst5, vec5, filt1, dst6, vec7, filt1, dst3,
2874 dst4, dst5, dst6);
2875
2876 DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r);
2877 DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l);
2878 DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r);
2879 DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l);
2880
2881 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
2882 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
2883 DUP4_ARG2(__lsx_vdp2_w_h, dst32_r, filt_h0, dst32_l, filt_h0, dst43_r,
2884 filt_h0, dst43_l, filt_h0, dst2_r, dst2_l, dst3_r, dst3_l);
2885 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l,
2886 dst32_l, filt_h1, dst1_r, dst43_r, filt_h1, dst1_l,
2887 dst43_l, filt_h1, dst0_r, dst0_l, dst1_r, dst1_l);
2888 DUP4_ARG3(__lsx_vdp2add_w_h, dst2_r, dst54_r, filt_h1, dst2_l,
2889 dst54_l, filt_h1, dst3_r, dst65_r, filt_h1, dst3_l,
2890 dst65_l, filt_h1, dst2_r, dst2_l, dst3_r, dst3_l);
2891
2892 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6,
2893 dst0_r, dst0_l, dst1_r, dst1_l);
2894 DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6,
2895 dst2_r, dst2_l, dst3_r, dst3_l);
2896
2897 DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l,
2898 dst2_r, dst3_l, dst3_r, dst0_r, dst1_r, dst2_r, dst3_r);
2899
2900 __lsx_vst(dst0_r, dst_tmp, 0);
2901 __lsx_vstx(dst1_r, dst_tmp, dst_stride_x);
2902 __lsx_vstx(dst2_r, dst_tmp, dst_stride_2x);
2903 __lsx_vstx(dst3_r, dst_tmp, dst_stride_3x);
2904 dst_tmp += dst_stride_2x;
2905
2906 dst10_r = dst54_r;
2907 dst10_l = dst54_l;
2908 dst21_r = dst65_r;
2909 dst21_l = dst65_l;
2910 dst2 = dst6;
2911 }
2912 src += 8;
2913 dst += 8;
2914 }
2915}
2916
2917static void hevc_hv_4t_8w_lsx(const uint8_t *src,
2918 int32_t src_stride,
2919 int16_t *dst,
2920 int32_t dst_stride,
2921 const int8_t *filter_x,
2922 const int8_t *filter_y,
2923 int32_t height)
2924{
2925
2926 if (2 == height) {
2927 hevc_hv_4t_8x2_lsx(src, src_stride, dst, dst_stride,
2928 filter_x, filter_y);
2929 } else if (4 == height) {
2930 hevc_hv_4t_8multx4_lsx(src, src_stride, dst, dst_stride,
2931 filter_x, filter_y, 1);
2932 } else if (6 == height) {
2933 hevc_hv_4t_8x6_lsx(src, src_stride, dst, dst_stride,
2934 filter_x, filter_y);
2935 } else if (0 == (height & 0x03)) {
2936 hevc_hv_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride,
2937 filter_x, filter_y, height, 1);
2938 }
2939}
2940
2941static void hevc_hv_4t_12w_lsx(const uint8_t *src,
2942 int32_t src_stride,
2943 int16_t *dst,
2944 int32_t dst_stride,
2945 const int8_t *filter_x,
2946 const int8_t *filter_y,
2947 int32_t height)
2948{
2949 uint32_t loop_cnt;
2950 const uint8_t *src_tmp;
2951 int16_t *dst_tmp;
2952 int32_t src_stride_2x = (src_stride << 1);
2953 int32_t dst_stride_x = (dst_stride << 1);
2954 int32_t src_stride_4x = (src_stride << 2);
2955 int32_t dst_stride_2x = (dst_stride << 2);
2956 int32_t src_stride_3x = src_stride_2x + src_stride;
2957 int32_t dst_stride_3x = dst_stride_2x + dst_stride_x;
2958
2959 __m128i src0, src1, src2, src3, src4, src5, src6;
2960 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2961 __m128i mask0, mask1;
2962 __m128i filt0, filt1, filt_h0, filt_h1, filter_vec, dst0;
2963 __m128i dst1, dst2, dst3, dst4, dst5, dst6;
2964 __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
2965 __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
2966 __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
2967
2968 src -= (src_stride + 1);
2969 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
2970
2971 filter_vec = __lsx_vld(filter_y, 0);
2972 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
2973 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
2974
2975 mask0 = __lsx_vld(ff_hevc_mask_arr, 0);
2976 mask1 = __lsx_vaddi_bu(mask0, 2);
2977
2978 src_tmp = src;
2979 dst_tmp = dst;
2980
2981 src0 = __lsx_vld(src_tmp, 0);
2982 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x, src1, src2);
2983 src_tmp += src_stride_3x;
2984
2985 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1);
2986 DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3);
2987 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5);
2988
2989 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, dst0, dst1);
2990 dst2 = __lsx_vdp2_h_bu_b(vec4, filt0);
2991 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1, dst0, dst1);
2992 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec5, filt1);
2993
2994 DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r);
2995 DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l);
2996
2997 for (loop_cnt = height >> 2; loop_cnt--;) {
2998 src3 = __lsx_vld(src_tmp, 0);
2999 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x, src4, src5);
3000 src6 = __lsx_vldx(src_tmp, src_stride_3x);
3001 src_tmp += src_stride_4x;
3002
3003 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1);
3004 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3);
3005 DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5);
3006 DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7);
3007
3008 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0,
3009 vec6, filt0, dst3, dst4, dst5, dst6);
3010 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst4, vec3, filt1,
3011 dst5, vec5, filt1, dst6, vec7, filt1, dst3, dst4, dst5, dst6);
3012
3013 DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r);
3014 DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l);
3015 DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r);
3016 DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l);
3017
3018 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
3019 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
3020 DUP4_ARG2(__lsx_vdp2_w_h, dst32_r, filt_h0, dst32_l, filt_h0, dst43_r,
3021 filt_h0, dst43_l, filt_h0, dst2_r, dst2_l, dst3_r, dst3_l);
3022 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
3023 filt_h1, dst1_r, dst43_r, filt_h1, dst1_l, dst43_l, filt_h1,
3024 dst0_r, dst0_l, dst1_r, dst1_l);
3025 DUP4_ARG3(__lsx_vdp2add_w_h, dst2_r, dst54_r, filt_h1, dst2_l, dst54_l,
3026 filt_h1, dst3_r, dst65_r, filt_h1, dst3_l, dst65_l, filt_h1,
3027 dst2_r, dst2_l, dst3_r, dst3_l);
3028 DUP2_ARG3(__lsx_vsrani_h_w, dst0_l, dst0_r, 6, dst1_l, dst1_r, 6, dst0_r, dst1_r);
3029 DUP2_ARG3(__lsx_vsrani_h_w, dst2_l, dst2_r, 6, dst3_l, dst3_r, 6, dst2_r, dst3_r);
3030 __lsx_vst(dst0_r, dst_tmp, 0);
3031 __lsx_vstx(dst1_r, dst_tmp, dst_stride_x);
3032 __lsx_vstx(dst2_r, dst_tmp, dst_stride_2x);
3033 __lsx_vstx(dst3_r, dst_tmp, dst_stride_3x);
3034 dst_tmp += dst_stride_2x;
3035
3036 dst10_r = dst54_r;
3037 dst10_l = dst54_l;
3038 dst21_r = dst65_r;
3039 dst21_l = dst65_l;
3040 dst2 = dst6;
3041 }
3042
3043 src_tmp = src + 8;
3044 dst_tmp = dst + 8;
3045
3046 src0 = __lsx_vld(src_tmp, 0);
3047 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x, src1, src2);
3048 src_tmp += src_stride_3x;
3049
3050 DUP2_ARG3(__lsx_vshuf_b, src0, src0, mask0, src0, src0, mask1, vec0, vec1);
3051 DUP2_ARG3(__lsx_vshuf_b, src1, src1, mask0, src1, src1, mask1, vec2, vec3);
3052 DUP2_ARG3(__lsx_vshuf_b, src2, src2, mask0, src2, src2, mask1, vec4, vec5);
3053
3054 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, dst0, dst1);
3055 dst2 = __lsx_vdp2_h_bu_b(vec4, filt0);
3056 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1, dst0, dst1);
3057 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec5, filt1);
3058
3059 DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r);
3060 DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l);
3061
3062 for (loop_cnt = height >> 2; loop_cnt--;) {
3063 src3 = __lsx_vld(src_tmp, 0);
3064 DUP2_ARG2(__lsx_vldx, src_tmp, src_stride, src_tmp, src_stride_2x, src4, src5);
3065 src6 = __lsx_vldx(src_tmp, src_stride_3x);
3066 src_tmp += src_stride_4x;
3067
3068 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1);
3069 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3);
3070 DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5);
3071 DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7);
3072
3073 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0,
3074 vec6, filt0, dst3, dst4, dst5, dst6);
3075 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst4, vec3, filt1,
3076 dst5, vec5, filt1, dst6, vec7, filt1, dst3, dst4, dst5, dst6);
3077
3078 DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r);
3079 DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l);
3080 DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r);
3081 DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l);
3082
3083 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
3084 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
3085 DUP4_ARG2(__lsx_vdp2_w_h, dst32_r, filt_h0, dst32_l, filt_h0, dst43_r,
3086 filt_h0, dst43_l, filt_h0, dst2_r, dst2_l, dst3_r, dst3_l);
3087 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
3088 filt_h1, dst1_r, dst43_r, filt_h1, dst1_l, dst43_l, filt_h1,
3089 dst0_r, dst0_l, dst1_r, dst1_l);
3090 DUP4_ARG3(__lsx_vdp2add_w_h, dst2_r, dst54_r, filt_h1, dst2_l, dst54_l,
3091 filt_h1, dst3_r, dst65_r, filt_h1, dst3_l, dst65_l, filt_h1,
3092 dst2_r, dst2_l, dst3_r, dst3_l);
3093 DUP2_ARG3(__lsx_vsrani_h_w, dst0_l, dst0_r, 6, dst1_l, dst1_r, 6, dst0_r, dst1_r);
3094 DUP2_ARG3(__lsx_vsrani_h_w, dst2_l, dst2_r, 6, dst3_l, dst3_r, 6, dst2_r, dst3_r);
3095 __lsx_vstelm_d(dst0_r, dst_tmp, 0, 0);
3096 dst_tmp += dst_stride;
3097 __lsx_vstelm_d(dst1_r, dst_tmp, 0, 0);
3098 dst_tmp += dst_stride;
3099 __lsx_vstelm_d(dst2_r, dst_tmp, 0, 0);
3100 dst_tmp += dst_stride;
3101 __lsx_vstelm_d(dst3_r, dst_tmp, 0, 0);
3102 dst_tmp += dst_stride;
3103
3104 dst10_r = dst54_r;
3105 dst10_l = dst54_l;
3106 dst21_r = dst65_r;
3107 dst21_l = dst65_l;
3108 dst2 = dst6;
3109 }
3110
3111}
3112
3113static void hevc_hv_4t_16w_lsx(const uint8_t *src,
3114 int32_t src_stride,
3115 int16_t *dst,
3116 int32_t dst_stride,
3117 const int8_t *filter_x,
3118 const int8_t *filter_y,
3119 int32_t height)
3120{
3121 if (4 == height) {
3122 hevc_hv_4t_8multx4_lsx(src, src_stride, dst, dst_stride,
3123 filter_x, filter_y, 2);
3124 } else {
3125 hevc_hv_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride,
3126 filter_x, filter_y, height, 2);
3127 }
3128}
3129
3130static void hevc_hv_4t_24w_lsx(const uint8_t *src,
3131 int32_t src_stride,
3132 int16_t *dst,
3133 int32_t dst_stride,
3134 const int8_t *filter_x,
3135 const int8_t *filter_y,
3136 int32_t height)
3137{
3138 hevc_hv_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride,
3139 filter_x, filter_y, height, 3);
3140}
3141
3142static void hevc_hv_4t_32w_lsx(const uint8_t *src,
3143 int32_t src_stride,
3144 int16_t *dst,
3145 int32_t dst_stride,
3146 const int8_t *filter_x,
3147 const int8_t *filter_y,
3148 int32_t height)
3149{
3150 hevc_hv_4t_8multx4mult_lsx(src, src_stride, dst, dst_stride,
3151 filter_x, filter_y, height, 4);
3152}
3153
3154#define MC_COPY(WIDTH) \
3155void ff_hevc_put_hevc_pel_pixels##WIDTH##_8_lsx(int16_t *dst, \
3156 const uint8_t *src, \
3157 ptrdiff_t src_stride, \
3158 int height, \
3159 intptr_t mx, \
3160 intptr_t my, \
3161 int width) \
3162{ \
3163 hevc_copy_##WIDTH##w_lsx(src, src_stride, dst, MAX_PB_SIZE, height); \
3164}
3165
3166MC_COPY(4);
3167MC_COPY(6);
3168MC_COPY(8);
3169MC_COPY(12);
3170MC_COPY(16);
3171MC_COPY(24);
3172MC_COPY(32);
3173MC_COPY(48);
3174MC_COPY(64);
3175
3176#undef MC_COPY
3177
3178#define MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \
3179void ff_hevc_put_hevc_##PEL##_##DIR##WIDTH##_8_lsx(int16_t *dst, \
3180 const uint8_t *src, \
3181 ptrdiff_t src_stride, \
3182 int height, \
3183 intptr_t mx, \
3184 intptr_t my, \
3185 int width) \
3186{ \
3187 const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR]; \
3188 \
3189 hevc_##DIR1##_##TAP##t_##WIDTH##w_lsx(src, src_stride, dst, \
3190 MAX_PB_SIZE, filter, height); \
3191}
3192
3193MC(qpel, h, 4, 8, hz, mx);
3194MC(qpel, h, 8, 8, hz, mx);
3195MC(qpel, h, 12, 8, hz, mx);
3196MC(qpel, h, 16, 8, hz, mx);
3197MC(qpel, h, 24, 8, hz, mx);
3198MC(qpel, h, 32, 8, hz, mx);
3199MC(qpel, h, 48, 8, hz, mx);
3200MC(qpel, h, 64, 8, hz, mx);
3201
3202MC(qpel, v, 4, 8, vt, my);
3203MC(qpel, v, 8, 8, vt, my);
3204MC(qpel, v, 12, 8, vt, my);
3205MC(qpel, v, 16, 8, vt, my);
3206MC(qpel, v, 24, 8, vt, my);
3207MC(qpel, v, 32, 8, vt, my);
3208MC(qpel, v, 48, 8, vt, my);
3209MC(qpel, v, 64, 8, vt, my);
3210
3211MC(epel, h, 32, 4, hz, mx);
3212
3213MC(epel, v, 16, 4, vt, my);
3214MC(epel, v, 24, 4, vt, my);
3215MC(epel, v, 32, 4, vt, my);
3216
3217#undef MC
3218
3219#define MC_HV(PEL, WIDTH, TAP) \
3220void ff_hevc_put_hevc_##PEL##_hv##WIDTH##_8_lsx(int16_t *dst, \
3221 const uint8_t *src, \
3222 ptrdiff_t src_stride, \
3223 int height, \
3224 intptr_t mx, \
3225 intptr_t my, \
3226 int width) \
3227{ \
3228 const int8_t *filter_x = ff_hevc_##PEL##_filters[mx]; \
3229 const int8_t *filter_y = ff_hevc_##PEL##_filters[my]; \
3230 \
3231 hevc_hv_##TAP##t_##WIDTH##w_lsx(src, src_stride, dst, MAX_PB_SIZE, \
3232 filter_x, filter_y, height); \
3233}
3234
3235MC_HV(qpel, 4, 8);
3236MC_HV(qpel, 8, 8);
3237MC_HV(qpel, 12, 8);
3238MC_HV(qpel, 16, 8);
3239MC_HV(qpel, 24, 8);
3240MC_HV(qpel, 32, 8);
3241MC_HV(qpel, 48, 8);
3242MC_HV(qpel, 64, 8);
3243
3244MC_HV(epel, 8, 4);
3245MC_HV(epel, 12, 4);
3246MC_HV(epel, 16, 4);
3247MC_HV(epel, 24, 4);
3248MC_HV(epel, 32, 4);
3249
3250#undef MC_HV
my
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t my
Definition dsp.h:57
mx
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t mx
Definition dsp.h:57
_src
uint8_t ptrdiff_t const uint8_t * _src
Definition dsp.h:56
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition dsp.h:87
int32_t
int32_t
Definition audioconvert.c:56
aligned
static int aligned(int val)
Definition dashdec.c:178
src2
const pixel * src2
Definition h264pred_template.c:421
ff_hevc_mask_arr
static const uint8_t ff_hevc_mask_arr[16 *2]
Definition hevc_mc_bi_lsx.c:26
hevc_hz_8t_12w_lsx
static void hevc_hz_8t_12w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:749
hevc_hv_8t_8multx1mult_lsx
static void hevc_hv_8t_8multx1mult_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t width)
Definition hevcdsp_lsx.c:1772
hevc_hz_8t_32w_lsx
static void hevc_hz_8t_32w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:955
hevc_vt_8t_16w_lsx
static void hevc_vt_8t_16w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1598
hevc_hv_8t_12w_lsx
static void hevc_hv_8t_12w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:1924
hevc_vt_8t_24w_lsx
static void hevc_vt_8t_24w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1606
hevc_vt_4t_24w_lsx
static void hevc_vt_4t_24w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:2314
hevc_hv_4t_8w_lsx
static void hevc_hv_4t_8w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:2917
MC_HV
#define MC_HV(PEL, WIDTH, TAP)
Definition hevcdsp_lsx.c:3219
hevc_hv_8t_32w_lsx
static void hevc_hv_8t_32w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:2179
hevc_vt_8t_48w_lsx
static void hevc_vt_8t_48w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1624
hevc_hz_4t_32w_lsx
static void hevc_hz_4t_32w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:2206
hevc_vt_4t_16w_lsx
static void hevc_vt_4t_16w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:2252
hevc_hv_8t_8w_lsx
static void hevc_hv_8t_8w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:1915
hevc_hv_8t_16w_lsx
static void hevc_hv_8t_16w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:2161
hevc_copy_12w_lsx
static void hevc_copy_12w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:200
hevc_hv_4t_16w_lsx
static void hevc_hv_4t_16w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:3113
hevc_hv_8t_48w_lsx
static void hevc_hv_8t_48w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:2188
hevc_vt_8t_16multx4mult_lsx
static void hevc_vt_8t_16multx4mult_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t width)
Definition hevcdsp_lsx.c:1474
hevc_copy_16w_lsx
static void hevc_copy_16w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:272
hevc_copy_8w_lsx
static void hevc_copy_8w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:149
hevc_hv_4t_12w_lsx
static void hevc_hv_4t_12w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:2941
MC
#define MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR)
Definition hevcdsp_lsx.c:3178
hevc_vt_8t_12w_lsx
static void hevc_vt_8t_12w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1372
hevc_copy_4w_lsx
static void hevc_copy_4w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:34
MC_COPY
#define MC_COPY(WIDTH)
Definition hevcdsp_lsx.c:3154
hevc_vt_8t_64w_lsx
static void hevc_vt_8t_64w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1632
hevc_copy_6w_lsx
static void hevc_copy_6w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:86
hevc_hz_8t_4w_lsx
static void hevc_hz_8t_4w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:598
hevc_hv_4t_32w_lsx
static void hevc_hv_4t_32w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:3142
hevc_hv_8t_64w_lsx
static void hevc_hv_8t_64w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:2197
hevc_hv_4t_8x6_lsx
static void hevc_hv_4t_8x6_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition hevcdsp_lsx.c:2681
hevc_hv_4t_24w_lsx
static void hevc_hv_4t_24w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:3130
hevc_hv_4t_8x2_lsx
static void hevc_hv_4t_8x2_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition hevcdsp_lsx.c:2514
hevc_hz_8t_8w_lsx
static void hevc_hz_8t_8w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:685
hevc_hv_8t_24w_lsx
static void hevc_hv_8t_24w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:2170
hevc_hz_8t_16w_lsx
static void hevc_hz_8t_16w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:825
hevc_copy_24w_lsx
static void hevc_copy_24w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:357
hevc_vt_4t_32w_lsx
static void hevc_vt_4t_32w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:2408
hevc_vt_8t_4w_lsx
static void hevc_vt_8t_4w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1178
hevc_hz_8t_64w_lsx
static void hevc_hz_8t_64w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1084
hevc_hv_4t_8multx4_lsx
static void hevc_hv_4t_8multx4_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t width8mult)
Definition hevcdsp_lsx.c:2581
hevc_hz_8t_24w_lsx
static void hevc_hz_8t_24w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:885
hevc_hv_4t_8multx4mult_lsx
static void hevc_hv_4t_8multx4mult_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t width8mult)
Definition hevcdsp_lsx.c:2790
hevc_copy_32w_lsx
static void hevc_copy_32w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:411
hevc_vt_8t_8w_lsx
static void hevc_vt_8t_8w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1298
hevc_hz_8t_48w_lsx
static void hevc_hz_8t_48w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1015
hevc_copy_48w_lsx
static void hevc_copy_48w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:471
hevc_hv_8t_4w_lsx
static void hevc_hv_8t_4w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition hevcdsp_lsx.c:1640
hevc_copy_64w_lsx
static void hevc_copy_64w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, int32_t height)
Definition hevcdsp_lsx.c:546
hevc_vt_8t_32w_lsx
static void hevc_vt_8t_32w_lsx(const uint8_t *src, int32_t src_stride, int16_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition hevcdsp_lsx.c:1616
hevcdsp_lsx.h
zero
static int zero(InterplayACMContext *s, unsigned ind, unsigned col)
Definition interplayacm.c:121
loongson_intrinsics.h
DUP4_ARG2
#define DUP4_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, _OUT0, _OUT1, _OUT2, _OUT3)
Definition loongson_intrinsics.h:76
DUP2_ARG3
#define DUP2_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _OUT0, _OUT1)
Definition loongson_intrinsics.h:64
DUP2_ARG2
#define DUP2_ARG2(_INS, _IN0, _IN1, _IN2, _IN3, _OUT0, _OUT1)
Definition loongson_intrinsics.h:58
DUP4_ARG3
#define DUP4_ARG3(_INS, _IN0, _IN1, _IN2, _IN3, _IN4, _IN5, _IN6, _IN7, _IN8, _IN9, _IN10, _IN11, _OUT0, _OUT1, _OUT2, _OUT3)
Definition loongson_intrinsics.h:83
filter
void(* filter)(uint8_t *src, ptrdiff_t stride, int qscale)
Definition h263dsp.c:29
src1
#define src1
Definition h264pred.c:141
src0
#define src0
Definition h264pred.c:140
src
#define src
Definition vp8dsp.c:248
height
#define height
Definition dsp.h:89
width
#define width
Definition dsp.h:89
h
h
Definition vp9dsp_template.c:2070
Generated on Tue Aug 18 2026 20:20:54 for FFmpeg by doxygen 1.13.2