28 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
29 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20
37 vec0 = __lsx_vsadd_h(in0, vec0);
38 vec1 = __lsx_vsadd_h(in1, vec1);
39 out = __lsx_vssrarni_bu_h(vec1, vec0, 7);
46 const int16_t *src1_ptr,
int32_t src2_stride,
51 int32_t src_stride_2x = (src_stride << 1);
52 int32_t dst_stride_2x = (dst_stride << 1);
53 int32_t src_stride_4x = (src_stride << 2);
54 int32_t dst_stride_4x = (dst_stride << 2);
55 int32_t src2_stride_2x = (src2_stride << 1);
56 int32_t src2_stride_4x = (src2_stride << 2);
57 int32_t src_stride_3x = src_stride_2x + src_stride;
58 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
59 int32_t src2_stride_3x = src2_stride_2x + src2_stride;
61 __m128i
zero = __lsx_vldi(0);
62 __m128i in0, in1, in2, in3;
63 __m128i tmp0, tmp1, tmp2, tmp3;
64 __m128i reg0, reg1, reg2, reg3;
65 __m128i dst0, dst1, dst2, dst3;
68 reg0 = __lsx_vldrepl_w(src0_ptr, 0);
69 reg1 = __lsx_vldrepl_w(src0_ptr + src_stride, 0);
70 reg2 = __lsx_vldrepl_w(src0_ptr + src_stride_2x, 0);
71 reg3 = __lsx_vldrepl_w(src0_ptr + src_stride_3x, 0);
72 src0_ptr += src_stride_4x;
73 DUP2_ARG2(__lsx_vilvl_w, reg1, reg0, reg3, reg2, tmp0, tmp1);
74 src0 = __lsx_vilvl_d(tmp1, tmp0);
75 reg0 = __lsx_vldrepl_w(src0_ptr, 0);
76 reg1 = __lsx_vldrepl_w(src0_ptr + src_stride, 0);
77 reg2 = __lsx_vldrepl_w(src0_ptr + src_stride_2x, 0);
78 reg3 = __lsx_vldrepl_w(src0_ptr + src_stride_3x, 0);
79 DUP2_ARG2(__lsx_vilvl_w, reg1, reg0, reg3, reg2, tmp0, tmp1);
80 src1 = __lsx_vilvl_d(tmp1, tmp0);
81 src0_ptr += src_stride_4x;
83 tmp0 = __lsx_vldrepl_d(src1_ptr, 0);
84 tmp1 = __lsx_vldrepl_d(src1_ptr + src2_stride, 0);
85 tmp2 = __lsx_vldrepl_d(src1_ptr + src2_stride_2x, 0);
86 tmp3 = __lsx_vldrepl_d(src1_ptr + src2_stride_3x, 0);
87 src1_ptr += src2_stride_4x;
88 DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, in0, in1);
89 tmp0 = __lsx_vldrepl_d(src1_ptr, 0);
90 tmp1 = __lsx_vldrepl_d(src1_ptr + src2_stride, 0);
91 tmp2 = __lsx_vldrepl_d(src1_ptr + src2_stride_2x, 0);
92 tmp3 = __lsx_vldrepl_d(src1_ptr + src2_stride_3x, 0);
93 src1_ptr += src2_stride_4x;
94 DUP2_ARG2(__lsx_vilvl_d, tmp1, tmp0, tmp3, tmp2, in2, in3);
97 DUP2_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst1, dst3);
100 __lsx_vstelm_w(dst0,
dst, 0, 0);
101 __lsx_vstelm_w(dst0,
dst + dst_stride, 0, 1);
102 __lsx_vstelm_w(dst0,
dst + dst_stride_2x, 0, 2);
103 __lsx_vstelm_w(dst0,
dst + dst_stride_3x, 0, 3);
104 dst += dst_stride_4x;
105 __lsx_vstelm_w(dst1,
dst, 0, 0);
106 __lsx_vstelm_w(dst1,
dst + dst_stride, 0, 1);
107 __lsx_vstelm_w(dst1,
dst + dst_stride_2x, 0, 2);
108 __lsx_vstelm_w(dst1,
dst + dst_stride_3x, 0, 3);
109 dst += dst_stride_4x;
112 reg0 = __lsx_vldrepl_w(src0_ptr, 0);
113 reg1 = __lsx_vldrepl_w(src0_ptr + src_stride, 0);
114 reg2 = __lsx_vldrepl_d(src1_ptr, 0);
115 reg3 = __lsx_vldrepl_d(src1_ptr + src2_stride, 0);
116 src0 = __lsx_vilvl_w(reg1, reg0);
117 in0 = __lsx_vilvl_d(reg3, reg2);
118 dst0 = __lsx_vsllwil_hu_bu(
src0, 6);
119 dst0 = __lsx_vsadd_h(dst0, in0);
120 dst0 = __lsx_vssrarni_bu_h(dst0, dst0, 7);
121 __lsx_vstelm_w(dst0,
dst, 0, 0);
122 __lsx_vstelm_w(dst0,
dst + dst_stride, 0, 1);
123 src0_ptr += src_stride_2x;
124 src1_ptr += src2_stride_2x;
125 dst += dst_stride_2x;
131 const int16_t *src1_ptr,
int32_t src2_stride,
136 int32_t src_stride_2x = (src_stride << 1);
137 int32_t dst_stride_2x = (dst_stride << 1);
138 int32_t src_stride_4x = (src_stride << 2);
139 int32_t dst_stride_4x = (dst_stride << 2);
140 int32_t src2_stride_x = (src2_stride << 1);
141 int32_t src2_stride_2x = (src2_stride << 2);
142 int32_t src_stride_3x = src_stride_2x + src_stride;
143 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
144 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
145 __m128i out0, out1, out2, out3;
146 __m128i
zero = __lsx_vldi(0);
148 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
149 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
150 __m128i reg0, reg1, reg2, reg3;
152 for (loop_cnt = (
height >> 3); loop_cnt--;) {
153 reg0 = __lsx_vldrepl_d(src0_ptr, 0);
154 reg1 = __lsx_vldrepl_d(src0_ptr + src_stride, 0);
155 reg2 = __lsx_vldrepl_d(src0_ptr + src_stride_2x, 0);
156 reg3 = __lsx_vldrepl_d(src0_ptr + src_stride_3x, 0);
158 src0_ptr += src_stride_4x;
159 reg0 = __lsx_vldrepl_d(src0_ptr, 0);
160 reg1 = __lsx_vldrepl_d(src0_ptr + src_stride, 0);
161 reg2 = __lsx_vldrepl_d(src0_ptr + src_stride_2x, 0);
162 reg3 = __lsx_vldrepl_d(src0_ptr + src_stride_3x, 0);
163 DUP2_ARG2(__lsx_vilvl_d, reg1, reg0, reg3, reg2,
src2, src3);
164 src0_ptr += src_stride_4x;
165 in0 = __lsx_vld(src1_ptr, 0);
166 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr,
167 src2_stride_2x, in1, in2);
168 in3 = __lsx_vldx(src1_ptr, src2_stride_3x);
169 src1_ptr += src2_stride_2x;
170 in4 = __lsx_vld(src1_ptr, 0);
171 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr,
172 src2_stride_2x, in5, in6);
173 in7 = __lsx_vldx(src1_ptr, src2_stride_3x);
174 src1_ptr += src2_stride_2x;
176 dst0, dst2, dst4, dst6);
178 dst1, dst3, dst5, dst7);
179 DUP4_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst5, 6, dst7, 6, dst1, dst3,
185 __lsx_vstelm_w(out0,
dst, 0, 0);
186 __lsx_vstelm_w(out0,
dst + dst_stride, 0, 2);
187 __lsx_vstelm_h(out0,
dst, 4, 2);
188 __lsx_vstelm_h(out0,
dst + dst_stride, 4, 6);
189 __lsx_vstelm_w(out1,
dst + dst_stride_2x, 0, 0);
190 __lsx_vstelm_w(out1,
dst + dst_stride_3x, 0, 2);
191 __lsx_vstelm_h(out1,
dst + dst_stride_2x, 4, 2);
192 __lsx_vstelm_h(out1,
dst + dst_stride_3x, 4, 6);
193 dst += dst_stride_4x;
194 __lsx_vstelm_w(out2,
dst, 0, 0);
195 __lsx_vstelm_w(out2,
dst + dst_stride, 0, 2);
196 __lsx_vstelm_h(out2,
dst, 4, 2);
197 __lsx_vstelm_h(out2,
dst + dst_stride, 4, 6);
198 __lsx_vstelm_w(out3,
dst + dst_stride_2x, 0, 0);
199 __lsx_vstelm_w(out3,
dst + dst_stride_3x, 0, 2);
200 __lsx_vstelm_h(out3,
dst + dst_stride_2x, 4, 2);
201 __lsx_vstelm_h(out3,
dst + dst_stride_3x, 4, 6);
202 dst += dst_stride_4x;
205 reg0 = __lsx_vldrepl_d(src0_ptr, 0);
206 reg1 = __lsx_vldrepl_d(src0_ptr + src_stride, 0);
207 src0 = __lsx_vilvl_d(reg1, reg0);
208 src0_ptr += src_stride_2x;
209 in0 = __lsx_vld(src1_ptr, 0);
210 in1 = __lsx_vldx(src1_ptr, src2_stride_x);
211 src1_ptr += src2_stride_x;
212 dst0 = __lsx_vsllwil_hu_bu(
src0, 6);
214 dst1 = __lsx_vslli_h(dst1, 6);
216 __lsx_vstelm_w(out0,
dst, 0, 0);
217 __lsx_vstelm_h(out0,
dst, 4, 2);
219 __lsx_vstelm_w(out0,
dst, 0, 2);
220 __lsx_vstelm_h(out0,
dst, 4, 6);
227 const int16_t *src1_ptr,
int32_t src2_stride,
232 int32_t src_stride_2x = (src_stride << 1);
233 int32_t dst_stride_2x = (dst_stride << 1);
234 int32_t src_stride_4x = (src_stride << 2);
235 int32_t dst_stride_4x = (dst_stride << 2);
236 int32_t src2_stride_x = (src2_stride << 1);
237 int32_t src2_stride_2x = (src2_stride << 2);
238 int32_t src_stride_3x = src_stride_2x + src_stride;
239 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
240 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
241 __m128i out0, out1, out2, out3;
243 __m128i
zero = __lsx_vldi(0);
244 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
245 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
246 __m128i reg0, reg1, reg2, reg3;
248 for (loop_cnt = (
height >> 3); loop_cnt--;) {
249 reg0 = __lsx_vldrepl_d(src0_ptr, 0);
250 reg1 = __lsx_vldrepl_d(src0_ptr + src_stride, 0);
251 reg2 = __lsx_vldrepl_d(src0_ptr + src_stride_2x, 0);
252 reg3 = __lsx_vldrepl_d(src0_ptr + src_stride_3x, 0);
254 src0_ptr += src_stride_4x;
255 reg0 = __lsx_vldrepl_d(src0_ptr, 0);
256 reg1 = __lsx_vldrepl_d(src0_ptr + src_stride, 0);
257 reg2 = __lsx_vldrepl_d(src0_ptr + src_stride_2x, 0);
258 reg3 = __lsx_vldrepl_d(src0_ptr + src_stride_3x, 0);
259 DUP2_ARG2(__lsx_vilvl_d, reg1, reg0, reg3, reg2,
src2, src3);
260 src0_ptr += src_stride_4x;
262 dst0, dst2, dst4, dst6);
264 src3, dst1, dst3, dst5, dst7);
265 DUP4_ARG2(__lsx_vslli_h, dst1, 6, dst3, 6, dst5, 6, dst7, 6, dst1,
267 in0 = __lsx_vld(src1_ptr, 0);
268 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr,
269 src2_stride_2x, in1, in2);
270 in3 = __lsx_vldx(src1_ptr, src2_stride_3x);
271 src1_ptr += src2_stride_2x;
272 in4 = __lsx_vld(src1_ptr, 0);
273 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr,
274 src2_stride_2x, in5, in6);
275 in7 = __lsx_vldx(src1_ptr, src2_stride_3x);
276 src1_ptr += src2_stride_2x;
281 __lsx_vstelm_d(out0,
dst, 0, 0);
282 __lsx_vstelm_d(out0,
dst + dst_stride, 0, 1);
283 __lsx_vstelm_d(out1,
dst + dst_stride_2x, 0, 0);
284 __lsx_vstelm_d(out1,
dst + dst_stride_3x, 0, 1);
285 dst += dst_stride_4x;
286 __lsx_vstelm_d(out2,
dst, 0, 0);
287 __lsx_vstelm_d(out2,
dst + dst_stride, 0, 1);
288 __lsx_vstelm_d(out3,
dst + dst_stride_2x, 0, 0);
289 __lsx_vstelm_d(out3,
dst + dst_stride_3x, 0, 1);
290 dst += dst_stride_4x;
293 reg0 = __lsx_vldrepl_d(src0_ptr, 0);
294 reg1 = __lsx_vldrepl_d(src0_ptr + src_stride, 0);
295 src0 = __lsx_vilvl_d(reg1, reg0);
296 in0 = __lsx_vld(src1_ptr, 0);
297 in1 = __lsx_vldx(src1_ptr, src2_stride_x);
298 dst0 = __lsx_vsllwil_hu_bu(
src0, 6);
300 dst1 = __lsx_vslli_h(dst1, 6);
302 __lsx_vstelm_d(out0,
dst, 0, 0);
303 __lsx_vstelm_d(out0,
dst + dst_stride, 0, 1);
304 src0_ptr += src_stride_2x;
305 src1_ptr += src2_stride_x;
306 dst += dst_stride_2x;
312 const int16_t *src1_ptr,
int32_t src2_stride,
316 int32_t src_stride_2x = (src_stride << 1);
317 int32_t dst_stride_2x = (dst_stride << 1);
318 int32_t src_stride_4x = (src_stride << 2);
319 int32_t dst_stride_4x = (dst_stride << 2);
320 int32_t src2_stride_x = (src2_stride << 1);
321 int32_t src2_stride_2x = (src2_stride << 2);
322 int32_t src_stride_3x = src_stride_2x + src_stride;
323 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
324 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
325 const int16_t *_src1 = src1_ptr + 8;
326 __m128i out0, out1, out2;
328 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
329 __m128i dst0, dst1, dst2, dst3, dst4, dst5;
331 for (loop_cnt = 4; loop_cnt--;) {
332 src0 = __lsx_vld(src0_ptr, 0);
333 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
335 src3 = __lsx_vldx(src0_ptr, src_stride_3x);
336 src0_ptr += src_stride_4x;
337 in0 = __lsx_vld(src1_ptr, 0);
338 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr,
339 src2_stride_2x, in1, in2);
340 in3 = __lsx_vldx(src1_ptr, src2_stride_3x);
341 src1_ptr += src2_stride_2x;
342 in4 = __lsx_vld(_src1, 0);
343 DUP2_ARG2(__lsx_vldx, _src1, src2_stride_x, _src1, src2_stride_2x,
345 in7 = __lsx_vldx(_src1, src2_stride_3x);
346 _src1 += src2_stride_2x;
348 DUP2_ARG2(__lsx_vilvl_d, in5, in4, in7, in6, in4, in5);
350 dst0, dst1, dst2, dst3)
356 __lsx_vstelm_d(out0,
dst, 0, 0);
357 __lsx_vstelm_d(out0,
dst + dst_stride, 0, 1);
358 __lsx_vstelm_d(out1,
dst + dst_stride_2x, 0, 0);
359 __lsx_vstelm_d(out1,
dst + dst_stride_3x, 0, 1);
360 __lsx_vstelm_w(out2,
dst, 8, 0);
361 __lsx_vstelm_w(out2,
dst + dst_stride, 8, 1);
362 __lsx_vstelm_w(out2,
dst + dst_stride_2x, 8, 2);
363 __lsx_vstelm_w(out2,
dst + dst_stride_3x, 8, 3);
364 dst += dst_stride_4x;
370 const int16_t *src1_ptr,
int32_t src2_stride,
374 int32_t src_stride_2x = (src_stride << 1);
375 int32_t dst_stride_2x = (dst_stride << 1);
376 int32_t src_stride_4x = (src_stride << 2);
377 int32_t dst_stride_4x = (dst_stride << 2);
378 int32_t src2_stride_x = (src2_stride << 1);
379 int32_t src2_stride_2x = (src2_stride << 2);
380 int32_t src_stride_3x = src_stride_2x + src_stride;
381 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
382 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
383 const int16_t *_src1 = src1_ptr + 8;
384 __m128i out0, out1, out2, out3;
386 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
387 __m128i dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, dst2_l, dst3_l;
390 for (loop_cnt = (
height >> 2); loop_cnt--;) {
391 src0 = __lsx_vld(src0_ptr, 0);
392 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
394 src3 = __lsx_vldx(src0_ptr, src_stride_3x);
395 src0_ptr += src_stride_4x;
396 in0 = __lsx_vld(src1_ptr, 0);
397 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr,
398 src2_stride_2x, in1, in2);
399 in3 = __lsx_vldx(src1_ptr, src2_stride_3x);
400 src1_ptr += src2_stride_2x;
401 in4 = __lsx_vld(_src1, 0);
402 DUP2_ARG2(__lsx_vldx, _src1, src2_stride_x, _src1, src2_stride_2x,
404 in7 = __lsx_vldx(_src1, src2_stride_3x);
405 _src1 += src2_stride_2x;
407 dst0_r, dst1_r, dst2_r, dst3_r)
409 dst0_l, dst1_l, dst2_l, dst3_l);
410 DUP4_ARG2(__lsx_vslli_h, dst0_l, 6, dst1_l, 6, dst2_l, 6, dst3_l, 6,
411 dst0_l, dst1_l, dst2_l, dst3_l);
417 __lsx_vst(out0,
dst, 0);
418 __lsx_vstx(out1,
dst, dst_stride);
419 __lsx_vstx(out2,
dst, dst_stride_2x);
420 __lsx_vstx(out3,
dst, dst_stride_3x);
421 dst += dst_stride_4x;
427 const int16_t *src1_ptr,
int32_t src2_stride,
438 const int16_t *src1_ptr,
int32_t src2_stride,
449 const int16_t *src1_ptr,
int32_t src2_stride,
460 const int16_t *src1_ptr,
int32_t src2_stride,
470 const int16_t *src1_ptr,
int32_t src2_stride,
475 const int32_t dst_stride_2x = (dst_stride << 1);
477 __m128i filt0, filt1, filt2, filt3;
478 __m128i mask1, mask2, mask3;
479 __m128i vec0, vec1, vec2, vec3;
480 __m128i dst0, dst1, dst2, dst3;
481 __m128i in0, in1, in2, in3;
486 filt0, filt1, filt2, filt3);
488 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
489 mask3 = __lsx_vaddi_bu(mask0, 6);
491 for (loop_cnt = (
height >> 1); loop_cnt--;) {
493 src0_ptr += src_stride;
495 src0_ptr += src_stride;
496 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in0, in1);
497 src1_ptr += src2_stride;
498 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in2, in3);
499 src1_ptr += src2_stride;
505 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
506 vec3, filt0, dst0, dst1, dst2, dst3);
511 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
512 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
517 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt2, dst1, vec1, filt2,
518 dst2, vec2, filt2, dst3, vec3, filt2, dst0, dst1, dst2, dst3);
523 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt3, dst1, vec1, filt3,
524 dst2, vec2, filt3, dst3, vec3, filt3, dst0, dst1, dst2, dst3);
528 __lsx_vst(dst0,
dst, 0);
529 __lsx_vstx(dst1,
dst, dst_stride);
530 dst += dst_stride_2x;
535 const int16_t *src1_ptr,
int32_t src2_stride,
541 __m128i filt0, filt1, filt2, filt3;
542 __m128i mask1, mask2, mask3, mask4, mask5, mask6, mask7;
543 __m128i vec0, vec1, vec2, vec3;
544 __m128i dst0, dst1, dst2;
545 __m128i in0, in1, in2;
550 filt0, filt1, filt2, filt3);
552 DUP4_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask0, 6, mask0, 8, mask1,
553 mask2, mask3, mask4);
554 DUP2_ARG2(__lsx_vaddi_bu, mask0, 10, mask0, 12, mask5, mask6);
555 mask7 = __lsx_vaddi_bu(mask0, 14);
557 for (loop_cnt =
height; loop_cnt--;) {
559 src0_ptr += src_stride;
560 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in0, in1);
561 in2 = __lsx_vld(src1_ptr, 32);
562 src1_ptr += src2_stride;
565 src1, mask0,
src0,
src0, mask1, vec0, vec1, vec2, vec3);
566 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, dst0, dst1);
567 dst2 = __lsx_vdp2_h_bu_b(vec2, filt0);
568 dst0 = __lsx_vdp2add_h_bu_b(dst0, vec3, filt1);
570 src0, mask2,
src1,
src0, mask6, vec0, vec1, vec2, vec3);
571 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst1, vec0, filt1, dst2, vec1, filt1,
572 dst0, vec2, filt2, dst1, vec3, filt2, dst1, dst2, dst0, dst1);
574 mask7,
src1,
src1, mask3, vec0, vec1, vec2, vec3);
575 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst2, vec0, filt2, dst0, vec1, filt3,
576 dst1, vec2, filt3, dst2, vec3, filt3, dst2, dst0, dst1, dst2);
579 dst2 = __lsx_vsadd_h(dst2, in2);
580 tmp1 = __lsx_vssrarni_bu_h(dst2, dst2, 7);
582 __lsx_vst(tmp0,
dst, 0);
583 __lsx_vstelm_d(tmp1,
dst, 16, 0);
589 const int16_t *src1_ptr,
int32_t src2_stride,
600 const int16_t *src1_ptr,
int32_t src2_stride,
611 const int16_t *src1_ptr,
int32_t src2_stride,
627 int32_t src_stride_2x = (src_stride << 1);
628 int32_t dst_stride_2x = (dst_stride << 1);
629 int32_t src_stride_4x = (src_stride << 2);
630 int32_t dst_stride_4x = (dst_stride << 2);
631 int32_t src2_stride_x = (src2_stride << 1);
632 int32_t src2_stride_2x = (src2_stride << 2);
633 int32_t src_stride_3x = src_stride_2x + src_stride;
634 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
635 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
637 __m128i src6, src7, src8, src9, src10;
638 __m128i in0, in1, in2, in3;
639 __m128i src10_r, src32_r, src54_r, src76_r, src98_r;
640 __m128i src21_r, src43_r, src65_r, src87_r, src109_r;
641 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
642 __m128i filt0, filt1, filt2, filt3;
644 src0_ptr -= src_stride_3x;
647 filt0, filt1, filt2, filt3);
649 src0 = __lsx_vld(src0_ptr, 0);
650 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
652 src3 = __lsx_vldx(src0_ptr, src_stride_3x);
653 src0_ptr += src_stride_4x;
654 src4 = __lsx_vld(src0_ptr, 0);
655 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
657 src0_ptr += src_stride_3x;
659 src10_r, src32_r, src54_r, src21_r);
660 DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r);
662 for (loop_cnt = (
height >> 2); loop_cnt--;) {
663 src7 = __lsx_vld(src0_ptr, 0);
664 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
666 src10 = __lsx_vldx(src0_ptr, src_stride_3x);
667 src0_ptr += src_stride_4x;
668 in0 = __lsx_vld(src1_ptr, 0);
669 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr, src2_stride_2x,
671 in3 = __lsx_vldx(src1_ptr, src2_stride_3x);
672 src1_ptr += src2_stride_2x;
673 DUP4_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src9, src8, src10, src9,
674 src76_r, src87_r, src98_r, src109_r);
676 DUP4_ARG2(__lsx_vdp2_h_bu_b, src10_r, filt0, src21_r, filt0, src32_r,
677 filt0, src43_r, filt0, dst0_r, dst1_r, dst2_r, dst3_r);
678 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst1_r, src43_r,
679 filt1, dst2_r, src54_r, filt1, dst3_r, src65_r, filt1,
680 dst0_r, dst1_r, dst2_r, dst3_r);
681 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src54_r, filt2, dst1_r, src65_r,
682 filt2, dst2_r, src76_r, filt2, dst3_r, src87_r, filt2,
683 dst0_r, dst1_r, dst2_r, dst3_r);
684 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src76_r, filt3, dst1_r, src87_r,
685 filt3, dst2_r, src98_r, filt3, dst3_r, src109_r, filt3,
686 dst0_r, dst1_r, dst2_r, dst3_r);
690 __lsx_vstelm_d(dst0_r,
dst, 0, 0);
691 __lsx_vstelm_d(dst0_r,
dst + dst_stride, 0, 1);
692 __lsx_vstelm_d(dst1_r,
dst + dst_stride_2x, 0, 0);
693 __lsx_vstelm_d(dst1_r,
dst + dst_stride_3x, 0, 1);
694 dst += dst_stride_4x;
709 const int16_t *src1_ptr,
int32_t src2_stride,
714 const uint8_t *src0_ptr_tmp;
715 const int16_t *src1_ptr_tmp;
719 int32_t src_stride_2x = (src_stride << 1);
720 int32_t dst_stride_2x = (dst_stride << 1);
721 int32_t src_stride_4x = (src_stride << 2);
722 int32_t src_stride_3x = src_stride_2x + src_stride;
723 __m128i
src0,
src1,
src2, src3, src4, src5, src6, src7, src8;
724 __m128i in0, in1, in2, in3;
725 __m128i src10_r, src32_r, src54_r, src76_r;
726 __m128i src21_r, src43_r, src65_r, src87_r;
727 __m128i dst0_r, dst1_r;
728 __m128i src10_l, src32_l, src54_l, src76_l;
729 __m128i src21_l, src43_l, src65_l, src87_l;
730 __m128i dst0_l, dst1_l;
731 __m128i filt0, filt1, filt2, filt3;
733 src0_ptr -= src_stride_3x;
736 filt0, filt1, filt2, filt3);
738 for (cnt = (
width >> 4); cnt--;) {
739 src0_ptr_tmp = src0_ptr;
740 src1_ptr_tmp = src1_ptr;
743 src0 = __lsx_vld(src0_ptr_tmp, 0);
744 DUP2_ARG2(__lsx_vldx, src0_ptr_tmp, src_stride, src0_ptr_tmp,
746 src3 = __lsx_vldx(src0_ptr_tmp, src_stride_3x);
747 src0_ptr_tmp += src_stride_4x;
748 src4 = __lsx_vld(src0_ptr_tmp, 0);
749 DUP2_ARG2(__lsx_vldx, src0_ptr_tmp, src_stride, src0_ptr_tmp,
750 src_stride_2x, src5, src6);
751 src0_ptr_tmp += src_stride_3x;
754 src10_r, src32_r, src54_r, src21_r);
755 DUP2_ARG2(__lsx_vilvl_b, src4, src3, src6, src5, src43_r, src65_r);
757 src10_l, src32_l, src54_l, src21_l);
758 DUP2_ARG2(__lsx_vilvh_b, src4, src3, src6, src5, src43_l, src65_l);
760 for (loop_cnt = (
height >> 1); loop_cnt--;) {
761 src7 = __lsx_vld(src0_ptr_tmp, 0);
762 src8 = __lsx_vldx(src0_ptr_tmp, src_stride);
763 src0_ptr_tmp += src_stride_2x;
764 DUP2_ARG2(__lsx_vld, src1_ptr_tmp, 0, src1_ptr_tmp, 16, in0, in2);
765 src1_ptr_tmp += src2_stride;
766 DUP2_ARG2(__lsx_vld, src1_ptr_tmp, 0, src1_ptr_tmp, 16, in1, in3);
767 src1_ptr_tmp += src2_stride;
769 DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r);
770 DUP2_ARG2(__lsx_vilvh_b, src7, src6, src8, src7, src76_l, src87_l);
772 DUP4_ARG2(__lsx_vdp2_h_bu_b, src10_r, filt0, src21_r, filt0, src10_l,
773 filt0, src21_l, filt0, dst0_r, dst1_r, dst0_l, dst1_l);
774 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst1_r,
775 src43_r, filt1, dst0_l, src32_l, filt1, dst1_l, src43_l,
776 filt1, dst0_r, dst1_r, dst0_l, dst1_l);
777 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src54_r, filt2, dst1_r,
778 src65_r, filt2, dst0_l, src54_l, filt2, dst1_l, src65_l,
779 filt2, dst0_r, dst1_r, dst0_l, dst1_l);
780 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src76_r, filt3, dst1_r,
781 src87_r, filt3, dst0_l, src76_l, filt3, dst1_l, src87_l,
782 filt3, dst0_r, dst1_r, dst0_l, dst1_l);
786 __lsx_vst(dst0_r, dst_tmp, 0);
787 __lsx_vstx(dst1_r, dst_tmp, dst_stride);
788 dst_tmp += dst_stride_2x;
812 const int16_t *src1_ptr,
int32_t src2_stride,
821 const int16_t *src1_ptr,
int32_t src2_stride,
832 const int16_t *src1_ptr,
int32_t src2_stride,
841 const int16_t *src1_ptr,
int32_t src2_stride,
850 const int16_t *src1_ptr,
int32_t src2_stride,
860 const int16_t *src1_ptr,
int32_t src2_stride,
862 const int8_t *filter_x,
const int8_t *filter_y,
867 const uint8_t *src0_ptr_tmp;
868 const int16_t *src1_ptr_tmp;
870 int32_t src_stride_2x = (src_stride << 1);
871 int32_t src_stride_4x = (src_stride << 2);
872 int32_t src_stride_3x = src_stride_2x + src_stride;
876 __m128i filt0, filt1, filt2, filt3;
877 __m128i filt_h0, filt_h1, filt_h2, filt_h3;
879 __m128i mask1, mask2, mask3;
880 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
881 __m128i vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
882 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
883 __m128i dst0_r, dst0_l;
884 __m128i dst10_r, dst32_r, dst54_r, dst76_r;
885 __m128i dst10_l, dst32_l, dst54_l, dst76_l;
887 src0_ptr -= src_stride_3x + 3;
889 DUP4_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filter_x, 4, filter_x,
890 6, filt0, filt1, filt2, filt3);
891 filt_h3 = __lsx_vld(filter_y, 0);
892 filt_h3 = __lsx_vsllwil_h_b(filt_h3, 0);
894 DUP4_ARG2(__lsx_vreplvei_w, filt_h3, 0, filt_h3, 1, filt_h3, 2, filt_h3, 3,
895 filt_h0, filt_h1, filt_h2, filt_h3);
897 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 4, mask1, mask2);
898 mask3 = __lsx_vaddi_bu(mask0, 6);
900 for (cnt =
width >> 3; cnt--;) {
901 src0_ptr_tmp = src0_ptr;
903 src1_ptr_tmp = src1_ptr;
905 src0 = __lsx_vld(src0_ptr_tmp, 0);
906 DUP2_ARG2(__lsx_vldx, src0_ptr_tmp, src_stride, src0_ptr_tmp,
908 src3 = __lsx_vldx(src0_ptr_tmp, src_stride_3x);
909 src0_ptr_tmp += src_stride_4x;
910 src4 = __lsx_vld(src0_ptr_tmp, 0);
911 DUP2_ARG2(__lsx_vldx, src0_ptr_tmp, src_stride, src0_ptr_tmp,
912 src_stride_2x, src5, src6);
913 src0_ptr_tmp += src_stride_3x;
917 src0, mask2,
src0,
src0, mask3, vec0, vec1, vec2, vec3);
919 src1, mask2,
src1,
src1, mask3, vec4, vec5, vec6, vec7);
921 src2, mask2,
src2,
src2, mask3, vec8, vec9, vec10, vec11);
922 DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src3,
923 src3, mask2, src3, src3, mask3, vec12, vec13, vec14, vec15);
924 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec4, filt0, vec8, filt0,
925 vec12, filt0, dst0, dst1, dst2, dst3);
926 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec5, filt1,
927 dst2, vec9, filt1, dst3, vec13, filt1, dst0, dst1, dst2, dst3);
928 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec2, filt2, dst1, vec6, filt2,
929 dst2, vec10, filt2, dst3, vec14, filt2, dst0, dst1, dst2, dst3);
930 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec3, filt3, dst1, vec7, filt3,
931 dst2, vec11, filt3, dst3, vec15, filt3, dst0, dst1, dst2, dst3);
933 DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, src4,
934 src4, mask2, src4, src4, mask3, vec0, vec1, vec2, vec3);
935 DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src5,
936 src5, mask2, src5, src5, mask3, vec4, vec5, vec6, vec7);
937 DUP4_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, src6,
938 src6, mask2, src6, src6, mask3, vec8, vec9, vec10, vec11);
939 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec4, filt0, dst4, dst5);
940 dst6 = __lsx_vdp2_h_bu_b(vec8, filt0);
941 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec1, filt1, dst5, vec5, filt1,
942 dst6, vec9, filt1, dst4, vec2, filt2, dst4, dst5, dst6, dst4);
943 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst5, vec6, filt2, dst6, vec10, filt2,
944 dst4, vec3, filt3, dst5, vec7, filt3, dst5, dst6, dst4, dst5);
945 dst6 = __lsx_vdp2add_h_bu_b(dst6, vec11, filt3);
947 for (loop_cnt =
height; loop_cnt--;) {
948 src7 = __lsx_vld(src0_ptr_tmp, 0);
949 src0_ptr_tmp += src_stride;
951 in0 = __lsx_vld(src1_ptr_tmp, 0);
952 src1_ptr_tmp += src2_stride;
954 DUP4_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, src7,
955 src7, mask2, src7, src7, mask3, vec0, vec1, vec2, vec3);
956 dst7 = __lsx_vdp2_h_bu_b(vec0, filt0);
957 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst7, vec1, filt1, dst7, vec2,
959 dst7 = __lsx_vdp2add_h_bu_b(dst7, vec3, filt3);
960 DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7,
961 dst6, dst10_r, dst32_r, dst54_r, dst76_r);
962 DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst3, dst2, dst5, dst4, dst7,
963 dst6, dst10_l, dst32_l, dst54_l, dst76_l);
965 DUP2_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0,
967 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l,
968 dst32_l, filt_h1, dst0_r, dst54_r, filt_h2, dst0_l,
969 dst54_l, filt_h2, dst0_r, dst0_l, dst0_r, dst0_l);
970 DUP2_ARG3(__lsx_vdp2add_w_h, dst0_r, dst76_r, filt_h3, dst0_l,
971 dst76_l, filt_h3, dst0_r, dst0_l);
972 dst0_r = __lsx_vsrli_w(dst0_r, 6);
973 dst0_l = __lsx_vsrli_w(dst0_l, 6);
975 tmp = __lsx_vpickev_h(dst0_l, dst0_r);
976 tmp = __lsx_vsadd_h(
tmp, in0);
977 tmp = __lsx_vmaxi_h(
tmp, 0);
979 __lsx_vstelm_d(
out, dst_tmp, 0, 0);
980 dst_tmp += dst_stride;
998 const int16_t *src1_ptr,
int32_t src2_stride,
1000 const int8_t *filter_x,
const int8_t *filter_y,
1004 dst, dst_stride, filter_x, filter_y,
height, 8);
1008 const int16_t *src1_ptr,
int32_t src2_stride,
1010 const int8_t *filter_x,
const int8_t *filter_y,
1014 dst, dst_stride, filter_x, filter_y,
height, 16);
1018 const int16_t *src1_ptr,
int32_t src2_stride,
1020 const int8_t *filter_x,
const int8_t *filter_y,
1024 dst, dst_stride, filter_x, filter_y,
height, 24);
1028 const int16_t *src1_ptr,
int32_t src2_stride,
1030 const int8_t *filter_x,
const int8_t *filter_y,
1034 dst, dst_stride, filter_x, filter_y,
height, 32);
1038 const int16_t *src1_ptr,
int32_t src2_stride,
1040 const int8_t *filter_x,
const int8_t *filter_y,
1044 dst, dst_stride, filter_x, filter_y,
height, 48);
1048 const int16_t *src1_ptr,
int32_t src2_stride,
1050 const int8_t *filter_x,
const int8_t *filter_y,
1054 dst, dst_stride, filter_x, filter_y,
height, 64);
1058 const int16_t *src1_ptr,
int32_t src2_stride,
1062 const int16_t *src1_ptr_tmp;
1065 int32_t dst_stride_2x = (dst_stride << 1);
1066 int32_t dst_stride_4x = (dst_stride << 2);
1067 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
1068 int32_t src2_stride_x = src2_stride << 1;
1069 int32_t src2_stride_2x = src2_stride << 2;
1070 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
1072 __m128i
src0,
src1,
src2, src3, src4, src5, src6, src7;
1073 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
1074 __m128i filt0, filt1;
1076 __m128i mask1, mask2, mask3;
1077 __m128i vec0, vec1, vec2, vec3;
1078 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1083 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 8, mask1, mask2);
1084 mask3 = __lsx_vaddi_bu(mask0, 10);
1087 src1_ptr_tmp = src1_ptr + 16;
1089 for (loop_cnt = (
height >> 2); loop_cnt--;) {
1091 src0_ptr += src_stride;
1092 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16,
src2, src3);
1093 src0_ptr += src_stride;
1094 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src4, src5);
1095 src0_ptr += src_stride;
1096 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src6, src7);
1097 src0_ptr += src_stride;
1099 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in0, in1);
1100 src1_ptr += src2_stride;
1101 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in2, in3);
1102 src1_ptr += src2_stride;
1103 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in4, in5);
1104 src1_ptr += src2_stride;
1105 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in6, in7);
1106 src1_ptr += src2_stride;
1109 src2, mask0, src3,
src2, mask2, vec0, vec1, vec2, vec3);
1110 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
1111 vec3, filt0, dst0, dst1, dst2, dst3);
1113 src2, mask1, src3,
src2, mask3, vec0, vec1, vec2, vec3);
1114 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
1115 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
1117 DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask0, src5, src4, mask2, src6,
1118 src6, mask0, src7, src6, mask2, vec0, vec1, vec2, vec3);
1119 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
1120 vec3, filt0, dst4, dst5, dst6, dst7);
1121 DUP4_ARG3(__lsx_vshuf_b, src4, src4, mask1, src5, src4, mask3, src6,
1122 src6, mask1, src7, src6, mask3, vec0, vec1, vec2, vec3);
1123 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst4, vec0, filt1, dst5, vec1, filt1,
1124 dst6, vec2, filt1, dst7, vec3, filt1, dst4, dst5, dst6, dst7);
1130 __lsx_vst(dst0,
dst, 0);
1131 __lsx_vstx(dst1,
dst, dst_stride);
1132 __lsx_vstx(dst2,
dst, dst_stride_2x);
1133 __lsx_vstx(dst3,
dst, dst_stride_3x);
1134 dst += dst_stride_4x;
1136 in0 = __lsx_vld(src1_ptr_tmp, 0);
1137 DUP2_ARG2(__lsx_vldx, src1_ptr_tmp, src2_stride_x, src1_ptr_tmp,
1138 src2_stride_2x, in1, in2);
1139 in3 = __lsx_vldx(src1_ptr_tmp, src2_stride_3x);
1140 src1_ptr_tmp += src2_stride_2x;
1143 src5, mask0, src7, src7, mask0, vec0, vec1, vec2, vec3);
1144 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
1145 vec3, filt0, dst0, dst1, dst2, dst3);
1147 src5, mask1, src7, src7, mask1, vec0, vec1, vec2, vec3);
1148 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
1149 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
1152 __lsx_vstelm_d(dst0, dst_tmp, 0, 0);
1153 __lsx_vstelm_d(dst0, dst_tmp + dst_stride, 0, 1);
1154 __lsx_vstelm_d(dst1, dst_tmp + dst_stride_2x, 0, 0);
1155 __lsx_vstelm_d(dst1, dst_tmp + dst_stride_3x, 0, 1);
1156 dst_tmp += dst_stride_4x;
1161 const int16_t *src1_ptr,
int32_t src2_stride,
1167 __m128i in0, in1, in2, in3;
1168 __m128i filt0, filt1;
1170 __m128i mask1, mask2, mask3;
1171 __m128i dst0, dst1, dst2, dst3;
1172 __m128i vec0, vec1, vec2, vec3;
1178 DUP2_ARG2(__lsx_vaddi_bu, mask0, 2, mask0, 8, mask1, mask2);
1179 mask3 = __lsx_vaddi_bu(mask0, 10);
1181 for (loop_cnt =
height; loop_cnt--;) {
1183 src2 = __lsx_vld(src0_ptr, 24);
1184 src0_ptr += src_stride;
1185 DUP4_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, src1_ptr, 32,
1186 src1_ptr, 48, in0, in1, in2, in3);
1187 src1_ptr += src2_stride;
1189 src1, mask0,
src2,
src2, mask0, vec0, vec1, vec2, vec3);
1190 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec1, filt0, vec2, filt0,
1191 vec3, filt0, dst0, dst1, dst2, dst3);
1193 src1, mask1,
src2,
src2, mask1, vec0, vec1, vec2, vec3);
1194 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec0, filt1, dst1, vec1, filt1,
1195 dst2, vec2, filt1, dst3, vec3, filt1, dst0, dst1, dst2, dst3);
1198 __lsx_vst(dst0,
dst, 0);
1199 __lsx_vst(dst1,
dst, 16);
1205 const int16_t *src1_ptr,
int32_t src2_stride,
1210 int32_t src_stride_2x = (src_stride << 1);
1211 int32_t dst_stride_2x = (dst_stride << 1);
1212 int32_t dst_stride_4x = (dst_stride << 2);
1213 int32_t src_stride_4x = (src_stride << 2);
1214 int32_t src2_stride_x = (src2_stride << 1);
1215 int32_t src2_stride_2x = (src2_stride << 2);
1216 int32_t src_stride_3x = src_stride_2x + src_stride;
1217 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
1218 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
1219 const int16_t *_src1 = src1_ptr + 8;
1221 __m128i in0, in1, in2, in3, in4, in5, in6, in7;
1222 __m128i src10_r, src32_r, src21_r, src43_r, src54_r, src65_r;
1223 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
1224 __m128i src10_l, src32_l, src54_l, src21_l, src43_l, src65_l;
1225 __m128i src2110, src4332, src6554;
1226 __m128i dst0_l, dst1_l, filt0, filt1;
1228 src0_ptr -= src_stride;
1231 src0 = __lsx_vld(src0_ptr, 0);
1232 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1234 src0_ptr += src_stride_3x;
1237 src2110 = __lsx_vilvl_d(src21_l, src10_l);
1239 for (loop_cnt = (
height >> 2); loop_cnt--;) {
1240 src3 = __lsx_vld(src0_ptr, 0);
1241 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1243 src6 = __lsx_vldx(src0_ptr, src_stride_3x);
1244 src0_ptr += src_stride_4x;
1245 in0 = __lsx_vld(src1_ptr, 0);
1246 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr,
1247 src2_stride_2x, in1, in2);
1248 in3 = __lsx_vldx(src1_ptr, src2_stride_3x);
1249 src1_ptr += src2_stride_2x;
1250 in4 = __lsx_vld(_src1, 0);
1251 DUP2_ARG2(__lsx_vldx, _src1, src2_stride_x, _src1, src2_stride_2x,
1253 in7 = __lsx_vldx(_src1, src2_stride_3x);
1254 _src1 += src2_stride_2x;
1255 DUP2_ARG2(__lsx_vilvl_d, in5, in4, in7, in6, in4, in5);
1257 DUP2_ARG2(__lsx_vilvl_b, src3,
src2, src4, src3, src32_r, src43_r);
1258 DUP2_ARG2(__lsx_vilvh_b, src3,
src2, src4, src3, src32_l, src43_l);
1259 src4332 = __lsx_vilvl_d(src43_l, src32_l);
1260 DUP2_ARG2(__lsx_vilvl_b, src5, src4, src6, src5, src54_r, src65_r);
1261 DUP2_ARG2(__lsx_vilvh_b, src5, src4, src6, src5, src54_l, src65_l);
1262 src6554 = __lsx_vilvl_d(src65_l, src54_l);
1264 DUP4_ARG2(__lsx_vdp2_h_bu_b, src10_r, filt0, src21_r, filt0, src2110,
1265 filt0, src32_r, filt0, dst0_r, dst1_r, dst0_l, dst2_r);
1266 DUP2_ARG2(__lsx_vdp2_h_bu_b, src43_r, filt0, src4332, filt0,
1268 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst1_r,
1269 src43_r, filt1, dst0_l, src4332, filt1, dst2_r, src54_r,
1270 filt1, dst0_r, dst1_r, dst0_l, dst2_r);
1271 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst3_r, src65_r, filt1, dst1_l,
1272 src6554, filt1, dst3_r, dst1_l);
1276 __lsx_vstelm_d(dst0_r,
dst, 0, 0);
1277 __lsx_vstelm_d(dst0_r,
dst + dst_stride, 0, 1);
1278 __lsx_vstelm_d(dst1_r,
dst + dst_stride_2x, 0, 0);
1279 __lsx_vstelm_d(dst1_r,
dst + dst_stride_3x, 0, 1);
1280 __lsx_vstelm_w(dst0_l,
dst, 8, 0);
1281 __lsx_vstelm_w(dst0_l,
dst + dst_stride, 8, 1);
1282 __lsx_vstelm_w(dst0_l,
dst + dst_stride_2x, 8, 2);
1283 __lsx_vstelm_w(dst0_l,
dst + dst_stride_3x, 8, 3);
1284 dst += dst_stride_4x;
1294 const int16_t *src1_ptr,
int32_t src2_stride,
1299 const int32_t src_stride_2x = (src_stride << 1);
1300 const int32_t dst_stride_2x = (dst_stride << 1);
1301 const int32_t src_stride_3x = src_stride_2x + src_stride;
1303 __m128i in0, in1, in2, in3;
1304 __m128i src10_r, src32_r, src21_r, src43_r;
1305 __m128i src10_l, src32_l, src21_l, src43_l;
1306 __m128i dst0_r, dst1_r, dst0_l, dst1_l;
1307 __m128i filt0, filt1;
1309 src0_ptr -= src_stride;
1312 src0 = __lsx_vld(src0_ptr, 0);
1313 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1315 src0_ptr += src_stride_3x;
1319 for (loop_cnt = (
height >> 2); loop_cnt--;) {
1320 src3 = __lsx_vld(src0_ptr, 0);
1321 src4 = __lsx_vldx(src0_ptr, src_stride);
1322 src0_ptr += src_stride_2x;
1323 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in0, in2);
1324 src1_ptr += src2_stride;
1325 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in1, in3);
1326 src1_ptr += src2_stride;
1327 DUP2_ARG2(__lsx_vilvl_b, src3,
src2, src4, src3, src32_r, src43_r);
1328 DUP2_ARG2(__lsx_vilvh_b, src3,
src2, src4, src3, src32_l, src43_l);
1330 DUP4_ARG2(__lsx_vdp2_h_bu_b, src10_r, filt0, src21_r, filt0, src10_l,
1331 filt0, src21_l, filt0, dst0_r, dst1_r, dst0_l, dst1_l);
1332 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst1_r, src43_r,
1333 filt1, dst0_l, src32_l, filt1, dst1_l, src43_l, filt1,
1334 dst0_r, dst1_r, dst0_l, dst1_l);
1338 __lsx_vst(dst0_r,
dst, 0);
1339 __lsx_vstx(dst1_r,
dst, dst_stride);
1340 dst += dst_stride_2x;
1342 src5 = __lsx_vld(src0_ptr, 0);
1343 src2 = __lsx_vldx(src0_ptr, src_stride);
1344 src0_ptr += src_stride_2x;
1345 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in0, in2);
1346 src1_ptr += src2_stride;
1347 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in1, in3);
1348 src1_ptr += src2_stride;
1349 DUP2_ARG2(__lsx_vilvl_b, src5, src4,
src2, src5, src10_r, src21_r);
1350 DUP2_ARG2(__lsx_vilvh_b, src5, src4,
src2, src5, src10_l, src21_l);
1352 DUP4_ARG2(__lsx_vdp2_h_bu_b, src32_r, filt0, src32_l, filt0, src43_r,
1353 filt0, src43_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
1354 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src10_r, filt1, dst0_l,
1355 src10_l, filt1, dst1_r, src21_r, filt1, dst1_l, src21_l,
1356 filt1, dst0_r, dst0_l, dst1_r, dst1_l);
1359 __lsx_vst(dst0_r,
dst, 0);
1360 __lsx_vstx(dst1_r,
dst, dst_stride);
1361 dst += dst_stride_2x;
1366 const int16_t *src1_ptr,
int32_t src2_stride,
1371 int32_t dst_stride_2x = dst_stride << 1;
1373 __m128i src6, src7, src8, src9, src10, src11;
1374 __m128i in0, in1, in2, in3, in4, in5;
1375 __m128i src10_r, src32_r, src76_r, src98_r;
1376 __m128i src21_r, src43_r, src87_r, src109_r;
1377 __m128i src10_l, src32_l, src21_l, src43_l;
1378 __m128i dst0_r, dst1_r, dst2_r, dst3_r;
1379 __m128i dst0_l, dst1_l;
1380 __m128i filt0, filt1;
1382 src0_ptr -= src_stride;
1386 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16,
src0, src6);
1387 src0_ptr += src_stride;
1388 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16,
src1, src7);
1389 src0_ptr += src_stride;
1390 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16,
src2, src8);
1391 src0_ptr += src_stride;
1395 DUP2_ARG2(__lsx_vilvl_b, src7, src6, src8, src7, src76_r, src87_r);
1397 for (loop_cnt = (
height >> 2); loop_cnt--;) {
1399 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src3, src9);
1400 src0_ptr += src_stride;
1401 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16, src4, src10);
1402 src0_ptr += src_stride;
1403 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in0, in2);
1404 in4 = __lsx_vld(src1_ptr, 32);
1405 src1_ptr += src2_stride;
1406 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr, 16, in1, in3);
1407 in5 = __lsx_vld(src1_ptr, 32);
1408 src1_ptr += src2_stride;
1409 DUP2_ARG2(__lsx_vilvl_b, src3,
src2, src4, src3, src32_r, src43_r);
1410 DUP2_ARG2(__lsx_vilvh_b, src3,
src2, src4, src3, src32_l, src43_l);
1412 DUP2_ARG2(__lsx_vilvl_b, src9, src8, src10, src9, src98_r, src109_r);
1414 DUP4_ARG2(__lsx_vdp2_h_bu_b, src10_r, filt0, src10_l, filt0, src21_r,
1415 filt0, src21_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
1416 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src32_r, filt1, dst0_l,
1417 src32_l, filt1, dst1_r, src43_r, filt1, dst1_l, src43_l, filt1,
1418 dst0_r, dst0_l, dst1_r, dst1_l);
1420 DUP2_ARG2(__lsx_vdp2_h_bu_b, src76_r, filt0, src87_r, filt0,
1422 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src98_r, filt1, dst3_r,
1423 src109_r, filt1, dst2_r, dst3_r);
1428 __lsx_vst(dst0_r,
dst, 0);
1429 __lsx_vstx(dst1_r,
dst, dst_stride);
1430 __lsx_vstelm_d(dst2_r,
dst, 16, 0);
1431 __lsx_vstelm_d(dst2_r,
dst + dst_stride, 16, 1);
1432 dst += dst_stride_2x;
1435 DUP4_ARG2(__lsx_vld, src0_ptr, 0, src1_ptr, 0, src1_ptr, 16, src1_ptr,
1436 32, src5, in0, in2, in4);
1437 src1_ptr += src2_stride;
1438 DUP4_ARG2(__lsx_vld, src0_ptr, 16, src1_ptr, 0, src1_ptr, 16, src1_ptr,
1439 32, src11, in1, in3, in5);
1440 src1_ptr += src2_stride;
1441 src0_ptr += src_stride;
1442 DUP2_ARG2(__lsx_vld, src0_ptr, 0, src0_ptr, 16,
src2, src8);
1443 src0_ptr += src_stride;
1444 DUP2_ARG2(__lsx_vilvl_b, src5, src4,
src2, src5, src10_r, src21_r);
1445 DUP2_ARG2(__lsx_vilvh_b, src5, src4,
src2, src5, src10_l, src21_l);
1447 DUP2_ARG2(__lsx_vilvl_b, src11, src10, src8, src11, src76_r, src87_r);
1449 DUP4_ARG2(__lsx_vdp2_h_bu_b, src32_r, filt0, src32_l, filt0, src43_r,
1450 filt0, src43_l, filt0, dst0_r, dst0_l, dst1_r, dst1_l);
1451 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0_r, src10_r, filt1, dst0_l,
1452 src10_l, filt1, dst1_r, src21_r, filt1, dst1_l, src21_l,
1453 filt1, dst0_r, dst0_l, dst1_r, dst1_l);
1456 DUP2_ARG2(__lsx_vdp2_h_bu_b, src98_r, filt0, src109_r, filt0,
1458 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst2_r, src76_r, filt1, dst3_r,
1459 src87_r, filt1, dst2_r, dst3_r);
1464 __lsx_vst(dst0_r,
dst, 0);
1465 __lsx_vstx(dst1_r,
dst, dst_stride);
1466 __lsx_vstelm_d(dst2_r,
dst, 16, 0);
1467 __lsx_vstelm_d(dst2_r,
dst + dst_stride, 16, 1);
1468 dst += dst_stride_2x;
1473 const int16_t *src1_ptr,
int32_t src2_stride,
1484 const int16_t *src1_ptr,
int32_t src2_stride,
1486 const int8_t *filter_x,
const int8_t *filter_y,
1489 int32_t src_stride_2x = (src_stride << 1);
1490 int32_t dst_stride_2x = (dst_stride << 1);
1491 int32_t src_stride_4x = (src_stride << 2);
1492 int32_t dst_stride_4x = (dst_stride << 2);
1493 int32_t src2_stride_2x = (src2_stride << 1);
1494 int32_t src2_stride_4x = (src2_stride << 2);
1495 int32_t src_stride_3x = src_stride_2x + src_stride;
1496 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
1497 int32_t src2_stride_3x = src2_stride_2x + src2_stride;
1500 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, mask1;
1501 __m128i filt0, filt1, filt_h0, filt_h1;
1502 __m128i dsth0, dsth1, dsth2, dsth3, dsth4, dsth5;
1503 __m128i dsth6, dsth7, dsth8, dsth9, dsth10;
1504 __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
1505 __m128i dst4_r, dst5_r, dst6_r, dst7_r;
1506 __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8;
1507 __m128i reg0, reg1, reg2, reg3;
1510 src0_ptr -= (src_stride + 1);
1511 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
1513 filt_h1 = __lsx_vld(filter_y, 0);
1514 filt_h1 = __lsx_vsllwil_h_b(filt_h1, 0);
1515 DUP2_ARG2(__lsx_vreplvei_w, filt_h1, 0, filt_h1, 1, filt_h0, filt_h1);
1517 mask1 = __lsx_vaddi_bu(mask0, 2);
1519 src0 = __lsx_vld(src0_ptr, 0);
1520 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1522 src0_ptr += src_stride_3x;
1528 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, dsth0, dsth1);
1529 dsth2 = __lsx_vdp2_h_bu_b(vec4, filt0);
1530 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dsth0, vec1, filt1, dsth1, vec3, filt1,
1532 dsth2 = __lsx_vdp2add_h_bu_b(dsth2, vec5, filt1);
1534 DUP2_ARG2(__lsx_vilvl_h, dsth1, dsth0, dsth2, dsth1, tmp0, tmp2);
1535 DUP2_ARG2(__lsx_vilvh_h, dsth1, dsth0, dsth2, dsth1, tmp1, tmp3);
1537 src3 = __lsx_vld(src0_ptr, 0);
1538 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1540 src6 = __lsx_vldx(src0_ptr, src_stride_3x);
1541 src0_ptr += src_stride_4x;
1542 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1);
1543 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3);
1544 DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5);
1545 DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7);
1547 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0, vec6,
1548 filt0, dsth3, dsth4, dsth5, dsth6);
1549 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dsth3, vec1, filt1, dsth4, vec3, filt1, dsth5,
1550 vec5, filt1, dsth6, vec7, filt1, dsth3, dsth4, dsth5, dsth6);
1552 src3 = __lsx_vld(src0_ptr, 0);
1553 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1555 src6 = __lsx_vldx(src0_ptr, src_stride_3x);
1557 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec0, vec1);
1558 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec2, vec3);
1559 DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec4, vec5);
1560 DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec6, vec7);
1562 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0, vec6,
1563 filt0, dsth7, dsth8, dsth9, dsth10);
1564 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dsth7, vec1, filt1, dsth8, vec3, filt1, dsth9,
1565 vec5, filt1, dsth10, vec7, filt1, dsth7, dsth8, dsth9, dsth10);
1567 DUP2_ARG2(__lsx_vilvl_h, dsth3, dsth2, dsth4, dsth3, tmp4, tmp6);
1568 DUP2_ARG2(__lsx_vilvh_h, dsth3, dsth2, dsth4, dsth3, tmp5, tmp7);
1569 DUP2_ARG2(__lsx_vilvl_h, dsth5, dsth4, dsth6, dsth5, dsth0, dsth2);
1570 DUP2_ARG2(__lsx_vilvh_h, dsth5, dsth4, dsth6, dsth5, dsth1, dsth3);
1571 DUP4_ARG2(__lsx_vdp2_w_h, tmp0, filt_h0, tmp2, filt_h0, tmp4, filt_h0,
1572 tmp6, filt_h0, dst0_r, dst1_r, dst2_r, dst3_r);
1573 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, tmp4, filt_h1, dst1_r, tmp6,
1574 filt_h1, dst2_r, dsth0, filt_h1, dst3_r, dsth2, filt_h1,
1575 dst0_r, dst1_r, dst2_r, dst3_r);
1576 DUP2_ARG2(__lsx_vpickev_d, tmp3, tmp1, tmp7, tmp5, tmp0, tmp8);
1577 dst0_l = __lsx_vdp2_w_h(tmp0, filt_h0);
1578 dst0_l = __lsx_vdp2add_w_h(dst0_l, tmp8, filt_h1);
1580 DUP2_ARG2(__lsx_vilvl_h, dsth7, dsth6, dsth8, dsth7, tmp0, tmp2);
1581 DUP2_ARG2(__lsx_vilvh_h, dsth7, dsth6, dsth8, dsth7, tmp1, tmp3);
1582 DUP2_ARG2(__lsx_vilvl_h, dsth9, dsth8, dsth10, dsth9, tmp4, tmp6);
1583 DUP2_ARG2(__lsx_vilvh_h, dsth9, dsth8, dsth10, dsth9, tmp5, tmp7);
1584 DUP4_ARG2(__lsx_vdp2_w_h, dsth0, filt_h0, dsth2, filt_h0, tmp0, filt_h0,
1585 tmp2, filt_h0, dst4_r, dst5_r, dst6_r, dst7_r);
1586 DUP4_ARG3(__lsx_vdp2add_w_h, dst4_r, tmp0, filt_h1, dst5_r, tmp2,
1587 filt_h1, dst6_r, tmp4, filt_h1, dst7_r, tmp6, filt_h1,
1588 dst4_r, dst5_r, dst6_r, dst7_r);
1589 DUP2_ARG2(__lsx_vpickev_d, dsth3, dsth1, tmp3, tmp1, tmp0, tmp1);
1590 tmp2 = __lsx_vpickev_d(tmp7, tmp5);
1592 DUP2_ARG2(__lsx_vdp2_w_h, tmp8, filt_h0, tmp0, filt_h0, dst1_l, dst2_l);
1593 dst3_l = __lsx_vdp2_w_h(tmp1, filt_h0);
1594 DUP2_ARG3(__lsx_vdp2add_w_h, dst1_l, tmp0, filt_h1, dst2_l, tmp1, filt_h1,
1596 dst3_l = __lsx_vdp2add_w_h(dst3_l, tmp2, filt_h1);
1598 DUP4_ARG2(__lsx_vsrai_d, dst0_r, 6, dst1_r, 6, dst2_r, 6, dst3_r, 6,
1599 dst0_r, dst1_r, dst2_r, dst3_r);
1600 DUP4_ARG2(__lsx_vsrai_d, dst4_r, 6, dst5_r, 6, dst6_r, 6, dst7_r, 6,
1601 dst4_r, dst5_r, dst6_r, dst7_r);
1602 DUP4_ARG2(__lsx_vsrai_d, dst0_l, 6, dst1_l, 6, dst2_l, 6, dst3_l, 6,
1603 dst0_l, dst1_l, dst2_l, dst3_l);
1604 DUP2_ARG2(__lsx_vpickev_h, dst1_r, dst0_r, dst3_r, dst2_r, tmp0, tmp1);
1605 DUP2_ARG2(__lsx_vpickev_h, dst5_r, dst4_r, dst7_r, dst6_r, tmp2, tmp3);
1606 DUP2_ARG2(__lsx_vpickev_h, dst1_l, dst0_l, dst3_l, dst2_l, tmp4, tmp5);
1608 reg0 = __lsx_vldrepl_d(src1_ptr, 0);
1609 reg1 = __lsx_vldrepl_d(src1_ptr + src2_stride, 0);
1610 dsth0 = __lsx_vilvl_d(reg1, reg0);
1611 reg0 = __lsx_vldrepl_d(src1_ptr + src2_stride_2x, 0);
1612 reg1 = __lsx_vldrepl_d(src1_ptr + src2_stride_3x, 0);
1613 dsth1 = __lsx_vilvl_d(reg1, reg0);
1614 src1_ptr += src2_stride_4x;
1615 reg0 = __lsx_vldrepl_d(src1_ptr, 0);
1616 reg1 = __lsx_vldrepl_d(src1_ptr + src2_stride, 0);
1617 dsth2 = __lsx_vilvl_d(reg1, reg0);
1618 reg0 = __lsx_vldrepl_d(src1_ptr + src2_stride_2x, 0);
1619 reg1 = __lsx_vldrepl_d(src1_ptr + src2_stride_3x, 0);
1620 dsth3 = __lsx_vilvl_d(reg1, reg0);
1622 DUP4_ARG2(__lsx_vsadd_h, dsth0, tmp0, dsth1, tmp1, dsth2, tmp2, dsth3,
1623 tmp3, tmp0, tmp1, tmp2, tmp3);
1624 DUP4_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp2, 0, tmp3, 0,
1625 tmp0, tmp1, tmp2, tmp3);
1626 DUP2_ARG3(__lsx_vssrlrni_bu_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1);
1628 __lsx_vstelm_w(out0,
dst, 0, 0);
1629 __lsx_vstelm_w(out0,
dst + dst_stride, 0, 1);
1630 __lsx_vstelm_w(out0,
dst + dst_stride_2x, 0, 2);
1631 __lsx_vstelm_w(out0,
dst + dst_stride_3x, 0, 3);
1632 dst += dst_stride_4x;
1633 __lsx_vstelm_w(out1,
dst, 0, 0);
1634 __lsx_vstelm_w(out1,
dst + dst_stride, 0, 1);
1635 __lsx_vstelm_w(out1,
dst + dst_stride_2x, 0, 2);
1636 __lsx_vstelm_w(out1,
dst + dst_stride_3x, 0, 3);
1637 dst -= dst_stride_4x;
1639 src1_ptr -= src2_stride_4x;
1641 reg0 = __lsx_vldrepl_w(src1_ptr, 8);
1642 reg1 = __lsx_vldrepl_w(src1_ptr + src2_stride, 8);
1643 reg2 = __lsx_vldrepl_w(src1_ptr + src2_stride_2x, 8);
1644 reg3 = __lsx_vldrepl_w(src1_ptr + src2_stride_3x, 8);
1645 DUP2_ARG2(__lsx_vilvl_w, reg1, reg0, reg3, reg2, tmp0, tmp1);
1646 dsth4 = __lsx_vilvl_d(tmp1, tmp0);
1647 src1_ptr += src2_stride_4x;
1649 reg0 = __lsx_vldrepl_w(src1_ptr, 8);
1650 reg1 = __lsx_vldrepl_w(src1_ptr + src2_stride, 8);
1651 reg2 = __lsx_vldrepl_w(src1_ptr + src2_stride_2x, 8);
1652 reg3 = __lsx_vldrepl_w(src1_ptr + src2_stride_3x, 8);
1653 DUP2_ARG2(__lsx_vilvl_w, reg1, reg0, reg3, reg2, tmp0, tmp1);
1654 dsth5 = __lsx_vilvl_d(tmp1, tmp0);
1655 DUP2_ARG2(__lsx_vsadd_h, dsth4, tmp4, dsth5, tmp5, tmp4, tmp5);
1656 DUP2_ARG2(__lsx_vmaxi_h, tmp4, 0, tmp5, 7, tmp4, tmp5);
1657 out0 = __lsx_vssrlrni_bu_h(tmp5, tmp4, 7);
1659 __lsx_vstelm_h(out0,
dst, 4, 0);
1660 __lsx_vstelm_h(out0,
dst + dst_stride, 4, 1);
1661 __lsx_vstelm_h(out0,
dst + dst_stride_2x, 4, 2);
1662 __lsx_vstelm_h(out0,
dst + dst_stride_3x, 4, 3);
1663 dst += dst_stride_4x;
1664 __lsx_vstelm_h(out0,
dst, 4, 4);
1665 __lsx_vstelm_h(out0,
dst + dst_stride, 4, 5);
1666 __lsx_vstelm_h(out0,
dst + dst_stride_2x, 4, 6);
1667 __lsx_vstelm_h(out0,
dst + dst_stride_3x, 4, 7);
1673 const int8_t *filter_x,
const int8_t *filter_y)
1675 int32_t src_stride_2x = (src_stride << 1);
1676 int32_t src_stride_4x = (src_stride << 2);
1677 int32_t src_stride_3x = src_stride_2x + src_stride;
1681 __m128i filt0, filt1;
1682 __m128i filt_h0, filt_h1;
1684 __m128i mask1, filter_vec;
1685 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
1686 __m128i dst0, dst1, dst2, dst3, dst4;
1687 __m128i dst0_r, dst0_l, dst1_r, dst1_l;
1688 __m128i dst10_r, dst32_r, dst21_r, dst43_r;
1689 __m128i dst10_l, dst32_l, dst21_l, dst43_l;
1693 src0_ptr -= (src_stride + 1);
1694 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
1696 filter_vec = __lsx_vld(filter_y, 0);
1697 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
1698 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
1700 mask1 = __lsx_vaddi_bu(mask0, 2);
1702 src0 = __lsx_vld(src0_ptr, 0);
1703 DUP4_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1704 src0_ptr, src_stride_3x, src0_ptr, src_stride_4x,
1707 DUP2_ARG2(__lsx_vld, src1_ptr, 0, src1_ptr + src2_stride, 0, in0, in1);
1712 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec6, vec7);
1713 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec8, vec9);
1715 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0, vec6,
1716 filt0, dst0, dst1, dst2, dst3);
1717 dst4 = __lsx_vdp2_h_bu_b(vec8, filt0);
1718 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1, dst2,
1719 vec5, filt1, dst3, vec7, filt1, dst0, dst1, dst2, dst3);
1720 dst4 = __lsx_vdp2add_h_bu_b(dst4, vec9, filt1);
1722 DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r);
1723 DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l);
1724 DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r);
1725 DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l);
1726 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
1727 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
1728 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
1729 filt_h1, dst1_r, dst43_r, filt_h1, dst1_l, dst43_l, filt_h1,
1730 dst0_r, dst0_l, dst1_r, dst1_l);
1731 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6,
1732 dst0_r, dst0_l, dst1_r, dst1_l);
1733 DUP2_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, tmp0, tmp1);
1734 DUP2_ARG2(__lsx_vsadd_h, in0, tmp0, in1, tmp1, tmp0, tmp1);
1735 DUP2_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp0, tmp1);
1736 out = __lsx_vssrlrni_bu_h(tmp1, tmp0, 7);
1737 __lsx_vstelm_d(
out,
dst, 0, 0);
1738 __lsx_vstelm_d(
out,
dst + dst_stride, 0, 1);
1743 const int16_t *src1_ptr,
int32_t src2_stride,
1745 const int8_t *filter_x,
const int8_t *filter_y,
1749 int32_t src_stride_2x = (src_stride << 1);
1750 int32_t dst_stride_2x = (dst_stride << 1);
1751 int32_t src_stride_4x = (src_stride << 2);
1752 int32_t src2_stride_x = (src2_stride << 1);
1753 int32_t src2_stride_2x = (src2_stride << 2);
1754 int32_t src_stride_3x = src_stride_2x + src_stride;
1755 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
1756 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
1759 __m128i
src0,
src1,
src2, src3, src4, src5, src6, mask0, mask1;
1760 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1761 __m128i filt0, filt1, filt_h0, filt_h1, filter_vec;
1762 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, tmp0, tmp1, tmp2, tmp3;
1763 __m128i in0, in1, in2, in3;
1764 __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
1765 __m128i dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
1766 __m128i dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
1768 src0_ptr -= (src_stride + 1);
1769 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
1771 filter_vec = __lsx_vld(filter_y, 0);
1772 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
1773 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
1776 mask1 = __lsx_vaddi_bu(mask0, 2);
1778 for (cnt = width8mult; cnt--;) {
1779 src0 = __lsx_vld(src0_ptr, 0);
1780 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1782 src3 = __lsx_vldx(src0_ptr, src_stride_3x);
1783 src0_ptr += src_stride_4x;
1784 src4 = __lsx_vld(src0_ptr, 0);
1785 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1787 src0_ptr += (8 - src_stride_4x);
1789 in0 = __lsx_vld(src1_ptr, 0);
1790 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr,
1791 src2_stride_2x, in1, in2);
1792 in3 = __lsx_vldx(src1_ptr, src2_stride_3x);
1802 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, dst0, dst1);
1803 dst2 = __lsx_vdp2_h_bu_b(vec4, filt0);
1804 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1,
1806 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec5, filt1);
1808 DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r);
1809 DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l);
1811 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1,
1813 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1,
1815 DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1,
1817 DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1,
1820 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0,
1821 vec6, filt0, dst3, dst4, dst5, dst6);
1822 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst4, vec3, filt1,
1823 dst5, vec5, filt1, dst6, vec7, filt1, dst3, dst4, dst5, dst6);
1825 DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r);
1826 DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l);
1827 DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r);
1828 DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l);
1830 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
1831 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
1832 DUP4_ARG2(__lsx_vdp2_w_h, dst32_r, filt_h0, dst32_l, filt_h0, dst43_r,
1833 filt_h0, dst43_l, filt_h0, dst2_r, dst2_l, dst3_r, dst3_l);
1834 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
1835 filt_h1, dst1_r, dst43_r, filt_h1, dst1_l, dst43_l, filt_h1,
1836 dst0_r, dst0_l, dst1_r, dst1_l);
1837 DUP4_ARG3(__lsx_vdp2add_w_h, dst2_r, dst54_r, filt_h1, dst2_l, dst54_l,
1838 filt_h1, dst3_r, dst65_r, filt_h1, dst3_l, dst65_l, filt_h1,
1839 dst2_r, dst2_l, dst3_r, dst3_l);
1841 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6,
1842 dst0_r, dst0_l, dst1_r, dst1_l);
1843 DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6,
1844 dst2_r, dst2_l, dst3_r, dst3_l);
1845 DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l,
1846 dst2_r, dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3);
1847 DUP4_ARG2(__lsx_vsadd_h, in0, tmp0, in1, tmp1, in2, tmp2, in3, tmp3,
1848 tmp0, tmp1, tmp2, tmp3);
1849 DUP4_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp2, 0, tmp3, 0,
1850 tmp0, tmp1, tmp2, tmp3);
1851 DUP2_ARG3(__lsx_vssrlrni_bu_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1);
1852 __lsx_vstelm_d(out0,
dst, 0, 0);
1853 __lsx_vstelm_d(out0,
dst + dst_stride, 0, 1);
1854 __lsx_vstelm_d(out1,
dst + dst_stride_2x, 0, 0);
1855 __lsx_vstelm_d(out1,
dst + dst_stride_3x, 0, 1);
1863 const int8_t *filter_x,
const int8_t *filter_y)
1865 int32_t src_stride_2x = (src_stride << 1);
1866 int32_t dst_stride_2x = (dst_stride << 1);
1867 int32_t src_stride_4x = (src_stride << 2);
1868 int32_t dst_stride_4x = (dst_stride << 2);
1869 int32_t src2_stride_x = (src2_stride << 1);
1870 int32_t src2_stride_2x = (src2_stride << 2);
1871 int32_t src_stride_3x = src_stride_2x + src_stride;
1872 int32_t dst_stride_3x = dst_stride_2x + dst_stride;
1873 int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
1875 __m128i out0, out1, out2;
1876 __m128i
src0,
src1,
src2, src3, src4, src5, src6, src7, src8;
1877 __m128i in0, in1, in2, in3, in4, in5;
1878 __m128i filt0, filt1;
1879 __m128i filt_h0, filt_h1;
1881 __m128i mask1, filter_vec;
1882 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
1883 __m128i vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17;
1884 __m128i tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
1885 __m128i dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
1886 __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
1887 __m128i dst4_r, dst4_l, dst5_r, dst5_l;
1888 __m128i dst10_r, dst32_r, dst10_l, dst32_l;
1889 __m128i dst21_r, dst43_r, dst21_l, dst43_l;
1890 __m128i dst54_r, dst54_l, dst65_r, dst65_l;
1891 __m128i dst76_r, dst76_l, dst87_r, dst87_l;
1893 src0_ptr -= (src_stride + 1);
1894 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
1896 filter_vec = __lsx_vld(filter_y, 0);
1897 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
1898 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
1900 mask1 = __lsx_vaddi_bu(mask0, 2);
1902 src0 = __lsx_vld(src0_ptr, 0);
1903 DUP2_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1905 src3 = __lsx_vldx(src0_ptr, src_stride_3x);
1906 src0_ptr += src_stride_4x;
1907 src4 = __lsx_vld(src0_ptr, 0);
1908 DUP4_ARG2(__lsx_vldx, src0_ptr, src_stride, src0_ptr, src_stride_2x,
1909 src0_ptr, src_stride_3x, src0_ptr, src_stride_4x,
1910 src5, src6, src7, src8);
1912 in0 = __lsx_vld(src1_ptr, 0);
1913 DUP2_ARG2(__lsx_vldx, src1_ptr, src2_stride_x, src1_ptr, src2_stride_2x,
1915 in3 = __lsx_vldx(src1_ptr, src2_stride_3x);
1916 src1_ptr += src2_stride_2x;
1917 in4 = __lsx_vld(src1_ptr, 0);
1918 in5 = __lsx_vldx(src1_ptr, src2_stride_x);
1923 DUP2_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, vec6, vec7);
1924 DUP2_ARG3(__lsx_vshuf_b, src4, src4, mask0, src4, src4, mask1, vec8, vec9);
1925 DUP2_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, vec10, vec11);
1926 DUP2_ARG3(__lsx_vshuf_b, src6, src6, mask0, src6, src6, mask1, vec12, vec13);
1927 DUP2_ARG3(__lsx_vshuf_b, src7, src7, mask0, src7, src7, mask1, vec14, vec15);
1928 DUP2_ARG3(__lsx_vshuf_b, src8, src8, mask0, src8, src8, mask1, vec16, vec17);
1930 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0, vec6,
1931 filt0, dst0, dst1, dst2, dst3);
1932 dst4 = __lsx_vdp2_h_bu_b(vec8, filt0);
1933 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec10, filt0, vec12, filt0, vec14, filt0,
1934 vec16, filt0, dst5, dst6, dst7, dst8);
1935 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1, dst2,
1936 vec5, filt1, dst3, vec7, filt1, dst0, dst1, dst2, dst3);
1937 dst4 = __lsx_vdp2add_h_bu_b(dst4, vec9, filt1);
1938 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst5, vec11, filt1, dst6, vec13, filt1,
1939 dst7, vec15, filt1, dst8, vec17, filt1, dst5, dst6, dst7, dst8);
1941 DUP4_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3,
1942 dst10_r, dst21_r, dst32_r, dst43_r);
1943 DUP4_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst3, dst2, dst4, dst3,
1944 dst10_l, dst21_l, dst32_l, dst43_l);
1945 DUP4_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7,
1946 dst54_r, dst65_r, dst76_r, dst87_r);
1947 DUP4_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst7, dst6, dst8, dst7,
1948 dst54_l, dst65_l, dst76_l, dst87_l);
1950 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
1951 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
1952 DUP4_ARG2(__lsx_vdp2_w_h, dst32_r, filt_h0, dst32_l, filt_h0, dst43_r,
1953 filt_h0, dst43_l, filt_h0, dst2_r, dst2_l, dst3_r, dst3_l);
1954 DUP4_ARG2(__lsx_vdp2_w_h, dst54_r, filt_h0, dst54_l, filt_h0, dst65_r,
1955 filt_h0, dst65_l, filt_h0, dst4_r, dst4_l, dst5_r, dst5_l);
1956 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l, dst32_l,
1957 filt_h1, dst1_r, dst43_r, filt_h1, dst1_l, dst43_l, filt_h1,
1958 dst0_r, dst0_l, dst1_r, dst1_l);
1959 DUP4_ARG3(__lsx_vdp2add_w_h, dst2_r, dst54_r, filt_h1, dst2_l, dst54_l,
1960 filt_h1, dst3_r, dst65_r, filt_h1, dst3_l, dst65_l, filt_h1,
1961 dst2_r, dst2_l, dst3_r, dst3_l);
1962 DUP4_ARG3(__lsx_vdp2add_w_h, dst4_r, dst76_r, filt_h1, dst4_l, dst76_l,
1963 filt_h1, dst5_r, dst87_r, filt_h1, dst5_l, dst87_l, filt_h1,
1964 dst4_r, dst4_l, dst5_r, dst5_l);
1966 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6,
1967 dst0_r, dst0_l, dst1_r, dst1_l);
1968 DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6,
1969 dst2_r, dst2_l, dst3_r, dst3_l);
1970 DUP4_ARG2(__lsx_vsrai_w, dst4_r, 6, dst4_l, 6, dst5_r, 6, dst5_l, 6,
1971 dst4_r, dst4_l, dst5_r, dst5_l);
1972 DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r,
1973 dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3);
1974 DUP2_ARG2(__lsx_vpickev_h, dst4_l, dst4_r, dst5_l, dst5_r, tmp4, tmp5);
1975 DUP4_ARG2(__lsx_vsadd_h, in0, tmp0, in1, tmp1, in2, tmp2, in3, tmp3,
1976 tmp0, tmp1, tmp2, tmp3);
1977 DUP2_ARG2(__lsx_vsadd_h, in4, tmp4, in5, tmp5, tmp4, tmp5);
1978 DUP4_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp2, 0, tmp3, 0,
1979 tmp0, tmp1, tmp2, tmp3);
1980 DUP2_ARG2(__lsx_vmaxi_h, tmp4, 0, tmp5, 0, tmp4, tmp5);
1981 DUP2_ARG3(__lsx_vssrlrni_bu_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1);
1982 out2 = __lsx_vssrlrni_bu_h(tmp5, tmp4, 7);
1983 __lsx_vstelm_d(out0,
dst, 0, 0);
1984 __lsx_vstelm_d(out0,
dst + dst_stride, 0, 1);
1985 __lsx_vstelm_d(out1,
dst + dst_stride_2x, 0, 0);
1986 __lsx_vstelm_d(out1,
dst + dst_stride_3x, 0, 1);
1987 dst += dst_stride_4x;
1988 __lsx_vstelm_d(out2,
dst, 0, 0);
1989 __lsx_vstelm_d(out2,
dst + dst_stride, 0, 1);
1994 const int16_t *src1_ptr,
int32_t src2_stride,
1996 const int8_t *filter_x,
const int8_t *filter_y,
1999 uint32_t loop_cnt, cnt;
2000 const uint8_t *src0_ptr_tmp;
2001 const int16_t *src1_ptr_tmp;
2003 const int32_t src_stride_2x = (src_stride << 1);
2004 const int32_t dst_stride_2x = (dst_stride << 1);
2005 const int32_t src_stride_4x = (src_stride << 2);
2006 const int32_t dst_stride_4x = (dst_stride << 2);
2007 const int32_t src2_stride_x = (src2_stride << 1);
2008 const int32_t src2_stride_2x = (src2_stride << 2);
2009 const int32_t src_stride_3x = src_stride_2x + src_stride;
2010 const int32_t dst_stride_3x = dst_stride_2x + dst_stride;
2011 const int32_t src2_stride_3x = src2_stride_2x + src2_stride_x;
2014 __m128i in0, in1, in2, in3;
2015 __m128i filt0, filt1;
2016 __m128i filt_h0, filt_h1;
2018 __m128i mask1, filter_vec;
2019 __m128i vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2020 __m128i dst0, dst1, dst2, dst3, dst4, dst5;
2021 __m128i dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
2022 __m128i tmp0, tmp1, tmp2, tmp3;
2023 __m128i dst10_r, dst32_r, dst21_r, dst43_r;
2024 __m128i dst10_l, dst32_l, dst21_l, dst43_l;
2025 __m128i dst54_r, dst54_l, dst65_r, dst65_l, dst6;
2027 src0_ptr -= (src_stride + 1);
2029 DUP2_ARG2(__lsx_vldrepl_h, filter_x, 0, filter_x, 2, filt0, filt1);
2031 filter_vec = __lsx_vld(filter_y, 0);
2032 filter_vec = __lsx_vsllwil_h_b(filter_vec, 0);
2034 DUP2_ARG2(__lsx_vreplvei_w, filter_vec, 0, filter_vec, 1, filt_h0, filt_h1);
2036 mask1 = __lsx_vaddi_bu(mask0, 2);
2038 for (cnt =
width >> 3; cnt--;) {
2039 src0_ptr_tmp = src0_ptr;
2041 src1_ptr_tmp = src1_ptr;
2043 src0 = __lsx_vld(src0_ptr_tmp, 0);
2044 DUP2_ARG2(__lsx_vldx, src0_ptr_tmp, src_stride, src0_ptr_tmp,
2046 src0_ptr_tmp += src_stride_3x;
2055 DUP2_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, dst0, dst1);
2056 dst2 = __lsx_vdp2_h_bu_b(vec4, filt0);
2057 DUP2_ARG3(__lsx_vdp2add_h_bu_b, dst0, vec1, filt1, dst1, vec3, filt1,
2059 dst2 = __lsx_vdp2add_h_bu_b(dst2, vec5, filt1);
2061 DUP2_ARG2(__lsx_vilvl_h, dst1, dst0, dst2, dst1, dst10_r, dst21_r);
2062 DUP2_ARG2(__lsx_vilvh_h, dst1, dst0, dst2, dst1, dst10_l, dst21_l);
2064 for (loop_cnt =
height >> 2; loop_cnt--;) {
2065 src3 = __lsx_vld(src0_ptr_tmp, 0);
2066 DUP2_ARG2(__lsx_vldx, src0_ptr_tmp, src_stride, src0_ptr_tmp,
2067 src_stride_2x, src4, src5);
2068 src6 = __lsx_vldx(src0_ptr_tmp, src_stride_3x);
2069 src0_ptr_tmp += src_stride_4x;
2070 in0 = __lsx_vld(src1_ptr_tmp, 0);
2071 DUP2_ARG2(__lsx_vldx, src1_ptr_tmp, src2_stride_x, src1_ptr_tmp,
2072 src2_stride_2x, in1, in2);
2073 in3 = __lsx_vldx(src1_ptr_tmp, src2_stride_3x);
2074 src1_ptr_tmp += src2_stride_2x;
2076 DUP4_ARG3(__lsx_vshuf_b, src3, src3, mask0, src3, src3, mask1, src4,
2077 src4, mask0, src4, src4, mask1, vec0, vec1, vec2, vec3);
2078 DUP4_ARG3(__lsx_vshuf_b, src5, src5, mask0, src5, src5, mask1, src6,
2079 src6, mask0, src6, src6, mask1, vec4, vec5, vec6, vec7);
2081 DUP4_ARG2(__lsx_vdp2_h_bu_b, vec0, filt0, vec2, filt0, vec4, filt0,
2082 vec6, filt0, dst3, dst4, dst5, dst6);
2083 DUP4_ARG3(__lsx_vdp2add_h_bu_b, dst3, vec1, filt1, dst4, vec3,
2084 filt1, dst5, vec5, filt1, dst6, vec7, filt1,
2085 dst3, dst4, dst5, dst6);
2087 DUP2_ARG2(__lsx_vilvl_h, dst3, dst2, dst4, dst3, dst32_r, dst43_r);
2088 DUP2_ARG2(__lsx_vilvh_h, dst3, dst2, dst4, dst3, dst32_l, dst43_l);
2089 DUP2_ARG2(__lsx_vilvl_h, dst5, dst4, dst6, dst5, dst54_r, dst65_r);
2090 DUP2_ARG2(__lsx_vilvh_h, dst5, dst4, dst6, dst5, dst54_l, dst65_l);
2092 DUP4_ARG2(__lsx_vdp2_w_h, dst10_r, filt_h0, dst10_l, filt_h0, dst21_r,
2093 filt_h0, dst21_l, filt_h0, dst0_r, dst0_l, dst1_r, dst1_l);
2094 DUP4_ARG2(__lsx_vdp2_w_h, dst32_r, filt_h0, dst32_l, filt_h0, dst43_r,
2095 filt_h0, dst43_l, filt_h0, dst2_r, dst2_l, dst3_r, dst3_l);
2096 DUP4_ARG3(__lsx_vdp2add_w_h, dst0_r, dst32_r, filt_h1, dst0_l,
2097 dst32_l, filt_h1, dst1_r, dst43_r, filt_h1, dst1_l,
2098 dst43_l, filt_h1, dst0_r, dst0_l, dst1_r, dst1_l);
2099 DUP4_ARG3(__lsx_vdp2add_w_h, dst2_r, dst54_r, filt_h1, dst2_l,
2100 dst54_l, filt_h1, dst3_r, dst65_r, filt_h1, dst3_l,
2101 dst65_l, filt_h1, dst2_r, dst2_l, dst3_r, dst3_l);
2103 DUP4_ARG2(__lsx_vsrai_w, dst0_r, 6, dst0_l, 6, dst1_r, 6, dst1_l, 6,
2104 dst0_r, dst0_l, dst1_r, dst1_l);
2105 DUP4_ARG2(__lsx_vsrai_w, dst2_r, 6, dst2_l, 6, dst3_r, 6, dst3_l, 6,
2106 dst2_r, dst2_l, dst3_r, dst3_l);
2107 DUP4_ARG2(__lsx_vpickev_h, dst0_l, dst0_r, dst1_l, dst1_r, dst2_l,
2108 dst2_r, dst3_l, dst3_r, tmp0, tmp1, tmp2, tmp3);
2109 DUP4_ARG2(__lsx_vsadd_h, in0, tmp0, in1, tmp1, in2, tmp2, in3, tmp3,
2110 tmp0, tmp1, tmp2, tmp3);
2111 DUP4_ARG2(__lsx_vmaxi_h, tmp0, 0, tmp1, 0, tmp2, 0, tmp3, 0, tmp0,
2113 DUP2_ARG3(__lsx_vssrlrni_bu_h, tmp1, tmp0, 7, tmp3, tmp2, 7, out0, out1);
2114 __lsx_vstelm_d(out0, dst_tmp, 0, 0);
2115 __lsx_vstelm_d(out0, dst_tmp + dst_stride, 0, 1);
2116 __lsx_vstelm_d(out1, dst_tmp + dst_stride_2x, 0, 0);
2117 __lsx_vstelm_d(out1, dst_tmp + dst_stride_3x, 0, 1);
2118 dst_tmp += dst_stride_4x;
2134 const int16_t *src1_ptr,
int32_t src2_stride,
2136 const int8_t *filter_x,
const int8_t *filter_y,
2141 dst, dst_stride, filter_x, filter_y);
2142 }
else if (4 ==
height) {
2144 dst, dst_stride, filter_x, filter_y, 1);
2145 }
else if (6 ==
height) {
2147 dst, dst_stride, filter_x, filter_y);
2150 dst, dst_stride, filter_x, filter_y,
height, 8);
2155 const int16_t *src1_ptr,
int32_t src2_stride,
2157 const int8_t *filter_x,
const int8_t *filter_y,
2162 dst, dst_stride, filter_x, filter_y, 2);
2165 dst, dst_stride, filter_x, filter_y,
height, 16);
2170 const int16_t *src1_ptr,
int32_t src2_stride,
2172 const int8_t *filter_x,
const int8_t *filter_y,
2176 dst, dst_stride, filter_x, filter_y,
height, 24);
2180 const int16_t *src1_ptr,
int32_t src2_stride,
2182 const int8_t *filter_x,
const int8_t *filter_y,
2186 dst, dst_stride, filter_x, filter_y,
height, 32);
2189 #define BI_MC_COPY(WIDTH) \
2190 void ff_hevc_put_hevc_bi_pel_pixels##WIDTH##_8_lsx(uint8_t *dst, \
2191 ptrdiff_t dst_stride, \
2192 const uint8_t *src, \
2193 ptrdiff_t src_stride, \
2194 const int16_t *src_16bit, \
2200 hevc_bi_copy_##WIDTH##w_lsx(src, src_stride, src_16bit, MAX_PB_SIZE, \
2201 dst, dst_stride, height); \
2216 #define BI_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \
2217 void ff_hevc_put_hevc_bi_##PEL##_##DIR##WIDTH##_8_lsx(uint8_t *dst, \
2218 ptrdiff_t dst_stride, \
2219 const uint8_t *src, \
2220 ptrdiff_t src_stride, \
2221 const int16_t *src_16bit, \
2227 const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR]; \
2229 hevc_##DIR1##_##TAP##t_##WIDTH##w_lsx(src, src_stride, src_16bit, \
2230 MAX_PB_SIZE, dst, dst_stride, \
2241 BI_MC(qpel, v, 16, 8, vt,
my);
2242 BI_MC(qpel, v, 24, 8, vt,
my);
2243 BI_MC(qpel, v, 32, 8, vt,
my);
2244 BI_MC(qpel, v, 48, 8, vt,
my);
2245 BI_MC(qpel, v, 64, 8, vt,
my);
2250 BI_MC(epel, v, 12, 4, vt,
my);
2251 BI_MC(epel, v, 16, 4, vt,
my);
2252 BI_MC(epel, v, 24, 4, vt,
my);
2253 BI_MC(epel, v, 32, 4, vt,
my);
2257 #define BI_MC_HV(PEL, WIDTH, TAP) \
2258 void ff_hevc_put_hevc_bi_##PEL##_hv##WIDTH##_8_lsx(uint8_t *dst, \
2259 ptrdiff_t dst_stride, \
2260 const uint8_t *src, \
2261 ptrdiff_t src_stride, \
2262 const int16_t *src_16bit, \
2268 const int8_t *filter_x = ff_hevc_##PEL##_filters[mx]; \
2269 const int8_t *filter_y = ff_hevc_##PEL##_filters[my]; \
2271 hevc_hv_##TAP##t_##WIDTH##w_lsx(src, src_stride, src_16bit, \
2272 MAX_PB_SIZE, dst, dst_stride, \
2273 filter_x, filter_y, height); \