FFmpeg
Loading...
Searching...
No Matches
hevc_mc_uniw_msa.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2015 - 2017 Manojkumar Bhosale (Manojkumar.Bhosale@imgtec.com)
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21#include "libavutil/mips/generic_macros_msa.h"
22#include "libavcodec/mips/hevcdsp_mips.h"
23#include "libavcodec/mips/hevc_macros_msa.h"
24
25static const uint8_t ff_hevc_mask_arr[16 * 2] __attribute__((aligned(0x40))) = {
26 /* 8 width cases */
27 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
28 /* 4 width cases */
29 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20
30};
31
32#define HEVC_UNIW_RND_CLIP2_MAX_SATU_H(in0_h, in1_h, wgt_w, offset_h, rnd_w, \
33 out0_h, out1_h) \
34{ \
35 v4i32 in0_r_m, in0_l_m, in1_r_m, in1_l_m; \
36 v8i16 zero = { 0 }; \
37 \
38 ILVRL_H2_SW(zero, in0_h, in0_r_m, in0_l_m); \
39 ILVRL_H2_SW(zero, in1_h, in1_r_m, in1_l_m); \
40 MUL4(in0_r_m, wgt_w, in0_l_m, wgt_w, in1_r_m, wgt_w, in1_l_m, wgt_w, \
41 in0_r_m, in0_l_m, in1_r_m, in1_l_m); \
42 SRAR_W4_SW(in0_r_m, in0_l_m, in1_r_m, in1_l_m, rnd_w); \
43 ADD4(in0_r_m, offset_h, in0_l_m, offset_h, in1_r_m, offset_h, in1_l_m, \
44 offset_h, in0_r_m, in0_l_m, in1_r_m, in1_l_m); \
45 CLIP_SW4_0_255(in0_r_m, in0_l_m, in1_r_m, in1_l_m); \
46 PCKEV_H2_SH(in0_l_m, in0_r_m, in1_l_m, in1_r_m, out0_h, out1_h); \
47}
48
49#define HEVC_UNIW_RND_CLIP4_MAX_SATU_H(in0_h, in1_h, in2_h, in3_h, wgt_w, \
50 offset_h, rnd_w, out0_h, out1_h, \
51 out2_h, out3_h) \
52{ \
53 HEVC_UNIW_RND_CLIP2_MAX_SATU_H(in0_h, in1_h, wgt_w, offset_h, rnd_w, \
54 out0_h, out1_h); \
55 HEVC_UNIW_RND_CLIP2_MAX_SATU_H(in2_h, in3_h, wgt_w, offset_h, rnd_w, \
56 out2_h, out3_h); \
57}
58
59#define HEVC_FILT_8TAP_4W_SH(in0, in1, in2, in3, filt0, filt1, \
60 filt2, filt3, dst0, dst1) \
61{ \
62 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7; \
63 ILVRL_B2_SH(zero, in0, tmp0, tmp4); \
64 ILVRL_B2_SH(zero, in1, tmp1, tmp5); \
65 ILVRL_B2_SH(zero, in2, tmp2, tmp6); \
66 ILVRL_B2_SH(zero, in3, tmp3, tmp7); \
67 dst0 = __msa_dotp_s_w((v8i16) tmp0, (v8i16) filt0); \
68 dst1 = __msa_dotp_s_w((v8i16) tmp4, (v8i16) filt0); \
69 DPADD_SH2_SW(tmp1, tmp5, filt1, filt1, dst0, dst1); \
70 DPADD_SH2_SW(tmp2, tmp6, filt2, filt2, dst0, dst1); \
71 DPADD_SH2_SW(tmp3, tmp7, filt3, filt3, dst0, dst1); \
72}
73
74static void hevc_uniwgt_copy_4w_msa(const uint8_t *src,
75 int32_t src_stride,
76 uint8_t *dst,
77 int32_t dst_stride,
78 int32_t height,
79 int32_t weight,
80 int32_t offset,
81 int32_t rnd_val)
82{
83 uint32_t loop_cnt, tp0, tp1, tp2, tp3;
84 v16i8 zero = { 0 };
85 v16u8 out0, out1;
86 v16i8 src0 = { 0 }, src1 = { 0 };
87 v8i16 dst0, dst1, dst2, dst3, offset_vec;
88 v4i32 weight_vec, rnd_vec;
89
90 weight_vec = __msa_fill_w(weight);
91 offset_vec = __msa_fill_w(offset);
92 rnd_vec = __msa_fill_w(rnd_val);
93
94 if (2 == height) {
95 v4i32 dst0_r, dst0_l;
96
97 LW2(src, src_stride, tp0, tp1);
98 INSERT_W2_SB(tp0, tp1, src0);
99 dst0 = (v8i16) __msa_ilvr_b(zero, src0);
100 dst0 <<= 6;
101
102 ILVRL_H2_SW(zero, dst0, dst0_r, dst0_l);
103 DOTP_SH2_SW(dst0_r, dst0_l, weight_vec, weight_vec, dst0_r, dst0_l);
104 SRAR_W2_SW(dst0_r, dst0_l, rnd_vec);
105 dst0_r += offset_vec;
106 dst0_l += offset_vec;
107 CLIP_SW2_0_255(dst0_r, dst0_l);
108 dst0 = __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
109 out0 = (v16u8) __msa_pckev_b((v16i8) dst0, (v16i8) dst0);
110 ST_W2(out0, 0, 1, dst, dst_stride);
111 } else if (4 == height) {
112 LW4(src, src_stride, tp0, tp1, tp2, tp3);
113 INSERT_W4_SB(tp0, tp1, tp2, tp3, src0);
114 ILVRL_B2_SH(zero, src0, dst0, dst1);
115 SLLI_2V(dst0, dst1, 6);
116 HEVC_UNIW_RND_CLIP2_MAX_SATU_H(dst0, dst1, weight_vec, offset_vec,
117 rnd_vec, dst0, dst1);
118 out0 = (v16u8) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
119 ST_W4(out0, 0, 1, 2, 3, dst, dst_stride);
120 } else if (0 == (height % 8)) {
121 for (loop_cnt = (height >> 3); loop_cnt--;) {
122 LW4(src, src_stride, tp0, tp1, tp2, tp3);
123 src += 4 * src_stride;
124 INSERT_W4_SB(tp0, tp1, tp2, tp3, src0);
125 LW4(src, src_stride, tp0, tp1, tp2, tp3);
126 src += 4 * src_stride;
127 INSERT_W4_SB(tp0, tp1, tp2, tp3, src1);
128 ILVRL_B2_SH(zero, src0, dst0, dst1);
129 ILVRL_B2_SH(zero, src1, dst2, dst3);
130 SLLI_4V(dst0, dst1, dst2, dst3, 6);
131 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
132 offset_vec, rnd_vec, dst0, dst1,
133 dst2, dst3);
134 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
135 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
136 dst += 8 * dst_stride;
137 }
138 }
139}
140
141static void hevc_uniwgt_copy_6w_msa(const uint8_t *src,
142 int32_t src_stride,
143 uint8_t *dst,
144 int32_t dst_stride,
145 int32_t height,
146 int32_t weight,
147 int32_t offset,
148 int32_t rnd_val)
149{
150 uint32_t loop_cnt;
151 int32_t res = height & 0x07;
152 uint64_t tp0, tp1, tp2, tp3;
153 v16i8 zero = { 0 };
154 v16u8 out0, out1, out2, out3;
155 v16i8 src0, src1, src2, src3;
156 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, offset_vec;
157 v4i32 weight_vec, rnd_vec;
158
159 weight_vec = __msa_fill_w(weight);
160 offset_vec = __msa_fill_w(offset);
161 rnd_vec = __msa_fill_w(rnd_val);
162
163 for (loop_cnt = (height >> 3); loop_cnt--;) {
164 LD4(src, src_stride, tp0, tp1, tp2, tp3);
165 src += (4 * src_stride);
166 INSERT_D2_SB(tp0, tp1, src0);
167 INSERT_D2_SB(tp2, tp3, src1);
168 LD4(src, src_stride, tp0, tp1, tp2, tp3);
169 src += (4 * src_stride);
170 INSERT_D2_SB(tp0, tp1, src2);
171 INSERT_D2_SB(tp2, tp3, src3);
172
173 ILVRL_B2_SH(zero, src0, dst0, dst1);
174 ILVRL_B2_SH(zero, src1, dst2, dst3);
175 ILVRL_B2_SH(zero, src2, dst4, dst5);
176 ILVRL_B2_SH(zero, src3, dst6, dst7);
177
178 SLLI_4V(dst0, dst1, dst2, dst3, 6);
179 SLLI_4V(dst4, dst5, dst6, dst7, 6);
180
181 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
182 offset_vec, rnd_vec, dst0, dst1, dst2,
183 dst3);
184 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst4, dst5, dst6, dst7, weight_vec,
185 offset_vec, rnd_vec, dst4, dst5, dst6,
186 dst7);
187 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
188 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
189
190 ST_W2(out0, 0, 2, dst, dst_stride);
191 ST_H2(out0, 2, 6, dst + 4, dst_stride);
192 ST_W2(out1, 0, 2, dst + 2 * dst_stride, dst_stride);
193 ST_H2(out1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
194 dst += (4 * dst_stride);
195 ST_W2(out2, 0, 2, dst, dst_stride);
196 ST_H2(out2, 2, 6, dst + 4, dst_stride);
197 ST_W2(out3, 0, 2, dst + 2 * dst_stride, dst_stride);
198 ST_H2(out3, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
199 dst += (4 * dst_stride);
200 }
201 if (res) {
202 LD4(src, src_stride, tp0, tp1, tp2, tp3);
203 src += (4 * src_stride);
204 INSERT_D2_SB(tp0, tp1, src0);
205 INSERT_D2_SB(tp2, tp3, src1);
206 LD4(src, src_stride, tp0, tp1, tp2, tp3);
207 src += (4 * src_stride);
208 INSERT_D2_SB(tp0, tp1, src2);
209 INSERT_D2_SB(tp2, tp3, src3);
210
211 ILVRL_B2_SH(zero, src0, dst0, dst1);
212 ILVRL_B2_SH(zero, src1, dst2, dst3);
213 ILVRL_B2_SH(zero, src2, dst4, dst5);
214 ILVRL_B2_SH(zero, src3, dst6, dst7);
215
216 SLLI_4V(dst0, dst1, dst2, dst3, 6);
217 SLLI_4V(dst4, dst5, dst6, dst7, 6);
218
219 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
220 offset_vec, rnd_vec, dst0, dst1, dst2,
221 dst3);
222 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst4, dst5, dst6, dst7, weight_vec,
223 offset_vec, rnd_vec, dst4, dst5, dst6,
224 dst7);
225 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
226 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
227
228 if (res == 2) {
229 ST_W2(out0, 0, 2, dst, dst_stride);
230 ST_H2(out0, 2, 6, dst + 4, dst_stride);
231 } else if (res == 4) {
232 ST_W2(out0, 0, 2, dst, dst_stride);
233 ST_H2(out0, 2, 6, dst + 4, dst_stride);
234 ST_W2(out1, 0, 2, dst + 2 * dst_stride, dst_stride);
235 ST_H2(out1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
236 } else {
237 ST_W2(out0, 0, 2, dst, dst_stride);
238 ST_H2(out0, 2, 6, dst + 4, dst_stride);
239 ST_W2(out1, 0, 2, dst + 2 * dst_stride, dst_stride);
240 ST_H2(out1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
241 dst += (4 * dst_stride);
242 ST_W2(out2, 0, 2, dst, dst_stride);
243 ST_H2(out2, 2, 6, dst + 4, dst_stride);
244 }
245 }
246}
247
248static void hevc_uniwgt_copy_8w_msa(const uint8_t *src,
249 int32_t src_stride,
250 uint8_t *dst,
251 int32_t dst_stride,
252 int32_t height,
253 int32_t weight,
254 int32_t offset,
255 int32_t rnd_val)
256{
257 uint32_t loop_cnt;
258 uint64_t tp0, tp1, tp2, tp3;
259 v16i8 src0 = { 0 }, src1 = { 0 }, src2 = { 0 }, src3 = { 0 };
260 v16i8 zero = { 0 };
261 v16u8 out0, out1, out2, out3;
262 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, offset_vec;
263 v4i32 weight_vec, rnd_vec;
264
265 weight_vec = __msa_fill_w(weight);
266 offset_vec = __msa_fill_w(offset);
267 rnd_vec = __msa_fill_w(rnd_val);
268
269 if (2 == height) {
270 LD2(src, src_stride, tp0, tp1);
271 INSERT_D2_SB(tp0, tp1, src0);
272 ILVRL_B2_SH(zero, src0, dst0, dst1);
273 SLLI_2V(dst0, dst1, 6);
274 HEVC_UNIW_RND_CLIP2_MAX_SATU_H(dst0, dst1, weight_vec, offset_vec,
275 rnd_vec, dst0, dst1);
276 out0 = (v16u8) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
277 ST_D2(out0, 0, 1, dst, dst_stride);
278 } else if (4 == height) {
279 LD4(src, src_stride, tp0, tp1, tp2, tp3);
280 INSERT_D2_SB(tp0, tp1, src0);
281 INSERT_D2_SB(tp2, tp3, src1);
282 ILVRL_B2_SH(zero, src0, dst0, dst1);
283 ILVRL_B2_SH(zero, src1, dst2, dst3);
284 SLLI_4V(dst0, dst1, dst2, dst3, 6);
285 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
286 offset_vec, rnd_vec, dst0, dst1, dst2,
287 dst3);
288 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
289 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
290 } else if (6 == height) {
291 LD4(src, src_stride, tp0, tp1, tp2, tp3);
292 src += 4 * src_stride;
293 INSERT_D2_SB(tp0, tp1, src0);
294 INSERT_D2_SB(tp2, tp3, src1);
295 LD2(src, src_stride, tp0, tp1);
296 INSERT_D2_SB(tp0, tp1, src2);
297 ILVRL_B2_SH(zero, src0, dst0, dst1);
298 ILVRL_B2_SH(zero, src1, dst2, dst3);
299 ILVRL_B2_SH(zero, src2, dst4, dst5);
300 SLLI_4V(dst0, dst1, dst2, dst3, 6);
301 SLLI_2V(dst4, dst5, 6);
302 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
303 offset_vec, rnd_vec, dst0, dst1, dst2,
304 dst3);
305 HEVC_UNIW_RND_CLIP2_MAX_SATU_H(dst4, dst5, weight_vec, offset_vec,
306 rnd_vec, dst4, dst5);
307 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
308 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
309 ST_D2(out2, 0, 1, dst + 4 * dst_stride, dst_stride);
310 } else if (0 == height % 8) {
311 for (loop_cnt = (height >> 3); loop_cnt--;) {
312 LD4(src, src_stride, tp0, tp1, tp2, tp3);
313 src += 4 * src_stride;
314 INSERT_D2_SB(tp0, tp1, src0);
315 INSERT_D2_SB(tp2, tp3, src1);
316 LD4(src, src_stride, tp0, tp1, tp2, tp3);
317 src += 4 * src_stride;
318 INSERT_D2_SB(tp0, tp1, src2);
319 INSERT_D2_SB(tp2, tp3, src3);
320
321 ILVRL_B2_SH(zero, src0, dst0, dst1);
322 ILVRL_B2_SH(zero, src1, dst2, dst3);
323 ILVRL_B2_SH(zero, src2, dst4, dst5);
324 ILVRL_B2_SH(zero, src3, dst6, dst7);
325 SLLI_4V(dst0, dst1, dst2, dst3, 6);
326 SLLI_4V(dst4, dst5, dst6, dst7, 6);
327 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
328 offset_vec, rnd_vec, dst0, dst1,
329 dst2, dst3);
330 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst4, dst5, dst6, dst7, weight_vec,
331 offset_vec, rnd_vec, dst4, dst5,
332 dst6, dst7);
333 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
334 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
335 ST_D8(out0, out1, out2, out3, 0, 1, 0, 1, 0, 1, 0, 1,
336 dst, dst_stride);
337 dst += (8 * dst_stride);
338 }
339 }
340}
341
342static void hevc_uniwgt_copy_12w_msa(const uint8_t *src,
343 int32_t src_stride,
344 uint8_t *dst,
345 int32_t dst_stride,
346 int32_t height,
347 int32_t weight,
348 int32_t offset,
349 int32_t rnd_val)
350{
351 uint32_t loop_cnt;
352 v16u8 out0, out1, out2;
353 v16i8 src0, src1, src2, src3;
354 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
355 v8i16 offset_vec;
356 v16i8 zero = { 0 };
357 v4i32 weight_vec, rnd_vec;
358
359 weight_vec = __msa_fill_w(weight);
360 offset_vec = __msa_fill_w(offset);
361 rnd_vec = __msa_fill_w(rnd_val);
362
363 for (loop_cnt = 4; loop_cnt--;) {
364 LD_SB4(src, src_stride, src0, src1, src2, src3);
365 src += (4 * src_stride);
366 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
367 dst0, dst1, dst2, dst3);
368
369 ILVL_W2_SB(src1, src0, src3, src2, src0, src1);
370 ILVR_B2_SH(zero, src0, zero, src1, dst4, dst5);
371 SLLI_4V(dst0, dst1, dst2, dst3, 6);
372 SLLI_2V(dst4, dst5, 6);
373 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
374 offset_vec, rnd_vec, dst0, dst1, dst2,
375 dst3);
376 HEVC_UNIW_RND_CLIP2_MAX_SATU_H(dst4, dst5, weight_vec, offset_vec,
377 rnd_vec, dst4, dst5);
378
379 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
380 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
381 ST_W4(out2, 0, 1, 2, 3, dst + 8, dst_stride);
382 dst += (4 * dst_stride);
383 }
384}
385
386static void hevc_uniwgt_copy_16w_msa(const uint8_t *src,
387 int32_t src_stride,
388 uint8_t *dst,
389 int32_t dst_stride,
390 int32_t height,
391 int32_t weight,
392 int32_t offset,
393 int32_t rnd_val)
394{
395 uint32_t loop_cnt;
396 v16u8 out0, out1, out2, out3;
397 v16i8 src0, src1, src2, src3;
398 v16i8 zero = { 0 };
399 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, offset_vec;
400 v4i32 weight_vec, rnd_vec;
401
402 weight_vec = __msa_fill_w(weight);
403 offset_vec = __msa_fill_w(offset);
404 rnd_vec = __msa_fill_w(rnd_val);
405
406 for (loop_cnt = height >> 2; loop_cnt--;) {
407 LD_SB4(src, src_stride, src0, src1, src2, src3);
408 src += (4 * src_stride);
409 ILVRL_B2_SH(zero, src0, dst0, dst1);
410 ILVRL_B2_SH(zero, src1, dst2, dst3);
411 ILVRL_B2_SH(zero, src2, dst4, dst5);
412 ILVRL_B2_SH(zero, src3, dst6, dst7);
413 SLLI_4V(dst0, dst1, dst2, dst3, 6);
414 SLLI_4V(dst4, dst5, dst6, dst7, 6);
415 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
416 offset_vec, rnd_vec, dst0, dst1, dst2,
417 dst3);
418 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst4, dst5, dst6, dst7, weight_vec,
419 offset_vec, rnd_vec, dst4, dst5, dst6,
420 dst7);
421 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
422 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
423 ST_UB4(out0, out1, out2, out3, dst, dst_stride);
424 dst += (4 * dst_stride);
425 }
426}
427
428static void hevc_uniwgt_copy_24w_msa(const uint8_t *src,
429 int32_t src_stride,
430 uint8_t *dst,
431 int32_t dst_stride,
432 int32_t height,
433 int32_t weight,
434 int32_t offset,
435 int32_t rnd_val)
436{
437 uint32_t loop_cnt;
438 v16u8 out0, out1, out2, out3, out4, out5;
439 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
440 v16i8 zero = { 0 };
441 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, offset_vec;
442 v8i16 dst8, dst9, dst10, dst11;
443 v4i32 weight_vec, rnd_vec;
444
445 weight_vec = __msa_fill_w(weight);
446 offset_vec = __msa_fill_w(offset);
447 rnd_vec = __msa_fill_w(rnd_val);
448
449 for (loop_cnt = (height >> 2); loop_cnt--;) {
450 LD_SB4(src, src_stride, src0, src1, src4, src5);
451 LD_SB4(src + 16, src_stride, src2, src3, src6, src7);
452 src += (4 * src_stride);
453
454 ILVRL_B2_SH(zero, src0, dst0, dst1);
455 ILVRL_B2_SH(zero, src1, dst2, dst3);
456 ILVR_B2_SH(zero, src2, zero, src3, dst4, dst5);
457 ILVRL_B2_SH(zero, src4, dst6, dst7);
458 ILVRL_B2_SH(zero, src5, dst8, dst9);
459 ILVR_B2_SH(zero, src6, zero, src7, dst10, dst11);
460 SLLI_4V(dst0, dst1, dst2, dst3, 6);
461 SLLI_4V(dst4, dst5, dst6, dst7, 6);
462 SLLI_4V(dst8, dst9, dst10, dst11, 6);
463 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
464 offset_vec, rnd_vec, dst0, dst1, dst2,
465 dst3);
466 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst4, dst5, dst6, dst7, weight_vec,
467 offset_vec, rnd_vec, dst4, dst5, dst6,
468 dst7);
469 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst8, dst9, dst10, dst11, weight_vec,
470 offset_vec, rnd_vec, dst8, dst9, dst10,
471 dst11);
472 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
473 PCKEV_B3_UB(dst7, dst6, dst9, dst8, dst11, dst10, out3, out4, out5);
474 ST_UB4(out0, out1, out3, out4, dst, dst_stride);
475 ST_D4(out2, out5, 0, 1, 0, 1, dst + 16, dst_stride);
476 dst += (4 * dst_stride);
477 }
478}
479
480static void hevc_uniwgt_copy_32w_msa(const uint8_t *src,
481 int32_t src_stride,
482 uint8_t *dst,
483 int32_t dst_stride,
484 int32_t height,
485 int32_t weight,
486 int32_t offset,
487 int32_t rnd_val)
488{
489 uint32_t loop_cnt;
490 v16u8 out0, out1, out2, out3;
491 v16i8 src0, src1, src2, src3;
492 v16i8 zero = { 0 };
493 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, offset_vec;
494 v4i32 weight_vec, rnd_vec;
495
496 weight_vec = __msa_fill_w(weight);
497 offset_vec = __msa_fill_w(offset);
498 rnd_vec = __msa_fill_w(rnd_val);
499
500 for (loop_cnt = (height >> 1); loop_cnt--;) {
501 LD_SB2(src, src_stride, src0, src1);
502 LD_SB2(src + 16, src_stride, src2, src3);
503 src += (2 * src_stride);
504
505 ILVRL_B2_SH(zero, src0, dst0, dst1);
506 ILVRL_B2_SH(zero, src1, dst2, dst3);
507 ILVRL_B2_SH(zero, src2, dst4, dst5);
508 ILVRL_B2_SH(zero, src3, dst6, dst7);
509 SLLI_4V(dst0, dst1, dst2, dst3, 6);
510 SLLI_4V(dst4, dst5, dst6, dst7, 6);
511 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
512 offset_vec, rnd_vec, dst0, dst1, dst2,
513 dst3);
514 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst4, dst5, dst6, dst7, weight_vec,
515 offset_vec, rnd_vec, dst4, dst5, dst6,
516 dst7);
517 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
518 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
519 ST_UB2(out0, out1, dst, dst_stride);
520 ST_UB2(out2, out3, dst + 16, dst_stride);
521 dst += (2 * dst_stride);
522 }
523}
524
525static void hevc_uniwgt_copy_48w_msa(const uint8_t *src,
526 int32_t src_stride,
527 uint8_t *dst,
528 int32_t dst_stride,
529 int32_t height,
530 int32_t weight,
531 int32_t offset,
532 int32_t rnd_val)
533{
534 uint32_t loop_cnt;
535 v16u8 out0, out1, out2, out3, out4, out5;
536 v16i8 src0, src1, src2, src3, src4, src5;
537 v16i8 zero = { 0 };
538 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, offset_vec;
539 v8i16 dst6, dst7, dst8, dst9, dst10, dst11;
540 v4i32 weight_vec, rnd_vec;
541
542 weight_vec = __msa_fill_w(weight);
543 offset_vec = __msa_fill_w(offset);
544 rnd_vec = __msa_fill_w(rnd_val);
545
546 for (loop_cnt = (height >> 1); loop_cnt--;) {
547 LD_SB3(src, 16, src0, src1, src2);
548 src += src_stride;
549 LD_SB3(src, 16, src3, src4, src5);
550 src += src_stride;
551
552 ILVRL_B2_SH(zero, src0, dst0, dst1);
553 ILVRL_B2_SH(zero, src1, dst2, dst3);
554 ILVRL_B2_SH(zero, src2, dst4, dst5);
555 ILVRL_B2_SH(zero, src3, dst6, dst7);
556 ILVRL_B2_SH(zero, src4, dst8, dst9);
557 ILVRL_B2_SH(zero, src5, dst10, dst11);
558 SLLI_4V(dst0, dst1, dst2, dst3, 6);
559 SLLI_4V(dst4, dst5, dst6, dst7, 6);
560 SLLI_4V(dst8, dst9, dst10, dst11, 6);
561 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
562 offset_vec, rnd_vec, dst0, dst1, dst2,
563 dst3);
564 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst4, dst5, dst6, dst7, weight_vec,
565 offset_vec, rnd_vec, dst4, dst5, dst6,
566 dst7);
567 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst8, dst9, dst10, dst11, weight_vec,
568 offset_vec, rnd_vec, dst8, dst9, dst10,
569 dst11);
570 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
571 PCKEV_B3_UB(dst7, dst6, dst9, dst8, dst11, dst10, out3, out4, out5);
572 ST_UB2(out0, out1, dst, 16);
573 ST_UB(out2, dst + 32);
574 dst += dst_stride;
575 ST_UB2(out3, out4, dst, 16);
576 ST_UB(out5, dst + 32);
577 dst += dst_stride;
578 }
579}
580
581static void hevc_uniwgt_copy_64w_msa(const uint8_t *src,
582 int32_t src_stride,
583 uint8_t *dst,
584 int32_t dst_stride,
585 int32_t height,
586 int32_t weight,
587 int32_t offset,
588 int32_t rnd_val)
589{
590 uint32_t loop_cnt;
591 v16u8 out0, out1, out2, out3, out4, out5, out6, out7;
592 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
593 v16i8 zero = { 0 };
594 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, offset_vec;
595 v8i16 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
596 v4i32 weight_vec, rnd_vec;
597
598 weight_vec = __msa_fill_w(weight);
599 offset_vec = __msa_fill_w(offset);
600 rnd_vec = __msa_fill_w(rnd_val);
601
602 for (loop_cnt = (height >> 1); loop_cnt--;) {
603 LD_SB4(src, 16, src0, src1, src2, src3);
604 src += src_stride;
605 LD_SB4(src, 16, src4, src5, src6, src7);
606 src += src_stride;
607
608 ILVRL_B2_SH(zero, src0, dst0, dst1);
609 ILVRL_B2_SH(zero, src1, dst2, dst3);
610 ILVRL_B2_SH(zero, src2, dst4, dst5);
611 ILVRL_B2_SH(zero, src3, dst6, dst7);
612 ILVRL_B2_SH(zero, src4, dst8, dst9);
613 ILVRL_B2_SH(zero, src5, dst10, dst11);
614 ILVRL_B2_SH(zero, src6, dst12, dst13);
615 ILVRL_B2_SH(zero, src7, dst14, dst15);
616 SLLI_4V(dst0, dst1, dst2, dst3, 6);
617 SLLI_4V(dst4, dst5, dst6, dst7, 6);
618 SLLI_4V(dst8, dst9, dst10, dst11, 6);
619 SLLI_4V(dst12, dst13, dst14, dst15, 6);
620 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst0, dst1, dst2, dst3, weight_vec,
621 offset_vec, rnd_vec, dst0, dst1, dst2,
622 dst3);
623 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst4, dst5, dst6, dst7, weight_vec,
624 offset_vec, rnd_vec, dst4, dst5, dst6,
625 dst7);
626 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst8, dst9, dst10, dst11, weight_vec,
627 offset_vec, rnd_vec, dst8, dst9, dst10,
628 dst11);
629 HEVC_UNIW_RND_CLIP4_MAX_SATU_H(dst12, dst13, dst14, dst15, weight_vec,
630 offset_vec, rnd_vec, dst12, dst13, dst14,
631 dst15);
632 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
633 PCKEV_B2_UB(dst5, dst4, dst7, dst6, out2, out3);
634 PCKEV_B2_UB(dst9, dst8, dst11, dst10, out4, out5);
635 PCKEV_B2_UB(dst13, dst12, dst15, dst14, out6, out7);
636 ST_UB4(out0, out1, out2, out3, dst, 16);
637 dst += dst_stride;
638 ST_UB4(out4, out5, out6, out7, dst, 16);
639 dst += dst_stride;
640 }
641}
642
643static void hevc_hz_uniwgt_8t_4w_msa(const uint8_t *src,
644 int32_t src_stride,
645 uint8_t *dst,
646 int32_t dst_stride,
647 const int8_t *filter,
648 int32_t height,
649 int32_t weight,
650 int32_t offset,
651 int32_t rnd_val)
652{
653 uint32_t loop_cnt;
654 uint32_t res = height & 0x07;
655 v16u8 out0, out1;
656 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
657 v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
658 v8i16 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
659 v16i8 mask0, mask1, mask2, mask3;
660 v8i16 filter_vec, filt0, filt1, filt2, filt3;
661 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
662 v4i32 weight_vec, rnd_vec, offset_vec;
663 v8i16 zero = { 0 };
664
665 src -= 3;
666
667 weight_vec = __msa_fill_w(weight);
668 rnd_vec = __msa_fill_w(rnd_val);
669 offset_vec = __msa_fill_w(offset);
670
671 filter_vec = LD_SH(filter);
672 UNPCK_R_SB_SH(filter_vec, filter_vec);
673 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
674
675 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
676 mask1 = mask0 + 2;
677 mask2 = mask0 + 4;
678 mask3 = mask0 + 6;
679
680 for (loop_cnt = (height >> 3); loop_cnt--;) {
681 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
682 src += (8 * src_stride);
683 VSHF_B4_SB(src0, src1, mask0, mask1, mask2, mask3,
684 vec0, vec1, vec2, vec3);
685 VSHF_B4_SB(src2, src3, mask0, mask1, mask2, mask3,
686 vec4, vec5, vec6, vec7);
687 VSHF_B4_SB(src4, src5, mask0, mask1, mask2, mask3,
688 vec8, vec9, vec10, vec11);
689 VSHF_B4_SB(src6, src7, mask0, mask1, mask2, mask3,
690 vec12, vec13, vec14, vec15);
691
692 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
693 filt3, dst0, dst1);
694 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
695 filt3, dst2, dst3);
696 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
697 filt3, dst4, dst5);
698 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
699 filt3, dst6, dst7);
700
701 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
702 weight_vec, dst0, dst1, dst2, dst3)
703 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
704 weight_vec, dst4, dst5, dst6, dst7);
705 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
706 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
707 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
708 offset_vec, dst0, dst1, dst2, dst3);
709 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
710 offset_vec, dst4, dst5, dst6, dst7);
711 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
712 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
713 vec0, vec1, vec2, vec3);
714 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
715
716 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
717 dst += (8 * dst_stride);
718 }
719 if (res) {
720 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
721 VSHF_B4_SB(src0, src1, mask0, mask1, mask2, mask3,
722 vec0, vec1, vec2, vec3);
723 VSHF_B4_SB(src2, src3, mask0, mask1, mask2, mask3,
724 vec4, vec5, vec6, vec7);
725 VSHF_B4_SB(src4, src5, mask0, mask1, mask2, mask3,
726 vec8, vec9, vec10, vec11);
727 VSHF_B4_SB(src6, src7, mask0, mask1, mask2, mask3,
728 vec12, vec13, vec14, vec15);
729
730 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
731 filt3, dst0, dst1);
732 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
733 filt3, dst2, dst3);
734 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
735 filt3, dst4, dst5);
736 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
737 filt3, dst6, dst7);
738
739 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
740 weight_vec, dst0, dst1, dst2, dst3)
741 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
742 weight_vec, dst4, dst5, dst6, dst7);
743 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
744 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
745 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
746 offset_vec, dst0, dst1, dst2, dst3);
747 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
748 offset_vec, dst4, dst5, dst6, dst7);
749 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
750 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
751 vec0, vec1, vec2, vec3);
752 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
753
754 if (res == 2) {
755 ST_W2(out0, 0, 1, dst, dst_stride);
756 } else if (res == 4) {
757 ST_W4(out0, 0, 1, 2, 3, dst, dst_stride);
758 } else {
759 ST_W4(out0, 0, 1, 2, 3, dst, dst_stride);
760 ST_W2(out1, 0, 1, dst + 4 * dst_stride, dst_stride);
761 }
762 }
763}
764
765static void hevc_hz_uniwgt_8t_8w_msa(const uint8_t *src,
766 int32_t src_stride,
767 uint8_t *dst,
768 int32_t dst_stride,
769 const int8_t *filter,
770 int32_t height,
771 int32_t weight,
772 int32_t offset,
773 int32_t rnd_val)
774{
775 uint32_t loop_cnt;
776 uint32_t res = height & 0x03;
777 v16u8 out0, out1;
778 v16i8 src0, src1, src2, src3;
779 v8i16 filt0, filt1, filt2, filt3;
780 v16i8 mask0, mask1, mask2, mask3;
781 v8i16 filter_vec;
782 v8i16 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
783 v8i16 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
784 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
785 v4i32 weight_vec, rnd_vec, offset_vec;
786 v8i16 zero = { 0 };
787
788 src -= 3;
789
790 weight_vec = __msa_fill_w(weight);
791 rnd_vec = __msa_fill_w(rnd_val);
792 offset_vec = __msa_fill_w(offset);
793
794 filter_vec = LD_SH(filter);
795 UNPCK_R_SB_SH(filter_vec, filter_vec);
796 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
797
798 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
799 mask1 = mask0 + 2;
800 mask2 = mask0 + 4;
801 mask3 = mask0 + 6;
802
803 for (loop_cnt = (height >> 2); loop_cnt--;) {
804 LD_SB4(src, src_stride, src0, src1, src2, src3);
805 src += (4 * src_stride);
806
807 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
808 vec0, vec1, vec2, vec3);
809 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
810 vec4, vec5, vec6, vec7);
811 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
812 vec8, vec9, vec10, vec11);
813 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
814 vec12, vec13, vec14, vec15);
815 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
816 filt3, dst0, dst1);
817 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
818 filt3, dst2, dst3);
819 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
820 filt3, dst4, dst5);
821 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
822 filt3, dst6, dst7);
823
824 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
825 weight_vec, dst0, dst1, dst2, dst3)
826 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
827 weight_vec, dst4, dst5, dst6, dst7);
828 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
829 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
830 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
831 offset_vec, dst0, dst1, dst2, dst3);
832 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
833 offset_vec, dst4, dst5, dst6, dst7);
834 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
835 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
836 vec0, vec1, vec2, vec3);
837 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
838 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
839 dst += (4 * dst_stride);
840 }
841 if (res) {
842 LD_SB2(src, src_stride, src0, src1);
843
844 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
845 vec0, vec1, vec2, vec3);
846 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
847 vec4, vec5, vec6, vec7);
848 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
849 filt3, dst0, dst1);
850 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
851 filt3, dst2, dst3);
852
853 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
854 weight_vec, dst0, dst1, dst2, dst3)
855 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
856 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
857 offset_vec, dst0, dst1, dst2, dst3);
858 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
859 PCKEV_H2_SH(dst1, dst0, dst3, dst2, vec0, vec1);
860 out0 = __msa_pckev_b((v16i8) vec1, (v16i8) vec0);
861 ST_D2(out0, 0, 1, dst, dst_stride);
862 }
863}
864
865static void hevc_hz_uniwgt_8t_12w_msa(const uint8_t *src,
866 int32_t src_stride,
867 uint8_t *dst,
868 int32_t dst_stride,
869 const int8_t *filter,
870 int32_t height,
871 int32_t weight,
872 int32_t offset,
873 int32_t rnd_val)
874{
875 uint32_t loop_cnt;
876 v16u8 out0, out1;
877 v8i16 filt0, filt1, filt2, filt3;
878 v16i8 src0, src1, src2, src3;
879 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
880 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
881 v8i16 filter_vec;
882 v4i32 dst0, dst1, dst2, dst3;
883 v4i32 dst00, dst01;
884 v4i32 weight_vec, rnd_vec, offset_vec;
885 v8i16 zero = { 0 };
886
887 src -= 3;
888
889 weight_vec = __msa_fill_w(weight);
890 rnd_vec = __msa_fill_w(rnd_val);
891 offset_vec = __msa_fill_w(offset);
892
893 filter_vec = LD_SH(filter);
894 UNPCK_R_SB_SH(filter_vec, filter_vec);
895 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
896
897 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
898 mask1 = mask0 + 2;
899 mask2 = mask0 + 4;
900 mask3 = mask0 + 6;
901 mask4 = LD_SB(&ff_hevc_mask_arr[16]);
902 mask5 = mask4 + 2;
903 mask6 = mask4 + 4;
904 mask7 = mask4 + 6;
905
906 for (loop_cnt = (height >> 1); loop_cnt--;) {
907 LD_SB2(src, src_stride, src0, src1);
908 LD_SB2(src + 8, src_stride, src2, src3);
909 src += (2 * src_stride);
910
911 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
912 vec0, vec1, vec2, vec3);
913 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
914 vec4, vec5, vec6, vec7);
915 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
916 filt3, dst0, dst1);
917 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
918 filt3, dst2, dst3);
919 VSHF_B4_SB(src2, src3, mask4, mask5, mask6, mask7,
920 vec0, vec1, vec2, vec3);
921 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
922 filt3, dst00, dst01);
923
924 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
925 weight_vec, dst0, dst1, dst2, dst3)
926 MUL2(dst00, weight_vec, dst01, weight_vec, dst00, dst01);
927 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
928 SRAR_W2_SW(dst00, dst01, rnd_vec);
929 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
930 offset_vec, dst0, dst1, dst2, dst3);
931 ADD2(dst00, offset_vec, dst01, offset_vec, dst00, dst01);
932 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
933 CLIP_SW2_0_255(dst00, dst01);
934 PCKEV_H2_SH(dst1, dst0, dst3, dst2, vec0, vec1);
935 vec2 = __msa_pckev_h((v8i16) dst01, (v8i16) dst00);
936 PCKEV_B2_UB(vec1, vec0, zero, vec2, out0, out1);
937
938 ST_D2(out0, 0, 1, dst, dst_stride);
939 ST_W2(out1, 0, 1, dst + 8, dst_stride);
940 dst += (2 * dst_stride);
941 }
942}
943
944static void hevc_hz_uniwgt_8t_16w_msa(const uint8_t *src,
945 int32_t src_stride,
946 uint8_t *dst,
947 int32_t dst_stride,
948 const int8_t *filter,
949 int32_t height,
950 int32_t weight,
951 int32_t offset,
952 int32_t rnd_val)
953{
954 uint32_t loop_cnt;
955 v16u8 out0, out1;
956 v16i8 src0, src1, src2, src3;
957 v8i16 filt0, filt1, filt2, filt3;
958 v16i8 mask0, mask1, mask2, mask3;
959 v8i16 filter_vec;
960 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
961 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
962 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
963 v4i32 weight_vec, rnd_vec, offset_vec;
964 v8i16 zero = { 0 };
965
966 src -= 3;
967
968 weight_vec = __msa_fill_w(weight);
969 rnd_vec = __msa_fill_w(rnd_val);
970 offset_vec = __msa_fill_w(offset);
971
972 filter_vec = LD_SH(filter);
973 UNPCK_R_SB_SH(filter_vec, filter_vec);
974 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
975
976 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
977 mask1 = mask0 + 2;
978 mask2 = mask0 + 4;
979 mask3 = mask0 + 6;
980
981 for (loop_cnt = (height >> 1); loop_cnt--;) {
982 LD_SB2(src, src_stride, src0, src2);
983 LD_SB2(src + 8, src_stride, src1, src3);
984 src += (2 * src_stride);
985
986 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
987 vec0, vec1, vec2, vec3);
988 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
989 vec4, vec5, vec6, vec7);
990 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
991 vec8, vec9, vec10, vec11);
992 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
993 vec12, vec13, vec14, vec15);
994 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
995 filt3, dst0, dst1);
996 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
997 filt3, dst2, dst3);
998 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
999 filt3, dst4, dst5);
1000 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1001 filt3, dst6, dst7);
1002
1003 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1004 weight_vec, dst0, dst1, dst2, dst3)
1005 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1006 weight_vec, dst4, dst5, dst6, dst7);
1007 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1008 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1009 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1010 offset_vec, dst0, dst1, dst2, dst3);
1011 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1012 offset_vec, dst4, dst5, dst6, dst7);
1013 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1014 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1015 vec0, vec1, vec2, vec3);
1016 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
1017
1018 ST_UB2(out0, out1, dst, dst_stride);
1019 dst += (2 * dst_stride);
1020 }
1021}
1022
1023static void hevc_hz_uniwgt_8t_24w_msa(const uint8_t *src,
1024 int32_t src_stride,
1025 uint8_t *dst,
1026 int32_t dst_stride,
1027 const int8_t *filter,
1028 int32_t height,
1029 int32_t weight,
1030 int32_t offset,
1031 int32_t rnd_val)
1032{
1033 uint32_t loop_cnt;
1034 v16u8 out0, out1, out2;
1035 v16i8 src0, src1, src2, src3;
1036 v8i16 filt0, filt1, filt2, filt3;
1037 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1038 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1039 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1040 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1041 v4i32 dst8, dst9, dst10, dst11;
1042 v8i16 filter_vec;
1043 v4i32 weight_vec, rnd_vec, offset_vec;
1044 v8i16 zero = { 0 };
1045
1046 src -= 3;
1047
1048 weight_vec = __msa_fill_w(weight);
1049 rnd_vec = __msa_fill_w(rnd_val);
1050 offset_vec = __msa_fill_w(offset);
1051
1052 filter_vec = LD_SH(filter);
1053 UNPCK_R_SB_SH(filter_vec, filter_vec);
1054 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
1055
1056 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1057 mask1 = mask0 + 2;
1058 mask2 = mask0 + 4;
1059 mask3 = mask0 + 6;
1060 mask4 = mask0 + 8;
1061 mask5 = mask0 + 10;
1062 mask6 = mask0 + 12;
1063 mask7 = mask0 + 14;
1064
1065 for (loop_cnt = 16; loop_cnt--;) {
1066 LD_SB2(src, 16, src0, src1);
1067 src += src_stride;
1068 LD_SB2(src, 16, src2, src3);
1069 src += src_stride;
1070 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1071 vec0, vec1, vec2, vec3);
1072 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
1073 vec4, vec5, vec6, vec7);
1074 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1075 vec8, vec9, vec10, vec11);
1076 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1077 vec12, vec13, vec14, vec15);
1078 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1079 filt3, dst0, dst1);
1080 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1081 filt3, dst2, dst3);
1082 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1083 filt3, dst4, dst5);
1084 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1085 filt3, dst6, dst7);
1086
1087 VSHF_B4_SB(src2, src3, mask4, mask5, mask6, mask7,
1088 vec0, vec1, vec2, vec3);
1089 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
1090 vec4, vec5, vec6, vec7);
1091 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1092 filt3, dst8, dst9);
1093 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1094 filt3, dst10, dst11);
1095
1096 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1097 weight_vec, dst0, dst1, dst2, dst3)
1098 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1099 weight_vec, dst4, dst5, dst6, dst7);
1100 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
1101 weight_vec, dst8, dst9, dst10, dst11)
1102 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1103 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1104 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
1105 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1106 offset_vec, dst0, dst1, dst2, dst3);
1107 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1108 offset_vec, dst4, dst5, dst6, dst7);
1109 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
1110 offset_vec, dst8, dst9, dst10, dst11);
1111 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1112 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
1113 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1114 vec0, vec1, vec2, vec3);
1115 PCKEV_H2_SH(dst9, dst8, dst11, dst10, vec4, vec5);
1116
1117 PCKEV_B3_UB(vec1, vec0, vec4, vec3, vec5, vec2, out0, out1, out2);
1118 ST_UB2(out0, out1, dst, dst_stride);
1119 ST_D2(out2, 0, 1, dst + 16, dst_stride);
1120 dst += (2 * dst_stride);
1121 }
1122}
1123
1124static void hevc_hz_uniwgt_8t_32w_msa(const uint8_t *src,
1125 int32_t src_stride,
1126 uint8_t *dst,
1127 int32_t dst_stride,
1128 const int8_t *filter,
1129 int32_t height,
1130 int32_t weight,
1131 int32_t offset,
1132 int32_t rnd_val)
1133{
1134 uint32_t loop_cnt;
1135 v16u8 out0, out1, out2, out3;
1136 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
1137 v8i16 filt0, filt1, filt2, filt3;
1138 v16i8 mask0, mask1, mask2, mask3;
1139 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1140 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1141 v8i16 filter_vec;
1142 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1143 v4i32 dst10, dst11, dst12, dst13, dst14, dst15, dst16, dst17;
1144 v4i32 weight_vec, rnd_vec, offset_vec;
1145 v8i16 zero = { 0 };
1146
1147 src -= 3;
1148
1149 weight_vec = __msa_fill_w(weight);
1150 rnd_vec = __msa_fill_w(rnd_val);
1151 offset_vec = __msa_fill_w(offset);
1152
1153 filter_vec = LD_SH(filter);
1154 UNPCK_R_SB_SH(filter_vec, filter_vec);
1155 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
1156
1157 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1158 mask1 = mask0 + 2;
1159 mask2 = mask0 + 4;
1160 mask3 = mask0 + 6;
1161
1162 for (loop_cnt = height >> 1; loop_cnt--;) {
1163 LD_SB4(src, 8, src0, src1, src2, src3);
1164 src += src_stride;
1165 LD_SB4(src, 8, src4, src5, src6, src7);
1166 src += src_stride;
1167
1168 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1169 vec0, vec1, vec2, vec3);
1170 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1171 vec4, vec5, vec6, vec7);
1172 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1173 vec8, vec9, vec10, vec11);
1174 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
1175 vec12, vec13, vec14, vec15);
1176 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1177 filt3, dst0, dst1);
1178 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1179 filt3, dst2, dst3);
1180 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1181 filt3, dst4, dst5);
1182 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1183 filt3, dst6, dst7);
1184
1185 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
1186 vec0, vec1, vec2, vec3);
1187 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3,
1188 vec4, vec5, vec6, vec7);
1189 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3,
1190 vec8, vec9, vec10, vec11);
1191 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3,
1192 vec12, vec13, vec14, vec15);
1193 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1194 filt3, dst10, dst11);
1195 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1196 filt3, dst12, dst13);
1197 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1198 filt3, dst14, dst15);
1199 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1200 filt3, dst16, dst17);
1201
1202 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1203 weight_vec, dst0, dst1, dst2, dst3)
1204 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1205 weight_vec, dst4, dst5, dst6, dst7);
1206 MUL4(dst10, weight_vec, dst11, weight_vec, dst12, weight_vec, dst13,
1207 weight_vec, dst10, dst11, dst12, dst13)
1208 MUL4(dst14, weight_vec, dst15, weight_vec, dst16, weight_vec, dst17,
1209 weight_vec, dst14, dst15, dst16, dst17);
1210 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1211 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1212 SRAR_W4_SW(dst10, dst11, dst12, dst13, rnd_vec);
1213 SRAR_W4_SW(dst14, dst15, dst16, dst17, rnd_vec);
1214 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1215 offset_vec, dst0, dst1, dst2, dst3);
1216 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1217 offset_vec, dst4, dst5, dst6, dst7);
1218 ADD4(dst10, offset_vec, dst11, offset_vec, dst12, offset_vec, dst13,
1219 offset_vec, dst10, dst11, dst12, dst13);
1220 ADD4(dst14, offset_vec, dst15, offset_vec, dst16, offset_vec, dst17,
1221 offset_vec, dst14, dst15, dst16, dst17);
1222 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1223 CLIP_SW8_0_255(dst10, dst11, dst12, dst13, dst14, dst15, dst16, dst17);
1224 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1225 vec0, vec1, vec2, vec3);
1226 PCKEV_H4_SH(dst11, dst10, dst13, dst12, dst15, dst14, dst17, dst16,
1227 vec4, vec5, vec6, vec7);
1228
1229 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
1230 PCKEV_B2_UB(vec5, vec4, vec7, vec6, out2, out3);
1231 ST_UB2(out0, out1, dst, 16);
1232 dst += dst_stride;
1233 ST_UB2(out2, out3, dst, 16);
1234 dst += dst_stride;
1235 }
1236}
1237
1238static void hevc_hz_uniwgt_8t_48w_msa(const uint8_t *src,
1239 int32_t src_stride,
1240 uint8_t *dst,
1241 int32_t dst_stride,
1242 const int8_t *filter,
1243 int32_t height,
1244 int32_t weight,
1245 int32_t offset,
1246 int32_t rnd_val)
1247{
1248 uint32_t loop_cnt;
1249 v16u8 out0, out1, out2;
1250 v16i8 src0, src1, src2, src3;
1251 v8i16 filt0, filt1, filt2, filt3;
1252 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1253 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1254 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1255 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1256 v4i32 dst8, dst9, dst10, dst11;
1257 v8i16 filter_vec;
1258 v4i32 weight_vec, rnd_vec, offset_vec;
1259 v8i16 zero = { 0 };
1260
1261 src -= 3;
1262
1263 weight_vec = __msa_fill_w(weight);
1264 rnd_vec = __msa_fill_w(rnd_val);
1265 offset_vec = __msa_fill_w(offset);
1266
1267 filter_vec = LD_SH(filter);
1268 UNPCK_R_SB_SH(filter_vec, filter_vec);
1269 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
1270
1271 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1272 mask1 = mask0 + 2;
1273 mask2 = mask0 + 4;
1274 mask3 = mask0 + 6;
1275 mask4 = mask0 + 8;
1276 mask5 = mask0 + 10;
1277 mask6 = mask0 + 12;
1278 mask7 = mask0 + 14;
1279
1280 for (loop_cnt = 64; loop_cnt--;) {
1281 LD_SB3(src, 16, src0, src1, src2);
1282 src3 = LD_SB(src + 40);
1283 src += src_stride;
1284
1285 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1286 vec0, vec1, vec2, vec3);
1287 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
1288 vec4, vec5, vec6, vec7);
1289 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1290 vec8, vec9, vec10, vec11);
1291 VSHF_B4_SB(src1, src2, mask4, mask5, mask6, mask7,
1292 vec12, vec13, vec14, vec15);
1293 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1294 filt3, dst0, dst1);
1295 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1296 filt3, dst2, dst3);
1297 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1298 filt3, dst4, dst5);
1299 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1300 filt3, dst6, dst7);
1301
1302 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1303 vec0, vec1, vec2, vec3);
1304 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
1305 vec4, vec5, vec6, vec7);
1306 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1307 filt3, dst8, dst9);
1308 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1309 filt3, dst10, dst11);
1310
1311 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1312 weight_vec, dst0, dst1, dst2, dst3)
1313 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1314 weight_vec, dst4, dst5, dst6, dst7);
1315 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
1316 weight_vec, dst8, dst9, dst10, dst11)
1317 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1318 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1319 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
1320 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1321 offset_vec, dst0, dst1, dst2, dst3);
1322 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1323 offset_vec, dst4, dst5, dst6, dst7);
1324 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
1325 offset_vec, dst8, dst9, dst10, dst11);
1326 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1327 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
1328 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1329 vec0, vec1, vec2, vec3);
1330 PCKEV_H2_SH(dst9, dst8, dst11, dst10, vec4, vec5);
1331 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
1332 out2 = __msa_pckev_b((v16i8) vec5, (v16i8) vec4);
1333 ST_UB2(out0, out1, dst, 16);
1334 ST_UB(out2, dst + 32);
1335 dst += dst_stride;
1336 }
1337}
1338
1339static void hevc_hz_uniwgt_8t_64w_msa(const uint8_t *src,
1340 int32_t src_stride,
1341 uint8_t *dst,
1342 int32_t dst_stride,
1343 const int8_t *filter,
1344 int32_t height,
1345 int32_t weight,
1346 int32_t offset,
1347 int32_t rnd_val)
1348{
1349 const uint8_t *src_tmp;
1350 uint8_t *dst_tmp;
1351 uint32_t loop_cnt, cnt;
1352 v16u8 out0, out1;
1353 v16i8 src0, src1, src2;
1354 v8i16 filt0, filt1, filt2, filt3;
1355 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1356 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1357 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1358 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1359 v8i16 filter_vec;
1360 v4i32 weight_vec, rnd_vec, offset_vec;
1361 v8i16 zero = { 0 };
1362
1363 src -= 3;
1364
1365 weight_vec = __msa_fill_w(weight);
1366 rnd_vec = __msa_fill_w(rnd_val);
1367 offset_vec = __msa_fill_w(offset);
1368
1369 filter_vec = LD_SH(filter);
1370 UNPCK_R_SB_SH(filter_vec, filter_vec);
1371 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
1372
1373 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1374 mask1 = mask0 + 2;
1375 mask2 = mask0 + 4;
1376 mask3 = mask0 + 6;
1377 mask4 = mask0 + 8;
1378 mask5 = mask0 + 10;
1379 mask6 = mask0 + 12;
1380 mask7 = mask0 + 14;
1381
1382 for (loop_cnt = height; loop_cnt--;) {
1383 src_tmp = src;
1384 dst_tmp = dst;
1385
1386 for (cnt = 2; cnt--;) {
1387 LD_SB2(src_tmp, 16, src0, src1);
1388 src2 = LD_SB(src_tmp + 24);
1389 src_tmp += 32;
1390
1391 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1392 vec0, vec1, vec2, vec3);
1393 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
1394 vec4, vec5, vec6, vec7);
1395 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1396 vec8, vec9, vec10, vec11);
1397 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1398 vec12, vec13, vec14, vec15);
1399 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1400 filt3, dst0, dst1);
1401 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1402 filt3, dst2, dst3);
1403 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1404 filt3, dst4, dst5);
1405 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1406 filt3, dst6, dst7);
1407 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1408 weight_vec, dst0, dst1, dst2, dst3)
1409 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1410 weight_vec, dst4, dst5, dst6, dst7);
1411 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1412 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1413 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1414 offset_vec, dst0, dst1, dst2, dst3);
1415 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1416 offset_vec, dst4, dst5, dst6, dst7);
1417 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1418 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1419 vec0, vec1, vec2, vec3);
1420 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
1421 ST_UB2(out0, out1, dst_tmp, 16);
1422 dst_tmp += 32;
1423 }
1424
1425 src += src_stride;
1426 dst += dst_stride;
1427 }
1428}
1429
1430static void hevc_vt_uniwgt_8t_4w_msa(const uint8_t *src,
1431 int32_t src_stride,
1432 uint8_t *dst,
1433 int32_t dst_stride,
1434 const int8_t *filter,
1435 int32_t height,
1436 int32_t weight,
1437 int32_t offset,
1438 int32_t rnd_val)
1439{
1440 int32_t loop_cnt;
1441 int32_t res = height & 0x07;
1442 v16u8 out0, out1;
1443 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1444 v16i8 src9, src10, src11, src12, src13, src14;
1445 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1446 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1447 v16i8 src1110_r, src1211_r, src1312_r, src1413_r;
1448 v16i8 src2110, src4332, src6554, src8776, src10998;
1449 v16i8 src12111110, src14131312;
1450 v8i16 filter_vec;
1451 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1452 v8i16 filt0, filt1, filt2, filt3;
1453 v8i16 vec0, vec1, vec2, vec3;
1454 v4i32 weight_vec, rnd_vec, offset_vec;
1455 v8i16 zero = { 0 };
1456
1457 src -= (3 * src_stride);
1458
1459
1460 weight_vec = __msa_fill_w(weight);
1461 rnd_vec = __msa_fill_w(rnd_val);
1462 offset_vec = __msa_fill_w(offset);
1463
1464 filter_vec = LD_SH(filter);
1465 UNPCK_R_SB_SH(filter_vec, filter_vec);
1466 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
1467
1468 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1469 src += (7 * src_stride);
1470
1471 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1472 src10_r, src32_r, src54_r, src21_r);
1473
1474 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1475
1476 ILVR_D3_SB(src21_r, src10_r, src43_r,
1477 src32_r, src65_r, src54_r, src2110, src4332, src6554);
1478
1479
1480 for (loop_cnt = (height >> 3); loop_cnt--;) {
1481 LD_SB8(src, src_stride,
1482 src7, src8, src9, src10, src11, src12, src13, src14);
1483 src += (8 * src_stride);
1484 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1485 src76_r, src87_r, src98_r, src109_r);
1486 ILVR_B4_SB(src11, src10, src12, src11, src13, src12, src14, src13,
1487 src1110_r, src1211_r, src1312_r, src1413_r);
1488 ILVR_D4_SB(src87_r, src76_r, src109_r, src98_r, src1211_r, src1110_r,
1489 src1413_r, src1312_r,
1490 src8776, src10998, src12111110, src14131312);
1491 HEVC_FILT_8TAP_4W_SH(src2110, src4332, src6554, src8776, filt0,
1492 filt1, filt2, filt3, dst0, dst1);
1493 HEVC_FILT_8TAP_4W_SH(src4332, src6554, src8776, src10998, filt0,
1494 filt1, filt2, filt3, dst2, dst3);
1495 HEVC_FILT_8TAP_4W_SH(src6554, src8776, src10998, src12111110,
1496 filt0, filt1, filt2, filt3, dst4, dst5);
1497 HEVC_FILT_8TAP_4W_SH(src8776, src10998, src12111110, src14131312,
1498 filt0, filt1, filt2, filt3, dst6, dst7);
1499 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1500 weight_vec, dst0, dst1, dst2, dst3)
1501 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1502 weight_vec, dst4, dst5, dst6, dst7);
1503 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1504 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1505 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1506 offset_vec, dst0, dst1, dst2, dst3);
1507 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1508 offset_vec, dst4, dst5, dst6, dst7);
1509 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1510 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1511 vec0, vec1, vec2, vec3);
1512 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
1513
1514
1515 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
1516 dst += (8 * dst_stride);
1517
1518 src2110 = src10998;
1519 src4332 = src12111110;
1520 src6554 = src14131312;
1521 src6 = src14;
1522 }
1523 if (res) {
1524 LD_SB8(src, src_stride,
1525 src7, src8, src9, src10, src11, src12, src13, src14);
1526 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1527 src76_r, src87_r, src98_r, src109_r);
1528 ILVR_B4_SB(src11, src10, src12, src11, src13, src12, src14, src13,
1529 src1110_r, src1211_r, src1312_r, src1413_r);
1530 ILVR_D4_SB(src87_r, src76_r, src109_r, src98_r, src1211_r, src1110_r,
1531 src1413_r, src1312_r,
1532 src8776, src10998, src12111110, src14131312);
1533 HEVC_FILT_8TAP_4W_SH(src2110, src4332, src6554, src8776, filt0,
1534 filt1, filt2, filt3, dst0, dst1);
1535 HEVC_FILT_8TAP_4W_SH(src4332, src6554, src8776, src10998, filt0,
1536 filt1, filt2, filt3, dst2, dst3);
1537 HEVC_FILT_8TAP_4W_SH(src6554, src8776, src10998, src12111110,
1538 filt0, filt1, filt2, filt3, dst4, dst5);
1539 HEVC_FILT_8TAP_4W_SH(src8776, src10998, src12111110, src14131312,
1540 filt0, filt1, filt2, filt3, dst6, dst7);
1541 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1542 weight_vec, dst0, dst1, dst2, dst3)
1543 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1544 weight_vec, dst4, dst5, dst6, dst7);
1545 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1546 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1547 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1548 offset_vec, dst0, dst1, dst2, dst3);
1549 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1550 offset_vec, dst4, dst5, dst6, dst7);
1551 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1552 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1553 vec0, vec1, vec2, vec3);
1554 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
1555
1556 if (res == 2) {
1557 ST_W2(out0, 0, 1, dst, dst_stride);
1558 } else if (res == 4) {
1559 ST_W4(out0, 0, 1, 2, 3, dst, dst_stride);
1560 } else {
1561 ST_W4(out0, 0, 1, 2, 3, dst, dst_stride);
1562 ST_W2(out1, 0, 1, dst + 4 * dst_stride, dst_stride);
1563 }
1564 }
1565}
1566
1567static void hevc_vt_uniwgt_8t_8w_msa(const uint8_t *src,
1568 int32_t src_stride,
1569 uint8_t *dst,
1570 int32_t dst_stride,
1571 const int8_t *filter,
1572 int32_t height,
1573 int32_t weight,
1574 int32_t offset,
1575 int32_t rnd_val)
1576{
1577 int32_t loop_cnt;
1578 int32_t res = height & 0x03;
1579 v16u8 out0, out1;
1580 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1581 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1582 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1583 v8i16 filt0, filt1, filt2, filt3;
1584 v8i16 filter_vec, vec0, vec1, vec2, vec3;
1585 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1586 v4i32 weight_vec, rnd_vec, offset_vec;
1587 v8i16 zero = { 0 };
1588
1589 src -= (3 * src_stride);
1590
1591 weight_vec = __msa_fill_w(weight);
1592 rnd_vec = __msa_fill_w(rnd_val);
1593 offset_vec = __msa_fill_w(offset);
1594
1595 filter_vec = LD_SH(filter);
1596 UNPCK_R_SB_SH(filter_vec, filter_vec);
1597 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
1598
1599 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1600 src += (7 * src_stride);
1601
1602 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1603 src10_r, src32_r, src54_r, src21_r);
1604 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1605
1606 for (loop_cnt = (height >> 2); loop_cnt--;) {
1607 LD_SB4(src, src_stride, src7, src8, src9, src10);
1608 src += (4 * src_stride);
1609 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1610 src76_r, src87_r, src98_r, src109_r);
1611 HEVC_FILT_8TAP_4W_SH(src10_r, src32_r, src54_r, src76_r, filt0,
1612 filt1, filt2, filt3, dst0, dst1);
1613 HEVC_FILT_8TAP_4W_SH(src21_r, src43_r, src65_r, src87_r, filt0,
1614 filt1, filt2, filt3, dst2, dst3);
1615 HEVC_FILT_8TAP_4W_SH(src32_r, src54_r, src76_r, src98_r,
1616 filt0, filt1, filt2, filt3, dst4, dst5);
1617 HEVC_FILT_8TAP_4W_SH(src43_r, src65_r, src87_r, src109_r,
1618 filt0, filt1, filt2, filt3, dst6, dst7);
1619 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1620 weight_vec, dst0, dst1, dst2, dst3)
1621 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1622 weight_vec, dst4, dst5, dst6, dst7);
1623 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1624 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1625 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1626 offset_vec, dst0, dst1, dst2, dst3);
1627 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1628 offset_vec, dst4, dst5, dst6, dst7);
1629 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1630 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1631 vec0, vec1, vec2, vec3);
1632 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
1633 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
1634 dst += (4 * dst_stride);
1635
1636 src10_r = src54_r;
1637 src32_r = src76_r;
1638 src54_r = src98_r;
1639 src21_r = src65_r;
1640 src43_r = src87_r;
1641 src65_r = src109_r;
1642 src6 = src10;
1643 }
1644 if (res) {
1645 LD_SB2(src, src_stride, src7, src8);
1646 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
1647 HEVC_FILT_8TAP_4W_SH(src10_r, src32_r, src54_r, src76_r, filt0,
1648 filt1, filt2, filt3, dst0, dst1);
1649 HEVC_FILT_8TAP_4W_SH(src21_r, src43_r, src65_r, src87_r, filt0,
1650 filt1, filt2, filt3, dst2, dst3);
1651 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1652 weight_vec, dst0, dst1, dst2, dst3)
1653 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1654 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1655 offset_vec, dst0, dst1, dst2, dst3);
1656 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
1657 PCKEV_H2_SH(dst1, dst0, dst3, dst2, vec0, vec1);
1658 out0 = __msa_pckev_b((v16i8) vec1, (v16i8) vec0);
1659 ST_D2(out0, 0, 1, dst, dst_stride);
1660 }
1661}
1662
1663static void hevc_vt_uniwgt_8t_12w_msa(const uint8_t *src,
1664 int32_t src_stride,
1665 uint8_t *dst,
1666 int32_t dst_stride,
1667 const int8_t *filter,
1668 int32_t height,
1669 int32_t weight,
1670 int32_t offset,
1671 int32_t rnd_val)
1672{
1673 int32_t loop_cnt;
1674 v16u8 out0, out1, out2;
1675 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1676 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1677 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1678 v16i8 src10_l, src32_l, src54_l, src76_l, src98_l;
1679 v16i8 src21_l, src43_l, src65_l, src87_l, src109_l;
1680 v16i8 src2110, src4332, src6554, src8776, src10998;
1681 v8i16 filt0, filt1, filt2, filt3;
1682 v4i32 dst0, dst1, dst2, dst3, dst4, dst5;
1683 v4i32 dst6, dst7, dst8, dst9, dst10, dst11;
1684 v8i16 filter_vec, vec0, vec1, vec2, vec3, vec4, vec5;
1685 v4i32 weight_vec, rnd_vec, offset_vec;
1686 v8i16 zero = { 0 };
1687
1688 src -= (3 * src_stride);
1689
1690 weight_vec = __msa_fill_w(weight);
1691 rnd_vec = __msa_fill_w(rnd_val);
1692 offset_vec = __msa_fill_w(offset);
1693
1694 filter_vec = LD_SH(filter);
1695 UNPCK_R_SB_SH(filter_vec, filter_vec);
1696 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
1697
1698 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1699 src += (7 * src_stride);
1700
1701 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1702 src10_r, src32_r, src54_r, src21_r);
1703 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1704 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1705 src10_l, src32_l, src54_l, src21_l);
1706 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1707 ILVR_D3_SB(src21_l, src10_l, src43_l, src32_l, src65_l, src54_l,
1708 src2110, src4332, src6554);
1709
1710 for (loop_cnt = 4; loop_cnt--;) {
1711 LD_SB4(src, src_stride, src7, src8, src9, src10);
1712 src += (4 * src_stride);
1713
1714 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1715 src76_r, src87_r, src98_r, src109_r);
1716 ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1717 src76_l, src87_l, src98_l, src109_l);
1718 ILVR_D2_SB(src87_l, src76_l, src109_l, src98_l, src8776, src10998);
1719
1720 HEVC_FILT_8TAP_4W_SH(src10_r, src32_r, src54_r, src76_r, filt0,
1721 filt1, filt2, filt3, dst0, dst1);
1722 HEVC_FILT_8TAP_4W_SH(src21_r, src43_r, src65_r, src87_r, filt0,
1723 filt1, filt2, filt3, dst2, dst3);
1724 HEVC_FILT_8TAP_4W_SH(src32_r, src54_r, src76_r, src98_r,
1725 filt0, filt1, filt2, filt3, dst4, dst5);
1726 HEVC_FILT_8TAP_4W_SH(src43_r, src65_r, src87_r, src109_r,
1727 filt0, filt1, filt2, filt3, dst6, dst7);
1728 HEVC_FILT_8TAP_4W_SH(src2110, src4332, src6554, src8776,
1729 filt0, filt1, filt2, filt3, dst8, dst9);
1730 HEVC_FILT_8TAP_4W_SH(src4332, src6554, src8776, src10998,
1731 filt0, filt1, filt2, filt3, dst10, dst11);
1732
1733 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1734 weight_vec, dst0, dst1, dst2, dst3)
1735 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1736 weight_vec, dst4, dst5, dst6, dst7);
1737 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
1738 weight_vec, dst8, dst9, dst10, dst11);
1739 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1740 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1741 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
1742 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1743 offset_vec, dst0, dst1, dst2, dst3);
1744 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1745 offset_vec, dst4, dst5, dst6, dst7);
1746 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
1747 offset_vec, dst8, dst9, dst10, dst11);
1748 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1749 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
1750 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1751 vec0, vec1, vec2, vec3);
1752 PCKEV_H2_SH(dst9, dst8, dst11, dst10, vec4, vec5);
1753 PCKEV_B3_UB(vec1, vec0, vec3, vec2, vec5, vec4, out0, out1, out2);
1754 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
1755 ST_W4(out2, 0, 1, 2, 3, dst + 8, dst_stride);
1756 dst += (4 * dst_stride);
1757
1758 src10_r = src54_r;
1759 src32_r = src76_r;
1760 src54_r = src98_r;
1761 src21_r = src65_r;
1762 src43_r = src87_r;
1763 src65_r = src109_r;
1764 src2110 = src6554;
1765 src4332 = src8776;
1766 src6554 = src10998;
1767 src6 = src10;
1768 }
1769}
1770
1771static void hevc_vt_uniwgt_8t_16multx4mult_msa(const uint8_t *src,
1772 int32_t src_stride,
1773 uint8_t *dst,
1774 int32_t dst_stride,
1775 const int8_t *filter,
1776 int32_t height,
1777 int32_t weight,
1778 int32_t offset,
1779 int32_t rnd_val,
1780 int32_t weightmul16)
1781{
1782 const uint8_t *src_tmp;
1783 uint8_t *dst_tmp;
1784 int32_t loop_cnt, cnt;
1785 int32_t res = height & 0x03;
1786 v16u8 out0, out1, out2, out3;
1787 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1788 v16i8 src10_r, src32_r, src54_r, src76_r;
1789 v16i8 src21_r, src43_r, src65_r, src87_r;
1790 v16i8 src10_l, src32_l, src54_l, src76_l;
1791 v16i8 src21_l, src43_l, src65_l, src87_l;
1792 v16i8 src98_r, src109_r, src98_l, src109_l;
1793 v8i16 filt0, filt1, filt2, filt3;
1794 v8i16 filter_vec, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1795 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
1796 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
1797 v4i32 weight_vec, rnd_vec, offset_vec;
1798 v8i16 zero = { 0 };
1799
1800 src -= (3 * src_stride);
1801
1802 weight_vec = __msa_fill_w(weight);
1803 rnd_vec = __msa_fill_w(rnd_val);
1804 offset_vec = __msa_fill_w(offset);
1805
1806
1807 filter_vec = LD_SH(filter);
1808 UNPCK_R_SB_SH(filter_vec, filter_vec);
1809 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
1810
1811 for (cnt = weightmul16; cnt--;) {
1812 src_tmp = src;
1813 dst_tmp = dst;
1814
1815 LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
1816 src_tmp += (7 * src_stride);
1817
1818 for (loop_cnt = (height >> 2); loop_cnt--;) {
1819 LD_SB4(src_tmp, src_stride, src7, src8, src9, src10);
1820 src_tmp += (4 * src_stride);
1821
1822 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1823 src10_r, src32_r, src54_r, src21_r);
1824 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1825 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1826 src10_l, src32_l, src54_l, src21_l);
1827 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1828 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1829 src76_r, src87_r, src98_r, src109_r);
1830 ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1831 src76_l, src87_l, src98_l, src109_l);
1832
1833 HEVC_FILT_8TAP_4W_SH(src10_r, src32_r, src54_r, src76_r, filt0,
1834 filt1, filt2, filt3, dst0, dst1);
1835 HEVC_FILT_8TAP_4W_SH(src10_l, src32_l, src54_l, src76_l, filt0,
1836 filt1, filt2, filt3, dst2, dst3);
1837 HEVC_FILT_8TAP_4W_SH(src21_r, src43_r, src65_r, src87_r, filt0,
1838 filt1, filt2, filt3, dst4, dst5);
1839 HEVC_FILT_8TAP_4W_SH(src21_l, src43_l, src65_l, src87_l, filt0,
1840 filt1, filt2, filt3, dst6, dst7);
1841 HEVC_FILT_8TAP_4W_SH(src32_r, src54_r, src76_r, src98_r, filt0,
1842 filt1, filt2, filt3, dst8, dst9);
1843 HEVC_FILT_8TAP_4W_SH(src32_l, src54_l, src76_l, src98_l, filt0,
1844 filt1, filt2, filt3, dst10, dst11);
1845 HEVC_FILT_8TAP_4W_SH(src43_r, src65_r, src87_r, src109_r, filt0,
1846 filt1, filt2, filt3, dst12, dst13);
1847 HEVC_FILT_8TAP_4W_SH(src43_l, src65_l, src87_l, src109_l, filt0,
1848 filt1, filt2, filt3, dst14, dst15);
1849
1850 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1851 weight_vec, dst0, dst1, dst2, dst3)
1852 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1853 weight_vec, dst4, dst5, dst6, dst7);
1854 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
1855 weight_vec, dst8, dst9, dst10, dst11);
1856 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec, dst15,
1857 weight_vec, dst12, dst13, dst14, dst15);
1858 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1859 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1860 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
1861 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
1862 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1863 offset_vec, dst0, dst1, dst2, dst3);
1864 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1865 offset_vec, dst4, dst5, dst6, dst7);
1866 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
1867 offset_vec, dst8, dst9, dst10, dst11);
1868 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec, dst15,
1869 offset_vec, dst12, dst13, dst14, dst15);
1870 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1871 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
1872 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1873 vec0, vec1, vec2, vec3);
1874 PCKEV_H4_SH(dst9, dst8, dst11, dst10, dst13, dst12, dst15,
1875 dst14, vec4, vec5, vec6, vec7);
1876 PCKEV_B4_UB(vec1, vec0, vec3, vec2, vec5, vec4, vec7, vec6,
1877 out0, out1, out2, out3);
1878
1879 ST_UB4(out0, out1, out2, out3, dst_tmp, dst_stride);
1880 dst_tmp += (4 * dst_stride);
1881
1882 src0 = src4;
1883 src1 = src5;
1884 src2 = src6;
1885 src3 = src7;
1886 src4 = src8;
1887 src5 = src9;
1888 src6 = src10;
1889 }
1890 if (res) {
1891 LD_SB2(src_tmp, src_stride, src7, src8);
1892
1893 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1894 src10_r, src32_r, src54_r, src21_r);
1895 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1896 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1897 src10_l, src32_l, src54_l, src21_l);
1898 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1899 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
1900 ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
1901
1902 HEVC_FILT_8TAP_4W_SH(src10_r, src32_r, src54_r, src76_r, filt0,
1903 filt1, filt2, filt3, dst0, dst1);
1904 HEVC_FILT_8TAP_4W_SH(src10_l, src32_l, src54_l, src76_l, filt0,
1905 filt1, filt2, filt3, dst2, dst3);
1906 HEVC_FILT_8TAP_4W_SH(src21_r, src43_r, src65_r, src87_r, filt0,
1907 filt1, filt2, filt3, dst4, dst5);
1908 HEVC_FILT_8TAP_4W_SH(src21_l, src43_l, src65_l, src87_l, filt0,
1909 filt1, filt2, filt3, dst6, dst7);
1910 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
1911 weight_vec, dst0, dst1, dst2, dst3)
1912 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec, dst7,
1913 weight_vec, dst4, dst5, dst6, dst7);
1914 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
1915 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
1916 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
1917 offset_vec, dst0, dst1, dst2, dst3);
1918 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
1919 offset_vec, dst4, dst5, dst6, dst7);
1920 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
1921 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
1922 vec0, vec1, vec2, vec3);
1923 PCKEV_B2_UB(vec1, vec0, vec3, vec2, out0, out1);
1924
1925 ST_UB2(out0, out1, dst_tmp, dst_stride);
1926 }
1927
1928 src += 16;
1929 dst += 16;
1930 }
1931}
1932
1933static void hevc_vt_uniwgt_8t_16w_msa(const uint8_t *src,
1934 int32_t src_stride,
1935 uint8_t *dst,
1936 int32_t dst_stride,
1937 const int8_t *filter,
1938 int32_t height,
1939 int32_t weight,
1940 int32_t offset,
1941 int32_t rnd_val)
1942{
1943 hevc_vt_uniwgt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1944 filter, height, weight,
1945 offset, rnd_val, 1);
1946}
1947
1948static void hevc_vt_uniwgt_8t_24w_msa(const uint8_t *src,
1949 int32_t src_stride,
1950 uint8_t *dst,
1951 int32_t dst_stride,
1952 const int8_t *filter,
1953 int32_t height,
1954 int32_t weight,
1955 int32_t offset,
1956 int32_t rnd_val)
1957{
1958 hevc_vt_uniwgt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1959 filter, 32, weight,
1960 offset, rnd_val, 1);
1961
1962 hevc_vt_uniwgt_8t_8w_msa(src + 16, src_stride, dst + 16, dst_stride,
1963 filter, 32, weight, offset, rnd_val);
1964}
1965
1966static void hevc_vt_uniwgt_8t_32w_msa(const uint8_t *src,
1967 int32_t src_stride,
1968 uint8_t *dst,
1969 int32_t dst_stride,
1970 const int8_t *filter,
1971 int32_t height,
1972 int32_t weight,
1973 int32_t offset,
1974 int32_t rnd_val)
1975{
1976 hevc_vt_uniwgt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1977 filter, height, weight,
1978 offset, rnd_val, 2);
1979}
1980
1981static void hevc_vt_uniwgt_8t_48w_msa(const uint8_t *src,
1982 int32_t src_stride,
1983 uint8_t *dst,
1984 int32_t dst_stride,
1985 const int8_t *filter,
1986 int32_t height,
1987 int32_t weight,
1988 int32_t offset,
1989 int32_t rnd_val)
1990{
1991 hevc_vt_uniwgt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
1992 filter, 64, weight,
1993 offset, rnd_val, 3);
1994}
1995
1996static void hevc_vt_uniwgt_8t_64w_msa(const uint8_t *src,
1997 int32_t src_stride,
1998 uint8_t *dst,
1999 int32_t dst_stride,
2000 const int8_t *filter,
2001 int32_t height,
2002 int32_t weight,
2003 int32_t offset,
2004 int32_t rnd_val)
2005{
2006 hevc_vt_uniwgt_8t_16multx4mult_msa(src, src_stride, dst, dst_stride,
2007 filter, height, weight,
2008 offset, rnd_val, 4);
2009}
2010
2011static void hevc_hv_uniwgt_8t_4w_msa(const uint8_t *src,
2012 int32_t src_stride,
2013 uint8_t *dst,
2014 int32_t dst_stride,
2015 const int8_t *filter_x,
2016 const int8_t *filter_y,
2017 int32_t height,
2018 int32_t weight,
2019 int32_t offset,
2020 int32_t rnd_val)
2021{
2022 uint32_t loop_cnt;
2023 v16u8 out;
2024 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2025 v8i16 filt0, filt1, filt2, filt3;
2026 v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
2027 v16i8 mask1, mask2, mask3;
2028 v8i16 filter_vec;
2029 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2030 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
2031 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9, dst10;
2032 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst98_r;
2033 v8i16 dst21_r, dst43_r, dst65_r, dst87_r, dst109_r;
2034 v4i32 dst0_r, dst1_r, dst2_r, dst3_r;
2035 v4i32 weight_vec, offset_vec, rnd_vec;
2036 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
2037 v8i16 zero = { 0 };
2038
2039 src -= ((3 * src_stride) + 3);
2040 filter_vec = LD_SH(filter_x);
2041 UNPCK_R_SB_SH(filter_vec, filter_vec);
2042 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
2043
2044 filter_vec = LD_SH(filter_y);
2045 UNPCK_R_SB_SH(filter_vec, filter_vec);
2046 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
2047
2048 mask1 = mask0 + 2;
2049 mask2 = mask0 + 4;
2050 mask3 = mask0 + 6;
2051
2052 weight_vec = __msa_fill_w(weight);
2053 offset_vec = __msa_fill_w(offset);
2054 rnd_vec = __msa_fill_w(rnd_val);
2055
2056 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
2057 src += (7 * src_stride);
2058
2059 /* row 0 row 1 row 2 row 3 */
2060 VSHF_B4_SB(src0, src3, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
2061 VSHF_B4_SB(src1, src4, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
2062 VSHF_B4_SB(src2, src5, mask0, mask1, mask2, mask3,
2063 vec8, vec9, vec10, vec11);
2064 VSHF_B4_SB(src3, src6, mask0, mask1, mask2, mask3,
2065 vec12, vec13, vec14, vec15);
2066 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2067 filt3, dst0, dst3);
2068 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2069 filt3, dst1, dst4);
2070 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2071 filt3, dst2, dst5);
2072 vec0 = __msa_ilvl_b((v16i8) zero, (v16i8) vec12);
2073 vec1 = __msa_ilvl_b((v16i8) zero, (v16i8) vec13);
2074 vec2 = __msa_ilvl_b((v16i8) zero, (v16i8) vec14);
2075 vec3 = __msa_ilvl_b((v16i8) zero, (v16i8) vec15);
2076 dst6 = __msa_dotp_s_w((v8i16) vec0, (v8i16) filt0);
2077 dst6 = __msa_dpadd_s_w((v4i32) dst6, (v8i16) vec1, (v8i16) filt1);
2078 dst6 = __msa_dpadd_s_w((v4i32) dst6, (v8i16) vec2, (v8i16) filt2);
2079 dst6 = __msa_dpadd_s_w((v4i32) dst6, (v8i16) vec3, (v8i16) filt3);
2080
2081 ILVEV_H2_SH(dst0, dst1, dst3, dst4, dst10_r, dst43_r);
2082 ILVEV_H2_SH(dst1, dst2, dst4, dst5, dst21_r, dst54_r);
2083 ILVEV_H2_SH(dst2, dst3, dst5, dst6, dst32_r, dst65_r);
2084
2085 for (loop_cnt = height >> 2; loop_cnt--;) {
2086 LD_SB4(src, src_stride, src7, src8, src9, src10);
2087 src += (4 * src_stride);
2088
2089 VSHF_B4_SB(src7, src9, mask0, mask1, mask2, mask3,
2090 vec0, vec1, vec2, vec3);
2091 VSHF_B4_SB(src8, src10, mask0, mask1, mask2, mask3,
2092 vec4, vec5, vec6, vec7);
2093 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2094 filt3, dst7, dst9);
2095 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2096 filt3, dst8, dst10);
2097
2098 dst76_r = __msa_ilvev_h((v8i16) dst7, (v8i16) dst6);
2099 ILVEV_H2_SH(dst7, dst8, dst9, dst10, dst87_r, dst109_r);
2100 dst98_r = __msa_ilvev_h((v8i16) dst9, (v8i16) dst8);
2101
2102 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0,
2103 filt_h1, filt_h2, filt_h3);
2104 dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r, filt_h0,
2105 filt_h1, filt_h2, filt_h3);
2106 dst2_r = HEVC_FILT_8TAP(dst32_r, dst54_r, dst76_r, dst98_r, filt_h0,
2107 filt_h1, filt_h2, filt_h3);
2108 dst3_r = HEVC_FILT_8TAP(dst43_r, dst65_r, dst87_r, dst109_r, filt_h0,
2109 filt_h1, filt_h2, filt_h3);
2110
2111 SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
2112 MUL2(dst0_r, weight_vec, dst1_r, weight_vec, dst0_r, dst1_r);
2113 MUL2(dst2_r, weight_vec, dst3_r, weight_vec, dst2_r, dst3_r);
2114 SRAR_W4_SW(dst0_r, dst1_r, dst2_r, dst3_r, rnd_vec);
2115 ADD2(dst0_r, offset_vec, dst1_r, offset_vec, dst0_r, dst1_r);
2116 ADD2(dst2_r, offset_vec, dst3_r, offset_vec, dst2_r, dst3_r);
2117 CLIP_SW4_0_255(dst0_r, dst1_r, dst2_r, dst3_r);
2118 PCKEV_H2_SW(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
2119 out = (v16u8) __msa_pckev_b((v16i8) dst1_r, (v16i8) dst0_r);
2120 ST_W4(out, 0, 1, 2, 3, dst, dst_stride);
2121 dst += (4 * dst_stride);
2122
2123 dst10_r = dst54_r;
2124 dst32_r = dst76_r;
2125 dst54_r = dst98_r;
2126 dst21_r = dst65_r;
2127 dst43_r = dst87_r;
2128 dst65_r = dst109_r;
2129 dst6 = dst10;
2130 }
2131}
2132
2133static void hevc_hv_uniwgt_8t_8multx2mult_msa(const uint8_t *src,
2134 int32_t src_stride,
2135 uint8_t *dst,
2136 int32_t dst_stride,
2137 const int8_t *filter_x,
2138 const int8_t *filter_y,
2139 int32_t height,
2140 int32_t weight,
2141 int32_t offset,
2142 int32_t rnd_val,
2143 int32_t width)
2144{
2145 uint32_t loop_cnt, cnt;
2146 const uint8_t *src_tmp;
2147 uint8_t *dst_tmp;
2148 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2149 v8i16 filt0, filt1, filt2, filt3;
2150 v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
2151 v16i8 mask1, mask2, mask3;
2152 v8i16 filter_vec;
2153 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2154 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
2155 v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
2156 v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
2157 v8i16 dst10_l, dst32_l, dst54_l, dst76_l;
2158 v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
2159 v8i16 dst21_l, dst43_l, dst65_l, dst87_l;
2160 v4i32 weight_vec, offset_vec, rnd_vec;
2161 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9, dst10;
2162 v4i32 dst11, dst12, dst13, dst14, dst15;
2163 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2164 v8i16 zero = { 0 };
2165
2166 src -= ((3 * src_stride) + 3);
2167
2168 weight_vec = __msa_fill_w(weight);
2169 offset_vec = __msa_fill_w(offset);
2170 rnd_vec = __msa_fill_w(rnd_val);
2171
2172
2173 filter_vec = LD_SH(filter_x);
2174 UNPCK_R_SB_SH(filter_vec, filter_vec);
2175 SPLATI_W4_SW(filter_vec, filt0, filt1, filt2, filt3);
2176
2177 filter_vec = LD_SH(filter_y);
2178 UNPCK_R_SB_SH(filter_vec, filter_vec);
2179 SPLATI_W4_SW(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
2180
2181 mask1 = mask0 + 2;
2182 mask2 = mask0 + 4;
2183 mask3 = mask0 + 6;
2184
2185 for (cnt = width >> 3; cnt--;) {
2186 src_tmp = src;
2187 dst_tmp = dst;
2188
2189 LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
2190 src_tmp += (7 * src_stride);
2191
2192 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
2193 vec0, vec1, vec2, vec3);
2194 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
2195 vec4, vec5, vec6, vec7);
2196 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
2197 vec8, vec9, vec10, vec11);
2198 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
2199 vec12, vec13, vec14, vec15);
2200 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2201 filt3, dst0, dst1);
2202 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2203 filt3, dst2, dst3);
2204 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2205 filt3, dst4, dst5);
2206 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1,
2207 filt2, filt3, dst6, dst7);
2208
2209 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
2210 vec0, vec1, vec2, vec3);
2211 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3,
2212 vec4, vec5, vec6, vec7);
2213 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3,
2214 vec8, vec9, vec10, vec11);
2215 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2216 filt3, dst8, dst9);
2217 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2218 filt3, dst10, dst11);
2219 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2220 filt3, dst12, dst13);
2221
2222 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst10_r, dst10_l);
2223 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst21_r, dst21_l);
2224 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst32_r, dst32_l);
2225 ILVEV_H2_SH(dst6, dst8, dst7, dst9, dst43_r, dst43_l);
2226 ILVEV_H2_SH(dst8, dst10, dst9, dst11, dst54_r, dst54_l);
2227 ILVEV_H2_SH(dst10, dst12, dst11, dst13, dst65_r, dst65_l);
2228
2229 for (loop_cnt = height >> 1; loop_cnt--;) {
2230 LD_SB2(src_tmp, src_stride, src7, src8);
2231 src_tmp += 2 * src_stride;
2232
2233 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3,
2234 vec0, vec1, vec2, vec3);
2235 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1,
2236 filt2, filt3, dst14, dst15);
2237
2238 ILVEV_H2_SH(dst12, dst14, dst13, dst15, dst76_r, dst76_l);
2239 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
2240 filt_h0, filt_h1, filt_h2, filt_h3);
2241 dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l,
2242 filt_h0, filt_h1, filt_h2, filt_h3);
2243 dst0_r >>= 6;
2244 dst0_l >>= 6;
2245
2246 /* row 8 */
2247 VSHF_B4_SB(src8, src8, mask0, mask1, mask2, mask3,
2248 vec0, vec1, vec2, vec3);
2249 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1,
2250 filt2, filt3, dst0, dst1);
2251
2252 ILVEV_H2_SH(dst14, dst0, dst15, dst1, dst87_r, dst87_l);
2253 dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r,
2254 filt_h0, filt_h1, filt_h2, filt_h3);
2255 dst1_l = HEVC_FILT_8TAP(dst21_l, dst43_l, dst65_l, dst87_l,
2256 filt_h0, filt_h1, filt_h2, filt_h3);
2257 dst1_r >>= 6;
2258 dst1_l >>= 6;
2259
2260 MUL2(dst0_r, weight_vec, dst0_l, weight_vec, dst0_r, dst0_l);
2261 MUL2(dst1_r, weight_vec, dst1_l, weight_vec, dst1_r, dst1_l);
2262 SRAR_W4_SW(dst0_r, dst1_r, dst0_l, dst1_l, rnd_vec);
2263 ADD2(dst0_r, offset_vec, dst0_l, offset_vec, dst0_r, dst0_l);
2264 ADD2(dst1_r, offset_vec, dst1_l, offset_vec, dst1_r, dst1_l);
2265 CLIP_SW4_0_255(dst0_r, dst1_r, dst0_l, dst1_l);
2266
2267 PCKEV_H2_SW(dst0_l, dst0_r, dst1_l, dst1_r, dst0_r, dst1_r);
2268 dst0_r = (v4i32) __msa_pckev_b((v16i8) dst1_r, (v16i8) dst0_r);
2269 ST_D2(dst0_r, 0, 1, dst_tmp, dst_stride);
2270 dst_tmp += (2 * dst_stride);
2271
2272 dst10_r = dst32_r;
2273 dst32_r = dst54_r;
2274 dst54_r = dst76_r;
2275 dst10_l = dst32_l;
2276 dst32_l = dst54_l;
2277 dst54_l = dst76_l;
2278 dst21_r = dst43_r;
2279 dst43_r = dst65_r;
2280 dst65_r = dst87_r;
2281 dst21_l = dst43_l;
2282 dst43_l = dst65_l;
2283 dst65_l = dst87_l;
2284 dst12 = dst0;
2285 dst13 = dst1;
2286 }
2287
2288 src += 8;
2289 dst += 8;
2290 }
2291}
2292
2293static void hevc_hv_uniwgt_8t_8w_msa(const uint8_t *src,
2294 int32_t src_stride,
2295 uint8_t *dst,
2296 int32_t dst_stride,
2297 const int8_t *filter_x,
2298 const int8_t *filter_y,
2299 int32_t height,
2300 int32_t weight,
2301 int32_t offset,
2302 int32_t rnd_val)
2303{
2304 hevc_hv_uniwgt_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
2305 filter_x, filter_y, height, weight,
2306 offset, rnd_val, 8);
2307}
2308
2309static void hevc_hv_uniwgt_8t_12w_msa(const uint8_t *src,
2310 int32_t src_stride,
2311 uint8_t *dst,
2312 int32_t dst_stride,
2313 const int8_t *filter_x,
2314 const int8_t *filter_y,
2315 int32_t height,
2316 int32_t weight,
2317 int32_t offset,
2318 int32_t rnd_val)
2319{
2320 uint32_t loop_cnt;
2321 const uint8_t *src_tmp;
2322 uint8_t *dst_tmp;
2323 v16u8 out;
2324 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2325 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
2326 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2327 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
2328 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2329 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
2330 v8i16 filt0, filt1, filt2, filt3, filt_h0, filt_h1, filt_h2, filt_h3;
2331 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst10_l, dst32_l, dst54_l;
2332 v8i16 dst98_r, dst21_r, dst43_r, dst65_r, dst87_r, dst109_r;
2333 v8i16 dst76_l, filter_vec;
2334 v4i32 dst0_r, dst0_l, dst1_r, dst2_r, dst3_r;
2335 v4i32 weight_vec, offset_vec, rnd_vec;
2336 v8i16 zero = { 0 };
2337
2338 src -= ((3 * src_stride) + 3);
2339
2340 filter_vec = LD_SH(filter_x);
2341 UNPCK_R_SB_SH(filter_vec, filter_vec);
2342 SPLATI_W4_SH(filter_vec, filt0, filt1, filt2, filt3);
2343
2344 filter_vec = LD_SH(filter_y);
2345 UNPCK_R_SB_SH(filter_vec, filter_vec);
2346
2347 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
2348
2349 weight_vec = __msa_fill_w(weight);
2350 offset_vec = __msa_fill_w(offset);
2351 rnd_vec = __msa_fill_w(rnd_val);
2352
2353 mask0 = LD_SB(ff_hevc_mask_arr);
2354 mask1 = mask0 + 2;
2355 mask2 = mask0 + 4;
2356 mask3 = mask0 + 6;
2357
2358 src_tmp = src;
2359 dst_tmp = dst;
2360
2361 LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
2362 src_tmp += (7 * src_stride);
2363
2364 /* row 0 row 1 row 2 row 3 */
2365 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
2366 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
2367 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
2368 vec11);
2369 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3, vec12, vec13, vec14,
2370 vec15);
2371 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2372 filt3, dst0, dst1);
2373 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2374 filt3, dst2, dst3);
2375 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2376 filt3, dst4, dst5);
2377 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1,
2378 filt2, filt3, dst6, dst7);
2379 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
2380 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
2381 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
2382 vec11);
2383 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2384 filt3, dst8, dst9);
2385 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2386 filt3, dst10, dst11);
2387 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2388 filt3, dst12, dst13);
2389
2390 for (loop_cnt = 16; loop_cnt--;) {
2391 src7 = LD_SB(src_tmp);
2392 src_tmp += src_stride;
2393
2394 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
2395 vec3);
2396 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2397 filt3, dst14, dst15);
2398 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst10_r, dst10_l);
2399 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst32_r, dst32_l);
2400 ILVEV_H2_SH(dst8, dst10, dst9, dst11, dst54_r, dst54_l);
2401 ILVEV_H2_SH(dst12, dst14, dst13, dst15, dst76_r, dst76_l);
2402
2403 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
2404 filt_h0, filt_h1, filt_h2, filt_h3);
2405 dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l,
2406 filt_h0, filt_h1, filt_h2, filt_h3);
2407 dst0_r >>= 6;
2408 dst0_l >>= 6;
2409
2410 MUL2(dst0_r, weight_vec, dst0_l, weight_vec, dst0_r, dst0_l);
2411 SRAR_W2_SW(dst0_r, dst0_l, rnd_vec);
2412 ADD2(dst0_r, offset_vec, dst0_l, offset_vec, dst0_r, dst0_l);
2413 CLIP_SW2_0_255(dst0_r, dst0_l);
2414 dst0_r = (v4i32) __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
2415 out = (v16u8) __msa_pckev_b((v16i8) dst0_r, (v16i8) dst0_r);
2416 ST_D1(out, 0, dst_tmp);
2417 dst_tmp += dst_stride;
2418
2419 dst0 = dst2;
2420 dst1 = dst3;
2421 dst2 = dst4;
2422 dst3 = dst5;
2423 dst4 = dst6;
2424 dst5 = dst7;
2425 dst6 = dst8;
2426 dst7 = dst9;
2427 dst8 = dst10;
2428 dst9 = dst11;
2429 dst10 = dst12;
2430 dst11 = dst13;
2431 dst12 = dst14;
2432 dst13 = dst15;
2433 }
2434
2435 src += 8;
2436 dst += 8;
2437
2438 mask4 = LD_SB(ff_hevc_mask_arr + 16);
2439 mask5 = mask4 + 2;
2440 mask6 = mask4 + 4;
2441 mask7 = mask4 + 6;
2442
2443 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
2444 src += (7 * src_stride);
2445
2446 VSHF_B4_SB(src0, src3, mask4, mask5, mask6, mask7, vec0, vec1, vec2, vec3);
2447 VSHF_B4_SB(src1, src4, mask4, mask5, mask6, mask7, vec4, vec5, vec6, vec7);
2448 VSHF_B4_SB(src2, src5, mask4, mask5, mask6, mask7, vec8, vec9, vec10,
2449 vec11);
2450 VSHF_B4_SB(src3, src6, mask4, mask5, mask6, mask7, vec12, vec13, vec14,
2451 vec15);
2452 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2453 filt3, dst0, dst3);
2454 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2455 filt3, dst1, dst4);
2456 HEVC_FILT_8TAP_4W_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2457 filt3, dst2, dst5);
2458 HEVC_FILT_8TAP_4W_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
2459 filt3, dst3, dst6);
2460 ILVEV_H2_SH(dst0, dst1, dst3, dst4, dst10_r, dst43_r);
2461 ILVEV_H2_SH(dst1, dst2, dst4, dst5, dst21_r, dst54_r);
2462 ILVEV_H2_SH(dst2, dst3, dst5, dst6, dst32_r, dst65_r);
2463
2464 for (loop_cnt = 4; loop_cnt--;) {
2465 LD_SB4(src, src_stride, src7, src8, src9, src10);
2466 src += (4 * src_stride);
2467
2468 VSHF_B4_SB(src7, src9, mask4, mask5, mask6, mask7, vec0, vec1, vec2,
2469 vec3);
2470 VSHF_B4_SB(src8, src10, mask4, mask5, mask6, mask7, vec4, vec5, vec6,
2471 vec7);
2472 HEVC_FILT_8TAP_4W_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2473 filt3, dst7, dst9);
2474 HEVC_FILT_8TAP_4W_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2475 filt3, dst8, dst10);
2476
2477 ILVEV_H2_SH(dst6, dst7, dst7, dst8, dst76_r, dst87_r);
2478 ILVEV_H2_SH(dst9, dst10, dst8, dst9, dst109_r, dst98_r);
2479
2480 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0,
2481 filt_h1, filt_h2, filt_h3);
2482 dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r, filt_h0,
2483 filt_h1, filt_h2, filt_h3);
2484 dst2_r = HEVC_FILT_8TAP(dst32_r, dst54_r, dst76_r, dst98_r, filt_h0,
2485 filt_h1, filt_h2, filt_h3);
2486 dst3_r = HEVC_FILT_8TAP(dst43_r, dst65_r, dst87_r, dst109_r, filt_h0,
2487 filt_h1, filt_h2, filt_h3);
2488
2489 SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
2490 MUL2(dst0_r, weight_vec, dst1_r, weight_vec, dst0_r, dst1_r);
2491 MUL2(dst2_r, weight_vec, dst3_r, weight_vec, dst2_r, dst3_r);
2492 SRAR_W4_SW(dst0_r, dst1_r, dst2_r, dst3_r, rnd_vec);
2493 ADD2(dst0_r, offset_vec, dst1_r, offset_vec, dst0_r, dst1_r);
2494 ADD2(dst2_r, offset_vec, dst3_r, offset_vec, dst2_r, dst3_r);
2495 CLIP_SW4_0_255(dst0_r, dst1_r, dst2_r, dst3_r);
2496 PCKEV_H2_SW(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
2497 out = (v16u8) __msa_pckev_b((v16i8) dst1_r, (v16i8) dst0_r);
2498 ST_W4(out, 0, 1, 2, 3, dst, dst_stride);
2499 dst += (4 * dst_stride);
2500
2501 dst10_r = dst54_r;
2502 dst32_r = dst76_r;
2503 dst54_r = dst98_r;
2504 dst21_r = dst65_r;
2505 dst43_r = dst87_r;
2506 dst65_r = dst109_r;
2507 dst6 = dst10;
2508 }
2509}
2510
2511static void hevc_hv_uniwgt_8t_16w_msa(const uint8_t *src,
2512 int32_t src_stride,
2513 uint8_t *dst,
2514 int32_t dst_stride,
2515 const int8_t *filter_x,
2516 const int8_t *filter_y,
2517 int32_t height,
2518 int32_t weight,
2519 int32_t offset,
2520 int32_t rnd_val)
2521{
2522 hevc_hv_uniwgt_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
2523 filter_x, filter_y, height, weight,
2524 offset, rnd_val, 16);
2525}
2526
2527static void hevc_hv_uniwgt_8t_24w_msa(const uint8_t *src,
2528 int32_t src_stride,
2529 uint8_t *dst,
2530 int32_t dst_stride,
2531 const int8_t *filter_x,
2532 const int8_t *filter_y,
2533 int32_t height,
2534 int32_t weight,
2535 int32_t offset,
2536 int32_t rnd_val)
2537{
2538 hevc_hv_uniwgt_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
2539 filter_x, filter_y, height, weight,
2540 offset, rnd_val, 24);
2541}
2542
2543static void hevc_hv_uniwgt_8t_32w_msa(const uint8_t *src,
2544 int32_t src_stride,
2545 uint8_t *dst,
2546 int32_t dst_stride,
2547 const int8_t *filter_x,
2548 const int8_t *filter_y,
2549 int32_t height,
2550 int32_t weight,
2551 int32_t offset,
2552 int32_t rnd_val)
2553{
2554 hevc_hv_uniwgt_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
2555 filter_x, filter_y, height, weight,
2556 offset, rnd_val, 32);
2557}
2558
2559static void hevc_hv_uniwgt_8t_48w_msa(const uint8_t *src,
2560 int32_t src_stride,
2561 uint8_t *dst,
2562 int32_t dst_stride,
2563 const int8_t *filter_x,
2564 const int8_t *filter_y,
2565 int32_t height,
2566 int32_t weight,
2567 int32_t offset,
2568 int32_t rnd_val)
2569{
2570 hevc_hv_uniwgt_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
2571 filter_x, filter_y, height, weight,
2572 offset, rnd_val, 48);
2573}
2574
2575static void hevc_hv_uniwgt_8t_64w_msa(const uint8_t *src,
2576 int32_t src_stride,
2577 uint8_t *dst,
2578 int32_t dst_stride,
2579 const int8_t *filter_x,
2580 const int8_t *filter_y,
2581 int32_t height,
2582 int32_t weight,
2583 int32_t offset,
2584 int32_t rnd_val)
2585{
2586 hevc_hv_uniwgt_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
2587 filter_x, filter_y, height, weight,
2588 offset, rnd_val, 64);
2589}
2590
2591static void hevc_hz_uniwgt_4t_4x2_msa(const uint8_t *src,
2592 int32_t src_stride,
2593 uint8_t *dst,
2594 int32_t dst_stride,
2595 const int8_t *filter,
2596 int32_t weight,
2597 int32_t offset,
2598 int32_t rnd_val)
2599{
2600 v16u8 out;
2601 v8i16 filt0, filt1, filter_vec;
2602 v16i8 src0, src1, vec0, vec1;
2603 v8i16 tmp0, tmp1, tmp2, tmp3;
2604 v16i8 mask1;
2605 v4i32 dst0, dst1;
2606 v4i32 weight_vec, rnd_vec, offset_vec;
2607 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2608 v8i16 zero = { 0 };
2609
2610 src -= 1;
2611
2612 filter_vec = LD_SH(filter);
2613 UNPCK_R_SB_SH(filter_vec, filter_vec);
2614 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
2615
2616 mask1 = mask0 + 2;
2617
2618 weight_vec = __msa_fill_w(weight);
2619 rnd_vec = __msa_fill_w(rnd_val);
2620 offset_vec = __msa_fill_w(offset);
2621
2622 LD_SB2(src, src_stride, src0, src1);
2623
2624 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
2625 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
2626 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
2627 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2628 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2629
2630 MUL2(dst0, weight_vec, dst1, weight_vec, dst0, dst1);
2631 SRAR_W2_SW(dst0, dst1, rnd_vec);
2632 ADD2(dst0, offset_vec, dst1, offset_vec, dst0, dst1);
2633 CLIP_SW2_0_255(dst0, dst1);
2634 vec0 = __msa_pckev_h((v8i16) dst1, (v8i16) dst0);
2635 out = (v16u8) __msa_pckev_b((v16i8) vec0, (v16i8) vec0);
2636 ST_W2(out, 0, 1, dst, dst_stride);
2637 dst += (4 * dst_stride);
2638}
2639
2640static void hevc_hz_uniwgt_4t_4x4_msa(const uint8_t *src,
2641 int32_t src_stride,
2642 uint8_t *dst,
2643 int32_t dst_stride,
2644 const int8_t *filter,
2645 int32_t weight,
2646 int32_t offset,
2647 int32_t rnd_val)
2648{
2649 v16u8 out;
2650 v8i16 filt0, filt1;
2651 v16i8 src0, src1, src2, src3;
2652 v16i8 mask1, vec0, vec1, vec2, vec3;
2653 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
2654 v4i32 dst0, dst1, dst2, dst3;
2655 v8i16 filter_vec;
2656 v4i32 weight_vec, rnd_vec, offset_vec;
2657 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2658 v8i16 zero = { 0 };
2659
2660 src -= 1;
2661
2662 /* rearranging filter */
2663 filter_vec = LD_SH(filter);
2664 UNPCK_R_SB_SH(filter_vec, filter_vec);
2665 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
2666
2667 weight_vec = __msa_fill_w(weight);
2668 rnd_vec = __msa_fill_w(rnd_val);
2669 offset_vec = __msa_fill_w(offset);
2670 mask1 = mask0 + 2;
2671
2672 LD_SB4(src, src_stride, src0, src1, src2, src3);
2673
2674 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
2675 VSHF_B2_SB(src2, src3, src2, src3, mask0, mask1, vec2, vec3);
2676 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
2677 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
2678 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
2679 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
2680 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2681 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2682 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
2683 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
2684
2685 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
2686 weight_vec, dst0, dst1, dst2, dst3);
2687 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
2688 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
2689 offset_vec, dst0, dst1, dst2, dst3);
2690 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
2691 tmp0 = __msa_pckev_h((v8i16) dst1, (v8i16) dst0);
2692 tmp1 = __msa_pckev_h((v8i16) dst3, (v8i16) dst2);
2693
2694 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
2695 ST_W4(out, 0, 1, 2, 3, dst, dst_stride);
2696}
2697
2698static void hevc_hz_uniwgt_4t_4x8multiple_msa(const uint8_t *src,
2699 int32_t src_stride,
2700 uint8_t *dst,
2701 int32_t dst_stride,
2702 const int8_t *filter,
2703 int32_t height,
2704 int32_t weight,
2705 int32_t offset,
2706 int32_t rnd_val)
2707{
2708 uint32_t loop_cnt;
2709 v16u8 out0, out1;
2710 v8i16 filt0, filt1;
2711 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2712 v16i8 mask1, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2713 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
2714 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2715 v8i16 filter_vec;
2716 v4i32 weight_vec, rnd_vec, offset_vec;
2717 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2718 v8i16 zero = { 0 };
2719
2720 src -= 1;
2721
2722 filter_vec = LD_SH(filter);
2723 UNPCK_R_SB_SH(filter_vec, filter_vec);
2724 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
2725
2726 weight_vec = __msa_fill_w(weight);
2727 rnd_vec = __msa_fill_w(rnd_val);
2728 offset_vec = __msa_fill_w(offset);
2729
2730 mask1 = mask0 + 2;
2731
2732 for (loop_cnt = (height >> 3); loop_cnt--;) {
2733 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
2734 src += (8 * src_stride);
2735
2736 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
2737 VSHF_B2_SB(src2, src3, src2, src3, mask0, mask1, vec2, vec3);
2738 VSHF_B2_SB(src4, src5, src4, src5, mask0, mask1, vec4, vec5);
2739 VSHF_B2_SB(src6, src7, src6, src7, mask0, mask1, vec6, vec7);
2740 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
2741 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
2742 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
2743 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
2744 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2745 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2746 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
2747 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
2748 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
2749 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
2750 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
2751 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
2752 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2753 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2754 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
2755 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
2756
2757 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
2758 weight_vec, dst0, dst1, dst2, dst3);
2759 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
2760 dst7, weight_vec, dst4, dst5, dst6, dst7);
2761 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
2762 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
2763 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
2764 offset_vec, dst0, dst1, dst2, dst3);
2765 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
2766 dst7, offset_vec, dst4, dst5, dst6, dst7);
2767 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
2768 tmp0 = __msa_pckev_h((v8i16) dst1, (v8i16) dst0);
2769 tmp1 = __msa_pckev_h((v8i16) dst3, (v8i16) dst2);
2770 tmp2 = __msa_pckev_h((v8i16) dst5, (v8i16) dst4);
2771 tmp3 = __msa_pckev_h((v8i16) dst7, (v8i16) dst6);
2772 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
2773 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
2774 dst += (8 * dst_stride);
2775 }
2776}
2777
2778static void hevc_hz_uniwgt_4t_4w_msa(const uint8_t *src,
2779 int32_t src_stride,
2780 uint8_t *dst,
2781 int32_t dst_stride,
2782 const int8_t *filter,
2783 int32_t height,
2784 int32_t weight,
2785 int32_t offset,
2786 int32_t rnd_val)
2787{
2788 if (2 == height) {
2789 hevc_hz_uniwgt_4t_4x2_msa(src, src_stride, dst, dst_stride,
2790 filter, weight, offset, rnd_val);
2791 } else if (4 == height) {
2792 hevc_hz_uniwgt_4t_4x4_msa(src, src_stride, dst, dst_stride,
2793 filter, weight, offset, rnd_val);
2794 } else if (8 == height || 16 == height) {
2795 hevc_hz_uniwgt_4t_4x8multiple_msa(src, src_stride, dst, dst_stride,
2796 filter, height, weight,
2797 offset, rnd_val);
2798 }
2799}
2800
2801static void hevc_hz_uniwgt_4t_6w_msa(const uint8_t *src,
2802 int32_t src_stride,
2803 uint8_t *dst,
2804 int32_t dst_stride,
2805 const int8_t *filter,
2806 int32_t height,
2807 int32_t weight,
2808 int32_t offset,
2809 int32_t rnd_val)
2810{
2811 v16u8 out0, out1, out2, out3;
2812 v8i16 filter_vec, filt0, filt1;
2813 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2814 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2815 v16i8 mask1;
2816 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2817 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
2818 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2819 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
2820 v4i32 weight_vec, rnd_vec, offset_vec;
2821 v8i16 zero = { 0 };
2822
2823 src -= 1;
2824
2825 filter_vec = LD_SH(filter);
2826 UNPCK_R_SB_SH(filter_vec, filter_vec);
2827 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
2828
2829 weight_vec = __msa_fill_w(weight);
2830 rnd_vec = __msa_fill_w(rnd_val);
2831 offset_vec = __msa_fill_w(offset);
2832
2833 mask1 = mask0 + 2;
2834
2835 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
2836 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2837 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
2838 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
2839 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
2840 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
2841 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
2842 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
2843 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
2844 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2845 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2846 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
2847 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
2848 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
2849 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
2850 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
2851 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
2852 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2853 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2854 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
2855 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
2856 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
2857 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec2, vec3);
2858 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec4, vec5);
2859 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec6, vec7);
2860 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
2861 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
2862 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
2863 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
2864 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2865 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2866 dst10 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
2867 dst11 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
2868 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
2869 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
2870 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
2871 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
2872 dst12 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2873 dst13 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2874 dst14 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
2875 dst15 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
2876
2877 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
2878 weight_vec, dst0, dst1, dst2, dst3);
2879 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
2880 dst7, weight_vec, dst4, dst5, dst6, dst7);
2881 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
2882 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
2883 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
2884 offset_vec, dst0, dst1, dst2, dst3);
2885 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
2886 dst7, offset_vec, dst4, dst5, dst6, dst7);
2887 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
2888
2889 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
2890 weight_vec, dst8, dst9, dst10, dst11);
2891 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
2892 dst15, weight_vec, dst12, dst13, dst14, dst15);
2893 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
2894 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
2895 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
2896 offset_vec, dst8, dst9, dst10, dst11);
2897 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
2898 dst15, offset_vec, dst12, dst13, dst14, dst15);
2899 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
2900
2901 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
2902 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
2903 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
2904 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
2905 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
2906 PCKEV_B2_UB(tmp5, tmp4, tmp7, tmp6, out2, out3);
2907 ST_W2(out0, 0, 2, dst, dst_stride);
2908 ST_H2(out0, 2, 6, dst + 4, dst_stride);
2909 ST_W2(out1, 0, 2, dst + 2 * dst_stride, dst_stride);
2910 ST_H2(out1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
2911 dst += (4 * dst_stride);
2912 ST_W2(out2, 0, 2, dst, dst_stride);
2913 ST_H2(out2, 2, 6, dst + 4, dst_stride);
2914 ST_W2(out3, 0, 2, dst + 2 * dst_stride, dst_stride);
2915 ST_H2(out3, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
2916}
2917
2918static void hevc_hz_uniwgt_4t_8x2_msa(const uint8_t *src,
2919 int32_t src_stride,
2920 uint8_t *dst,
2921 int32_t dst_stride,
2922 const int8_t *filter,
2923 int32_t weight,
2924 int32_t offset,
2925 int32_t rnd_val)
2926{
2927 v16u8 out;
2928 v8i16 filter_vec, filt0, filt1;
2929 v16i8 src0, src1;
2930 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2931 v16i8 mask1;
2932 v16i8 vec0, vec1, vec2, vec3;
2933 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
2934 v4i32 weight_vec, rnd_vec, offset_vec;
2935 v4i32 dst0, dst1, dst2, dst3;
2936 v8i16 zero = { 0 };
2937
2938 src -= 1;
2939
2940 filter_vec = LD_SH(filter);
2941 UNPCK_R_SB_SH(filter_vec, filter_vec);
2942 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
2943
2944 weight_vec = __msa_fill_w(weight);
2945 rnd_vec = __msa_fill_w(rnd_val);
2946 offset_vec = __msa_fill_w(offset);
2947 mask1 = mask0 + 2;
2948
2949 LD_SB2(src, src_stride, src0, src1);
2950
2951 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2952 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
2953 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
2954 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
2955 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
2956 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
2957 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
2958 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
2959 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
2960 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
2961
2962 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
2963 weight_vec, dst0, dst1, dst2, dst3);
2964 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
2965 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
2966 offset_vec, dst0, dst1, dst2, dst3);
2967 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
2968 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
2969
2970 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
2971 ST_D2(out, 0, 1, dst, dst_stride);
2972}
2973
2974static void hevc_hz_uniwgt_4t_8x4_msa(const uint8_t *src,
2975 int32_t src_stride,
2976 uint8_t *dst,
2977 int32_t dst_stride,
2978 const int8_t *filter,
2979 int32_t weight,
2980 int32_t offset,
2981 int32_t rnd_val)
2982{
2983 v16u8 out0, out1;
2984 v16i8 src0, src1, src2, src3;
2985 v16i8 mask0, mask1, vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2986 v8i16 filter_vec, filt0, filt1;
2987 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
2988 v4i32 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
2989 v4i32 weight_vec, rnd_vec, offset_vec;
2990 v8i16 zero = { 0 };
2991
2992 src -= 1;
2993
2994 filter_vec = LD_SH(filter);
2995 UNPCK_R_SB_SH(filter_vec, filter_vec);
2996 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
2997
2998 weight_vec = __msa_fill_w(weight);
2999 rnd_vec = __msa_fill_w(rnd_val);
3000 offset_vec = __msa_fill_w(offset);
3001
3002 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3003 mask1 = mask0 + 2;
3004
3005 LD_SB4(src, src_stride, src0, src1, src2, src3);
3006 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3007 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3008 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3009 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3010 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3011 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3012 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3013 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3014 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3015 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3016 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3017 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3018 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3019 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3020 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3021 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3022 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3023 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3024 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3025 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3026
3027 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3028 weight_vec, dst0, dst1, dst2, dst3);
3029 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
3030 dst7, weight_vec, dst4, dst5, dst6, dst7);
3031 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3032 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
3033 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3034 offset_vec, dst0, dst1, dst2, dst3);
3035 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
3036 dst7, offset_vec, dst4, dst5, dst6, dst7);
3037 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
3038
3039 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3040 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
3041
3042 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
3043 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
3044}
3045
3046static void hevc_hz_uniwgt_4t_8x6_msa(const uint8_t *src,
3047 int32_t src_stride,
3048 uint8_t *dst,
3049 int32_t dst_stride,
3050 const int8_t *filter,
3051 int32_t weight,
3052 int32_t offset,
3053 int32_t rnd_val)
3054{
3055 v16u8 out0, out1, out2;
3056 v8i16 filter_vec, filt0, filt1;
3057 v16i8 src0, src1, src2, src3, src4, src5;
3058 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3059 v16i8 mask1;
3060 v16i8 vec11;
3061 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
3062 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3063 v4i32 dst8, dst9, dst10, dst11;
3064 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
3065 v4i32 weight_vec, rnd_vec, offset_vec;
3066 v8i16 zero = { 0 };
3067
3068 src -= 1;
3069
3070 filter_vec = LD_SH(filter);
3071 UNPCK_R_SB_SH(filter_vec, filter_vec);
3072 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3073
3074 weight_vec = __msa_fill_w(weight);
3075 rnd_vec = __msa_fill_w(rnd_val);
3076 offset_vec = __msa_fill_w(offset);
3077
3078 mask1 = mask0 + 2;
3079
3080 LD_SB6(src, src_stride, src0, src1, src2, src3, src4, src5);
3081
3082 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3083 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3084 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3085 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3086 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
3087 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec10, vec11);
3088 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3089 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3090 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3091 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3092 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3093 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3094 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3095 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3096 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3097 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3098 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3099 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3100 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3101 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3102 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3103 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3104 ILVRL_B2_SH(zero, vec8, tmp0, tmp1);
3105 ILVRL_B2_SH(zero, vec9, tmp2, tmp3);
3106 ILVRL_B2_SH(zero, vec10, tmp4, tmp5);
3107 ILVRL_B2_SH(zero, vec11, tmp6, tmp7);
3108 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3109 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3110 dst10 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3111 dst11 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3112
3113 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3114 weight_vec, dst0, dst1, dst2, dst3);
3115 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
3116 dst7, weight_vec, dst4, dst5, dst6, dst7);
3117 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3118 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
3119 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3120 offset_vec, dst0, dst1, dst2, dst3);
3121 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
3122 dst7, offset_vec, dst4, dst5, dst6, dst7);
3123 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
3124
3125 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
3126 weight_vec, dst8, dst9, dst10, dst11);
3127 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
3128 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
3129 offset_vec, dst8, dst9, dst10, dst11);
3130 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
3131
3132 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3133 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
3134 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
3135
3136 PCKEV_B3_UB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, out0, out1, out2);
3137 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
3138 ST_D2(out2, 0, 1, dst + 4 * dst_stride, dst_stride);
3139}
3140
3141static void hevc_hz_uniwgt_4t_8x8multiple_msa(const uint8_t *src,
3142 int32_t src_stride,
3143 uint8_t *dst,
3144 int32_t dst_stride,
3145 const int8_t *filter,
3146 int32_t height,
3147 int32_t weight,
3148 int32_t offset,
3149 int32_t rnd_val)
3150{
3151 uint32_t loop_cnt;
3152 v8i16 filter_vec, filt0, filt1;
3153 v16u8 out0, out1, out2, out3;
3154 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
3155 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3156 v16i8 mask1;
3157 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3158 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
3159 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3160 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
3161 v4i32 weight_vec, rnd_vec, offset_vec;
3162 v8i16 zero = { 0 };
3163
3164 src -= 1;
3165
3166 filter_vec = LD_SH(filter);
3167 UNPCK_R_SB_SH(filter_vec, filter_vec);
3168 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3169
3170 weight_vec = __msa_fill_w(weight);
3171 rnd_vec = __msa_fill_w(rnd_val);
3172 offset_vec = __msa_fill_w(offset);
3173
3174 mask1 = mask0 + 2;
3175
3176 for (loop_cnt = (height >> 3); loop_cnt--;) {
3177 LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
3178 src += (8 * src_stride);
3179
3180 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3181 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3182 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3183 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3184
3185 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3186 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3187 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3188 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3189 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3190 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3191 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3192 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3193 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3194 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3195 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3196 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3197 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3198 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3199 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3200 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3201 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
3202 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec2, vec3);
3203 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec4, vec5);
3204 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec6, vec7);
3205 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3206 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3207 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3208 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3209 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3210 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3211 dst10 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3212 dst11 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3213 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3214 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3215 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3216 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3217 dst12 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3218 dst13 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3219 dst14 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3220 dst15 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3221
3222 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3223 weight_vec, dst0, dst1, dst2, dst3);
3224 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
3225 dst7, weight_vec, dst4, dst5, dst6, dst7);
3226 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3227 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
3228 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3229 offset_vec, dst0, dst1, dst2, dst3);
3230 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
3231 dst7, offset_vec, dst4, dst5, dst6, dst7);
3232 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
3233
3234 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
3235 weight_vec, dst8, dst9, dst10, dst11);
3236 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
3237 dst15, weight_vec, dst12, dst13, dst14, dst15);
3238 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
3239 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
3240 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
3241 offset_vec, dst8, dst9, dst10, dst11);
3242 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
3243 dst15, offset_vec, dst12, dst13, dst14, dst15);
3244 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
3245
3246 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3247 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
3248 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
3249 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
3250
3251 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
3252 PCKEV_B2_UB(tmp5, tmp4, tmp7, tmp6, out2, out3);
3253 ST_D8(out0, out1, out2, out3, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
3254 dst += (8 * dst_stride);
3255 }
3256}
3257
3258static void hevc_hz_uniwgt_4t_8w_msa(const uint8_t *src,
3259 int32_t src_stride,
3260 uint8_t *dst,
3261 int32_t dst_stride,
3262 const int8_t *filter,
3263 int32_t height,
3264 int32_t weight,
3265 int32_t offset,
3266 int32_t rnd_val)
3267{
3268 if (2 == height) {
3269 hevc_hz_uniwgt_4t_8x2_msa(src, src_stride, dst, dst_stride,
3270 filter, weight, offset, rnd_val);
3271 } else if (4 == height) {
3272 hevc_hz_uniwgt_4t_8x4_msa(src, src_stride, dst, dst_stride,
3273 filter, weight, offset, rnd_val);
3274 } else if (6 == height) {
3275 hevc_hz_uniwgt_4t_8x6_msa(src, src_stride, dst, dst_stride,
3276 filter, weight, offset, rnd_val);
3277 } else {
3278 hevc_hz_uniwgt_4t_8x8multiple_msa(src, src_stride, dst, dst_stride,
3279 filter, height, weight, offset,
3280 rnd_val);
3281 }
3282}
3283
3284static void hevc_hz_uniwgt_4t_12w_msa(const uint8_t *src,
3285 int32_t src_stride,
3286 uint8_t *dst,
3287 int32_t dst_stride,
3288 const int8_t *filter,
3289 int32_t height,
3290 int32_t weight,
3291 int32_t offset,
3292 int32_t rnd_val)
3293{
3294 uint32_t loop_cnt;
3295 v16u8 out0, out1, out2;
3296 v8i16 filter_vec, filt0, filt1;
3297 v16i8 src0, src1, src2, src3;
3298 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3299 v16i8 mask2 = { 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
3300 };
3301 v16i8 mask1;
3302 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
3303 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3304 v4i32 dst8, dst9, dst10, dst11;
3305 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
3306 v16i8 mask3, vec11;
3307 v4i32 weight_vec, rnd_vec, offset_vec;
3308 v8i16 zero = { 0 };
3309
3310 src -= 1;
3311
3312 filter_vec = LD_SH(filter);
3313 UNPCK_R_SB_SH(filter_vec, filter_vec);
3314 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3315
3316 weight_vec = __msa_fill_w(weight);
3317 rnd_vec = __msa_fill_w(rnd_val);
3318 offset_vec = __msa_fill_w(offset);
3319
3320 mask1 = mask0 + 2;
3321 mask3 = mask2 + 2;
3322
3323 for (loop_cnt = 4; loop_cnt--;) {
3324 LD_SB4(src, src_stride, src0, src1, src2, src3);
3325 src += (4 * src_stride);
3326
3327 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3328 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3329 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3330 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3331 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec8, vec9);
3332 VSHF_B2_SB(src2, src3, src2, src3, mask2, mask3, vec10, vec11);
3333
3334 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3335 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3336 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3337 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3338 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3339 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3340 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3341 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3342 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3343 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3344 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3345 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3346 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3347 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3348 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3349 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3350 ILVRL_B2_SH(zero, vec8, tmp0, tmp1);
3351 ILVRL_B2_SH(zero, vec9, tmp2, tmp3);
3352 ILVRL_B2_SH(zero, vec10, tmp4, tmp5);
3353 ILVRL_B2_SH(zero, vec11, tmp6, tmp7);
3354 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3355 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3356 dst10 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3357 dst11 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3358
3359 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3360 weight_vec, dst0, dst1, dst2, dst3);
3361 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
3362 dst7, weight_vec, dst4, dst5, dst6, dst7);
3363 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3364 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
3365 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3366 offset_vec, dst0, dst1, dst2, dst3);
3367 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
3368 dst7, offset_vec, dst4, dst5, dst6, dst7);
3369 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
3370
3371 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
3372 weight_vec, dst8, dst9, dst10, dst11);
3373 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
3374 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
3375 offset_vec, dst8, dst9, dst10, dst11);
3376 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
3377
3378 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3379 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
3380 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
3381
3382 PCKEV_B3_UB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, out0, out1, out2);
3383 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
3384 ST_W4(out2, 0, 1, 2, 3, dst + 8, dst_stride);
3385 dst += (4 * dst_stride);
3386 }
3387}
3388
3389static void hevc_hz_uniwgt_4t_16w_msa(const uint8_t *src,
3390 int32_t src_stride,
3391 uint8_t *dst,
3392 int32_t dst_stride,
3393 const int8_t *filter,
3394 int32_t height,
3395 int32_t weight,
3396 int32_t offset,
3397 int32_t rnd_val)
3398{
3399 uint32_t loop_cnt;
3400 v16u8 out0, out1, out2, out3;
3401 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
3402 v8i16 filter_vec, filt0, filt1;
3403 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3404 v16i8 mask1;
3405 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3406 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3407 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
3408 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
3409 v4i32 weight_vec, rnd_vec, offset_vec;
3410 v8i16 zero = { 0 };
3411
3412 src -= 1;
3413
3414 filter_vec = LD_SH(filter);
3415 UNPCK_R_SB_SH(filter_vec, filter_vec);
3416 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3417
3418 weight_vec = __msa_fill_w(weight);
3419 rnd_vec = __msa_fill_w(rnd_val);
3420 offset_vec = __msa_fill_w(offset);
3421
3422 mask1 = mask0 + 2;
3423
3424 for (loop_cnt = (height >> 2); loop_cnt--;) {
3425 LD_SB4(src, src_stride, src0, src2, src4, src6);
3426 LD_SB4(src + 8, src_stride, src1, src3, src5, src7);
3427 src += (4 * src_stride);
3428
3429 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3430 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3431 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3432 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3433 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3434 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3435 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3436 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3437 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3438 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3439 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3440 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3441 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3442 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3443 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3444 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3445 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3446 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3447 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3448 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3449
3450 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
3451 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec2, vec3);
3452 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec4, vec5);
3453 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec6, vec7);
3454 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3455 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3456 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3457 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3458 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3459 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3460 dst10 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3461 dst11 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3462 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3463 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3464 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3465 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3466 dst12 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3467 dst13 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3468 dst14 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3469 dst15 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3470
3471 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3472 weight_vec, dst0, dst1, dst2, dst3);
3473 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
3474 dst7, weight_vec, dst4, dst5, dst6, dst7);
3475 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3476 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
3477 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3478 offset_vec, dst0, dst1, dst2, dst3);
3479 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
3480 dst7, offset_vec, dst4, dst5, dst6, dst7);
3481 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
3482
3483 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
3484 weight_vec, dst8, dst9, dst10, dst11);
3485 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
3486 dst15, weight_vec, dst12, dst13, dst14, dst15);
3487 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
3488 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
3489 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
3490 offset_vec, dst8, dst9, dst10, dst11);
3491 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
3492 dst15, offset_vec, dst12, dst13, dst14, dst15);
3493 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
3494
3495 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3496 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
3497 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
3498 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
3499 PCKEV_B4_UB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6,
3500 out0, out1, out2, out3);
3501
3502 ST_UB4(out0, out1, out2, out3, dst, dst_stride);
3503 dst += (4 * dst_stride);
3504 }
3505}
3506
3507static void hevc_hz_uniwgt_4t_24w_msa(const uint8_t *src,
3508 int32_t src_stride,
3509 uint8_t *dst,
3510 int32_t dst_stride,
3511 const int8_t *filter,
3512 int32_t height,
3513 int32_t weight,
3514 int32_t offset,
3515 int32_t rnd_val)
3516{
3517 uint32_t loop_cnt;
3518 v16u8 out0, out1, out2;
3519 v16i8 src0, src1, src2, src3;
3520 v8i16 filter_vec, filt0, filt1;
3521 v16i8 mask0, mask1, mask2, mask3;
3522 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3523 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
3524 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3525 v4i32 dst8, dst9, dst10, dst11;
3526 v4i32 weight_vec, rnd_vec, offset_vec;
3527 v8i16 zero = { 0 };
3528
3529 src -= 1;
3530
3531 filter_vec = LD_SH(filter);
3532 UNPCK_R_SB_SH(filter_vec, filter_vec);
3533 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3534
3535 weight_vec = __msa_fill_w(weight);
3536 rnd_vec = __msa_fill_w(rnd_val);
3537 offset_vec = __msa_fill_w(offset);
3538
3539 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3540 mask1 = mask0 + 2;
3541 mask2 = mask0 + 8;
3542 mask3 = mask0 + 10;
3543
3544 for (loop_cnt = 16; loop_cnt--;) {
3545 LD_SB2(src, src_stride, src0, src2);
3546 LD_SB2(src + 16, src_stride, src1, src3);
3547 src += (2 * src_stride);
3548
3549 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3550 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec2, vec3);
3551 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3552 VSHF_B2_SB(src2, src3, src2, src3, mask2, mask3, vec6, vec7);
3553 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3554 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3555 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3556 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3557 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3558 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3559 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3560 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3561 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3562 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3563 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3564 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3565 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3566 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3567 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3568 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3569
3570 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
3571 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec2, vec3);
3572 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3573 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3574 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3575 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3576 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3577 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3578 dst10 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3579 dst11 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3580
3581 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3582 weight_vec, dst0, dst1, dst2, dst3);
3583 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
3584 dst7, weight_vec, dst4, dst5, dst6, dst7);
3585 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3586 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
3587 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3588 offset_vec, dst0, dst1, dst2, dst3);
3589 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
3590 dst7, offset_vec, dst4, dst5, dst6, dst7);
3591 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
3592
3593 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
3594 weight_vec, dst8, dst9, dst10, dst11);
3595 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
3596 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
3597 offset_vec, dst8, dst9, dst10, dst11);
3598 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
3599
3600 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3601 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
3602 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
3603
3604 PCKEV_B3_UB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, out0, out1, out2);
3605 ST_UB2(out0, out1, dst, dst_stride);
3606 ST_D2(out2, 0, 1, dst + 16, dst_stride);
3607 dst += (2 * dst_stride);
3608 }
3609}
3610
3611static void hevc_hz_uniwgt_4t_32w_msa(const uint8_t *src,
3612 int32_t src_stride,
3613 uint8_t *dst,
3614 int32_t dst_stride,
3615 const int8_t *filter,
3616 int32_t height,
3617 int32_t weight,
3618 int32_t offset,
3619 int32_t rnd_val)
3620{
3621 uint32_t loop_cnt;
3622 v16u8 out0, out1, out2, out3;
3623 v16i8 src0, src1, src2, src3, src4, src5;
3624 v8i16 filter_vec, filt0, filt1;
3625 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3626 v16i8 mask1, mask2, mask3;
3627 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3628 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
3629 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3630 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
3631 v4i32 weight_vec, rnd_vec, offset_vec;
3632 v8i16 zero = { 0 };
3633
3634 src -= 1;
3635
3636 filter_vec = LD_SH(filter);
3637 UNPCK_R_SB_SH(filter_vec, filter_vec);
3638 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3639
3640 weight_vec = __msa_fill_w(weight);
3641 rnd_vec = __msa_fill_w(rnd_val);
3642 offset_vec = __msa_fill_w(offset);
3643
3644 mask1 = mask0 + 2;
3645 mask2 = mask0 + 8;
3646 mask3 = mask0 + 10;
3647
3648 for (loop_cnt = (height >> 1); loop_cnt--;) {
3649 LD_SB2(src, 16, src0, src1);
3650 src2 = LD_SB(src + 24);
3651 src += src_stride;
3652 LD_SB2(src, 16, src3, src4);
3653 src5 = LD_SB(src + 24);
3654 src += src_stride;
3655 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3656 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec2, vec3);
3657 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec4, vec5);
3658 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec6, vec7);
3659 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3660 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3661 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3662 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3663 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3664 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3665 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3666 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3667 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3668 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3669 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3670 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3671 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3672 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3673 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3674 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3675
3676 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3677 VSHF_B2_SB(src3, src4, src3, src4, mask2, mask3, vec2, vec3);
3678 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec4, vec5);
3679 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec6, vec7);
3680 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
3681 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
3682 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
3683 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
3684 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3685 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3686 dst10 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3687 dst11 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3688 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
3689 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
3690 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
3691 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
3692 dst12 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3693 dst13 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3694 dst14 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3695 dst15 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3696
3697 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3698 weight_vec, dst0, dst1, dst2, dst3);
3699 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
3700 dst7, weight_vec, dst4, dst5, dst6, dst7);
3701 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3702 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
3703 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3704 offset_vec, dst0, dst1, dst2, dst3);
3705 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
3706 dst7, offset_vec, dst4, dst5, dst6, dst7);
3707 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
3708
3709 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
3710 weight_vec, dst8, dst9, dst10, dst11);
3711 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
3712 dst15, weight_vec, dst12, dst13, dst14, dst15);
3713 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
3714 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
3715 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
3716 offset_vec, dst8, dst9, dst10, dst11);
3717 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
3718 dst15, offset_vec, dst12, dst13, dst14, dst15);
3719 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
3720
3721 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3722 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
3723 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
3724 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
3725 PCKEV_B4_UB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, tmp7, tmp6,
3726 out0, out1, out2, out3);
3727 ST_UB2(out0, out1, dst, 16);
3728 dst += dst_stride;
3729 ST_UB2(out2, out3, dst, 16);
3730 dst += dst_stride;
3731 }
3732}
3733
3734static void hevc_vt_uniwgt_4t_4x2_msa(const uint8_t *src,
3735 int32_t src_stride,
3736 uint8_t *dst,
3737 int32_t dst_stride,
3738 const int8_t *filter,
3739 int32_t weight,
3740 int32_t offset,
3741 int32_t rnd_val)
3742{
3743 v16u8 out;
3744 v16i8 src0, src1, src2, src3, src4;
3745 v16i8 src10_r, src32_r, src21_r, src43_r;
3746 v16i8 src2110, src4332;
3747 v8i16 dst0;
3748 v4i32 dst0_r, dst0_l;
3749 v8i16 filter_vec, filt0, filt1;
3750 v8i16 tmp0, tmp1, tmp2, tmp3;
3751 v4i32 weight_vec, rnd_vec, offset_vec;
3752 v8i16 zero = { 0 };
3753
3754 src -= src_stride;
3755
3756 weight_vec = __msa_fill_w(weight);
3757 rnd_vec = __msa_fill_w(rnd_val);
3758 offset_vec = __msa_fill_w(offset);
3759
3760 filter_vec = LD_SH(filter);
3761 UNPCK_R_SB_SH(filter_vec, filter_vec);
3762 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3763
3764 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
3765 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3766 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3767 ILVR_D2_SB(src21_r, src10_r, src43_r, src32_r, src2110, src4332);
3768 ILVRL_B2_SH(zero, src2110, tmp0, tmp1);
3769 ILVRL_B2_SH(zero, src4332, tmp2, tmp3);
3770
3771 dst0_r = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3772 dst0_l = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3773
3774 MUL2(dst0_r, weight_vec, dst0_l, weight_vec, dst0_r, dst0_l);
3775 SRAR_W2_SW(dst0_r, dst0_l, rnd_vec);
3776 ADD2(dst0_r, offset_vec, dst0_l, offset_vec, dst0_r, dst0_l);
3777 CLIP_SW2_0_255(dst0_r, dst0_l);
3778 dst0 = __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
3779 out = (v16u8) __msa_pckev_b((v16i8) dst0, (v16i8) dst0);
3780 ST_W2(out, 0, 1, dst, dst_stride);
3781}
3782
3783static void hevc_vt_uniwgt_4t_4x4_msa(const uint8_t *src,
3784 int32_t src_stride,
3785 uint8_t *dst,
3786 int32_t dst_stride,
3787 const int8_t *filter,
3788 int32_t weight,
3789 int32_t offset,
3790 int32_t rnd_val)
3791{
3792 v16u8 out;
3793 v16i8 src0, src1, src2, src3, src4, src5, src6;
3794 v16i8 src10_r, src32_r, src54_r, src21_r, src43_r, src65_r;
3795 v16i8 src2110, src4332, src6554;
3796 v4i32 dst0, dst1, dst2, dst3;
3797 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
3798 v8i16 filter_vec, filt0, filt1;
3799 v4i32 weight_vec, rnd_vec, offset_vec;
3800 v8i16 zero = { 0 };
3801
3802 src -= src_stride;
3803
3804 weight_vec = __msa_fill_w(weight);
3805 rnd_vec = __msa_fill_w(rnd_val);
3806 offset_vec = __msa_fill_w(offset);
3807
3808 filter_vec = LD_SH(filter);
3809 UNPCK_R_SB_SH(filter_vec, filter_vec);
3810 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3811
3812 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
3813 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3814 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
3815 src32_r, src43_r, src54_r, src65_r);
3816 ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r,
3817 src2110, src4332, src6554);
3818
3819 ILVRL_B2_SH(zero, src2110, tmp0, tmp1);
3820 ILVRL_B2_SH(zero, src4332, tmp2, tmp3);
3821 ILVRL_B2_SH(zero, src6554, tmp4, tmp5);
3822
3823 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3824 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3825 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp4, filt0, filt1);
3826 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp5, filt0, filt1);
3827
3828 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3829 weight_vec, dst0, dst1, dst2, dst3);
3830 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3831 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3832 offset_vec, dst0, dst1, dst2, dst3);
3833 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
3834
3835 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3836 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
3837 ST_W4(out, 0, 1, 2, 3, dst, dst_stride);
3838}
3839
3840static void hevc_vt_uniwgt_4t_4x8multiple_msa(const uint8_t *src,
3841 int32_t src_stride,
3842 uint8_t *dst,
3843 int32_t dst_stride,
3844 const int8_t *filter,
3845 int32_t height,
3846 int32_t weight,
3847 int32_t offset,
3848 int32_t rnd_val)
3849{
3850 int32_t loop_cnt;
3851 v16u8 out0, out1;
3852 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3853 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
3854 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
3855 v16i8 src2110, src4332, src6554, src8776;
3856 v16i8 src10998;
3857 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3858 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
3859 v8i16 filter_vec, filt0, filt1;
3860 v4i32 weight_vec, rnd_vec, offset_vec;
3861 v8i16 zero = { 0 };
3862
3863 src -= src_stride;
3864
3865 weight_vec = __msa_fill_w(weight);
3866 rnd_vec = __msa_fill_w(rnd_val);
3867 offset_vec = __msa_fill_w(offset);
3868
3869 filter_vec = LD_SH(filter);
3870 UNPCK_R_SB_SH(filter_vec, filter_vec);
3871 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3872
3873 LD_SB3(src, src_stride, src0, src1, src2);
3874 src += (3 * src_stride);
3875 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3876 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
3877
3878 for (loop_cnt = (height >> 3); loop_cnt--;) {
3879 LD_SB8(src, src_stride,
3880 src3, src4, src5, src6, src7, src8, src9, src10);
3881 src += (8 * src_stride);
3882 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
3883 src32_r, src43_r, src54_r, src65_r);
3884 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3885 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
3886 ILVR_D4_SB(src43_r, src32_r, src65_r, src54_r, src87_r, src76_r,
3887 src109_r, src98_r, src4332, src6554, src8776, src10998);
3888
3889 ILVRL_B2_SH(zero, src2110, tmp0, tmp1);
3890 ILVRL_B2_SH(zero, src4332, tmp2, tmp3);
3891 ILVRL_B2_SH(zero, src6554, tmp4, tmp5);
3892 ILVRL_B2_SH(zero, src8776, tmp6, tmp7);
3893 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
3894 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
3895 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp4, filt0, filt1);
3896 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp5, filt0, filt1);
3897 ILVRL_B2_SH(zero, src10998, tmp0, tmp1);
3898 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
3899 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
3900 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp0, filt0, filt1);
3901 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp1, filt0, filt1);
3902
3903 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
3904 weight_vec, dst0, dst1, dst2, dst3);
3905 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
3906 dst7, weight_vec, dst4, dst5, dst6, dst7);
3907 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
3908 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
3909 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
3910 offset_vec, dst0, dst1, dst2, dst3);
3911 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
3912 dst7, offset_vec, dst4, dst5, dst6, dst7);
3913 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
3914 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3915 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
3916
3917 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
3918 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
3919 dst += (8 * dst_stride);
3920
3921 src2 = src10;
3922 src2110 = src10998;
3923 }
3924}
3925
3926static void hevc_vt_uniwgt_4t_4w_msa(const uint8_t *src,
3927 int32_t src_stride,
3928 uint8_t *dst,
3929 int32_t dst_stride,
3930 const int8_t *filter,
3931 int32_t height,
3932 int32_t weight,
3933 int32_t offset,
3934 int32_t rnd_val)
3935{
3936 if (2 == height) {
3937 hevc_vt_uniwgt_4t_4x2_msa(src, src_stride, dst, dst_stride,
3938 filter, weight, offset, rnd_val);
3939 } else if (4 == height) {
3940 hevc_vt_uniwgt_4t_4x4_msa(src, src_stride, dst, dst_stride,
3941 filter, weight, offset, rnd_val);
3942 } else if (0 == (height % 8)) {
3943 hevc_vt_uniwgt_4t_4x8multiple_msa(src, src_stride, dst, dst_stride,
3944 filter, height, weight, offset,
3945 rnd_val);
3946 }
3947}
3948
3949static void hevc_vt_uniwgt_4t_6w_msa(const uint8_t *src,
3950 int32_t src_stride,
3951 uint8_t *dst,
3952 int32_t dst_stride,
3953 const int8_t *filter,
3954 int32_t height,
3955 int32_t weight,
3956 int32_t offset,
3957 int32_t rnd_val)
3958{
3959 v16u8 out0, out1, out2, out3;
3960 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3961 v16i8 src10_r, src32_r, src21_r, src43_r;
3962 v16i8 src54_r, src65_r, src76_r, src87_r, src98_r, src109_r;
3963 v8i16 filter_vec, filt0, filt1;
3964 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
3965 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3966 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
3967 v4i32 weight_vec, rnd_vec, offset_vec;
3968 v8i16 zero = { 0 };
3969
3970 src -= src_stride;
3971
3972 weight_vec = __msa_fill_w(weight);
3973 rnd_vec = __msa_fill_w(rnd_val);
3974 offset_vec = __msa_fill_w(offset);
3975
3976 filter_vec = LD_SH(filter);
3977 UNPCK_R_SB_SH(filter_vec, filter_vec);
3978 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
3979
3980 LD_SB3(src, src_stride, src0, src1, src2);
3981 src += (3 * src_stride);
3982 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9, src10);
3983 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3984 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3985 ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
3986 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3987 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
3988
3989 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
3990 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
3991 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
3992 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
3993 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
3994 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
3995 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
3996 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
3997 ILVRL_B2_SH(zero, src54_r, tmp0, tmp1);
3998 ILVRL_B2_SH(zero, src65_r, tmp2, tmp3);
3999 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4000 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4001 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4002 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4003 ILVRL_B2_SH(zero, src76_r, tmp4, tmp5);
4004 ILVRL_B2_SH(zero, src87_r, tmp6, tmp7);
4005 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4006 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4007 dst10 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4008 dst11 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4009 ILVRL_B2_SH(zero, src98_r, tmp0, tmp1);
4010 ILVRL_B2_SH(zero, src109_r, tmp2, tmp3);
4011 dst12 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4012 dst13 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4013 dst14 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4014 dst15 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4015
4016 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4017 weight_vec, dst0, dst1, dst2, dst3);
4018 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4019 dst7, weight_vec, dst4, dst5, dst6, dst7);
4020 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4021 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4022 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4023 offset_vec, dst0, dst1, dst2, dst3);
4024 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4025 dst7, offset_vec, dst4, dst5, dst6, dst7);
4026 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4027
4028 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
4029 weight_vec, dst8, dst9, dst10, dst11);
4030 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
4031 dst15, weight_vec, dst12, dst13, dst14, dst15);
4032 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
4033 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
4034 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
4035 offset_vec, dst8, dst9, dst10, dst11);
4036 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
4037 dst15, offset_vec, dst12, dst13, dst14, dst15);
4038 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
4039
4040 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4041 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
4042 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
4043 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
4044
4045 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
4046 PCKEV_B2_UB(tmp5, tmp4, tmp7, tmp6, out2, out3);
4047 ST_W2(out0, 0, 2, dst, dst_stride);
4048 ST_H2(out0, 2, 6, dst + 4, dst_stride);
4049 ST_W2(out1, 0, 2, dst + 2 * dst_stride, dst_stride);
4050 ST_H2(out1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
4051 dst += (4 * dst_stride);
4052 ST_W2(out2, 0, 2, dst, dst_stride);
4053 ST_H2(out2, 2, 6, dst + 4, dst_stride);
4054 ST_W2(out3, 0, 2, dst + 2 * dst_stride, dst_stride);
4055 ST_H2(out3, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
4056}
4057
4058static void hevc_vt_uniwgt_4t_8x2_msa(const uint8_t *src,
4059 int32_t src_stride,
4060 uint8_t *dst,
4061 int32_t dst_stride,
4062 const int8_t *filter,
4063 int32_t weight,
4064 int32_t offset,
4065 int32_t rnd_val)
4066{
4067 v16u8 out;
4068 v16i8 src0, src1, src2, src3, src4;
4069 v16i8 src10_r, src32_r, src21_r, src43_r;
4070 v4i32 dst0, dst1, dst2, dst3;
4071 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4072 v8i16 filter_vec, filt0, filt1;
4073 v4i32 weight_vec, rnd_vec, offset_vec;
4074 v8i16 zero = { 0 };
4075
4076 src -= src_stride;
4077
4078 weight_vec = __msa_fill_w(weight);
4079 rnd_vec = __msa_fill_w(rnd_val);
4080 offset_vec = __msa_fill_w(offset);
4081
4082 filter_vec = LD_SH(filter);
4083 UNPCK_R_SB_SH(filter_vec, filter_vec);
4084 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4085
4086 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
4087 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4088 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4089
4090 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
4091 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
4092 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
4093 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
4094
4095 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4096 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4097 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4098 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4099 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4100 weight_vec, dst0, dst1, dst2, dst3);
4101 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4102 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4103 offset_vec, dst0, dst1, dst2, dst3);
4104 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
4105 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4106
4107 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
4108 ST_D2(out, 0, 1, dst, dst_stride);
4109}
4110
4111static void hevc_vt_uniwgt_4t_8x4_msa(const uint8_t *src,
4112 int32_t src_stride,
4113 uint8_t *dst,
4114 int32_t dst_stride,
4115 const int8_t *filter,
4116 int32_t weight,
4117 int32_t offset,
4118 int32_t rnd_val)
4119{
4120 v16u8 out0, out1;
4121 v16i8 src0, src1, src2, src3, src4;
4122 v16i8 src10_r, src32_r, src21_r, src43_r;
4123 v16i8 src5, src6, src54_r, src65_r;
4124 v8i16 filter_vec, filt0, filt1;
4125 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
4126 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4127 v4i32 weight_vec, rnd_vec, offset_vec;
4128 v8i16 zero = { 0 };
4129
4130 src -= src_stride;
4131
4132 weight_vec = __msa_fill_w(weight);
4133 rnd_vec = __msa_fill_w(rnd_val);
4134 offset_vec = __msa_fill_w(offset);
4135
4136 filter_vec = LD_SH(filter);
4137 UNPCK_R_SB_SH(filter_vec, filter_vec);
4138 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4139
4140 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
4141 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4142 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4143 ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
4144
4145 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
4146 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
4147 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
4148 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
4149 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4150 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4151 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4152 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4153 ILVRL_B2_SH(zero, src54_r, tmp0, tmp1);
4154 ILVRL_B2_SH(zero, src65_r, tmp2, tmp3);
4155 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4156 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4157 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4158 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4159
4160 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4161 weight_vec, dst0, dst1, dst2, dst3);
4162 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4163 dst7, weight_vec, dst4, dst5, dst6, dst7);
4164 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4165 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4166 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4167 offset_vec, dst0, dst1, dst2, dst3);
4168 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4169 dst7, offset_vec, dst4, dst5, dst6, dst7);
4170 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4171 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4172 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
4173 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
4174 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
4175}
4176
4177static void hevc_vt_uniwgt_4t_8x6_msa(const uint8_t *src,
4178 int32_t src_stride,
4179 uint8_t *dst,
4180 int32_t dst_stride,
4181 const int8_t *filter,
4182 int32_t weight,
4183 int32_t offset,
4184 int32_t rnd_val)
4185{
4186 v16u8 out0, out1, out2;
4187 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
4188 v16i8 src10_r, src32_r, src54_r, src76_r;
4189 v16i8 src21_r, src43_r, src65_r, src87_r;
4190 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
4191 v4i32 dst8, dst9, dst10, dst11;
4192 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4193 v8i16 filter_vec, filt0, filt1;
4194 v4i32 weight_vec, rnd_vec, offset_vec;
4195 v8i16 zero = { 0 };
4196
4197 src -= src_stride;
4198 weight_vec = __msa_fill_w(weight);
4199 rnd_vec = __msa_fill_w(rnd_val);
4200 offset_vec = __msa_fill_w(offset);
4201
4202 filter_vec = LD_SH(filter);
4203 UNPCK_R_SB_SH(filter_vec, filter_vec);
4204 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4205
4206 LD_SB3(src, src_stride, src0, src1, src2);
4207 src += (3 * src_stride);
4208 LD_SB6(src, src_stride, src3, src4, src5, src6, src7, src8);
4209
4210 ILVR_B4_SB(src1, src0, src2, src1, src3, src2, src4, src3, src10_r, src21_r,
4211 src32_r, src43_r);
4212 ILVR_B4_SB(src5, src4, src6, src5, src7, src6, src8, src7, src54_r, src65_r,
4213 src76_r, src87_r);
4214
4215 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
4216 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
4217 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
4218 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
4219 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4220 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4221 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4222 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4223 ILVRL_B2_SH(zero, src54_r, tmp0, tmp1);
4224 ILVRL_B2_SH(zero, src65_r, tmp2, tmp3);
4225 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4226 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4227 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4228 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4229 ILVRL_B2_SH(zero, src76_r, tmp4, tmp5);
4230 ILVRL_B2_SH(zero, src87_r, tmp6, tmp7);
4231 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4232 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4233 dst10 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4234 dst11 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4235
4236 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4237 weight_vec, dst0, dst1, dst2, dst3);
4238 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4239 dst7, weight_vec, dst4, dst5, dst6, dst7);
4240 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4241 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4242 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4243 offset_vec, dst0, dst1, dst2, dst3);
4244 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4245 dst7, offset_vec, dst4, dst5, dst6, dst7);
4246 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4247 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
4248 weight_vec, dst8, dst9, dst10, dst11);
4249 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
4250 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
4251 offset_vec, dst8, dst9, dst10, dst11);
4252 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
4253
4254 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4255 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
4256 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
4257 PCKEV_B3_UB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, out0, out1, out2);
4258 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
4259 ST_D2(out2, 0, 1, dst + 4 * dst_stride, dst_stride);
4260}
4261
4262static void hevc_vt_uniwgt_4t_8x8mult_msa(const uint8_t *src,
4263 int32_t src_stride,
4264 uint8_t *dst,
4265 int32_t dst_stride,
4266 const int8_t *filter,
4267 int32_t height,
4268 int32_t weight,
4269 int32_t offset,
4270 int32_t rnd_val)
4271{
4272 int32_t loop_cnt;
4273 v16u8 out0, out1, out2, out3;
4274 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
4275 v16i8 src10_r, src32_r, src21_r, src43_r;
4276 v16i8 src54_r, src65_r, src76_r, src87_r, src98_r, src109_r;
4277 v8i16 filter_vec, filt0, filt1;
4278 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4279 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
4280 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
4281 v4i32 weight_vec, rnd_vec, offset_vec;
4282 v8i16 zero = { 0 };
4283
4284 src -= src_stride;
4285
4286 weight_vec = __msa_fill_w(weight);
4287 rnd_vec = __msa_fill_w(rnd_val);
4288 offset_vec = __msa_fill_w(offset);
4289
4290 filter_vec = LD_SH(filter);
4291 UNPCK_R_SB_SH(filter_vec, filter_vec);
4292 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4293
4294 LD_SB3(src, src_stride, src0, src1, src2);
4295 src += (3 * src_stride);
4296 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4297
4298 for (loop_cnt = (height >> 3); loop_cnt--;) {
4299 LD_SB8(src, src_stride,
4300 src3, src4, src5, src6, src7, src8, src9, src10);
4301 src += (8 * src_stride);
4302 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4303 ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
4304 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
4305 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
4306
4307 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
4308 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
4309 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
4310 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
4311 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4312 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4313 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4314 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4315 ILVRL_B2_SH(zero, src54_r, tmp0, tmp1);
4316 ILVRL_B2_SH(zero, src65_r, tmp2, tmp3);
4317 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4318 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4319 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4320 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4321 ILVRL_B2_SH(zero, src76_r, tmp4, tmp5);
4322 ILVRL_B2_SH(zero, src87_r, tmp6, tmp7);
4323 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4324 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4325 dst10 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4326 dst11 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4327 ILVRL_B2_SH(zero, src98_r, tmp0, tmp1);
4328 ILVRL_B2_SH(zero, src109_r, tmp2, tmp3);
4329 dst12 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4330 dst13 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4331 dst14 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4332 dst15 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4333
4334 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4335 weight_vec, dst0, dst1, dst2, dst3);
4336 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4337 dst7, weight_vec, dst4, dst5, dst6, dst7);
4338 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4339 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4340 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4341 offset_vec, dst0, dst1, dst2, dst3);
4342 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4343 dst7, offset_vec, dst4, dst5, dst6, dst7);
4344 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4345
4346 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
4347 weight_vec, dst8, dst9, dst10, dst11);
4348 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
4349 dst15, weight_vec, dst12, dst13, dst14, dst15);
4350 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
4351 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
4352 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
4353 offset_vec, dst8, dst9, dst10, dst11);
4354 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
4355 dst15, offset_vec, dst12, dst13, dst14, dst15);
4356 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
4357
4358 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4359 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
4360 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
4361 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
4362
4363 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
4364 PCKEV_B2_UB(tmp5, tmp4, tmp7, tmp6, out2, out3);
4365 ST_D8(out0, out1, out2, out3, 0, 1, 0, 1, 0, 1, 0, 1, dst, dst_stride);
4366 dst += (8 * dst_stride);
4367
4368 src2 = src10;
4369 src10_r = src98_r;
4370 src21_r = src109_r;
4371 }
4372}
4373
4374static void hevc_vt_uniwgt_4t_8w_msa(const uint8_t *src,
4375 int32_t src_stride,
4376 uint8_t *dst,
4377 int32_t dst_stride,
4378 const int8_t *filter,
4379 int32_t height,
4380 int32_t weight,
4381 int32_t offset,
4382 int32_t rnd_val)
4383{
4384 if (2 == height) {
4385 hevc_vt_uniwgt_4t_8x2_msa(src, src_stride, dst, dst_stride,
4386 filter, weight, offset, rnd_val);
4387 } else if (4 == height) {
4388 hevc_vt_uniwgt_4t_8x4_msa(src, src_stride, dst, dst_stride,
4389 filter, weight, offset, rnd_val);
4390 } else if (6 == height) {
4391 hevc_vt_uniwgt_4t_8x6_msa(src, src_stride, dst, dst_stride,
4392 filter, weight, offset, rnd_val);
4393 } else {
4394 hevc_vt_uniwgt_4t_8x8mult_msa(src, src_stride, dst, dst_stride,
4395 filter, height, weight, offset,
4396 rnd_val);
4397 }
4398}
4399
4400static void hevc_vt_uniwgt_4t_12w_msa(const uint8_t *src,
4401 int32_t src_stride,
4402 uint8_t *dst,
4403 int32_t dst_stride,
4404 const int8_t *filter,
4405 int32_t height,
4406 int32_t weight,
4407 int32_t offset,
4408 int32_t rnd_val)
4409{
4410 int32_t loop_cnt;
4411 v16u8 out0, out1, out2, out3, out4, out5;
4412 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
4413 v16i8 src10_r, src32_r, src21_r, src43_r;
4414 v16i8 src10_l, src32_l, src54_l, src21_l, src43_l, src65_l;
4415 v16i8 src2110, src4332;
4416 v16i8 src54_r, src76_r, src98_r, src65_r, src87_r, src109_r;
4417 v16i8 src76_l, src98_l, src87_l, src109_l, src6554, src8776, src10998;
4418 v8i16 filter_vec, filt0, filt1;
4419 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
4420 v4i32 dst9, dst10, dst11;
4421 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8, tmp9;
4422 v4i32 weight_vec, rnd_vec, offset_vec;
4423 v8i16 zero = { 0 };
4424
4425 src -= (1 * src_stride);
4426
4427 weight_vec = __msa_fill_w(weight);
4428 rnd_vec = __msa_fill_w(rnd_val);
4429 offset_vec = __msa_fill_w(offset);
4430
4431 filter_vec = LD_SH(filter);
4432 UNPCK_R_SB_SH(filter_vec, filter_vec);
4433 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4434
4435 LD_SB3(src, src_stride, src0, src1, src2);
4436 src += (3 * src_stride);
4437 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4438 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
4439 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_l, (v2i64) src10_l);
4440
4441 for (loop_cnt = 2; loop_cnt--;) {
4442 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9, src10);
4443 src += (8 * src_stride);
4444 ILVRL_B2_SB(src3, src2, src32_r, src32_l);
4445 ILVRL_B2_SB(src4, src3, src43_r, src43_l);
4446 ILVRL_B2_SB(src5, src4, src54_r, src54_l);
4447 ILVRL_B2_SB(src6, src5, src65_r, src65_l);
4448 src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_l, (v2i64) src32_l);
4449 src6554 = (v16i8) __msa_ilvr_d((v2i64) src65_l, (v2i64) src54_l);
4450
4451 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
4452 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
4453 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
4454 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
4455 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4456 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4457 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4458 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4459 ILVRL_B2_SH(zero, src54_r, tmp0, tmp1);
4460 ILVRL_B2_SH(zero, src65_r, tmp2, tmp3);
4461 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4462 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4463 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4464 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4465 ILVRL_B2_SH(zero, src2110, tmp4, tmp5);
4466 ILVRL_B2_SH(zero, src4332, tmp6, tmp7);
4467 ILVRL_B2_SH(zero, src6554, tmp8, tmp9);
4468 dst8 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
4469 dst9 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
4470 dst10 = HEVC_FILT_4TAP_SW(tmp6, tmp8, filt0, filt1);
4471 dst11 = HEVC_FILT_4TAP_SW(tmp7, tmp9, filt0, filt1);
4472
4473 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4474 weight_vec, dst0, dst1, dst2, dst3);
4475 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4476 dst7, weight_vec, dst4, dst5, dst6, dst7);
4477 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4478 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4479 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4480 offset_vec, dst0, dst1, dst2, dst3);
4481 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4482 dst7, offset_vec, dst4, dst5, dst6, dst7);
4483 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4484
4485 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
4486 weight_vec, dst8, dst9, dst10, dst11);
4487 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
4488 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
4489 offset_vec, dst8, dst9, dst10, dst11);
4490 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
4491
4492 PCKEV_H2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
4493 PCKEV_H2_SH(dst5, dst4, dst7, dst6, dst2, dst3);
4494 PCKEV_H2_SH(dst9, dst8, dst11, dst10, dst4, dst5);
4495
4496 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
4497 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
4498 ST_W4(out2, 0, 1, 2, 3, dst + 8, dst_stride);
4499 dst += (4 * dst_stride);
4500
4501 ILVRL_B2_SB(src7, src6, src76_r, src76_l);
4502 ILVRL_B2_SB(src8, src7, src87_r, src87_l);
4503 ILVRL_B2_SB(src9, src8, src98_r, src98_l);
4504 ILVRL_B2_SB(src10, src9, src109_r, src109_l);
4505 src8776 = (v16i8) __msa_ilvr_d((v2i64) src87_l, (v2i64) src76_l);
4506 src10998 = (v16i8) __msa_ilvr_d((v2i64) src109_l, (v2i64) src98_l);
4507
4508 ILVRL_B2_SH(zero, src76_r, tmp4, tmp5);
4509 ILVRL_B2_SH(zero, src87_r, tmp6, tmp7);
4510 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4511 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4512 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4513 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4514 ILVRL_B2_SH(zero, src98_r, tmp0, tmp1);
4515 ILVRL_B2_SH(zero, src109_r, tmp2, tmp3);
4516 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4517 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4518 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4519 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4520 ILVRL_B2_SH(zero, src8776, tmp4, tmp5);
4521 ILVRL_B2_SH(zero, src10998, tmp6, tmp7);
4522 ILVRL_B2_SH(zero, src6554, tmp8, tmp9);
4523 dst8 = HEVC_FILT_4TAP_SW(tmp8, tmp4, filt0, filt1);
4524 dst9 = HEVC_FILT_4TAP_SW(tmp9, tmp5, filt0, filt1);
4525 dst10 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
4526 dst11 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
4527
4528 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4529 weight_vec, dst0, dst1, dst2, dst3);
4530 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4531 dst7, weight_vec, dst4, dst5, dst6, dst7);
4532 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4533 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4534 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4535 offset_vec, dst0, dst1, dst2, dst3);
4536 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4537 dst7, offset_vec, dst4, dst5, dst6, dst7);
4538 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4539
4540 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
4541 weight_vec, dst8, dst9, dst10, dst11);
4542 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
4543 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
4544 offset_vec, dst8, dst9, dst10, dst11);
4545 CLIP_SW4_0_255(dst8, dst9, dst10, dst11);
4546
4547 PCKEV_H2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
4548 PCKEV_H2_SH(dst5, dst4, dst7, dst6, dst2, dst3);
4549 PCKEV_H2_SH(dst9, dst8, dst11, dst10, dst4, dst5);
4550
4551 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out3, out4, out5);
4552 ST_D4(out3, out4, 0, 1, 0, 1, dst, dst_stride);
4553 ST_W4(out5, 0, 1, 2, 3, dst + 8, dst_stride);
4554 dst += (4 * dst_stride);
4555
4556 src2 = src10;
4557 src10_r = src98_r;
4558 src21_r = src109_r;
4559 src2110 = src10998;
4560 }
4561}
4562
4563static void hevc_vt_uniwgt_4t_16w_msa(const uint8_t *src,
4564 int32_t src_stride,
4565 uint8_t *dst,
4566 int32_t dst_stride,
4567 const int8_t *filter,
4568 int32_t height,
4569 int32_t weight,
4570 int32_t offset,
4571 int32_t rnd_val)
4572{
4573 int32_t loop_cnt;
4574 v16u8 out0, out1, out2, out3;
4575 v16i8 src0, src1, src2, src3, src4, src5;
4576 v16i8 src10_r, src32_r, src21_r, src43_r;
4577 v16i8 src10_l, src32_l, src21_l, src43_l;
4578 v16i8 src54_r, src54_l, src65_r, src65_l, src6;
4579 v8i16 filter_vec, filt0, filt1;
4580 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4581 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
4582 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
4583 v4i32 weight_vec, rnd_vec, offset_vec;
4584 v8i16 zero = { 0 };
4585
4586 src -= src_stride;
4587
4588 weight_vec = __msa_fill_w(weight);
4589 rnd_vec = __msa_fill_w(rnd_val);
4590 offset_vec = __msa_fill_w(offset);
4591
4592 filter_vec = LD_SH(filter);
4593 UNPCK_R_SB_SH(filter_vec, filter_vec);
4594 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4595
4596 LD_SB3(src, src_stride, src0, src1, src2);
4597 src += (3 * src_stride);
4598 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4599 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
4600
4601 for (loop_cnt = (height >> 2); loop_cnt--;) {
4602 LD_SB4(src, src_stride, src3, src4, src5, src6);
4603 src += (4 * src_stride);
4604 ILVRL_B2_SB(src3, src2, src32_r, src32_l);
4605 ILVRL_B2_SB(src4, src3, src43_r, src43_l);
4606 ILVRL_B2_SB(src5, src4, src54_r, src54_l);
4607 ILVRL_B2_SB(src6, src5, src65_r, src65_l);
4608
4609 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
4610 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
4611 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
4612 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
4613 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4614 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4615 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4616 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4617 ILVRL_B2_SH(zero, src54_r, tmp0, tmp1);
4618 ILVRL_B2_SH(zero, src65_r, tmp2, tmp3);
4619 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4620 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4621 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4622 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4623 ILVRL_B2_SH(zero, src10_l, tmp0, tmp1);
4624 ILVRL_B2_SH(zero, src21_l, tmp2, tmp3);
4625 ILVRL_B2_SH(zero, src32_l, tmp4, tmp5);
4626 ILVRL_B2_SH(zero, src43_l, tmp6, tmp7);
4627 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4628 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4629 dst10 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4630 dst11 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4631 ILVRL_B2_SH(zero, src54_l, tmp0, tmp1);
4632 ILVRL_B2_SH(zero, src65_l, tmp2, tmp3);
4633 dst12 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4634 dst13 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4635 dst14 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4636 dst15 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4637
4638 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4639 weight_vec, dst0, dst1, dst2, dst3);
4640 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4641 dst7, weight_vec, dst4, dst5, dst6, dst7);
4642 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4643 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4644 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4645 offset_vec, dst0, dst1, dst2, dst3);
4646 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4647 dst7, offset_vec, dst4, dst5, dst6, dst7);
4648 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4649
4650 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
4651 weight_vec, dst8, dst9, dst10, dst11);
4652 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
4653 dst15, weight_vec, dst12, dst13, dst14, dst15);
4654 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
4655 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
4656 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
4657 offset_vec, dst8, dst9, dst10, dst11);
4658 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
4659 dst15, offset_vec, dst12, dst13, dst14, dst15);
4660 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
4661
4662 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4663 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
4664 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
4665 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
4666 PCKEV_B4_UB(tmp4, tmp0, tmp5, tmp1, tmp6, tmp2, tmp7, tmp3, out0, out1,
4667 out2, out3);
4668 ST_UB4(out0, out1, out2, out3, dst, dst_stride);
4669 dst += (4 * dst_stride);
4670
4671 src2 = src6;
4672 src10_r = src54_r;
4673 src21_r = src65_r;
4674 src10_l = src54_l;
4675 src21_l = src65_l;
4676 }
4677}
4678
4679static void hevc_vt_uniwgt_4t_24w_msa(const uint8_t *src,
4680 int32_t src_stride,
4681 uint8_t *dst,
4682 int32_t dst_stride,
4683 const int8_t *filter,
4684 int32_t height,
4685 int32_t weight,
4686 int32_t offset,
4687 int32_t rnd_val)
4688{
4689 uint32_t loop_cnt;
4690 v16u8 out0, out1, out2, out3, out4, out5;
4691 v16i8 src0, src1, src2, src3, src4, src5;
4692 v16i8 src6, src7, src8, src9, src10, src11, src12, src13;
4693 v16i8 src10_r, src32_r, src54_r, src21_r, src43_r, src65_r;
4694 v16i8 src10_l, src32_l, src54_l, src21_l, src43_l, src65_l;
4695 v16i8 src87_r, src98_r, src109_r, src1110_r, src1211_r, src1312_r;
4696 v8i16 filter_vec, filt0, filt1;
4697 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4698 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
4699 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
4700 v4i32 dst16, dst17, dst18, dst19, dst20, dst21, dst22, dst23;
4701 v4i32 weight_vec, rnd_vec, offset_vec;
4702 v8i16 zero = { 0 };
4703
4704 src -= src_stride;
4705
4706 weight_vec = __msa_fill_w(weight);
4707 rnd_vec = __msa_fill_w(rnd_val);
4708 offset_vec = __msa_fill_w(offset);
4709
4710 filter_vec = LD_SH(filter);
4711 UNPCK_R_SB_SH(filter_vec, filter_vec);
4712 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4713
4714 LD_SB3(src, src_stride, src0, src1, src2);
4715 LD_SB3(src + 16, src_stride, src7, src8, src9);
4716 src += (3 * src_stride);
4717 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4718 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
4719 ILVR_B2_SB(src8, src7, src9, src8, src87_r, src98_r);
4720
4721 for (loop_cnt = 8; loop_cnt--;) {
4722 LD_SB4(src, src_stride, src3, src4, src5, src6);
4723 LD_SB4(src + 16, src_stride, src10, src11, src12, src13);
4724 src += (4 * src_stride);
4725 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4726 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
4727 ILVRL_B2_SB(src5, src4, src54_r, src54_l);
4728 ILVRL_B2_SB(src6, src5, src65_r, src65_l);
4729 ILVR_B2_SB(src10, src9, src11, src10, src109_r, src1110_r);
4730 ILVR_B2_SB(src12, src11, src13, src12, src1211_r, src1312_r);
4731
4732 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
4733 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
4734 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
4735 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
4736 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4737 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4738 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4739 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4740 ILVRL_B2_SH(zero, src54_r, tmp0, tmp1);
4741 ILVRL_B2_SH(zero, src65_r, tmp2, tmp3);
4742 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4743 dst5 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4744 dst6 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4745 dst7 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4746 ILVRL_B2_SH(zero, src10_l, tmp0, tmp1);
4747 ILVRL_B2_SH(zero, src21_l, tmp2, tmp3);
4748 ILVRL_B2_SH(zero, src32_l, tmp4, tmp5);
4749 ILVRL_B2_SH(zero, src43_l, tmp6, tmp7);
4750 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4751 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4752 dst10 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4753 dst11 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4754 ILVRL_B2_SH(zero, src54_l, tmp0, tmp1);
4755 ILVRL_B2_SH(zero, src65_l, tmp2, tmp3);
4756 dst12 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4757 dst13 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4758 dst14 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4759 dst15 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4760 ILVRL_B2_SH(zero, src87_r, tmp0, tmp1);
4761 ILVRL_B2_SH(zero, src98_r, tmp2, tmp3);
4762 ILVRL_B2_SH(zero, src109_r, tmp4, tmp5);
4763 ILVRL_B2_SH(zero, src1110_r, tmp6, tmp7);
4764 dst16 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4765 dst17 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4766 dst18 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4767 dst19 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4768 ILVRL_B2_SH(zero, src1211_r, tmp0, tmp1);
4769 ILVRL_B2_SH(zero, src1312_r, tmp2, tmp3);
4770 dst20 = HEVC_FILT_4TAP_SW(tmp4, tmp0, filt0, filt1);
4771 dst21 = HEVC_FILT_4TAP_SW(tmp5, tmp1, filt0, filt1);
4772 dst22 = HEVC_FILT_4TAP_SW(tmp6, tmp2, filt0, filt1);
4773 dst23 = HEVC_FILT_4TAP_SW(tmp7, tmp3, filt0, filt1);
4774
4775 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4776 weight_vec, dst0, dst1, dst2, dst3);
4777 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4778 dst7, weight_vec, dst4, dst5, dst6, dst7);
4779 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4780 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4781 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4782 offset_vec, dst0, dst1, dst2, dst3);
4783 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4784 dst7, offset_vec, dst4, dst5, dst6, dst7);
4785 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4786
4787 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
4788 weight_vec, dst8, dst9, dst10, dst11);
4789 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
4790 dst15, weight_vec, dst12, dst13, dst14, dst15);
4791 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
4792 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
4793 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
4794 offset_vec, dst8, dst9, dst10, dst11);
4795 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
4796 dst15, offset_vec, dst12, dst13, dst14, dst15);
4797 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
4798
4799 MUL4(dst16, weight_vec, dst17, weight_vec, dst18, weight_vec, dst19,
4800 weight_vec, dst16, dst17, dst18, dst19);
4801 MUL4(dst20, weight_vec, dst21, weight_vec, dst22, weight_vec,
4802 dst23, weight_vec, dst20, dst21, dst22, dst23);
4803 SRAR_W4_SW(dst16, dst17, dst18, dst19, rnd_vec);
4804 SRAR_W4_SW(dst20, dst21, dst22, dst23, rnd_vec);
4805 ADD4(dst16, offset_vec, dst17, offset_vec, dst18, offset_vec, dst19,
4806 offset_vec, dst16, dst17, dst18, dst19);
4807 ADD4(dst20, offset_vec, dst21, offset_vec, dst22, offset_vec,
4808 dst23, offset_vec, dst20, dst21, dst22, dst23);
4809 CLIP_SW8_0_255(dst16, dst17, dst18, dst19, dst20, dst21, dst22, dst23);
4810
4811 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4812 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
4813 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
4814 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
4815 PCKEV_B4_UB(tmp4, tmp0, tmp5, tmp1, tmp6, tmp2, tmp7, tmp3, out0, out1,
4816 out2, out3);
4817
4818 PCKEV_H2_SH(dst17, dst16, dst19, dst18, tmp0, tmp1);
4819 PCKEV_H2_SH(dst21, dst20, dst23, dst22, tmp2, tmp3);
4820
4821 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out4, out5);
4822 ST_UB4(out0, out1, out2, out3, dst, dst_stride);
4823 ST_D4(out4, out5, 0, 1, 0, 1, dst + 16, dst_stride);
4824 dst += (4 * dst_stride);
4825
4826 src2 = src6;
4827 src9 = src13;
4828 src10_r = src54_r;
4829 src21_r = src65_r;
4830 src10_l = src54_l;
4831 src21_l = src65_l;
4832 src87_r = src1211_r;
4833 src98_r = src1312_r;
4834 }
4835}
4836
4837static void hevc_vt_uniwgt_4t_32w_msa(const uint8_t *src,
4838 int32_t src_stride,
4839 uint8_t *dst,
4840 int32_t dst_stride,
4841 const int8_t *filter,
4842 int32_t height,
4843 int32_t weight,
4844 int32_t offset,
4845 int32_t rnd_val)
4846{
4847 uint32_t loop_cnt;
4848 v16u8 out0, out1, out2, out3;
4849 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
4850 v16i8 src10_r, src32_r, src76_r, src98_r;
4851 v16i8 src21_r, src43_r, src65_r, src87_r;
4852 v16i8 src10_l, src32_l, src76_l, src98_l;
4853 v16i8 src21_l, src43_l, src65_l, src87_l;
4854 v8i16 filter_vec, filt0, filt1;
4855 v4i32 weight_vec, rnd_vec, offset_vec;
4856 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
4857 v4i32 dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15;
4858 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4859 v8i16 zero = { 0 };
4860
4861 src -= src_stride;
4862
4863 weight_vec = __msa_fill_w(weight);
4864 rnd_vec = __msa_fill_w(rnd_val);
4865 offset_vec = __msa_fill_w(offset);
4866
4867 filter_vec = LD_SH(filter);
4868 UNPCK_R_SB_SH(filter_vec, filter_vec);
4869 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4870
4871 LD_SB3(src, src_stride, src0, src1, src2);
4872 LD_SB3(src + 16, src_stride, src5, src6, src7);
4873 src += (3 * src_stride);
4874 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4875 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
4876 ILVR_B2_SB(src6, src5, src7, src6, src65_r, src76_r);
4877 ILVL_B2_SB(src6, src5, src7, src6, src65_l, src76_l);
4878
4879 for (loop_cnt = (height >> 1); loop_cnt--;) {
4880 LD_SB2(src, src_stride, src3, src4);
4881 LD_SB2(src + 16, src_stride, src8, src9);
4882 src += (2 * src_stride);
4883 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4884 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
4885 ILVRL_B2_SB(src8, src7, src87_r, src87_l);
4886 ILVRL_B2_SB(src9, src8, src98_r, src98_l);
4887
4888 ILVRL_B2_SH(zero, src10_r, tmp0, tmp1);
4889 ILVRL_B2_SH(zero, src21_r, tmp2, tmp3);
4890 ILVRL_B2_SH(zero, src32_r, tmp4, tmp5);
4891 ILVRL_B2_SH(zero, src43_r, tmp6, tmp7);
4892 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4893 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4894 dst2 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4895 dst3 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4896
4897 ILVRL_B2_SH(zero, src10_l, tmp0, tmp1);
4898 ILVRL_B2_SH(zero, src21_l, tmp2, tmp3);
4899 ILVRL_B2_SH(zero, src32_l, tmp4, tmp5);
4900 ILVRL_B2_SH(zero, src43_l, tmp6, tmp7);
4901 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4902 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4903 dst6 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4904 dst7 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4905
4906 ILVRL_B2_SH(zero, src65_r, tmp0, tmp1);
4907 ILVRL_B2_SH(zero, src76_r, tmp2, tmp3);
4908 ILVRL_B2_SH(zero, src87_r, tmp4, tmp5);
4909 ILVRL_B2_SH(zero, src98_r, tmp6, tmp7);
4910 dst8 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4911 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4912 dst10 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4913 dst11 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4914 ILVRL_B2_SH(zero, src65_l, tmp0, tmp1);
4915 ILVRL_B2_SH(zero, src76_l, tmp2, tmp3);
4916 ILVRL_B2_SH(zero, src87_l, tmp4, tmp5);
4917 ILVRL_B2_SH(zero, src98_l, tmp6, tmp7);
4918 dst12 = HEVC_FILT_4TAP_SW(tmp0, tmp4, filt0, filt1);
4919 dst13 = HEVC_FILT_4TAP_SW(tmp1, tmp5, filt0, filt1);
4920 dst14 = HEVC_FILT_4TAP_SW(tmp2, tmp6, filt0, filt1);
4921 dst15 = HEVC_FILT_4TAP_SW(tmp3, tmp7, filt0, filt1);
4922
4923 MUL4(dst0, weight_vec, dst1, weight_vec, dst2, weight_vec, dst3,
4924 weight_vec, dst0, dst1, dst2, dst3);
4925 MUL4(dst4, weight_vec, dst5, weight_vec, dst6, weight_vec,
4926 dst7, weight_vec, dst4, dst5, dst6, dst7);
4927 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4928 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4929 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
4930 offset_vec, dst0, dst1, dst2, dst3);
4931 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec,
4932 dst7, offset_vec, dst4, dst5, dst6, dst7);
4933 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
4934
4935 MUL4(dst8, weight_vec, dst9, weight_vec, dst10, weight_vec, dst11,
4936 weight_vec, dst8, dst9, dst10, dst11);
4937 MUL4(dst12, weight_vec, dst13, weight_vec, dst14, weight_vec,
4938 dst15, weight_vec, dst12, dst13, dst14, dst15);
4939 SRAR_W4_SW(dst8, dst9, dst10, dst11, rnd_vec);
4940 SRAR_W4_SW(dst12, dst13, dst14, dst15, rnd_vec);
4941 ADD4(dst8, offset_vec, dst9, offset_vec, dst10, offset_vec, dst11,
4942 offset_vec, dst8, dst9, dst10, dst11);
4943 ADD4(dst12, offset_vec, dst13, offset_vec, dst14, offset_vec,
4944 dst15, offset_vec, dst12, dst13, dst14, dst15);
4945 CLIP_SW8_0_255(dst8, dst9, dst10, dst11, dst12, dst13, dst14, dst15);
4946
4947 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4948 PCKEV_H2_SH(dst5, dst4, dst7, dst6, tmp2, tmp3);
4949 PCKEV_H2_SH(dst9, dst8, dst11, dst10, tmp4, tmp5);
4950 PCKEV_H2_SH(dst13, dst12, dst15, dst14, tmp6, tmp7);
4951 PCKEV_B4_UB(tmp2, tmp0, tmp3, tmp1, tmp6, tmp4, tmp7, tmp5, out0, out1,
4952 out2, out3);
4953 ST_UB2(out0, out2, dst, 16);
4954 dst += dst_stride;
4955 ST_UB2(out1, out3, dst, 16);
4956 dst += dst_stride;
4957
4958 src2 = src4;
4959 src7 = src9;
4960 src10_r = src32_r;
4961 src21_r = src43_r;
4962 src10_l = src32_l;
4963 src21_l = src43_l;
4964 src65_r = src87_r;
4965 src76_r = src98_r;
4966 src65_l = src87_l;
4967 src76_l = src98_l;
4968 }
4969}
4970
4971static void hevc_hv_uniwgt_4t_4x2_msa(const uint8_t *src,
4972 int32_t src_stride,
4973 uint8_t *dst,
4974 int32_t dst_stride,
4975 const int8_t *filter_x,
4976 const int8_t *filter_y,
4977 int32_t weight,
4978 int32_t offset,
4979 int32_t rnd_val)
4980{
4981 v16u8 out;
4982 v16i8 src0, src1, src2, src3, src4;
4983 v8i16 filt0, filt1;
4984 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
4985 v16i8 mask1;
4986 v8i16 filt_h0, filt_h1, filter_vec, tmp;
4987 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
4988 v8i16 dst10, dst21, dst32, dst43;
4989 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4990 v4i32 dst0, dst1, dst2, dst3, dst4;
4991 v4i32 weight_vec, rnd_vec, offset_vec;
4992 v8i16 zero = { 0 };
4993
4994 src -= (src_stride + 1);
4995
4996 filter_vec = LD_SH(filter_x);
4997 UNPCK_R_SB_SH(filter_vec, filter_vec);
4998 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
4999
5000 filter_vec = LD_SH(filter_y);
5001 UNPCK_R_SB_SH(filter_vec, filter_vec);
5002 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5003
5004 mask1 = mask0 + 2;
5005
5006 weight_vec = __msa_fill_w(weight);
5007 rnd_vec = __msa_fill_w(rnd_val);
5008 offset_vec = __msa_fill_w(offset);
5009
5010 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
5011 VSHF_B2_SB(src0, src2, src0, src2, mask0, mask1, vec0, vec1);
5012 VSHF_B2_SB(src1, src3, src1, src3, mask0, mask1, vec2, vec3);
5013 VSHF_B2_SB(src2, src4, src2, src4, mask0, mask1, vec4, vec5);
5014 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5015 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5016 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5017 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5018 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5019 dst2 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5020 dst1 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5021 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5022 ILVL_B2_SH(zero, vec4, zero, vec5, tmp1, tmp3);
5023 dst4 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5024
5025 ILVEV_H2_SH(dst0, dst1, dst2, dst3, dst10, dst32);
5026 ILVEV_H2_SH(dst1, dst2, dst3, dst4, dst21, dst43);
5027 dst0 = HEVC_FILT_4TAP(dst10, dst32, filt_h0, filt_h1);
5028 dst1 = HEVC_FILT_4TAP(dst21, dst43, filt_h0, filt_h1);
5029 dst0 >>= 6;
5030 dst1 >>= 6;
5031 MUL2(dst0, weight_vec, dst1, weight_vec, dst0, dst1);
5032 SRAR_W2_SW(dst0, dst1, rnd_vec);
5033 ADD2(dst0, offset_vec, dst1, offset_vec, dst0, dst1);
5034 CLIP_SW2_0_255(dst0, dst1);
5035 tmp = __msa_pckev_h((v8i16) dst1, (v8i16) dst0);
5036 out = (v16u8) __msa_pckev_b((v16i8) tmp, (v16i8) tmp);
5037 ST_W2(out, 0, 1, dst, dst_stride);
5038}
5039
5040static void hevc_hv_uniwgt_4t_4x4_msa(const uint8_t *src,
5041 int32_t src_stride,
5042 uint8_t *dst,
5043 int32_t dst_stride,
5044 const int8_t *filter_x,
5045 const int8_t *filter_y,
5046 int32_t weight,
5047 int32_t offset,
5048 int32_t rnd_val)
5049{
5050 v16u8 out;
5051 v16i8 src0, src1, src2, src3, src4, src5, src6;
5052 v8i16 filt0, filt1;
5053 v8i16 filt_h0, filt_h1, filter_vec;
5054 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
5055 v16i8 mask1;
5056 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5057 v8i16 dst10, dst32, dst54, dst21, dst43, dst65;
5058 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6;
5059 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5060 v4i32 weight_vec, rnd_vec, offset_vec;
5061 v8i16 zero = { 0 };
5062
5063 src -= (src_stride + 1);
5064
5065 filter_vec = LD_SH(filter_x);
5066 UNPCK_R_SB_SH(filter_vec, filter_vec);
5067 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
5068
5069 filter_vec = LD_SH(filter_y);
5070 UNPCK_R_SB_SH(filter_vec, filter_vec);
5071 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5072
5073 mask1 = mask0 + 2;
5074
5075 weight_vec = __msa_fill_w(weight);
5076 rnd_vec = __msa_fill_w(rnd_val);
5077 offset_vec = __msa_fill_w(offset);
5078
5079 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
5080 VSHF_B2_SB(src0, src3, src0, src3, mask0, mask1, vec0, vec1);
5081 VSHF_B2_SB(src1, src4, src1, src4, mask0, mask1, vec2, vec3);
5082 VSHF_B2_SB(src2, src5, src2, src5, mask0, mask1, vec4, vec5);
5083 VSHF_B2_SB(src3, src6, src3, src6, mask0, mask1, vec6, vec7);
5084
5085 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5086 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5087 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5088 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5089 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5090 dst3 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5091 dst1 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5092 dst4 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5093 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5094 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5095 ILVL_B2_SH(zero, vec6, zero, vec7, tmp5, tmp7);
5096 dst2 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5097 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5098 dst6 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5099 ILVEV_H2_SH(dst0, dst1, dst3, dst4, dst10, dst43);
5100 ILVEV_H2_SH(dst1, dst2, dst4, dst5, dst21, dst54);
5101 ILVEV_H2_SH(dst2, dst3, dst5, dst6, dst32, dst65);
5102 dst0 = HEVC_FILT_4TAP(dst10, dst32, filt_h0, filt_h1);
5103 dst1 = HEVC_FILT_4TAP(dst21, dst43, filt_h0, filt_h1);
5104 dst2 = HEVC_FILT_4TAP(dst32, dst54, filt_h0, filt_h1);
5105 dst3 = HEVC_FILT_4TAP(dst43, dst65, filt_h0, filt_h1);
5106 SRA_4V(dst0, dst1, dst2, dst3, 6);
5107 MUL2(dst0, weight_vec, dst1, weight_vec, dst0, dst1);
5108 MUL2(dst2, weight_vec, dst3, weight_vec, dst2, dst3);
5109 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5110 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
5111 offset_vec, dst0, dst1, dst2, dst3);
5112 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5113 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
5114 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
5115 ST_W4(out, 0, 1, 2, 3, dst, dst_stride);
5116}
5117
5118static void hevc_hv_uniwgt_4t_4multx8mult_msa(const uint8_t *src,
5119 int32_t src_stride,
5120 uint8_t *dst,
5121 int32_t dst_stride,
5122 const int8_t *filter_x,
5123 const int8_t *filter_y,
5124 int32_t height,
5125 int32_t weight,
5126 int32_t offset,
5127 int32_t rnd_val)
5128{
5129 uint32_t loop_cnt;
5130 v16u8 out0, out1;
5131 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
5132 v8i16 filt0, filt1;
5133 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
5134 v16i8 mask1;
5135 v8i16 filter_vec, filt_h0, filt_h1;
5136 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5137 v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
5138 v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
5139 v8i16 dst98_r, dst109_r;
5140 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5141 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9, dst10;
5142 v4i32 offset_vec, weight_vec, rnd_vec;
5143 v8i16 zero = { 0 };
5144
5145 src -= (src_stride + 1);
5146
5147 filter_vec = LD_SH(filter_x);
5148 UNPCK_R_SB_SH(filter_vec, filter_vec);
5149 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
5150
5151 filter_vec = LD_SH(filter_y);
5152 UNPCK_R_SB_SH(filter_vec, filter_vec);
5153 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5154
5155 mask1 = mask0 + 2;
5156
5157 weight_vec = __msa_fill_w(weight);
5158 rnd_vec = __msa_fill_w(rnd_val);
5159 offset_vec = __msa_fill_w(offset);
5160
5161 LD_SB3(src, src_stride, src0, src1, src2);
5162 src += (3 * src_stride);
5163
5164 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
5165 VSHF_B2_SB(src1, src2, src1, src2, mask0, mask1, vec2, vec3);
5166 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5167 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5168 ILVL_B2_SH(zero, vec2, zero, vec3, tmp5, tmp7);
5169 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5170 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5171 dst2 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5172
5173 ILVEV_H2_SH(dst0, dst1, dst1, dst2, dst10_r, dst21_r);
5174
5175 for (loop_cnt = height >> 3; loop_cnt--;) {
5176 LD_SB8(src, src_stride,
5177 src3, src4, src5, src6, src7, src8, src9, src10);
5178 src += (8 * src_stride);
5179
5180 VSHF_B2_SB(src3, src7, src3, src7, mask0, mask1, vec0, vec1);
5181 VSHF_B2_SB(src4, src8, src4, src8, mask0, mask1, vec2, vec3);
5182 VSHF_B2_SB(src5, src9, src5, src9, mask0, mask1, vec4, vec5);
5183 VSHF_B2_SB(src6, src10, src6, src10, mask0, mask1, vec6, vec7);
5184
5185 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5186 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5187 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5188 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5189 dst3 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5190 dst7 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5191 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5192 dst8 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5193 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5194 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5195 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
5196 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
5197 dst5 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5198 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5199 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5200 dst10 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5201
5202 dst32_r = __msa_ilvev_h(dst3, dst2);
5203 ILVEV_H2_SH(dst3, dst4, dst7, dst8, dst43_r, dst87_r);
5204 ILVEV_H2_SH(dst4, dst5, dst8, dst9, dst54_r, dst98_r);
5205 ILVEV_H2_SH(dst5, dst6, dst9, dst10, dst65_r, dst109_r);
5206 dst76_r = __msa_ilvev_h(dst7, dst6);
5207 dst0 = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5208 dst1 = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5209 dst2 = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5210 dst3 = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5211 dst4 = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
5212 dst5 = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
5213 dst6 = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
5214 dst7 = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
5215 SRA_4V(dst0, dst1, dst2, dst3, 6);
5216 SRA_4V(dst4, dst5, dst6, dst7, 6);
5217 MUL2(dst0, weight_vec, dst1, weight_vec, dst0, dst1);
5218 MUL2(dst2, weight_vec, dst3, weight_vec, dst2, dst3);
5219 MUL2(dst4, weight_vec, dst5, weight_vec, dst4, dst5);
5220 MUL2(dst6, weight_vec, dst7, weight_vec, dst6, dst7);
5221 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5222 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
5223 ADD4(dst0, offset_vec, dst1, offset_vec, dst2, offset_vec, dst3,
5224 offset_vec, dst0, dst1, dst2, dst3);
5225 ADD4(dst4, offset_vec, dst5, offset_vec, dst6, offset_vec, dst7,
5226 offset_vec, dst4, dst5, dst6, dst7);
5227 CLIP_SW8_0_255(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7);
5228 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, tmp0, tmp1,
5229 tmp2, tmp3);
5230 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5231 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
5232 dst += (8 * dst_stride);
5233
5234 dst10_r = dst98_r;
5235 dst21_r = dst109_r;
5236 dst2 = dst10;
5237 }
5238}
5239
5240static void hevc_hv_uniwgt_4t_4w_msa(const uint8_t *src,
5241 int32_t src_stride,
5242 uint8_t *dst,
5243 int32_t dst_stride,
5244 const int8_t *filter_x,
5245 const int8_t *filter_y,
5246 int32_t height,
5247 int32_t weight,
5248 int32_t offset,
5249 int32_t rnd_val)
5250{
5251 if (2 == height) {
5252 hevc_hv_uniwgt_4t_4x2_msa(src, src_stride, dst, dst_stride,
5253 filter_x, filter_y, weight,
5254 offset, rnd_val);
5255 } else if (4 == height) {
5256 hevc_hv_uniwgt_4t_4x4_msa(src, src_stride, dst, dst_stride,
5257 filter_x,filter_y, weight,
5258 offset, rnd_val);
5259 } else if (0 == (height % 8)) {
5260 hevc_hv_uniwgt_4t_4multx8mult_msa(src, src_stride, dst, dst_stride,
5261 filter_x, filter_y, height, weight,
5262 offset, rnd_val);
5263 }
5264}
5265
5266static void hevc_hv_uniwgt_4t_6w_msa(const uint8_t *src,
5267 int32_t src_stride,
5268 uint8_t *dst,
5269 int32_t dst_stride,
5270 const int8_t *filter_x,
5271 const int8_t *filter_y,
5272 int32_t height,
5273 int32_t weight,
5274 int32_t offset,
5275 int32_t rnd_val)
5276{
5277 v16u8 out0, out1, out2;
5278 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
5279 v8i16 filt0, filt1;
5280 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
5281 v16i8 mask1;
5282 v8i16 filt_h0, filt_h1, filter_vec;
5283 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5284 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9;
5285 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5286 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst98_r, dst21_r, dst43_r;
5287 v8i16 dst65_r, dst87_r, dst109_r, dst10_l, dst32_l, dst54_l, dst76_l;
5288 v8i16 dst98_l, dst21_l, dst43_l, dst65_l, dst87_l, dst109_l;
5289 v8i16 dst1021_l, dst3243_l, dst5465_l, dst7687_l, dst98109_l;
5290 v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r, dst6_r, dst7_r;
5291 v4i32 dst0_l, dst1_l, dst2_l, dst3_l, weight_vec, rnd_vec, offset_vec;
5292 v8i16 zero = { 0 };
5293
5294 src -= (src_stride + 1);
5295
5296 filter_vec = LD_SH(filter_x);
5297 UNPCK_R_SB_SH(filter_vec, filter_vec);
5298 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
5299
5300 filter_vec = LD_SH(filter_y);
5301 UNPCK_R_SB_SH(filter_vec, filter_vec);
5302 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5303
5304 mask1 = mask0 + 2;
5305
5306 weight_vec = __msa_fill_w(weight);
5307 rnd_vec = __msa_fill_w(rnd_val);
5308 offset_vec = __msa_fill_w(offset);
5309
5310 LD_SB3(src, src_stride, src0, src1, src2);
5311 src += (3 * src_stride);
5312
5313 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5314 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5315 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5316 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5317 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5318 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5319 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5320 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5321 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5322 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5323 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5324 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5325 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5326 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5327 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5328
5329 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst10_r, dst10_l);
5330 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst21_r, dst21_l);
5331
5332 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9, src10);
5333 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
5334 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
5335 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
5336 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
5337 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5338 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5339 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5340 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5341 dst6 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5342 dst7 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5343 dst8 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5344 dst9 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5345 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst32_r, dst32_l);
5346 ILVEV_H2_SH(dst6, dst8, dst7, dst9, dst43_r, dst43_l);
5347 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5348 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5349 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
5350 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
5351 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5352 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5353 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5354 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5355 ILVEV_H2_SH(dst8, dst0, dst9, dst1, dst54_r, dst54_l);
5356 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst65_r, dst65_l);
5357 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
5358 VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec2, vec3);
5359 VSHF_B2_SB(src9, src9, src9, src9, mask0, mask1, vec4, vec5);
5360 VSHF_B2_SB(src10, src10, src10, src10, mask0, mask1, vec6, vec7);
5361 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5362 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5363 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5364 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5365 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5366 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5367 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5368 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5369 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst76_r, dst76_l);
5370 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst87_r, dst87_l);
5371 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5372 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5373 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
5374 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
5375 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5376 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5377 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5378 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5379 ILVEV_H2_SH(dst6, dst0, dst7, dst1, dst98_r, dst98_l);
5380 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst109_r, dst109_l);
5381 PCKEV_D2_SH(dst21_l, dst10_l, dst43_l, dst32_l, dst1021_l, dst3243_l);
5382 PCKEV_D2_SH(dst65_l, dst54_l, dst87_l, dst76_l, dst5465_l, dst7687_l);
5383 dst98109_l = (v8i16) __msa_pckev_d((v2i64) dst109_l, (v2i64) dst98_l);
5384 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5385 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5386 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5387 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5388 dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
5389 dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
5390 dst6_r = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
5391 dst7_r = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
5392 dst0_l = HEVC_FILT_4TAP(dst1021_l, dst3243_l, filt_h0, filt_h1);
5393 dst1_l = HEVC_FILT_4TAP(dst3243_l, dst5465_l, filt_h0, filt_h1);
5394 dst2_l = HEVC_FILT_4TAP(dst5465_l, dst7687_l, filt_h0, filt_h1);
5395 dst3_l = HEVC_FILT_4TAP(dst7687_l, dst98109_l, filt_h0, filt_h1);
5396 SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
5397 SRA_4V(dst4_r, dst5_r, dst6_r, dst7_r, 6);
5398 SRA_4V(dst0_l, dst1_l, dst2_l, dst3_l, 6);
5399 MUL2(dst0_r, weight_vec, dst1_r, weight_vec, dst0_r, dst1_r);
5400 MUL2(dst2_r, weight_vec, dst3_r, weight_vec, dst2_r, dst3_r);
5401 MUL2(dst4_r, weight_vec, dst5_r, weight_vec, dst4_r, dst5_r);
5402 MUL2(dst6_r, weight_vec, dst7_r, weight_vec, dst6_r, dst7_r);
5403 MUL2(dst0_l, weight_vec, dst1_l, weight_vec, dst0_l, dst1_l);
5404 MUL2(dst2_l, weight_vec, dst3_l, weight_vec, dst2_l, dst3_l);
5405 SRAR_W4_SW(dst0_r, dst1_r, dst2_r, dst3_r, rnd_vec);
5406 SRAR_W4_SW(dst4_r, dst5_r, dst6_r, dst7_r, rnd_vec);
5407 SRAR_W4_SW(dst0_l, dst1_l, dst2_l, dst3_l, rnd_vec);
5408 ADD4(dst0_r, offset_vec, dst1_r, offset_vec, dst2_r, offset_vec, dst3_r,
5409 offset_vec, dst0_r, dst1_r, dst2_r, dst3_r);
5410 ADD4(dst4_r, offset_vec, dst5_r, offset_vec, dst6_r, offset_vec, dst7_r,
5411 offset_vec, dst4_r, dst5_r, dst6_r, dst7_r);
5412 ADD4(dst0_l, offset_vec, dst1_l, offset_vec, dst2_l, offset_vec, dst3_l,
5413 offset_vec, dst0_l, dst1_l, dst2_l, dst3_l);
5414 CLIP_SW8_0_255(dst1_r, dst0_r, dst3_r, dst2_r,
5415 dst4_r, dst5_r, dst6_r, dst7_r);
5416 CLIP_SW4_0_255(dst0_l, dst1_l, dst2_l, dst3_l);
5417 PCKEV_H2_SH(dst1_r, dst0_r, dst3_r, dst2_r, tmp0, tmp1);
5418 PCKEV_H2_SH(dst5_r, dst4_r, dst7_r, dst6_r, tmp2, tmp3);
5419 PCKEV_H2_SH(dst1_l, dst0_l, dst3_l, dst2_l, tmp4, tmp5);
5420 PCKEV_B3_UB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, out0, out1, out2);
5421 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
5422 ST_H8(out2, 0, 1, 2, 3, 4, 5, 6, 7, dst + 4, dst_stride);
5423}
5424
5425static void hevc_hv_uniwgt_4t_8x2_msa(const uint8_t *src,
5426 int32_t src_stride,
5427 uint8_t *dst,
5428 int32_t dst_stride,
5429 const int8_t *filter_x,
5430 const int8_t *filter_y,
5431 int32_t weight,
5432 int32_t offset,
5433 int32_t rnd_val)
5434{
5435 v16u8 out;
5436 v16i8 src0, src1, src2, src3, src4;
5437 v8i16 filt0, filt1;
5438 v8i16 filt_h0, filt_h1, filter_vec;
5439 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
5440 v16i8 mask1;
5441 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
5442 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
5443 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5444 v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
5445 v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
5446 v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
5447 v4i32 weight_vec, rnd_vec, offset_vec;
5448 v8i16 zero = { 0 };
5449
5450 src -= (src_stride + 1);
5451
5452 filter_vec = LD_SH(filter_x);
5453 UNPCK_R_SB_SH(filter_vec, filter_vec);
5454 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
5455
5456 filter_vec = LD_SH(filter_y);
5457 UNPCK_R_SB_SH(filter_vec, filter_vec);
5458 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5459
5460 mask1 = mask0 + 2;
5461
5462 weight_vec = __msa_fill_w(weight);
5463 rnd_vec = __msa_fill_w(rnd_val);
5464 offset_vec = __msa_fill_w(offset);
5465
5466 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
5467 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5468 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5469 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5470 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
5471 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
5472
5473 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5474 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5475 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5476 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5477 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5478 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5479 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5480 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5481 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5482 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5483 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
5484 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
5485 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5486 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5487 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5488 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5489
5490 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst10_r, dst10_l);
5491 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst21_r, dst21_l);
5492 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst32_r, dst32_l);
5493 ILVRL_B2_SH(zero, vec8, tmp0, tmp1);
5494 ILVRL_B2_SH(zero, vec9, tmp2, tmp3);
5495 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5496 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5497 ILVEV_H2_SH(dst6, dst0, dst7, dst1, dst43_r, dst43_l);
5498
5499 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5500 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5501 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5502 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5503 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5504 MUL2(dst0_r, weight_vec, dst1_r, weight_vec, dst0_r, dst1_r);
5505 MUL2(dst0_l, weight_vec, dst1_l, weight_vec, dst0_l, dst1_l);
5506 SRAR_W4_SW(dst0_r, dst0_l, dst1_r, dst1_l, rnd_vec);
5507 ADD4(dst0_r, offset_vec, dst0_l, offset_vec, dst1_r, offset_vec,
5508 dst1_l, offset_vec, dst0_r, dst0_l, dst1_r, dst1_l);
5509 CLIP_SW4_0_255(dst0_r, dst0_l, dst1_r, dst1_l);
5510 PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, tmp0, tmp1);
5511 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
5512 ST_D2(out, 0, 1, dst, dst_stride);
5513}
5514
5515static void hevc_hv_uniwgt_4t_8multx4_msa(const uint8_t *src,
5516 int32_t src_stride,
5517 uint8_t *dst,
5518 int32_t dst_stride,
5519 const int8_t *filter_x,
5520 const int8_t *filter_y,
5521 int32_t width8mult,
5522 int32_t weight,
5523 int32_t offset,
5524 int32_t rnd_val)
5525{
5526 uint32_t cnt;
5527 v16u8 out0, out1;
5528 v16i8 src0, src1, src2, src3, src4, src5, src6, mask0, mask1;
5529 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5530 v8i16 filt0, filt1, filt_h0, filt_h1, filter_vec;
5531 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
5532 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5533 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
5534 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
5535 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
5536 v4i32 weight_vec, rnd_vec, offset_vec;
5537 v8i16 zero = { 0 };
5538
5539 src -= (src_stride + 1);
5540
5541 filter_vec = LD_SH(filter_x);
5542 UNPCK_R_SB_SH(filter_vec, filter_vec);
5543 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
5544
5545 filter_vec = LD_SH(filter_y);
5546 UNPCK_R_SB_SH(filter_vec, filter_vec);
5547 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5548
5549 mask0 = LD_SB(ff_hevc_mask_arr);
5550 mask1 = mask0 + 2;
5551
5552 weight_vec = __msa_fill_w(weight);
5553 rnd_vec = __msa_fill_w(rnd_val);
5554 offset_vec = __msa_fill_w(offset);
5555
5556 for (cnt = width8mult; cnt--;) {
5557 LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
5558 src += 8;
5559 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5560 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5561 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5562 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5563 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5564 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5565 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5566 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5567 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5568 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5569 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5570 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5571 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5572 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5573 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5574 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst10_r, dst10_l);
5575 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst21_r, dst21_l);
5576 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
5577 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
5578 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
5579 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
5580 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5581 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5582 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5583 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5584 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5585 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5586 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5587 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5588 ILVEV_H2_SH(dst4, dst0, dst5, dst1, dst32_r, dst32_l);
5589 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst43_r, dst43_l);
5590 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5591 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5592 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
5593 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
5594 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5595 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5596 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5597 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5598 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst54_r, dst54_l);
5599 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst65_r, dst65_l);
5600
5601 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5602 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5603 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5604 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5605 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5606 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
5607 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5608 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
5609 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5610 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
5611 MUL2(dst0_r, weight_vec, dst1_r, weight_vec, dst0_r, dst1_r);
5612 MUL2(dst2_r, weight_vec, dst3_r, weight_vec, dst2_r, dst3_r);
5613 MUL2(dst0_l, weight_vec, dst1_l, weight_vec, dst0_l, dst1_l);
5614 MUL2(dst2_l, weight_vec, dst3_l, weight_vec, dst2_l, dst3_l);
5615 SRAR_W4_SW(dst0_r, dst0_l, dst1_r, dst1_l, rnd_vec);
5616 SRAR_W4_SW(dst2_r, dst2_l, dst3_r, dst3_l, rnd_vec);
5617 ADD2(dst0_r, offset_vec, dst1_r, offset_vec, dst0_r, dst1_r);
5618 ADD2(dst2_r, offset_vec, dst3_r, offset_vec, dst2_r, dst3_r);
5619 ADD2(dst0_l, offset_vec, dst1_l, offset_vec, dst0_l, dst1_l);
5620 ADD2(dst2_l, offset_vec, dst3_l, offset_vec, dst2_l, dst3_l);
5621 CLIP_SW8_0_255(dst0_r, dst0_l, dst1_r, dst1_l,
5622 dst2_r, dst2_l, dst3_r, dst3_l);
5623 PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
5624 dst3_r, tmp0, tmp1, tmp2, tmp3);
5625 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5626 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
5627 dst += 8;
5628 }
5629}
5630
5631static void hevc_hv_uniwgt_4t_8x6_msa(const uint8_t *src,
5632 int32_t src_stride,
5633 uint8_t *dst,
5634 int32_t dst_stride,
5635 const int8_t *filter_x,
5636 const int8_t *filter_y,
5637 int32_t weight,
5638 int32_t offset,
5639 int32_t rnd_val)
5640{
5641 v16u8 out0, out1, out2;
5642 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
5643 v8i16 filt0, filt1;
5644 v8i16 filt_h0, filt_h1, filter_vec;
5645 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
5646 v16i8 mask1;
5647 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
5648 v16i8 vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17;
5649 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
5650 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
5651 v4i32 dst4_r, dst4_l, dst5_r, dst5_l, weight_vec, rnd_vec, offset_vec;
5652 v8i16 dst10_r, dst32_r, dst10_l, dst32_l;
5653 v8i16 dst21_r, dst43_r, dst21_l, dst43_l;
5654 v8i16 dst54_r, dst54_l, dst65_r, dst65_l;
5655 v8i16 dst76_r, dst76_l, dst87_r, dst87_l;
5656 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5657 v8i16 zero = { 0 };
5658
5659 src -= (src_stride + 1);
5660
5661 filter_vec = LD_SH(filter_x);
5662 UNPCK_R_SB_SH(filter_vec, filter_vec);
5663 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
5664
5665 filter_vec = LD_SH(filter_y);
5666 UNPCK_R_SB_SH(filter_vec, filter_vec);
5667 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5668
5669 mask1 = mask0 + 2;
5670
5671 weight_vec = __msa_fill_w(weight);
5672 rnd_vec = __msa_fill_w(rnd_val);
5673 offset_vec = __msa_fill_w(offset);
5674
5675 LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
5676 src += (5 * src_stride);
5677 LD_SB4(src, src_stride, src5, src6, src7, src8);
5678 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5679 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5680 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5681 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
5682 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
5683 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec10, vec11);
5684 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec12, vec13);
5685 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec14, vec15);
5686 VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec16, vec17);
5687
5688 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5689 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5690 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5691 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5692 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5693 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5694 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5695 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5696 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5697 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5698 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
5699 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
5700 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5701 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5702 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5703 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5704 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst10_r, dst10_l);
5705 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst21_r, dst21_l);
5706 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst32_r, dst32_l);
5707 ILVRL_B2_SH(zero, vec8, tmp0, tmp1);
5708 ILVRL_B2_SH(zero, vec9, tmp2, tmp3);
5709 ILVRL_B2_SH(zero, vec10, tmp4, tmp5);
5710 ILVRL_B2_SH(zero, vec11, tmp6, tmp7);
5711 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5712 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5713 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5714 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5715 ILVEV_H2_SH(dst6, dst0, dst7, dst1, dst43_r, dst43_l);
5716 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst54_r, dst54_l);
5717 ILVRL_B2_SH(zero, vec12, tmp0, tmp1);
5718 ILVRL_B2_SH(zero, vec13, tmp2, tmp3);
5719 ILVRL_B2_SH(zero, vec14, tmp4, tmp5);
5720 ILVRL_B2_SH(zero, vec15, tmp6, tmp7);
5721 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5722 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5723 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5724 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5725 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst65_r, dst65_l);
5726 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst76_r, dst76_l);
5727 ILVRL_B2_SH(zero, vec16, tmp0, tmp1);
5728 ILVRL_B2_SH(zero, vec17, tmp2, tmp3);
5729 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5730 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5731 ILVEV_H2_SH(dst6, dst0, dst7, dst1, dst87_r, dst87_l);
5732 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5733 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5734 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5735 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5736 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5737 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
5738 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5739 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
5740 dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
5741 dst4_l = HEVC_FILT_4TAP(dst54_l, dst76_l, filt_h0, filt_h1);
5742 dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
5743 dst5_l = HEVC_FILT_4TAP(dst65_l, dst87_l, filt_h0, filt_h1);
5744 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5745 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
5746 SRA_4V(dst4_r, dst4_l, dst5_r, dst5_l, 6);
5747 MUL2(dst0_r, weight_vec, dst1_r, weight_vec, dst0_r, dst1_r);
5748 MUL2(dst2_r, weight_vec, dst3_r, weight_vec, dst2_r, dst3_r);
5749 MUL2(dst4_r, weight_vec, dst5_r, weight_vec, dst4_r, dst5_r);
5750 MUL2(dst0_l, weight_vec, dst1_l, weight_vec, dst0_l, dst1_l);
5751 MUL2(dst2_l, weight_vec, dst3_l, weight_vec, dst2_l, dst3_l);
5752 MUL2(dst4_l, weight_vec, dst5_l, weight_vec, dst4_l, dst5_l);
5753 SRAR_W4_SW(dst0_r, dst0_l, dst1_r, dst1_l, rnd_vec);
5754 SRAR_W4_SW(dst2_r, dst2_l, dst3_r, dst3_l, rnd_vec);
5755 SRAR_W4_SW(dst4_r, dst4_l, dst5_r, dst5_l, rnd_vec);
5756 ADD2(dst0_r, offset_vec, dst1_r, offset_vec, dst0_r, dst1_r);
5757 ADD2(dst2_r, offset_vec, dst3_r, offset_vec, dst2_r, dst3_r);
5758 ADD2(dst4_r, offset_vec, dst5_r, offset_vec, dst4_r, dst5_r);
5759 ADD2(dst0_l, offset_vec, dst1_l, offset_vec, dst0_l, dst1_l);
5760 ADD2(dst2_l, offset_vec, dst3_l, offset_vec, dst2_l, dst3_l);
5761 ADD2(dst4_l, offset_vec, dst5_l, offset_vec, dst4_l, dst5_l);
5762 CLIP_SW8_0_255(dst0_r, dst1_r, dst2_r, dst3_r,
5763 dst4_r, dst5_r, dst0_l, dst1_l);
5764 CLIP_SW4_0_255(dst2_l, dst3_l, dst4_l, dst5_l);
5765 PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l, dst3_r,
5766 tmp0, tmp1, tmp2, tmp3);
5767 PCKEV_H2_SH(dst4_l, dst4_r, dst5_l, dst5_r, tmp4, tmp5);
5768 PCKEV_B3_UB(tmp1, tmp0, tmp3, tmp2, tmp5, tmp4, out0, out1, out2);
5769 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
5770 ST_D2(out2, 0, 1, dst + 4 * dst_stride, dst_stride);
5771}
5772
5773static void hevc_hv_uniwgt_4t_8multx4mult_msa(const uint8_t *src,
5774 int32_t src_stride,
5775 uint8_t *dst,
5776 int32_t dst_stride,
5777 const int8_t *filter_x,
5778 const int8_t *filter_y,
5779 int32_t height,
5780 int32_t weight,
5781 int32_t offset,
5782 int32_t rnd_val,
5783 int32_t width8mult)
5784{
5785 uint32_t loop_cnt, cnt;
5786 const uint8_t *src_tmp;
5787 uint8_t *dst_tmp;
5788 v16u8 out0, out1;
5789 v16i8 src0, src1, src2, src3, src4, src5, src6;
5790 v8i16 filt0, filt1;
5791 v8i16 filt_h0, filt_h1, filter_vec;
5792 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
5793 v16i8 mask1;
5794 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5795 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
5796 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5797 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
5798 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
5799 v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
5800 v4i32 dst2_r, dst2_l, dst3_r, dst3_l;
5801 v4i32 weight_vec, rnd_vec, offset_vec;
5802 v8i16 zero = { 0 };
5803
5804 src -= (src_stride + 1);
5805
5806 filter_vec = LD_SH(filter_x);
5807 UNPCK_R_SB_SH(filter_vec, filter_vec);
5808 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
5809
5810 filter_vec = LD_SH(filter_y);
5811 UNPCK_R_SB_SH(filter_vec, filter_vec);
5812 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5813
5814 mask1 = mask0 + 2;
5815
5816 weight_vec = __msa_fill_w(weight);
5817 rnd_vec = __msa_fill_w(rnd_val);
5818 offset_vec = __msa_fill_w(offset);
5819
5820 for (cnt = width8mult; cnt--;) {
5821 src_tmp = src;
5822 dst_tmp = dst;
5823
5824 LD_SB3(src_tmp, src_stride, src0, src1, src2);
5825 src_tmp += (3 * src_stride);
5826
5827 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5828 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5829 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5830
5831 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5832 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5833 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5834 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5835 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5836 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5837 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5838 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5839 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5840 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5841 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5842 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5843 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst10_r, dst10_l);
5844 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst21_r, dst21_l);
5845
5846 for (loop_cnt = height >> 2; loop_cnt--;) {
5847 LD_SB4(src_tmp, src_stride, src3, src4, src5, src6);
5848 src_tmp += (4 * src_stride);
5849
5850 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
5851 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
5852 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
5853 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
5854
5855 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
5856 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
5857 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
5858 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
5859 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5860 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5861 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5862 dst3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5863 ILVEV_H2_SH(dst4, dst0, dst5, dst1, dst32_r, dst32_l);
5864 ILVEV_H2_SH(dst0, dst2, dst1, dst3, dst43_r, dst43_l);
5865 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
5866 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
5867 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
5868 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
5869 dst4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
5870 dst5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
5871 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
5872 dst7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
5873 ILVEV_H2_SH(dst2, dst4, dst3, dst5, dst54_r, dst54_l);
5874 ILVEV_H2_SH(dst4, dst6, dst5, dst7, dst65_r, dst65_l);
5875 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5876 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5877 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5878 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5879 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5880 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
5881 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5882 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
5883 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5884 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
5885 MUL2(dst0_r, weight_vec, dst1_r, weight_vec, dst0_r, dst1_r);
5886 MUL2(dst2_r, weight_vec, dst3_r, weight_vec, dst2_r, dst3_r);
5887 MUL2(dst0_l, weight_vec, dst1_l, weight_vec, dst0_l, dst1_l);
5888 MUL2(dst2_l, weight_vec, dst3_l, weight_vec, dst2_l, dst3_l);
5889 SRAR_W4_SW(dst0_r, dst0_l, dst1_r, dst1_l, rnd_vec);
5890 SRAR_W4_SW(dst2_r, dst2_l, dst3_r, dst3_l, rnd_vec);
5891 ADD2(dst0_r, offset_vec, dst1_r, offset_vec, dst0_r, dst1_r);
5892 ADD2(dst2_r, offset_vec, dst3_r, offset_vec, dst2_r, dst3_r);
5893 ADD2(dst0_l, offset_vec, dst1_l, offset_vec, dst0_l, dst1_l);
5894 ADD2(dst2_l, offset_vec, dst3_l, offset_vec, dst2_l, dst3_l);
5895 CLIP_SW8_0_255(dst0_r, dst0_l, dst1_r, dst1_l,
5896 dst2_r, dst2_l, dst3_r, dst3_l);
5897 PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
5898 dst3_r, tmp0, tmp1, tmp2, tmp3);
5899 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5900 ST_D4(out0, out1, 0, 1, 0, 1, dst_tmp, dst_stride);
5901 dst_tmp += (4 * dst_stride);
5902
5903 dst10_r = dst54_r;
5904 dst10_l = dst54_l;
5905 dst21_r = dst65_r;
5906 dst21_l = dst65_l;
5907 dst4 = dst6;
5908 dst5 = dst7;
5909 }
5910
5911 src += 8;
5912 dst += 8;
5913 }
5914}
5915
5916static void hevc_hv_uniwgt_4t_8w_msa(const uint8_t *src,
5917 int32_t src_stride,
5918 uint8_t *dst,
5919 int32_t dst_stride,
5920 const int8_t *filter_x,
5921 const int8_t *filter_y,
5922 int32_t height,
5923 int32_t weight,
5924 int32_t offset,
5925 int32_t rnd_val)
5926{
5927
5928 if (2 == height) {
5929 hevc_hv_uniwgt_4t_8x2_msa(src, src_stride, dst, dst_stride,
5930 filter_x, filter_y, weight,
5931 offset, rnd_val);
5932 } else if (4 == height) {
5933 hevc_hv_uniwgt_4t_8multx4_msa(src, src_stride, dst, dst_stride,
5934 filter_x, filter_y, 1, weight,
5935 offset, rnd_val);
5936 } else if (6 == height) {
5937 hevc_hv_uniwgt_4t_8x6_msa(src, src_stride, dst, dst_stride,
5938 filter_x, filter_y, weight,
5939 offset, rnd_val);
5940 } else if (0 == (height % 4)) {
5941 hevc_hv_uniwgt_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
5942 filter_x, filter_y, height, weight,
5943 offset, rnd_val, 1);
5944 }
5945}
5946
5947static void hevc_hv_uniwgt_4t_12w_msa(const uint8_t *src,
5948 int32_t src_stride,
5949 uint8_t *dst,
5950 int32_t dst_stride,
5951 const int8_t *filter_x,
5952 const int8_t *filter_y,
5953 int32_t height,
5954 int32_t weight,
5955 int32_t offset,
5956 int32_t rnd_val)
5957{
5958 uint32_t loop_cnt;
5959 const uint8_t *src_tmp;
5960 uint8_t *dst_tmp;
5961 v16u8 out0, out1;
5962 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
5963 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5964 v16i8 mask0, mask1, mask2, mask3;
5965 v8i16 filt0, filt1, filt_h0, filt_h1, filter_vec;
5966 v4i32 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6, dsth7;
5967 v8i16 dst76_r, dst98_r, dst87_r, dst109_r;
5968 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
5969 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
5970 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
5971 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, weight_vec, rnd_vec;
5972 v4i32 dst8, dst9, dst10, offset_vec;
5973 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5974 v8i16 zero = { 0 };
5975
5976 src -= (src_stride + 1);
5977
5978 filter_vec = LD_SH(filter_x);
5979 UNPCK_R_SB_SH(filter_vec, filter_vec);
5980 SPLATI_W2_SH(filter_vec, 0, filt0, filt1);
5981
5982 filter_vec = LD_SH(filter_y);
5983 UNPCK_R_SB_SH(filter_vec, filter_vec);
5984 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5985
5986 mask0 = LD_SB(ff_hevc_mask_arr);
5987 mask1 = mask0 + 2;
5988
5989 weight_vec = __msa_fill_w(weight);
5990 rnd_vec = __msa_fill_w(rnd_val);
5991 offset_vec = __msa_fill_w(offset);
5992
5993 src_tmp = src;
5994 dst_tmp = dst;
5995
5996 LD_SB3(src_tmp, src_stride, src0, src1, src2);
5997 src_tmp += (3 * src_stride);
5998 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5999 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
6000 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
6001
6002 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
6003 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
6004 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
6005 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
6006 dsth0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
6007 dsth1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
6008 dsth2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
6009 dsth3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
6010 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
6011 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
6012 dsth4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
6013 dsth5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
6014 ILVEV_H2_SH(dsth0, dsth2, dsth1, dsth3, dst10_r, dst10_l);
6015 ILVEV_H2_SH(dsth2, dsth4, dsth3, dsth5, dst21_r, dst21_l);
6016
6017 for (loop_cnt = 4; loop_cnt--;) {
6018 LD_SB4(src_tmp, src_stride, src3, src4, src5, src6);
6019 src_tmp += (4 * src_stride);
6020 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
6021 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
6022 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
6023 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
6024
6025 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
6026 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
6027 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
6028 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
6029 dsth0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
6030 dsth1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
6031 dsth2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
6032 dsth3 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
6033 ILVEV_H2_SH(dsth4, dsth0, dsth5, dsth1, dst32_r, dst32_l);
6034 ILVEV_H2_SH(dsth0, dsth2, dsth1, dsth3, dst43_r, dst43_l);
6035 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
6036 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
6037 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
6038 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
6039 dsth4 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
6040 dsth5 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
6041 dsth6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
6042 dsth7 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
6043 ILVEV_H2_SH(dsth2, dsth4, dsth3, dsth5, dst54_r, dst54_l);
6044 ILVEV_H2_SH(dsth4, dsth6, dsth5, dsth7, dst65_r, dst65_l);
6045
6046 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
6047 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
6048 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
6049 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
6050 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
6051 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
6052 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
6053 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
6054 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
6055 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
6056 MUL2(dst0_r, weight_vec, dst1_r, weight_vec, dst0_r, dst1_r);
6057 MUL2(dst2_r, weight_vec, dst3_r, weight_vec, dst2_r, dst3_r);
6058 MUL2(dst0_l, weight_vec, dst1_l, weight_vec, dst0_l, dst1_l);
6059 MUL2(dst2_l, weight_vec, dst3_l, weight_vec, dst2_l, dst3_l);
6060 SRAR_W4_SW(dst0_r, dst0_l, dst1_r, dst1_l, rnd_vec);
6061 SRAR_W4_SW(dst2_r, dst2_l, dst3_r, dst3_l, rnd_vec);
6062 ADD2(dst0_r, offset_vec, dst1_r, offset_vec, dst0_r, dst1_r);
6063 ADD2(dst2_r, offset_vec, dst3_r, offset_vec, dst2_r, dst3_r);
6064 ADD2(dst0_l, offset_vec, dst1_l, offset_vec, dst0_l, dst1_l);
6065 ADD2(dst2_l, offset_vec, dst3_l, offset_vec, dst2_l, dst3_l);
6066 CLIP_SW8_0_255(dst0_r, dst0_l, dst1_r, dst1_l,
6067 dst2_r, dst2_l, dst3_r, dst3_l);
6068 PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
6069 dst3_r, tmp0, tmp1, tmp2, tmp3);
6070 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
6071 ST_D4(out0, out1, 0, 1, 0, 1, dst_tmp, dst_stride);
6072 dst_tmp += (4 * dst_stride);
6073
6074 dst10_r = dst54_r;
6075 dst10_l = dst54_l;
6076 dst21_r = dst65_r;
6077 dst21_l = dst65_l;
6078 dsth4 = dsth6;
6079 dsth5 = dsth7;
6080 }
6081
6082 src += 8;
6083 dst += 8;
6084
6085 mask2 = LD_SB(ff_hevc_mask_arr + 16);
6086 mask3 = mask2 + 2;
6087
6088 LD_SB3(src, src_stride, src0, src1, src2);
6089 src += (3 * src_stride);
6090 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
6091 VSHF_B2_SB(src1, src2, src1, src2, mask2, mask3, vec2, vec3);
6092
6093 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
6094 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
6095 ILVL_B2_SH(zero, vec2, zero, vec3, tmp4, tmp6);
6096 dst0 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
6097 dst1 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
6098 dst2 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
6099
6100 ILVEV_H2_SH(dst0, dst1, dst1, dst2, dst10_r, dst21_r);
6101
6102 for (loop_cnt = 2; loop_cnt--;) {
6103 LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9,
6104 src10);
6105 src += (8 * src_stride);
6106 VSHF_B2_SB(src3, src7, src3, src7, mask2, mask3, vec0, vec1);
6107 VSHF_B2_SB(src4, src8, src4, src8, mask2, mask3, vec2, vec3);
6108 VSHF_B2_SB(src5, src9, src5, src9, mask2, mask3, vec4, vec5);
6109 VSHF_B2_SB(src6, src10, src6, src10, mask2, mask3, vec6, vec7);
6110
6111 ILVRL_B2_SH(zero, vec0, tmp0, tmp1);
6112 ILVRL_B2_SH(zero, vec1, tmp2, tmp3);
6113 ILVRL_B2_SH(zero, vec2, tmp4, tmp5);
6114 ILVRL_B2_SH(zero, vec3, tmp6, tmp7);
6115 dst3 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
6116 dst7 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
6117 dst4 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
6118 dst8 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
6119 ILVRL_B2_SH(zero, vec4, tmp0, tmp1);
6120 ILVRL_B2_SH(zero, vec5, tmp2, tmp3);
6121 ILVRL_B2_SH(zero, vec6, tmp4, tmp5);
6122 ILVRL_B2_SH(zero, vec7, tmp6, tmp7);
6123 dst5 = HEVC_FILT_4TAP_SW(tmp0, tmp2, filt0, filt1);
6124 dst9 = HEVC_FILT_4TAP_SW(tmp1, tmp3, filt0, filt1);
6125 dst6 = HEVC_FILT_4TAP_SW(tmp4, tmp6, filt0, filt1);
6126 dst10 = HEVC_FILT_4TAP_SW(tmp5, tmp7, filt0, filt1);
6127
6128 dst32_r = __msa_ilvev_h(dst3, dst2);
6129 ILVEV_H2_SH(dst3, dst4, dst7, dst8, dst43_r, dst87_r);
6130 ILVEV_H2_SH(dst4, dst5, dst8, dst9, dst54_r, dst98_r);
6131 ILVEV_H2_SH(dst5, dst6, dst9, dst10, dst65_r, dst109_r);
6132 dst76_r = __msa_ilvev_h(dst7, dst6);
6133 dst0 = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
6134 dst1 = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
6135 dst2 = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
6136 dst3 = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
6137 dst4 = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
6138 dst5 = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
6139 dst6 = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
6140 dst7 = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
6141 SRA_4V(dst0, dst1, dst2, dst3, 6);
6142 SRA_4V(dst4, dst5, dst6, dst7, 6);
6143 MUL2(dst0, weight_vec, dst1, weight_vec, dst0, dst1);
6144 MUL2(dst2, weight_vec, dst3, weight_vec, dst2, dst3);
6145 MUL2(dst4, weight_vec, dst5, weight_vec, dst4, dst5);
6146 MUL2(dst6, weight_vec, dst7, weight_vec, dst6, dst7);
6147 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
6148 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
6149 ADD2(dst0, offset_vec, dst1, offset_vec, dst0, dst1);
6150 ADD2(dst2, offset_vec, dst3, offset_vec, dst2, dst3);
6151 ADD2(dst4, offset_vec, dst5, offset_vec, dst4, dst5);
6152 ADD2(dst6, offset_vec, dst7, offset_vec, dst6, dst7);
6153 CLIP_SW8_0_255(dst0, dst1, dst2, dst3,
6154 dst4, dst5, dst6, dst7);
6155 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, tmp0, tmp1,
6156 tmp2, tmp3);
6157 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
6158 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
6159 dst += (8 * dst_stride);
6160
6161 dst10_r = dst98_r;
6162 dst21_r = dst109_r;
6163 dst2 = dst10;
6164 }
6165}
6166
6167static void hevc_hv_uniwgt_4t_16w_msa(const uint8_t *src,
6168 int32_t src_stride,
6169 uint8_t *dst,
6170 int32_t dst_stride,
6171 const int8_t *filter_x,
6172 const int8_t *filter_y,
6173 int32_t height,
6174 int32_t weight,
6175 int32_t offset,
6176 int32_t rnd_val)
6177{
6178 if (4 == height) {
6179 hevc_hv_uniwgt_4t_8multx4_msa(src, src_stride, dst, dst_stride,
6180 filter_x, filter_y, 2, weight, offset,
6181 rnd_val);
6182 } else {
6183 hevc_hv_uniwgt_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
6184 filter_x, filter_y, height, weight,
6185 offset, rnd_val, 2);
6186 }
6187}
6188
6189static void hevc_hv_uniwgt_4t_24w_msa(const uint8_t *src,
6190 int32_t src_stride,
6191 uint8_t *dst,
6192 int32_t dst_stride,
6193 const int8_t *filter_x,
6194 const int8_t *filter_y,
6195 int32_t height,
6196 int32_t weight,
6197 int32_t offset,
6198 int32_t rnd_val)
6199{
6200 hevc_hv_uniwgt_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
6201 filter_x, filter_y, height, weight,
6202 offset, rnd_val, 3);
6203}
6204
6205static void hevc_hv_uniwgt_4t_32w_msa(const uint8_t *src,
6206 int32_t src_stride,
6207 uint8_t *dst,
6208 int32_t dst_stride,
6209 const int8_t *filter_x,
6210 const int8_t *filter_y,
6211 int32_t height,
6212 int32_t weight,
6213 int32_t offset,
6214 int32_t rnd_val)
6215{
6216 hevc_hv_uniwgt_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
6217 filter_x, filter_y, height, weight,
6218 offset, rnd_val, 4);
6219}
6220
6221#define UNIWGT_MC_COPY(WIDTH) \
6222void ff_hevc_put_hevc_uni_w_pel_pixels##WIDTH##_8_msa(uint8_t *dst, \
6223 ptrdiff_t dst_stride, \
6224 const uint8_t *src, \
6225 ptrdiff_t src_stride, \
6226 int height, \
6227 int denom, \
6228 int weight, \
6229 int offset, \
6230 intptr_t mx, \
6231 intptr_t my, \
6232 int width) \
6233{ \
6234 int shift = denom + 14 - 8; \
6235 hevc_uniwgt_copy_##WIDTH##w_msa(src, src_stride, dst, dst_stride, \
6236 height, weight, offset, shift); \
6237}
6238
6239UNIWGT_MC_COPY(4);
6240UNIWGT_MC_COPY(6);
6241UNIWGT_MC_COPY(8);
6242UNIWGT_MC_COPY(12);
6243UNIWGT_MC_COPY(16);
6244UNIWGT_MC_COPY(24);
6245UNIWGT_MC_COPY(32);
6246UNIWGT_MC_COPY(48);
6247UNIWGT_MC_COPY(64);
6248
6249#undef UNIWGT_MC_COPY
6250
6251#define UNI_W_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \
6252void ff_hevc_put_hevc_uni_w_##PEL##_##DIR##WIDTH##_8_msa(uint8_t *dst, \
6253 ptrdiff_t \
6254 dst_stride, \
6255 const uint8_t *src, \
6256 ptrdiff_t \
6257 src_stride, \
6258 int height, \
6259 int denom, \
6260 int weight, \
6261 int offset, \
6262 intptr_t mx, \
6263 intptr_t my, \
6264 int width) \
6265{ \
6266 const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR]; \
6267 int shift = denom + 14 - 8; \
6268 \
6269 hevc_##DIR1##_uniwgt_##TAP##t_##WIDTH##w_msa(src, src_stride, dst, \
6270 dst_stride, filter, height, \
6271 weight, offset, shift); \
6272}
6273
6274UNI_W_MC(qpel, h, 4, 8, hz, mx);
6275UNI_W_MC(qpel, h, 8, 8, hz, mx);
6276UNI_W_MC(qpel, h, 12, 8, hz, mx);
6277UNI_W_MC(qpel, h, 16, 8, hz, mx);
6278UNI_W_MC(qpel, h, 24, 8, hz, mx);
6279UNI_W_MC(qpel, h, 32, 8, hz, mx);
6280UNI_W_MC(qpel, h, 48, 8, hz, mx);
6281UNI_W_MC(qpel, h, 64, 8, hz, mx);
6282
6283UNI_W_MC(qpel, v, 4, 8, vt, my);
6284UNI_W_MC(qpel, v, 8, 8, vt, my);
6285UNI_W_MC(qpel, v, 12, 8, vt, my);
6286UNI_W_MC(qpel, v, 16, 8, vt, my);
6287UNI_W_MC(qpel, v, 24, 8, vt, my);
6288UNI_W_MC(qpel, v, 32, 8, vt, my);
6289UNI_W_MC(qpel, v, 48, 8, vt, my);
6290UNI_W_MC(qpel, v, 64, 8, vt, my);
6291
6292UNI_W_MC(epel, h, 4, 4, hz, mx);
6293UNI_W_MC(epel, h, 6, 4, hz, mx);
6294UNI_W_MC(epel, h, 8, 4, hz, mx);
6295UNI_W_MC(epel, h, 12, 4, hz, mx);
6296UNI_W_MC(epel, h, 16, 4, hz, mx);
6297UNI_W_MC(epel, h, 24, 4, hz, mx);
6298UNI_W_MC(epel, h, 32, 4, hz, mx);
6299
6300UNI_W_MC(epel, v, 4, 4, vt, my);
6301UNI_W_MC(epel, v, 6, 4, vt, my);
6302UNI_W_MC(epel, v, 8, 4, vt, my);
6303UNI_W_MC(epel, v, 12, 4, vt, my);
6304UNI_W_MC(epel, v, 16, 4, vt, my);
6305UNI_W_MC(epel, v, 24, 4, vt, my);
6306UNI_W_MC(epel, v, 32, 4, vt, my);
6307
6308#undef UNI_W_MC
6309
6310#define UNI_W_MC_HV(PEL, WIDTH, TAP) \
6311void ff_hevc_put_hevc_uni_w_##PEL##_hv##WIDTH##_8_msa(uint8_t *dst, \
6312 ptrdiff_t dst_stride, \
6313 const uint8_t *src, \
6314 ptrdiff_t src_stride, \
6315 int height, \
6316 int denom, \
6317 int weight, \
6318 int offset, \
6319 intptr_t mx, \
6320 intptr_t my, \
6321 int width) \
6322{ \
6323 const int8_t *filter_x = ff_hevc_##PEL##_filters[mx]; \
6324 const int8_t *filter_y = ff_hevc_##PEL##_filters[my]; \
6325 int shift = denom + 14 - 8; \
6326 \
6327 hevc_hv_uniwgt_##TAP##t_##WIDTH##w_msa(src, src_stride, dst, dst_stride, \
6328 filter_x, filter_y, height, \
6329 weight, offset, shift); \
6330}
6331
6332UNI_W_MC_HV(qpel, 4, 8);
6333UNI_W_MC_HV(qpel, 8, 8);
6334UNI_W_MC_HV(qpel, 12, 8);
6335UNI_W_MC_HV(qpel, 16, 8);
6336UNI_W_MC_HV(qpel, 24, 8);
6337UNI_W_MC_HV(qpel, 32, 8);
6338UNI_W_MC_HV(qpel, 48, 8);
6339UNI_W_MC_HV(qpel, 64, 8);
6340
6341UNI_W_MC_HV(epel, 4, 4);
6342UNI_W_MC_HV(epel, 6, 4);
6343UNI_W_MC_HV(epel, 8, 4);
6344UNI_W_MC_HV(epel, 12, 4);
6345UNI_W_MC_HV(epel, 16, 4);
6346UNI_W_MC_HV(epel, 24, 4);
6347UNI_W_MC_HV(epel, 32, 4);
6348
6349#undef UNI_W_MC_HV
my
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t my
Definition dsp.h:57
mx
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t mx
Definition dsp.h:57
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition dsp.h:87
int32_t
int32_t
Definition audioconvert.c:56
aligned
static int aligned(int val)
Definition dashdec.c:178
generic_macros_msa.h
ST_UB4
#define ST_UB4(...)
Definition generic_macros_msa.h:374
ILVR_B2_SB
#define ILVR_B2_SB(...)
Definition generic_macros_msa.h:1338
PCKEV_H2_SW
#define PCKEV_H2_SW(...)
Definition generic_macros_msa.h:1760
LW2
#define LW2(psrc, stride, out0, out1)
Definition generic_macros_msa.h:210
SPLATI_W4_SH
#define SPLATI_W4_SH(...)
Definition generic_macros_msa.h:1700
CLIP_SW2_0_255
#define CLIP_SW2_0_255(in0, in1)
Definition generic_macros_msa.h:972
LD4
#define LD4(psrc, stride, out0, out1, out2, out3)
Definition generic_macros_msa.h:228
PCKEV_B4_UB
#define PCKEV_B4_UB(...)
Definition generic_macros_msa.h:1739
ST_D4
#define ST_D4(in0, in1, idx0, idx1, idx2, idx3, pdst, stride)
Definition generic_macros_msa.h:499
ILVR_D3_SB
#define ILVR_D3_SB(...)
Definition generic_macros_msa.h:1452
SLLI_2V
#define SLLI_2V(in0, in1, shift)
Definition generic_macros_msa.h:1916
ILVR_B4_SB
#define ILVR_B4_SB(...)
Definition generic_macros_msa.h:1360
SPLATI_W2_SH
#define SPLATI_W2_SH(...)
Definition generic_macros_msa.h:1692
PCKEV_B2_UB
#define PCKEV_B2_UB(...)
Definition generic_macros_msa.h:1720
VSHF_B4_SB
#define VSHF_B4_SB(...)
Definition generic_macros_msa.h:680
ILVRL_B2_SH
#define ILVRL_B2_SH(...)
Definition generic_macros_msa.h:1498
ILVR_B4_SH
#define ILVR_B4_SH(...)
Definition generic_macros_msa.h:1362
MUL4
#define MUL4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3)
Definition generic_macros_msa.h:2106
SRAR_W2_SW
#define SRAR_W2_SW(...)
Definition generic_macros_msa.h:2034
LD_SB7
#define LD_SB7(...)
Definition generic_macros_msa.h:327
SPLATI_W4_SW
#define SPLATI_W4_SW(...)
Definition generic_macros_msa.h:1701
INSERT_W4_SB
#define INSERT_W4_SB(...)
Definition generic_macros_msa.h:1154
ILVL_B4_SB
#define ILVL_B4_SB(...)
Definition generic_macros_msa.h:1274
ILVR_D4_SB
#define ILVR_D4_SB(...)
Definition generic_macros_msa.h:1460
LD_SB3
#define LD_SB3(...)
Definition generic_macros_msa.h:289
LD2
#define LD2(psrc, stride, out0, out1)
Definition generic_macros_msa.h:223
INSERT_W2_SB
#define INSERT_W2_SB(...)
Definition generic_macros_msa.h:1144
UNPCK_R_SB_SH
#define UNPCK_R_SB_SH(in, out)
Definition generic_macros_msa.h:2156
ST_D8
#define ST_D8(in0, in1, in2, in3, idx0, idx1, idx2, idx3, idx4, idx5, idx6, idx7, pdst, stride)
Definition generic_macros_msa.h:511
ADD2
#define ADD2(in0, in1, in2, in3, out0, out1)
Definition generic_macros_msa.h:2118
PCKEV_B3_UB
#define PCKEV_B3_UB(...)
Definition generic_macros_msa.h:1729
LD_SB5
#define LD_SB5(...)
Definition generic_macros_msa.h:308
PCKEV_D2_SH
#define PCKEV_D2_SH(...)
Definition generic_macros_msa.h:1789
PCKEV_H2_SH
#define PCKEV_H2_SH(...)
Definition generic_macros_msa.h:1759
ILVR_D2_SB
#define ILVR_D2_SB(...)
Definition generic_macros_msa.h:1444
ILVEV_H2_SH
#define ILVEV_H2_SH(...)
Definition generic_macros_msa.h:1208
SRA_4V
#define SRA_4V(in0, in1, in2, in3, shift)
Definition generic_macros_msa.h:1939
LD_SB8
#define LD_SB8(...)
Definition generic_macros_msa.h:336
ST_H8
#define ST_H8(in, idx0, idx1, idx2, idx3, idx4, idx5, idx6, idx7, pdst, stride)
Definition generic_macros_msa.h:429
SLLI_4V
#define SLLI_4V(in0, in1, in2, in3, shift)
Definition generic_macros_msa.h:1921
ST_UB2
#define ST_UB2(...)
Definition generic_macros_msa.h:363
ST_W2
#define ST_W2(in, idx0, idx1, pdst, stride)
Definition generic_macros_msa.h:450
CLIP_SW4_0_255
#define CLIP_SW4_0_255(in0, in1, in2, in3)
Definition generic_macros_msa.h:978
INSERT_D2_SB
#define INSERT_D2_SB(...)
Definition generic_macros_msa.h:1170
LD_SB4
#define LD_SB4(...)
Definition generic_macros_msa.h:297
ST_W8
#define ST_W8(in0, in1, idx0, idx1, idx2, idx3, idx4, idx5, idx6, idx7, pdst, stride)
Definition generic_macros_msa.h:470
LW4
#define LW4(psrc, stride, out0, out1, out2, out3)
Definition generic_macros_msa.h:202
LD_SB6
#define LD_SB6(...)
Definition generic_macros_msa.h:316
VSHF_B2_SB
#define VSHF_B2_SB(...)
Definition generic_macros_msa.h:662
ILVRL_B2_SB
#define ILVRL_B2_SB(...)
Definition generic_macros_msa.h:1496
ILVR_B2_SH
#define ILVR_B2_SH(...)
Definition generic_macros_msa.h:1340
CLIP_SW8_0_255
#define CLIP_SW8_0_255(in0, in1, in2, in3, in4, in5, in6, in7)
Definition generic_macros_msa.h:984
LD_SB
#define LD_SB(...)
Definition generic_macros_msa.h:33
ILVRL_H2_SW
#define ILVRL_H2_SW(...)
Definition generic_macros_msa.h:1509
MUL2
#define MUL2(in0, in1, in2, in3, out0, out1)
Definition generic_macros_msa.h:2101
ST_D1
#define ST_D1(in, idx, pdst)
Definition generic_macros_msa.h:485
LD_SH
#define LD_SH(...)
Definition generic_macros_msa.h:35
LD_SB2
#define LD_SB2(...)
Definition generic_macros_msa.h:278
ST_H2
#define ST_H2(in, idx0, idx1, pdst, stride)
Definition generic_macros_msa.h:409
ADD4
#define ADD4(in0, in1, in2, in3, in4, in5, in6, in7, out0, out1, out2, out3)
Definition generic_macros_msa.h:2123
ST_D2
#define ST_D2(in, idx0, idx1, pdst, stride)
Definition generic_macros_msa.h:491
SRAR_W4_SW
#define SRAR_W4_SW(...)
Definition generic_macros_msa.h:2041
DOTP_SH2_SW
#define DOTP_SH2_SW(...)
Definition generic_macros_msa.h:803
ST_UB
#define ST_UB(...)
Definition generic_macros_msa.h:40
PCKEV_H4_SH
#define PCKEV_H4_SH(...)
Definition generic_macros_msa.h:1768
ILVL_B2_SB
#define ILVL_B2_SB(...)
Definition generic_macros_msa.h:1263
ILVL_W2_SB
#define ILVL_W2_SB(...)
Definition generic_macros_msa.h:1319
ILVL_B2_SH
#define ILVL_B2_SH(...)
Definition generic_macros_msa.h:1265
src2
const pixel * src2
Definition h264pred_template.c:421
hevc_macros_msa.h
HEVC_FILT_4TAP
#define HEVC_FILT_4TAP(in0, in1, filt0, filt1)
Definition hevc_macros_msa.h:64
HEVC_FILT_8TAP
#define HEVC_FILT_8TAP(in0, in1, in2, in3, filt0, filt1, filt2, filt3)
Definition hevc_macros_msa.h:35
HEVC_FILT_4TAP_SW
#define HEVC_FILT_4TAP_SW(in0, in1, filt0, filt1)
Definition hevc_macros_msa.h:55
ff_hevc_mask_arr
static const uint8_t ff_hevc_mask_arr[16 *2]
Definition hevc_mc_bi_lsx.c:26
hevc_vt_uniwgt_4t_8x6_msa
static void hevc_vt_uniwgt_4t_8x6_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4177
hevc_uniwgt_copy_16w_msa
static void hevc_uniwgt_copy_16w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:386
hevc_hv_uniwgt_8t_8multx2mult_msa
static void hevc_hv_uniwgt_8t_8multx2mult_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val, int32_t width)
Definition hevc_mc_uniw_msa.c:2133
hevc_hv_uniwgt_4t_4x4_msa
static void hevc_hv_uniwgt_4t_4x4_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5040
hevc_vt_uniwgt_4t_8w_msa
static void hevc_vt_uniwgt_4t_8w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4374
hevc_vt_uniwgt_8t_12w_msa
static void hevc_vt_uniwgt_8t_12w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1663
HEVC_FILT_8TAP_4W_SH
#define HEVC_FILT_8TAP_4W_SH(in0, in1, in2, in3, filt0, filt1, filt2, filt3, dst0, dst1)
Definition hevc_mc_uniw_msa.c:59
hevc_hz_uniwgt_4t_4x4_msa
static void hevc_hz_uniwgt_4t_4x4_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2640
hevc_hv_uniwgt_4t_16w_msa
static void hevc_hv_uniwgt_4t_16w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:6167
hevc_hv_uniwgt_4t_8x2_msa
static void hevc_hv_uniwgt_4t_8x2_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5425
hevc_hz_uniwgt_4t_6w_msa
static void hevc_hz_uniwgt_4t_6w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2801
hevc_hv_uniwgt_4t_12w_msa
static void hevc_hv_uniwgt_4t_12w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5947
hevc_hz_uniwgt_8t_64w_msa
static void hevc_hz_uniwgt_8t_64w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1339
UNIWGT_MC_COPY
#define UNIWGT_MC_COPY(WIDTH)
Definition hevc_mc_uniw_msa.c:6221
hevc_hv_uniwgt_8t_64w_msa
static void hevc_hv_uniwgt_8t_64w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2575
hevc_hz_uniwgt_8t_16w_msa
static void hevc_hz_uniwgt_8t_16w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:944
hevc_vt_uniwgt_8t_16multx4mult_msa
static void hevc_vt_uniwgt_8t_16multx4mult_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val, int32_t weightmul16)
Definition hevc_mc_uniw_msa.c:1771
hevc_hv_uniwgt_4t_8x6_msa
static void hevc_hv_uniwgt_4t_8x6_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5631
hevc_uniwgt_copy_8w_msa
static void hevc_uniwgt_copy_8w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:248
hevc_hv_uniwgt_4t_32w_msa
static void hevc_hv_uniwgt_4t_32w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:6205
hevc_vt_uniwgt_8t_8w_msa
static void hevc_vt_uniwgt_8t_8w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1567
hevc_hz_uniwgt_8t_48w_msa
static void hevc_hz_uniwgt_8t_48w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1238
hevc_vt_uniwgt_4t_6w_msa
static void hevc_vt_uniwgt_4t_6w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3949
UNI_W_MC_HV
#define UNI_W_MC_HV(PEL, WIDTH, TAP)
Definition hevc_mc_uniw_msa.c:6310
hevc_vt_uniwgt_8t_4w_msa
static void hevc_vt_uniwgt_8t_4w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1430
hevc_vt_uniwgt_8t_64w_msa
static void hevc_vt_uniwgt_8t_64w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1996
hevc_hv_uniwgt_4t_8multx4mult_msa
static void hevc_hv_uniwgt_4t_8multx4mult_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val, int32_t width8mult)
Definition hevc_mc_uniw_msa.c:5773
hevc_vt_uniwgt_4t_24w_msa
static void hevc_vt_uniwgt_4t_24w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4679
hevc_hz_uniwgt_4t_4x2_msa
static void hevc_hz_uniwgt_4t_4x2_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2591
hevc_vt_uniwgt_4t_4x2_msa
static void hevc_vt_uniwgt_4t_4x2_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3734
hevc_vt_uniwgt_8t_32w_msa
static void hevc_vt_uniwgt_8t_32w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1966
hevc_uniwgt_copy_64w_msa
static void hevc_uniwgt_copy_64w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:581
hevc_uniwgt_copy_32w_msa
static void hevc_uniwgt_copy_32w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:480
hevc_vt_uniwgt_8t_48w_msa
static void hevc_vt_uniwgt_8t_48w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1981
hevc_vt_uniwgt_4t_32w_msa
static void hevc_vt_uniwgt_4t_32w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4837
HEVC_UNIW_RND_CLIP2_MAX_SATU_H
#define HEVC_UNIW_RND_CLIP2_MAX_SATU_H(in0_h, in1_h, wgt_w, offset_h, rnd_w, out0_h, out1_h)
Definition hevc_mc_uniw_msa.c:32
hevc_hv_uniwgt_8t_8w_msa
static void hevc_hv_uniwgt_8t_8w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2293
hevc_vt_uniwgt_4t_4x8multiple_msa
static void hevc_vt_uniwgt_4t_4x8multiple_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3840
hevc_vt_uniwgt_4t_4x4_msa
static void hevc_vt_uniwgt_4t_4x4_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3783
hevc_uniwgt_copy_12w_msa
static void hevc_uniwgt_copy_12w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:342
hevc_hv_uniwgt_4t_8multx4_msa
static void hevc_hv_uniwgt_4t_8multx4_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t width8mult, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5515
hevc_hz_uniwgt_4t_8x4_msa
static void hevc_hz_uniwgt_4t_8x4_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2974
hevc_hv_uniwgt_4t_4x2_msa
static void hevc_hv_uniwgt_4t_4x2_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4971
hevc_hz_uniwgt_8t_24w_msa
static void hevc_hz_uniwgt_8t_24w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1023
hevc_hv_uniwgt_8t_4w_msa
static void hevc_hv_uniwgt_8t_4w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2011
hevc_uniwgt_copy_24w_msa
static void hevc_uniwgt_copy_24w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:428
hevc_uniwgt_copy_4w_msa
static void hevc_uniwgt_copy_4w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:74
hevc_uniwgt_copy_6w_msa
static void hevc_uniwgt_copy_6w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:141
UNI_W_MC
#define UNI_W_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR)
Definition hevc_mc_uniw_msa.c:6251
hevc_hz_uniwgt_4t_32w_msa
static void hevc_hz_uniwgt_4t_32w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3611
hevc_vt_uniwgt_4t_12w_msa
static void hevc_vt_uniwgt_4t_12w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4400
hevc_hz_uniwgt_8t_8w_msa
static void hevc_hz_uniwgt_8t_8w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:765
hevc_hv_uniwgt_4t_24w_msa
static void hevc_hv_uniwgt_4t_24w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:6189
hevc_vt_uniwgt_4t_4w_msa
static void hevc_vt_uniwgt_4t_4w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3926
HEVC_UNIW_RND_CLIP4_MAX_SATU_H
#define HEVC_UNIW_RND_CLIP4_MAX_SATU_H(in0_h, in1_h, in2_h, in3_h, wgt_w, offset_h, rnd_w, out0_h, out1_h, out2_h, out3_h)
Definition hevc_mc_uniw_msa.c:49
hevc_hv_uniwgt_8t_48w_msa
static void hevc_hv_uniwgt_8t_48w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2559
hevc_hv_uniwgt_8t_24w_msa
static void hevc_hv_uniwgt_8t_24w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2527
hevc_hz_uniwgt_4t_8x6_msa
static void hevc_hz_uniwgt_4t_8x6_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3046
hevc_vt_uniwgt_8t_16w_msa
static void hevc_vt_uniwgt_8t_16w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1933
hevc_hv_uniwgt_8t_12w_msa
static void hevc_hv_uniwgt_8t_12w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2309
hevc_hz_uniwgt_4t_24w_msa
static void hevc_hz_uniwgt_4t_24w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3507
hevc_vt_uniwgt_4t_8x8mult_msa
static void hevc_vt_uniwgt_4t_8x8mult_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4262
hevc_hv_uniwgt_4t_4w_msa
static void hevc_hv_uniwgt_4t_4w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5240
hevc_hz_uniwgt_4t_12w_msa
static void hevc_hz_uniwgt_4t_12w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3284
hevc_vt_uniwgt_8t_24w_msa
static void hevc_vt_uniwgt_8t_24w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1948
hevc_hz_uniwgt_8t_12w_msa
static void hevc_hz_uniwgt_8t_12w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:865
hevc_uniwgt_copy_48w_msa
static void hevc_uniwgt_copy_48w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:525
hevc_hz_uniwgt_4t_4x8multiple_msa
static void hevc_hz_uniwgt_4t_4x8multiple_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2698
hevc_vt_uniwgt_4t_16w_msa
static void hevc_vt_uniwgt_4t_16w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4563
hevc_hz_uniwgt_8t_4w_msa
static void hevc_hz_uniwgt_8t_4w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:643
hevc_hz_uniwgt_4t_8x2_msa
static void hevc_hz_uniwgt_4t_8x2_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2918
hevc_hz_uniwgt_4t_4w_msa
static void hevc_hz_uniwgt_4t_4w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2778
hevc_hz_uniwgt_4t_8w_msa
static void hevc_hz_uniwgt_4t_8w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3258
hevc_hv_uniwgt_8t_16w_msa
static void hevc_hv_uniwgt_8t_16w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2511
hevc_hz_uniwgt_4t_16w_msa
static void hevc_hz_uniwgt_4t_16w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3389
hevc_hz_uniwgt_8t_32w_msa
static void hevc_hz_uniwgt_8t_32w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:1124
hevc_hv_uniwgt_8t_32w_msa
static void hevc_hv_uniwgt_8t_32w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:2543
hevc_hv_uniwgt_4t_8w_msa
static void hevc_hv_uniwgt_4t_8w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5916
hevc_vt_uniwgt_4t_8x4_msa
static void hevc_vt_uniwgt_4t_8x4_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4111
hevc_hv_uniwgt_4t_6w_msa
static void hevc_hv_uniwgt_4t_6w_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5266
hevc_vt_uniwgt_4t_8x2_msa
static void hevc_vt_uniwgt_4t_8x2_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:4058
hevc_hv_uniwgt_4t_4multx8mult_msa
static void hevc_hv_uniwgt_4t_4multx8mult_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:5118
hevc_hz_uniwgt_4t_8x8multiple_msa
static void hevc_hz_uniwgt_4t_8x8multiple_msa(const uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight, int32_t offset, int32_t rnd_val)
Definition hevc_mc_uniw_msa.c:3141
hevcdsp_mips.h
zero
static int zero(InterplayACMContext *s, unsigned ind, unsigned col)
Definition interplayacm.c:121
offset
unsigned offset
Definition libaomenc.c:763
weight
const h264_weight_func weight
Definition h264dsp_init.c:33
tmp
static uint8_t tmp[40]
Definition aes_ctr.c:52
filter
void(* filter)(uint8_t *src, ptrdiff_t stride, int qscale)
Definition h263dsp.c:29
src1
#define src1
Definition h264pred.c:141
src0
#define src0
Definition h264pred.c:140
src
#define src
Definition vp8dsp.c:248
out
static FILE * out
Definition movenc.c:55
height
#define height
Definition dsp.h:89
width
#define width
Definition dsp.h:89
ST_W4
#define ST_W4(in, idx0, idx1, idx2, idx3, pdst, stride)
Definition vp8_lpf_lsx.c:234
h
h
Definition vp9dsp_template.c:2070
Generated on Sun Aug 16 2026 20:20:49 for FFmpeg by doxygen 1.13.2