FFmpeg
Loading...
Searching...
No Matches
hevc_mc_biw_msa.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2015 - 2017 Manojkumar Bhosale (Manojkumar.Bhosale@imgtec.com)
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21#include "libavutil/mips/generic_macros_msa.h"
22#include "libavcodec/mips/hevcdsp_mips.h"
23#include "libavcodec/mips/hevc_macros_msa.h"
24
25static const uint8_t ff_hevc_mask_arr[16 * 2] __attribute__((aligned(0x40))) = {
26 /* 8 width cases */
27 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
28 0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20
29};
30
31#define HEVC_BIW_RND_CLIP2(in0, in1, vec0, vec1, wgt, rnd, offset, \
32 out0, out1) \
33{ \
34 v4i32 out0_r, out1_r, out0_l, out1_l; \
35 \
36 ILVR_H2_SW(in0, vec0, in1, vec1, out0_r, out1_r); \
37 ILVL_H2_SW(in0, vec0, in1, vec1, out0_l, out1_l); \
38 \
39 out0_r = __msa_dpadd_s_w(offset, (v8i16) out0_r, (v8i16) wgt); \
40 out1_r = __msa_dpadd_s_w(offset, (v8i16) out1_r, (v8i16) wgt); \
41 out0_l = __msa_dpadd_s_w(offset, (v8i16) out0_l, (v8i16) wgt); \
42 out1_l = __msa_dpadd_s_w(offset, (v8i16) out1_l, (v8i16) wgt); \
43 \
44 SRAR_W4_SW(out0_r, out1_r, out0_l, out1_l, rnd); \
45 CLIP_SW4_0_255(out0_l, out0_r, out1_l, out1_r); \
46 PCKEV_H2_SH(out0_l, out0_r, out1_l, out1_r, out0, out1); \
47}
48
49#define HEVC_BIW_RND_CLIP4(in0, in1, in2, in3, vec0, vec1, vec2, vec3, \
50 wgt, rnd, offset, out0, out1, out2, out3) \
51{ \
52 HEVC_BIW_RND_CLIP2(in0, in1, vec0, vec1, wgt, rnd, offset, out0, out1); \
53 HEVC_BIW_RND_CLIP2(in2, in3, vec2, vec3, wgt, rnd, offset, out2, out3); \
54}
55
56#define HEVC_BIW_RND_CLIP2_MAX_SATU(in0, in1, vec0, vec1, wgt, rnd, \
57 offset, out0, out1) \
58{ \
59 v4i32 out0_r, out1_r, out0_l, out1_l; \
60 \
61 ILVR_H2_SW(in0, vec0, in1, vec1, out0_r, out1_r); \
62 ILVL_H2_SW(in0, vec0, in1, vec1, out0_l, out1_l); \
63 out0_r = __msa_dpadd_s_w(offset, (v8i16) out0_r, (v8i16) wgt); \
64 out1_r = __msa_dpadd_s_w(offset, (v8i16) out1_r, (v8i16) wgt); \
65 out0_l = __msa_dpadd_s_w(offset, (v8i16) out0_l, (v8i16) wgt); \
66 out1_l = __msa_dpadd_s_w(offset, (v8i16) out1_l, (v8i16) wgt); \
67 SRAR_W4_SW(out0_r, out1_r, out0_l, out1_l, rnd); \
68 CLIP_SW4_0_255(out0_r, out1_r, out0_l, out1_l); \
69 PCKEV_H2_SH(out0_l, out0_r, out1_l, out1_r, out0, out1); \
70}
71
72#define HEVC_BIW_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, vec0, vec1, vec2, \
73 vec3, wgt, rnd, offset, out0, out1, \
74 out2, out3) \
75{ \
76 HEVC_BIW_RND_CLIP2_MAX_SATU(in0, in1, vec0, vec1, wgt, rnd, offset, \
77 out0, out1); \
78 HEVC_BIW_RND_CLIP2_MAX_SATU(in2, in3, vec2, vec3, wgt, rnd, offset, \
79 out2, out3); \
80}
81
82static void hevc_biwgt_copy_4w_msa(const uint8_t *src0_ptr,
83 int32_t src_stride,
84 const int16_t *src1_ptr,
85 int32_t src2_stride,
86 uint8_t *dst,
87 int32_t dst_stride,
88 int32_t height,
89 int32_t weight0,
90 int32_t weight1,
91 int32_t offset,
92 int32_t rnd_val)
93{
94 uint32_t loop_cnt, tp0, tp1, tp2, tp3;
95 uint64_t tpd0, tpd1, tpd2, tpd3;
96 int32_t weight;
97 v16u8 out0, out1;
98 v16i8 zero = { 0 };
99 v16i8 src0 = { 0 }, src1 = { 0 };
100 v8i16 in0 = { 0 }, in1 = { 0 }, in2 = { 0 }, in3 = { 0 };
101 v8i16 dst0, dst1, dst2, dst3, weight_vec;
102 v4i32 dst0_r, dst0_l, offset_vec, rnd_vec;
103
104 offset = offset << rnd_val;
105 weight0 = weight0 & 0x0000FFFF;
106 weight = weight0 | (weight1 << 16);
107
108 offset_vec = __msa_fill_w(offset);
109 weight_vec = (v8i16) __msa_fill_w(weight);
110 rnd_vec = __msa_fill_w(rnd_val + 1);
111
112 if (2 == height) {
113 LW2(src0_ptr, src_stride, tp0, tp1);
114 INSERT_W2_SB(tp0, tp1, src0);
115 LD2(src1_ptr, src2_stride, tpd0, tpd1);
116 INSERT_D2_SH(tpd0, tpd1, in0);
117
118 dst0 = (v8i16) __msa_ilvr_b(zero, src0);
119 dst0 <<= 6;
120
121 ILVRL_H2_SW(dst0, in0, dst0_r, dst0_l);
122 dst0_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_r, weight_vec);
123 dst0_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_l, weight_vec);
124 SRAR_W2_SW(dst0_r, dst0_l, rnd_vec);
125 CLIP_SW2_0_255(dst0_r, dst0_l);
126 dst0 = (v8i16) __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
127 out0 = (v16u8) __msa_pckev_b((v16i8) dst0, (v16i8) dst0);
128 ST_W2(out0, 0, 1, dst, dst_stride);
129 } else if (4 == height) {
130 LW4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
131 INSERT_W4_SB(tp0, tp1, tp2, tp3, src0);
132 LD4(src1_ptr, src2_stride, tpd0, tpd1, tpd2, tpd3);
133 INSERT_D2_SH(tpd0, tpd1, in0);
134 INSERT_D2_SH(tpd2, tpd3, in1);
135 ILVRL_B2_SH(zero, src0, dst0, dst1);
136 SLLI_2V(dst0, dst1, 6);
137 HEVC_BIW_RND_CLIP2_MAX_SATU(dst0, dst1, in0, in1, weight_vec, rnd_vec,
138 offset_vec, dst0, dst1);
139 out0 = (v16u8) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
140 ST_W4(out0, 0, 1, 2, 3, dst, dst_stride);
141 } else if (0 == height % 8) {
142 for (loop_cnt = (height >> 3); loop_cnt--;) {
143 LW4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
144 src0_ptr += 4 * src_stride;
145 INSERT_W4_SB(tp0, tp1, tp2, tp3, src0);
146 LW4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
147 src0_ptr += 4 * src_stride;
148 INSERT_W4_SB(tp0, tp1, tp2, tp3, src1);
149 LD4(src1_ptr, src2_stride, tpd0, tpd1, tpd2, tpd3);
150 src1_ptr += (4 * src2_stride);
151 INSERT_D2_SH(tpd0, tpd1, in0);
152 INSERT_D2_SH(tpd2, tpd3, in1);
153 LD4(src1_ptr, src2_stride, tpd0, tpd1, tpd2, tpd3);
154 src1_ptr += (4 * src2_stride);
155 INSERT_D2_SH(tpd0, tpd1, in2);
156 INSERT_D2_SH(tpd2, tpd3, in3);
157 ILVRL_B2_SH(zero, src0, dst0, dst1);
158 ILVRL_B2_SH(zero, src1, dst2, dst3);
159 SLLI_4V(dst0, dst1, dst2, dst3, 6);
160 HEVC_BIW_RND_CLIP4_MAX_SATU(dst0, dst1, dst2, dst3, in0, in1, in2,
161 in3, weight_vec, rnd_vec, offset_vec,
162 dst0, dst1, dst2, dst3);
163 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
164 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
165 dst += (8 * dst_stride);
166 }
167 }
168}
169
170static void hevc_biwgt_copy_6w_msa(const uint8_t *src0_ptr,
171 int32_t src_stride,
172 const int16_t *src1_ptr,
173 int32_t src2_stride,
174 uint8_t *dst,
175 int32_t dst_stride,
176 int32_t height,
177 int32_t weight0,
178 int32_t weight1,
179 int32_t offset,
180 int32_t rnd_val)
181{
182 uint32_t loop_cnt;
183 int32_t res = height & 0x03;
184 int32_t weight;
185 uint64_t tp0, tp1, tp2, tp3;
186 v16u8 out0, out1;
187 v16i8 zero = { 0 };
188 v16i8 src0 = { 0 }, src1 = { 0 };
189 v8i16 in0, in1, in2, in3;
190 v8i16 dst0, dst1, dst2, dst3;
191 v4i32 offset_vec, weight_vec, rnd_vec;
192
193 offset = offset << rnd_val;
194 weight0 = weight0 & 0x0000FFFF;
195 weight = weight0 | (weight1 << 16);
196
197 weight_vec = __msa_fill_w(weight);
198 offset_vec = __msa_fill_w(offset);
199 rnd_vec = __msa_fill_w(rnd_val + 1);
200
201 for (loop_cnt = (height >> 2); loop_cnt--;) {
202 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
203 src0_ptr += (4 * src_stride);
204 INSERT_D2_SB(tp0, tp1, src0);
205 INSERT_D2_SB(tp2, tp3, src1);
206 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
207 src1_ptr += (4 * src2_stride);
208 ILVRL_B2_SH(zero, src0, dst0, dst1);
209 ILVRL_B2_SH(zero, src1, dst2, dst3);
210 SLLI_4V(dst0, dst1, dst2, dst3, 6);
211 HEVC_BIW_RND_CLIP4_MAX_SATU(dst0, dst1, dst2, dst3,
212 in0, in1, in2, in3,
213 weight_vec, rnd_vec, offset_vec,
214 dst0, dst1, dst2, dst3);
215 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
216 ST_W2(out0, 0, 2, dst, dst_stride);
217 ST_H2(out0, 2, 6, dst + 4, dst_stride);
218 ST_W2(out1, 0, 2, dst + 2 * dst_stride, dst_stride);
219 ST_H2(out1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
220 dst += (4 * dst_stride);
221 }
222 if (res) {
223 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
224 src0_ptr += (4 * src_stride);
225 INSERT_D2_SB(tp0, tp1, src0);
226 INSERT_D2_SB(tp2, tp3, src1);
227 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
228 src1_ptr += (4 * src2_stride);
229 ILVRL_B2_SH(zero, src0, dst0, dst1);
230 ILVRL_B2_SH(zero, src1, dst2, dst3);
231 SLLI_4V(dst0, dst1, dst2, dst3, 6);
232 HEVC_BIW_RND_CLIP4_MAX_SATU(dst0, dst1, dst2, dst3,
233 in0, in1, in2, in3,
234 weight_vec, rnd_vec, offset_vec,
235 dst0, dst1, dst2, dst3);
236
237 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
238 ST_W2(out0, 0, 2, dst, dst_stride);
239 ST_H2(out0, 2, 6, dst + 4, dst_stride);
240 ST_W2(out1, 0, 2, dst + 2 * dst_stride, dst_stride);
241 ST_H2(out1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
242 }
243}
244
245static void hevc_biwgt_copy_8w_msa(const uint8_t *src0_ptr,
246 int32_t src_stride,
247 const int16_t *src1_ptr,
248 int32_t src2_stride,
249 uint8_t *dst,
250 int32_t dst_stride,
251 int32_t height,
252 int32_t weight0,
253 int32_t weight1,
254 int32_t offset,
255 int32_t rnd_val)
256{
257 uint64_t tp0, tp1, tp2, tp3;
258 int32_t weight;
259 v16u8 out0, out1, out2;
260 v16i8 zero = { 0 };
261 v16i8 src0 = { 0 }, src1 = { 0 }, src2 = { 0 };
262 v8i16 in0, in1, in2, in3, in4, in5;
263 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
264 v4i32 offset_vec, weight_vec, rnd_vec;
265
266 offset = offset << rnd_val;
267 weight0 = weight0 & 0x0000FFFF;
268 weight = weight0 | (weight1 << 16);
269
270 offset_vec = __msa_fill_w(offset);
271 weight_vec = __msa_fill_w(weight);
272 rnd_vec = __msa_fill_w(rnd_val + 1);
273
274 if (2 == height) {
275 LD2(src0_ptr, src_stride, tp0, tp1);
276 INSERT_D2_SB(tp0, tp1, src0);
277 LD_SH2(src1_ptr, src2_stride, in0, in1);
278 ILVRL_B2_SH(zero, src0, dst0, dst1);
279 SLLI_2V(dst0, dst1, 6);
280
281 HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
282 weight_vec, rnd_vec, offset_vec,
283 dst0, dst1);
284
285 out0 = (v16u8) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
286 ST_D2(out0, 0, 1, dst, dst_stride);
287 } else if (6 == height) {
288 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
289 src0_ptr += 4 * src_stride;
290 INSERT_D2_SB(tp0, tp1, src0);
291 INSERT_D2_SB(tp2, tp3, src1);
292 LD2(src0_ptr, src_stride, tp0, tp1);
293 INSERT_D2_SB(tp0, tp1, src2);
294 ILVRL_B2_SH(zero, src0, dst0, dst1);
295 ILVRL_B2_SH(zero, src1, dst2, dst3);
296 ILVRL_B2_SH(zero, src2, dst4, dst5);
297 LD_SH6(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5);
298 SLLI_4V(dst0, dst1, dst2, dst3, 6);
299 SLLI_2V(dst4, dst5, 6);
300 HEVC_BIW_RND_CLIP4_MAX_SATU(dst0, dst1, dst2, dst3, in0, in1, in2, in3,
301 weight_vec, rnd_vec, offset_vec, dst0, dst1,
302 dst2, dst3);
303 HEVC_BIW_RND_CLIP2_MAX_SATU(dst4, dst5, in4, in5, weight_vec, rnd_vec,
304 offset_vec, dst4, dst5);
305 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
306 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
307 ST_D2(out2, 0, 1, dst + 4 * dst_stride, dst_stride);
308 } else if (0 == height % 4) {
309 uint32_t loop_cnt;
310
311 for (loop_cnt = (height >> 2); loop_cnt--;) {
312 LD4(src0_ptr, src_stride, tp0, tp1, tp2, tp3);
313 src0_ptr += (4 * src_stride);
314 INSERT_D2_SB(tp0, tp1, src0);
315 INSERT_D2_SB(tp2, tp3, src1);
316 ILVRL_B2_SH(zero, src0, dst0, dst1);
317 ILVRL_B2_SH(zero, src1, dst2, dst3);
318 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
319 src1_ptr += (4 * src2_stride);
320
321 SLLI_4V(dst0, dst1, dst2, dst3, 6);
322 HEVC_BIW_RND_CLIP4_MAX_SATU(dst0, dst1, dst2, dst3, in0, in1, in2,
323 in3, weight_vec, rnd_vec, offset_vec,
324 dst0, dst1, dst2, dst3);
325 PCKEV_B2_UB(dst1, dst0, dst3, dst2, out0, out1);
326 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
327 dst += (4 * dst_stride);
328 }
329 }
330}
331
332static void hevc_biwgt_copy_12w_msa(const uint8_t *src0_ptr,
333 int32_t src_stride,
334 const int16_t *src1_ptr,
335 int32_t src2_stride,
336 uint8_t *dst,
337 int32_t dst_stride,
338 int32_t height,
339 int32_t weight0,
340 int32_t weight1,
341 int32_t offset,
342 int32_t rnd_val)
343{
344 uint32_t loop_cnt;
345 int32_t weight;
346 v16i8 zero = { 0 };
347 v16u8 out0, out1, out2;
348 v16i8 src0, src1, src2, src3;
349 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
350 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
351 v4i32 offset_vec, weight_vec, rnd_vec;
352
353 offset = offset << rnd_val;
354 weight0 = weight0 & 0x0000FFFF;
355 weight = weight0 | (weight1 << 16);
356
357 offset_vec = __msa_fill_w(offset);
358 weight_vec = __msa_fill_w(weight);
359 rnd_vec = __msa_fill_w(rnd_val + 1);
360
361 for (loop_cnt = (height >> 2); loop_cnt--;) {
362 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
363 src0_ptr += (4 * src_stride);
364 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
365 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
366 src1_ptr += (4 * src2_stride);
367
368 ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
369 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3,
370 dst0, dst1, dst2, dst3);
371
372 SLLI_4V(dst0, dst1, dst2, dst3, 6);
373 ILVL_W2_SB(src1, src0, src3, src2, src0, src1);
374 ILVR_B2_SH(zero, src0, zero, src1, dst4, dst5);
375
376 dst4 <<= 6;
377 dst5 <<= 6;
378 HEVC_BIW_RND_CLIP4_MAX_SATU(dst0, dst1, dst2, dst3, in0, in1, in2, in3,
379 weight_vec, rnd_vec, offset_vec, dst0, dst1,
380 dst2, dst3);
381 HEVC_BIW_RND_CLIP2_MAX_SATU(dst4, dst5, in4, in5, weight_vec, rnd_vec,
382 offset_vec, dst4, dst5);
383 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
384 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
385 ST_W4(out2, 0, 1, 2, 3, dst + 8, dst_stride);
386 dst += (4 * dst_stride);
387 }
388}
389
390static void hevc_biwgt_copy_16w_msa(const uint8_t *src0_ptr,
391 int32_t src_stride,
392 const int16_t *src1_ptr,
393 int32_t src2_stride,
394 uint8_t *dst,
395 int32_t dst_stride,
396 int32_t height,
397 int32_t weight0,
398 int32_t weight1,
399 int32_t offset,
400 int32_t rnd_val)
401{
402 uint32_t loop_cnt;
403 int32_t weight;
404 v16u8 out0, out1, out2, out3;
405 v16i8 zero = { 0 };
406 v16i8 src0, src1, src2, src3;
407 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
408 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
409 v4i32 offset_vec, weight_vec, rnd_vec;
410
411 offset = offset << rnd_val;
412 weight0 = weight0 & 0x0000FFFF;
413 weight = weight0 | (weight1 << 16);
414
415 offset_vec = __msa_fill_w(offset);
416 weight_vec = __msa_fill_w(weight);
417 rnd_vec = __msa_fill_w(rnd_val + 1);
418
419 for (loop_cnt = (height >> 2); loop_cnt--;) {
420 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
421 src0_ptr += (4 * src_stride);
422 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
423 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
424 src1_ptr += (4 * src2_stride);
425 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, tmp0, tmp1,
426 tmp2, tmp3);
427 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, tmp4, tmp5,
428 tmp6, tmp7);
429 SLLI_4V(tmp0, tmp1, tmp2, tmp3, 6);
430 SLLI_4V(tmp4, tmp5, tmp6, tmp7, 6);
431 HEVC_BIW_RND_CLIP4_MAX_SATU(tmp0, tmp1, tmp4, tmp5, in0, in1, in4, in5,
432 weight_vec, rnd_vec, offset_vec, tmp0, tmp1,
433 tmp4, tmp5);
434 HEVC_BIW_RND_CLIP4_MAX_SATU(tmp2, tmp3, tmp6, tmp7, in2, in3, in6, in7,
435 weight_vec, rnd_vec, offset_vec, tmp2, tmp3,
436 tmp6, tmp7);
437 PCKEV_B2_UB(tmp4, tmp0, tmp5, tmp1, out0, out1);
438 PCKEV_B2_UB(tmp6, tmp2, tmp7, tmp3, out2, out3);
439 ST_UB4(out0, out1, out2, out3, dst, dst_stride);
440 dst += (4 * dst_stride);
441 }
442}
443
444static void hevc_biwgt_copy_24w_msa(const uint8_t *src0_ptr,
445 int32_t src_stride,
446 const int16_t *src1_ptr,
447 int32_t src2_stride,
448 uint8_t *dst,
449 int32_t dst_stride,
450 int32_t height,
451 int32_t weight0,
452 int32_t weight1,
453 int32_t offset,
454 int32_t rnd_val)
455{
456 uint32_t loop_cnt;
457 int32_t weight;
458 v16u8 out0, out1, out2, out3, out4, out5;
459 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, zero = { 0 };
460 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8, dst9, dst10;
461 v8i16 in0, in1, in2, in3, in4, in5, in6, in7, in8, in9, in10, in11, dst11;
462 v4i32 offset_vec, weight_vec, rnd_vec;
463
464 offset = offset << rnd_val;
465 weight0 = weight0 & 0x0000FFFF;
466 weight = weight0 | (weight1 << 16);
467
468 offset_vec = __msa_fill_w(offset);
469 weight_vec = __msa_fill_w(weight);
470 rnd_vec = __msa_fill_w(rnd_val + 1);
471
472 for (loop_cnt = 8; loop_cnt--;) {
473 LD_SB4(src0_ptr, src_stride, src0, src1, src4, src5);
474 LD_SB4(src0_ptr + 16, src_stride, src2, src3, src6, src7);
475 src0_ptr += (4 * src_stride);
476 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
477 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
478 LD_SH4(src1_ptr + 16, src2_stride, in8, in9, in10, in11);
479 src1_ptr += (4 * src2_stride);
480
481 ILVRL_B2_SH(zero, src0, dst0, dst1);
482 ILVRL_B2_SH(zero, src1, dst2, dst3);
483 ILVR_B2_SH(zero, src2, zero, src3, dst4, dst5);
484 ILVRL_B2_SH(zero, src4, dst6, dst7);
485 ILVRL_B2_SH(zero, src5, dst8, dst9);
486 ILVR_B2_SH(zero, src6, zero, src7, dst10, dst11);
487 SLLI_4V(dst0, dst1, dst2, dst3, 6);
488 SLLI_4V(dst4, dst5, dst6, dst7, 6);
489 SLLI_4V(dst8, dst9, dst10, dst11, 6);
490 HEVC_BIW_RND_CLIP4_MAX_SATU(dst0, dst1, dst2, dst3, in0, in4, in1, in5,
491 weight_vec, rnd_vec, offset_vec, dst0, dst1,
492 dst2, dst3);
493 HEVC_BIW_RND_CLIP4_MAX_SATU(dst4, dst5, dst6, dst7, in8, in9, in2, in6,
494 weight_vec, rnd_vec, offset_vec, dst4, dst5,
495 dst6, dst7);
496 HEVC_BIW_RND_CLIP4_MAX_SATU(dst8, dst9, dst10, dst11, in3, in7, in10,
497 in11, weight_vec, rnd_vec, offset_vec,
498 dst8, dst9, dst10, dst11);
499 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
500 PCKEV_B3_UB(dst7, dst6, dst9, dst8, dst11, dst10, out3, out4, out5);
501 ST_UB4(out0, out1, out3, out4, dst, dst_stride);
502 ST_D4(out2, out5, 0, 1, 0, 1, dst + 16, dst_stride);
503 dst += (4 * dst_stride);
504 }
505}
506
507static void hevc_biwgt_copy_32w_msa(const uint8_t *src0_ptr,
508 int32_t src_stride,
509 const int16_t *src1_ptr,
510 int32_t src2_stride,
511 uint8_t *dst,
512 int32_t dst_stride,
513 int32_t height,
514 int32_t weight0,
515 int32_t weight1,
516 int32_t offset,
517 int32_t rnd_val)
518{
519 uint32_t loop_cnt;
520 int32_t weight;
521 v16u8 out0, out1, out2, out3;
522 v16i8 zero = { 0 };
523 v16i8 src0, src1, src2, src3;
524 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
525 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
526 v4i32 offset_vec, weight_vec, rnd_vec;
527
528 offset = offset << rnd_val;
529 weight0 = weight0 & 0x0000FFFF;
530 weight = weight0 | (weight1 << 16);
531
532 offset_vec = __msa_fill_w(offset);
533 weight_vec = __msa_fill_w(weight);
534 rnd_vec = __msa_fill_w(rnd_val + 1);
535
536 for (loop_cnt = (height >> 1); loop_cnt--;) {
537 LD_SB2(src0_ptr, 16, src0, src1);
538 src0_ptr += src_stride;
539 LD_SB2(src0_ptr, 16, src2, src3);
540 src0_ptr += src_stride;
541 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
542 src1_ptr += src2_stride;
543 LD_SH4(src1_ptr, 8, in4, in5, in6, in7);
544 src1_ptr += src2_stride;
545
546 ILVRL_B2_SH(zero, src0, tmp0, tmp4);
547 ILVRL_B2_SH(zero, src1, tmp1, tmp5);
548 ILVRL_B2_SH(zero, src2, tmp2, tmp6);
549 ILVRL_B2_SH(zero, src3, tmp3, tmp7);
550 SLLI_4V(tmp0, tmp1, tmp2, tmp3, 6);
551 SLLI_4V(tmp4, tmp5, tmp6, tmp7, 6);
552 HEVC_BIW_RND_CLIP4_MAX_SATU(tmp0, tmp4, tmp1, tmp5, in0, in1, in2, in3,
553 weight_vec, rnd_vec, offset_vec, tmp0, tmp4,
554 tmp1, tmp5);
555 HEVC_BIW_RND_CLIP4_MAX_SATU(tmp2, tmp6, tmp3, tmp7, in4, in5, in6, in7,
556 weight_vec, rnd_vec, offset_vec, tmp2, tmp6,
557 tmp3, tmp7);
558 PCKEV_B2_UB(tmp4, tmp0, tmp5, tmp1, out0, out1);
559 PCKEV_B2_UB(tmp6, tmp2, tmp7, tmp3, out2, out3);
560 ST_UB2(out0, out1, dst, 16);
561 dst += dst_stride;
562 ST_UB2(out2, out3, dst, 16);
563 dst += dst_stride;
564 }
565}
566
567static void hevc_biwgt_copy_48w_msa(const uint8_t *src0_ptr,
568 int32_t src_stride,
569 const int16_t *src1_ptr,
570 int32_t src2_stride,
571 uint8_t *dst,
572 int32_t dst_stride,
573 int32_t height,
574 int32_t weight0,
575 int32_t weight1,
576 int32_t offset,
577 int32_t rnd_val)
578{
579 uint32_t loop_cnt;
580 int32_t weight;
581 v16u8 out0, out1, out2;
582 v16i8 src0, src1, src2;
583 v16i8 zero = { 0 };
584 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, in0, in1, in2, in3, in4, in5;
585 v4i32 offset_vec, weight_vec, rnd_vec;
586
587 offset = offset << rnd_val;
588 weight0 = weight0 & 0x0000FFFF;
589 weight = weight0 | (weight1 << 16);
590
591 offset_vec = __msa_fill_w(offset);
592 weight_vec = __msa_fill_w(weight);
593 rnd_vec = __msa_fill_w(rnd_val + 1);
594
595 for (loop_cnt = 64; loop_cnt--;) {
596 LD_SB3(src0_ptr, 16, src0, src1, src2);
597 src0_ptr += src_stride;
598 LD_SH6(src1_ptr, 8, in0, in1, in2, in3, in4, in5);
599 src1_ptr += src2_stride;
600
601 ILVRL_B2_SH(zero, src0, dst0, dst1);
602 ILVRL_B2_SH(zero, src1, dst2, dst3);
603 ILVRL_B2_SH(zero, src2, dst4, dst5);
604 SLLI_4V(dst0, dst1, dst2, dst3, 6);
605 SLLI_2V(dst4, dst5, 6);
606 HEVC_BIW_RND_CLIP4_MAX_SATU(dst0, dst1, dst2, dst3, in0, in1, in2, in3,
607 weight_vec, rnd_vec, offset_vec, dst0, dst1,
608 dst2, dst3);
609 HEVC_BIW_RND_CLIP2_MAX_SATU(dst4, dst5, in4, in5, weight_vec, rnd_vec,
610 offset_vec, dst4, dst5);
611 PCKEV_B3_UB(dst1, dst0, dst3, dst2, dst5, dst4, out0, out1, out2);
612 ST_UB2(out0, out1, dst, 16);
613 ST_UB(out2, dst + 32);
614 dst += dst_stride;
615 }
616}
617
618static void hevc_biwgt_copy_64w_msa(const uint8_t *src0_ptr,
619 int32_t src_stride,
620 const int16_t *src1_ptr,
621 int32_t src2_stride,
622 uint8_t *dst,
623 int32_t dst_stride,
624 int32_t height,
625 int32_t weight0,
626 int32_t weight1,
627 int32_t offset,
628 int32_t rnd_val)
629{
630 uint32_t loop_cnt;
631 int32_t weight;
632 v16u8 out0, out1, out2, out3;
633 v16i8 zero = { 0 };
634 v16i8 src0, src1, src2, src3;
635 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
636 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
637 v4i32 offset_vec, weight_vec, rnd_vec;
638
639 offset = offset << rnd_val;
640 weight0 = weight0 & 0x0000FFFF;
641 weight = weight0 | (weight1 << 16);
642
643 offset_vec = __msa_fill_w(offset);
644 weight_vec = __msa_fill_w(weight);
645 rnd_vec = __msa_fill_w(rnd_val + 1);
646
647 for (loop_cnt = height; loop_cnt--;) {
648 LD_SB4(src0_ptr, 16, src0, src1, src2, src3);
649 src0_ptr += src_stride;
650 LD_SH8(src1_ptr, 8, in0, in1, in2, in3, in4, in5, in6, in7);
651 src1_ptr += src2_stride;
652
653 ILVR_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, tmp0, tmp1,
654 tmp2, tmp3);
655 ILVL_B4_SH(zero, src0, zero, src1, zero, src2, zero, src3, tmp4, tmp5,
656 tmp6, tmp7);
657 SLLI_4V(tmp0, tmp1, tmp2, tmp3, 6);
658 SLLI_4V(tmp4, tmp5, tmp6, tmp7, 6);
659 HEVC_BIW_RND_CLIP4_MAX_SATU(tmp0, tmp4, tmp1, tmp5, in0, in1, in2, in3,
660 weight_vec, rnd_vec, offset_vec, tmp0, tmp4,
661 tmp1, tmp5);
662 HEVC_BIW_RND_CLIP4_MAX_SATU(tmp2, tmp6, tmp3, tmp7, in4, in5, in6, in7,
663 weight_vec, rnd_vec, offset_vec, tmp2, tmp6,
664 tmp3, tmp7);
665 PCKEV_B2_UB(tmp4, tmp0, tmp5, tmp1, out0, out1);
666 PCKEV_B2_UB(tmp6, tmp2, tmp7, tmp3, out2, out3);
667 ST_UB4(out0, out1, out2, out3, dst, 16);
668 dst += dst_stride;
669 }
670}
671
672static void hevc_hz_biwgt_8t_4w_msa(const uint8_t *src0_ptr,
673 int32_t src_stride,
674 const int16_t *src1_ptr,
675 int32_t src2_stride,
676 uint8_t *dst,
677 int32_t dst_stride,
678 const int8_t *filter,
679 int32_t height,
680 int32_t weight0,
681 int32_t weight1,
682 int32_t offset,
683 int32_t rnd_val)
684{
685 uint32_t loop_cnt;
686 int32_t weight, constant;
687 v8i16 filt0, filt1, filt2, filt3;
688 v16i8 src0, src1, src2, src3;
689 v16i8 mask1, mask2, mask3;
690 v16i8 vec0, vec1, vec2, vec3;
691 v8i16 dst0, dst1;
692 v8i16 in0, in1, in2, in3;
693 v8i16 filter_vec, out0, out1;
694 v4i32 weight_vec, offset_vec, rnd_vec;
695 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
696
697 src0_ptr -= 3;
698 filter_vec = LD_SH(filter);
699 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
700
701 mask1 = mask0 + 2;
702 mask2 = mask0 + 4;
703 mask3 = mask0 + 6;
704
705 offset = offset << rnd_val;
706 weight0 = weight0 & 0x0000FFFF;
707 weight = weight0 | (weight1 << 16);
708 constant = 128 * weight1;
709 constant <<= 6;
710 offset += constant;
711
712 offset_vec = __msa_fill_w(offset);
713 weight_vec = __msa_fill_w(weight);
714 rnd_vec = __msa_fill_w(rnd_val + 1);
715
716 for (loop_cnt = (height >> 2); loop_cnt--;) {
717 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
718 src0_ptr += (4 * src_stride);
719 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
720 src1_ptr += (4 * src2_stride);
721 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
722 XORI_B4_128_SB(src0, src1, src2, src3);
723
724 VSHF_B4_SB(src0, src1, mask0, mask1, mask2, mask3,
725 vec0, vec1, vec2, vec3);
726 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
727 filt3);
728 VSHF_B4_SB(src2, src3, mask0, mask1, mask2, mask3,
729 vec0, vec1, vec2, vec3);
730 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
731 filt3);
732
733 HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
734 weight_vec, rnd_vec, offset_vec,
735 out0, out1);
736
737 out0 = (v8i16) __msa_pckev_b((v16i8) out1, (v16i8) out0);
738 ST_W4(out0, 0, 1, 2, 3, dst, dst_stride);
739 dst += (4 * dst_stride);
740 }
741}
742
743static void hevc_hz_biwgt_8t_8w_msa(const uint8_t *src0_ptr,
744 int32_t src_stride,
745 const int16_t *src1_ptr,
746 int32_t src2_stride,
747 uint8_t *dst,
748 int32_t dst_stride,
749 const int8_t *filter,
750 int32_t height,
751 int32_t weight0,
752 int32_t weight1,
753 int32_t offset,
754 int32_t rnd_val)
755{
756 uint32_t loop_cnt;
757 int32_t weight, constant;
758 v8i16 filt0, filt1, filt2, filt3;
759 v16i8 src0, src1, src2, src3;
760 v16i8 mask1, mask2, mask3;
761 v16i8 vec0, vec1, vec2, vec3;
762 v8i16 dst0, dst1, dst2, dst3;
763 v8i16 in0, in1, in2, in3;
764 v8i16 filter_vec, out0, out1, out2, out3;
765 v4i32 weight_vec, offset_vec, rnd_vec;
766 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
767
768 src0_ptr -= 3;
769 offset = offset << rnd_val;
770 weight0 = weight0 & 0x0000FFFF;
771 weight = weight0 | (weight1 << 16);
772 constant = 128 * weight1;
773 constant <<= 6;
774 offset += constant;
775
776 offset_vec = __msa_fill_w(offset);
777 weight_vec = __msa_fill_w(weight);
778 rnd_vec = __msa_fill_w(rnd_val + 1);
779
780 filter_vec = LD_SH(filter);
781 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
782
783 mask1 = mask0 + 2;
784 mask2 = mask0 + 4;
785 mask3 = mask0 + 6;
786
787 for (loop_cnt = (height >> 2); loop_cnt--;) {
788 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
789 src0_ptr += (4 * src_stride);
790 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
791 src1_ptr += (4 * src2_stride);
792 XORI_B4_128_SB(src0, src1, src2, src3);
793
794 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
795 vec0, vec1, vec2, vec3);
796 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
797 filt3);
798 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
799 vec0, vec1, vec2, vec3);
800 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
801 filt3);
802 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
803 vec0, vec1, vec2, vec3);
804 dst2 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
805 filt3);
806 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
807 vec0, vec1, vec2, vec3);
808 dst3 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
809 filt3);
810
811 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
812 in0, in1, in2, in3,
813 weight_vec, rnd_vec, offset_vec,
814 out0, out1, out2, out3);
815
816 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
817 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
818 dst += (4 * dst_stride);
819 }
820}
821
822static void hevc_hz_biwgt_8t_12w_msa(const uint8_t *src0_ptr,
823 int32_t src_stride,
824 const int16_t *src1_ptr,
825 int32_t src2_stride,
826 uint8_t *dst,
827 int32_t dst_stride,
828 const int8_t *filter,
829 int32_t height,
830 int32_t weight0,
831 int32_t weight1,
832 int32_t offset,
833 int32_t rnd_val)
834{
835 uint32_t loop_cnt;
836 int32_t weight, constant;
837 v16i8 src0, src1, src2, src3, vec0, vec1, vec2, vec3;
838 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
839 v8i16 filt0, filt1, filt2, filt3, out0, out1, out2, out3;
840 v8i16 dst0, dst1, dst2, dst3, in0, in1, in2, in3, filter_vec;
841 v4i32 weight_vec, offset_vec, rnd_vec;
842
843 src0_ptr -= 3;
844
845 weight0 = weight0 & 0x0000FFFF;
846 weight = weight0 | (weight1 << 16);
847 constant = 128 * weight1;
848 constant <<= 6;
849 offset = offset << rnd_val;
850 offset += constant;
851
852 offset_vec = __msa_fill_w(offset);
853 weight_vec = __msa_fill_w(weight);
854 rnd_vec = __msa_fill_w(rnd_val + 1);
855
856 filter_vec = LD_SH(filter);
857 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
858
859 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
860 mask1 = mask0 + 2;
861 mask2 = mask0 + 4;
862 mask3 = mask0 + 6;
863 mask4 = LD_SB(&ff_hevc_mask_arr[16]);
864 mask5 = mask4 + 2;
865 mask6 = mask4 + 4;
866 mask7 = mask4 + 6;
867
868 for (loop_cnt = 4; loop_cnt--;) {
869 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
870 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
871 XORI_B4_128_SB(src0, src1, src2, src3);
872 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
873 vec3);
874 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
875 filt3);
876 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
877 vec3);
878 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
879 filt3);
880 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
881 vec3);
882 dst2 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
883 filt3);
884 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
885 vec3);
886 dst3 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
887 filt3);
888 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3, in0, in1, in2, in3,
889 weight_vec, rnd_vec, offset_vec, out0, out1, out2,
890 out3);
891 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
892 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
893
894 LD_SB4(src0_ptr + 8, src_stride, src0, src1, src2, src3);
895 src0_ptr += (4 * src_stride);
896 LD_SH4(src1_ptr + 8, src2_stride, in0, in1, in2, in3);
897 src1_ptr += (4 * src2_stride);
898 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
899 XORI_B4_128_SB(src0, src1, src2, src3);
900 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7, vec0, vec1, vec2,
901 vec3);
902 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
903 filt3);
904 VSHF_B4_SB(src2, src3, mask4, mask5, mask6, mask7, vec0, vec1, vec2,
905 vec3);
906 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
907 filt3);
908 HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1, weight_vec, rnd_vec,
909 offset_vec, out0, out1);
910 out0 = (v8i16) __msa_pckev_b((v16i8) out1, (v16i8) out0);
911 ST_W4(out0, 0, 1, 2, 3, dst + 8, dst_stride);
912 dst += (4 * dst_stride);
913 }
914}
915
916static void hevc_hz_biwgt_8t_16w_msa(const uint8_t *src0_ptr,
917 int32_t src_stride,
918 const int16_t *src1_ptr,
919 int32_t src2_stride,
920 uint8_t *dst,
921 int32_t dst_stride,
922 const int8_t *filter,
923 int32_t height,
924 int32_t weight0,
925 int32_t weight1,
926 int32_t offset,
927 int32_t rnd_val)
928{
929 uint32_t loop_cnt;
930 int32_t weight, constant;
931 v16i8 src0, src1, src2, src3;
932 v8i16 in0, in1, in2, in3;
933 v8i16 filt0, filt1, filt2, filt3;
934 v16i8 mask1, mask2, mask3;
935 v8i16 filter_vec, out0, out1, out2, out3;
936 v16i8 vec0, vec1, vec2, vec3;
937 v8i16 dst0, dst1, dst2, dst3;
938 v4i32 weight_vec, offset_vec, rnd_vec;
939 v16i8 mask0 = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 };
940
941 src0_ptr -= 3;
942 offset = offset << rnd_val;
943 weight0 = weight0 & 0x0000FFFF;
944 weight = weight0 | (weight1 << 16);
945 constant = 128 * weight1;
946 constant <<= 6;
947 offset += constant;
948
949 offset_vec = __msa_fill_w(offset);
950 weight_vec = __msa_fill_w(weight);
951 rnd_vec = __msa_fill_w(rnd_val + 1);
952
953 filter_vec = LD_SH(filter);
954 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
955
956 mask1 = mask0 + 2;
957 mask2 = mask0 + 4;
958 mask3 = mask0 + 6;
959
960 for (loop_cnt = (height >> 1); loop_cnt--;) {
961 LD_SB2(src0_ptr, 8, src0, src1);
962 src0_ptr += src_stride;
963 LD_SB2(src0_ptr, 8, src2, src3);
964 src0_ptr += src_stride;
965 LD_SH2(src1_ptr, 8, in0, in1);
966 src1_ptr += src2_stride;
967 LD_SH2(src1_ptr, 8, in2, in3);
968 src1_ptr += src2_stride;
969 XORI_B4_128_SB(src0, src1, src2, src3);
970
971 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
972 vec0, vec1, vec2, vec3);
973 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
974 filt3);
975 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
976 vec0, vec1, vec2, vec3);
977 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
978 filt3);
979 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
980 vec0, vec1, vec2, vec3);
981 dst2 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
982 filt3);
983 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
984 vec0, vec1, vec2, vec3);
985 dst3 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
986 filt3);
987
988 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
989 in0, in1, in2, in3,
990 weight_vec, rnd_vec, offset_vec,
991 out0, out1, out2, out3);
992
993 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
994 ST_SH2(out0, out1, dst, dst_stride);
995 dst += (2 * dst_stride);
996 }
997}
998
999static void hevc_hz_biwgt_8t_24w_msa(const uint8_t *src0_ptr,
1000 int32_t src_stride,
1001 const int16_t *src1_ptr,
1002 int32_t src2_stride,
1003 uint8_t *dst,
1004 int32_t dst_stride,
1005 const int8_t *filter,
1006 int32_t height,
1007 int32_t weight0,
1008 int32_t weight1,
1009 int32_t offset,
1010 int32_t rnd_val)
1011{
1012 uint32_t loop_cnt;
1013 uint64_t dst_val0;
1014 int32_t weight, constant;
1015 v16i8 src0, src1;
1016 v8i16 in0, in1, in2;
1017 v8i16 filt0, filt1, filt2, filt3;
1018 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1019 v16i8 vec0, vec1, vec2, vec3;
1020 v8i16 dst0, dst1, dst2;
1021 v4i32 dst2_r, dst2_l;
1022 v8i16 filter_vec, out0, out1, out2;
1023 v4i32 weight_vec, offset_vec, rnd_vec;
1024 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1025
1026 src0_ptr = src0_ptr - 3;
1027 offset = offset << rnd_val;
1028 weight0 = weight0 & 0x0000FFFF;
1029 weight = weight0 | (weight1 << 16);
1030 constant = 128 * weight1;
1031 constant <<= 6;
1032 offset += constant;
1033
1034 offset_vec = __msa_fill_w(offset);
1035 weight_vec = __msa_fill_w(weight);
1036 rnd_vec = __msa_fill_w(rnd_val + 1);
1037
1038 filter_vec = LD_SH(filter);
1039 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1040
1041 mask1 = mask0 + 2;
1042 mask2 = mask0 + 4;
1043 mask3 = mask0 + 6;
1044 mask4 = mask0 + 8;
1045 mask5 = mask0 + 10;
1046 mask6 = mask0 + 12;
1047 mask7 = mask0 + 14;
1048
1049 LD_SB2(src0_ptr, 16, src0, src1);
1050 src0_ptr += src_stride;
1051 LD_SH2(src1_ptr, 8, in0, in1);
1052 in2 = LD_SH(src1_ptr + 16);
1053 src1_ptr += src2_stride;
1054 XORI_B2_128_SB(src0, src1);
1055
1056 for (loop_cnt = 31; loop_cnt--;) {
1057 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1058 vec0, vec1, vec2, vec3);
1059 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1060 filt3);
1061 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
1062 vec0, vec1, vec2, vec3);
1063 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1064 filt3);
1065 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1066 vec0, vec1, vec2, vec3);
1067 dst2 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1068 filt3);
1069
1070 HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
1071 weight_vec, rnd_vec, offset_vec,
1072 out0, out1);
1073
1074 ILVRL_H2_SW(dst2, in2, dst2_r, dst2_l);
1075 dst2_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_r,
1076 (v8i16) weight_vec);
1077 dst2_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_l,
1078 (v8i16) weight_vec);
1079 SRAR_W2_SW(dst2_r, dst2_l, rnd_vec);
1080 CLIP_SW2_0_255(dst2_r, dst2_l);
1081 out2 = __msa_pckev_h((v8i16) dst2_l, (v8i16) dst2_r);
1082
1083 LD_SB2(src0_ptr, 16, src0, src1);
1084 src0_ptr += src_stride;
1085 LD_SH2(src1_ptr, 8, in0, in1);
1086 in2 = LD_SH(src1_ptr + 16);
1087 src1_ptr += src2_stride;
1088 XORI_B2_128_SB(src0, src1);
1089 PCKEV_B2_SH(out1, out0, out2, out2, out0, out2);
1090 dst_val0 = __msa_copy_u_d((v2i64) out2, 0);
1091 ST_SH(out0, dst);
1092 SD(dst_val0, dst + 16);
1093 dst += dst_stride;
1094 }
1095
1096 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1097 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1098 filt3);
1099 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7, vec0, vec1, vec2, vec3);
1100 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1101 filt3);
1102 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1103 dst2 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1104 filt3);
1105 HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1, weight_vec, rnd_vec, offset_vec,
1106 out0, out1);
1107 ILVRL_H2_SW(dst2, in2, dst2_r, dst2_l);
1108 dst2_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_r, (v8i16) weight_vec);
1109 dst2_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_l, (v8i16) weight_vec);
1110 SRAR_W2_SW(dst2_r, dst2_l, rnd_vec);
1111 CLIP_SW2_0_255(dst2_r, dst2_l);
1112 out2 = __msa_pckev_h((v8i16) dst2_l, (v8i16) dst2_r);
1113 PCKEV_B2_SH(out1, out0, out2, out2, out0, out2);
1114 dst_val0 = __msa_copy_u_d((v2i64) out2, 0);
1115 ST_SH(out0, dst);
1116 SD(dst_val0, dst + 16);
1117 dst += dst_stride;
1118}
1119
1120static void hevc_hz_biwgt_8t_32w_msa(const uint8_t *src0_ptr,
1121 int32_t src_stride,
1122 const int16_t *src1_ptr,
1123 int32_t src2_stride,
1124 uint8_t *dst,
1125 int32_t dst_stride,
1126 const int8_t *filter,
1127 int32_t height,
1128 int32_t weight0,
1129 int32_t weight1,
1130 int32_t offset,
1131 int32_t rnd_val)
1132{
1133 uint32_t loop_cnt;
1134 int32_t weight, constant;
1135 v16i8 src0, src1, src2;
1136 v8i16 in0, in1, in2, in3;
1137 v8i16 filt0, filt1, filt2, filt3;
1138 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1139 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1140 v16i8 vec0, vec1, vec2, vec3;
1141 v8i16 dst0, dst1, dst2, dst3;
1142 v8i16 filter_vec, out0, out1, out2, out3;
1143 v4i32 weight_vec, offset_vec, rnd_vec;
1144
1145 src0_ptr -= 3;
1146 offset = offset << rnd_val;
1147 weight0 = weight0 & 0x0000FFFF;
1148 weight = weight0 | (weight1 << 16);
1149 constant = 128 * weight1;
1150 constant <<= 6;
1151 offset += constant;
1152
1153 offset_vec = __msa_fill_w(offset);
1154 weight_vec = __msa_fill_w(weight);
1155 rnd_vec = __msa_fill_w(rnd_val + 1);
1156
1157 filter_vec = LD_SH(filter);
1158 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1159
1160 mask1 = mask0 + 2;
1161 mask2 = mask0 + 4;
1162 mask3 = mask0 + 6;
1163 mask4 = mask0 + 8;
1164 mask5 = mask0 + 10;
1165 mask6 = mask0 + 12;
1166 mask7 = mask0 + 14;
1167
1168 for (loop_cnt = height; loop_cnt--;) {
1169 LD_SB2(src0_ptr, 16, src0, src1);
1170 src2 = LD_SB(src0_ptr + 24);
1171 src0_ptr += src_stride;
1172 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
1173 src1_ptr += src2_stride;
1174
1175 XORI_B3_128_SB(src0, src1, src2);
1176
1177 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1178 vec0, vec1, vec2, vec3);
1179 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1180 filt3);
1181 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
1182 vec0, vec1, vec2, vec3);
1183 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1184 filt3);
1185 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1186 vec0, vec1, vec2, vec3);
1187 dst2 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1188 filt3);
1189 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1190 vec0, vec1, vec2, vec3);
1191 dst3 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1192 filt3);
1193
1194 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
1195 in0, in1, in2, in3,
1196 weight_vec, rnd_vec, offset_vec,
1197 out0, out1, out2, out3);
1198
1199 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
1200 ST_SH2(out0, out1, dst, 16);
1201 dst += dst_stride;
1202 }
1203}
1204
1205static void hevc_hz_biwgt_8t_48w_msa(const uint8_t *src0_ptr,
1206 int32_t src_stride,
1207 const int16_t *src1_ptr,
1208 int32_t src2_stride,
1209 uint8_t *dst,
1210 int32_t dst_stride,
1211 const int8_t *filter,
1212 int32_t height,
1213 int32_t weight0,
1214 int32_t weight1,
1215 int32_t offset,
1216 int32_t rnd_val)
1217{
1218 uint32_t loop_cnt;
1219 int32_t weight, constant;
1220 v16i8 src0, src1, src2, src3, src4;
1221 v8i16 in0, in1, in2, in3;
1222 v8i16 filt0, filt1, filt2, filt3;
1223 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1224 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1225 v16i8 vec0, vec1, vec2, vec3;
1226 v8i16 dst0, dst1, dst2, dst3;
1227 v8i16 filter_vec, out0, out1, out2, out3;
1228 v4i32 weight_vec, offset_vec, rnd_vec;
1229
1230 src0_ptr -= 3;
1231 offset = offset << rnd_val;
1232 weight0 = weight0 & 0x0000FFFF;
1233 weight = weight0 | (weight1 << 16);
1234 constant = 128 * weight1;
1235 constant <<= 6;
1236 offset += constant;
1237
1238 offset_vec = __msa_fill_w(offset);
1239 weight_vec = __msa_fill_w(weight);
1240 rnd_vec = __msa_fill_w(rnd_val + 1);
1241
1242 filter_vec = LD_SH(filter);
1243 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1244
1245 mask1 = mask0 + 2;
1246 mask2 = mask0 + 4;
1247 mask3 = mask0 + 6;
1248 mask4 = mask0 + 8;
1249 mask5 = mask0 + 10;
1250 mask6 = mask0 + 12;
1251 mask7 = mask0 + 14;
1252
1253 for (loop_cnt = 64; loop_cnt--;) {
1254 LD_SB2(src0_ptr, 16, src0, src1);
1255 src2 = LD_SB(src0_ptr + 24);
1256 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
1257 XORI_B3_128_SB(src0, src1, src2);
1258 LD_SB2(src0_ptr + 32, 8, src3, src4);
1259 src0_ptr += src_stride;
1260 XORI_B2_128_SB(src3, src4);
1261
1262 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1263 vec0, vec1, vec2, vec3);
1264 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1265 filt3);
1266 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
1267 vec0, vec1, vec2, vec3);
1268 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1269 filt3);
1270 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1271 vec0, vec1, vec2, vec3);
1272 dst2 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1273 filt3);
1274 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1275 vec0, vec1, vec2, vec3);
1276 dst3 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1277 filt3);
1278
1279 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3, in0, in1, in2, in3,
1280 weight_vec, rnd_vec, offset_vec,
1281 out0, out1, out2, out3);
1282
1283 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
1284 ST_SH2(out0, out1, dst, 16);
1285
1286 LD_SH2(src1_ptr + 32, 8, in2, in3);
1287 src1_ptr += src2_stride;
1288
1289 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
1290 vec0, vec1, vec2, vec3);
1291 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1292 filt3);
1293 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
1294 vec0, vec1, vec2, vec3);
1295 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1296 filt3);
1297
1298 HEVC_BIW_RND_CLIP2(dst0, dst1, in2, in3,
1299 weight_vec, rnd_vec, offset_vec,
1300 out0, out1);
1301
1302 out0 = (v8i16) __msa_pckev_b((v16i8) out1, (v16i8) out0);
1303 ST_SH(out0, dst + 32);
1304 dst += dst_stride;
1305 }
1306}
1307
1308static void hevc_hz_biwgt_8t_64w_msa(const uint8_t *src0_ptr,
1309 int32_t src_stride,
1310 const int16_t *src1_ptr,
1311 int32_t src2_stride,
1312 uint8_t *dst,
1313 int32_t dst_stride,
1314 const int8_t *filter,
1315 int32_t height,
1316 int32_t weight0,
1317 int32_t weight1,
1318 int32_t offset,
1319 int32_t rnd_val)
1320{
1321 const uint8_t *src0_ptr_tmp;
1322 uint8_t *dst_tmp;
1323 const int16_t *src1_ptr_tmp;
1324 uint32_t loop_cnt, cnt;
1325 int32_t weight, constant;
1326 v16i8 src0, src1, src2;
1327 v8i16 in0, in1, in2, in3;
1328 v8i16 filt0, filt1, filt2, filt3;
1329 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
1330 v16i8 mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1331 v16i8 vec0, vec1, vec2, vec3;
1332 v8i16 dst0, dst1, dst2, dst3;
1333 v8i16 filter_vec, out0, out1, out2, out3;
1334 v4i32 weight_vec, offset_vec, rnd_vec;
1335
1336 src0_ptr -= 3;
1337 offset = offset << rnd_val;
1338 weight0 = weight0 & 0x0000FFFF;
1339 weight = weight0 | (weight1 << 16);
1340 constant = 128 * weight1;
1341 constant <<= 6;
1342 offset += constant;
1343
1344 offset_vec = __msa_fill_w(offset);
1345 weight_vec = __msa_fill_w(weight);
1346 rnd_vec = __msa_fill_w(rnd_val + 1);
1347
1348 filter_vec = LD_SH(filter);
1349 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1350
1351 mask1 = mask0 + 2;
1352 mask2 = mask0 + 4;
1353 mask3 = mask0 + 6;
1354 mask4 = mask0 + 8;
1355 mask5 = mask0 + 10;
1356 mask6 = mask0 + 12;
1357 mask7 = mask0 + 14;
1358
1359 for (loop_cnt = height; loop_cnt--;) {
1360 src0_ptr_tmp = src0_ptr;
1361 dst_tmp = dst;
1362 src1_ptr_tmp = src1_ptr;
1363
1364 for (cnt = 2; cnt--;) {
1365 LD_SB2(src0_ptr_tmp, 16, src0, src1);
1366 src2 = LD_SB(src0_ptr_tmp + 24);
1367 src0_ptr_tmp += 32;
1368 LD_SH4(src1_ptr_tmp, 8, in0, in1, in2, in3);
1369 src1_ptr_tmp += 32;
1370 XORI_B3_128_SB(src0, src1, src2);
1371
1372 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1373 vec0, vec1, vec2, vec3);
1374 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
1375 filt2, filt3);
1376 VSHF_B4_SB(src0, src1, mask4, mask5, mask6, mask7,
1377 vec0, vec1, vec2, vec3);
1378 dst1 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
1379 filt2, filt3);
1380 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1381 vec0, vec1, vec2, vec3);
1382 dst2 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
1383 filt2, filt3);
1384 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1385 vec0, vec1, vec2, vec3);
1386 dst3 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
1387 filt2, filt3);
1388
1389 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
1390 in0, in1, in2, in3,
1391 weight_vec, rnd_vec, offset_vec,
1392 out0, out1, out2, out3);
1393
1394 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
1395 ST_SH2(out0, out1, dst_tmp, 16);
1396 dst_tmp += 32;
1397 }
1398
1399 src0_ptr += src_stride;
1400 src1_ptr += src2_stride;
1401 dst += dst_stride;
1402
1403 }
1404}
1405
1406static void hevc_vt_biwgt_8t_4w_msa(const uint8_t *src0_ptr,
1407 int32_t src_stride,
1408 const int16_t *src1_ptr,
1409 int32_t src2_stride,
1410 uint8_t *dst,
1411 int32_t dst_stride,
1412 const int8_t *filter,
1413 int32_t height,
1414 int32_t weight0,
1415 int32_t weight1,
1416 int32_t offset,
1417 int32_t rnd_val)
1418{
1419 uint32_t loop_cnt;
1420 int32_t res = height & 0x07;
1421 int32_t weight;
1422 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1423 v16i8 src11, src12, src13, src14;
1424 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
1425 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1426 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1427 v16i8 src1110_r, src1211_r, src1312_r, src1413_r;
1428 v16i8 src2110, src4332, src6554, src8776, src10998;
1429 v16i8 src12111110, src14131312;
1430 v8i16 dst10, dst32, dst54, dst76;
1431 v8i16 filt0, filt1, filt2, filt3;
1432 v8i16 filter_vec, out0, out1, out2, out3;
1433 v4i32 weight_vec, weight1_vec, offset_vec, rnd_vec, const_vec;
1434
1435 src0_ptr -= (3 * src_stride);
1436 offset = offset << rnd_val;
1437 weight0 = weight0 & 0x0000FFFF;
1438 weight = weight0 | (weight1 << 16);
1439
1440 const_vec = __msa_ldi_w(128);
1441 const_vec <<= 6;
1442 offset_vec = __msa_fill_w(offset);
1443 weight_vec = __msa_fill_w(weight);
1444 rnd_vec = __msa_fill_w(rnd_val + 1);
1445 weight1_vec = __msa_fill_w(weight1);
1446 offset_vec += const_vec * weight1_vec;
1447
1448 filter_vec = LD_SH(filter);
1449 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1450
1451 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
1452 src0_ptr += (7 * src_stride);
1453
1454 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1455 src10_r, src32_r, src54_r, src21_r);
1456 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1457 ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r,
1458 src2110, src4332, src6554);
1459 XORI_B3_128_SB(src2110, src4332, src6554);
1460
1461 for (loop_cnt = (height >> 3); loop_cnt--;) {
1462 LD_SB8(src0_ptr, src_stride,
1463 src7, src8, src9, src10, src11, src12, src13, src14);
1464 src0_ptr += (8 * src_stride);
1465 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
1466 src1_ptr += (8 * src2_stride);
1467
1468 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
1469 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
1470 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1471 src76_r, src87_r, src98_r, src109_r);
1472 ILVR_B4_SB(src11, src10, src12, src11, src13, src12, src14, src13,
1473 src1110_r, src1211_r, src1312_r, src1413_r);
1474 ILVR_D4_SB(src87_r, src76_r, src109_r, src98_r, src1211_r, src1110_r,
1475 src1413_r, src1312_r,
1476 src8776, src10998, src12111110, src14131312);
1477 XORI_B4_128_SB(src8776, src10998, src12111110, src14131312);
1478
1479 DOTP_SB4_SH(src2110, src4332, src6554, src8776, filt0, filt0, filt0,
1480 filt0, dst10, dst32, dst54, dst76);
1481 DPADD_SB4_SH(src4332, src6554, src8776, src10998, filt1, filt1, filt1,
1482 filt1, dst10, dst32, dst54, dst76);
1483 DPADD_SB4_SH(src6554, src8776, src10998, src12111110, filt2, filt2,
1484 filt2, filt2, dst10, dst32, dst54, dst76);
1485 DPADD_SB4_SH(src8776, src10998, src12111110, src14131312, filt3, filt3,
1486 filt3, filt3, dst10, dst32, dst54, dst76);
1487
1488 HEVC_BIW_RND_CLIP4(dst10, dst32, dst54, dst76,
1489 in0, in1, in2, in3,
1490 weight_vec, rnd_vec, offset_vec,
1491 out0, out1, out2, out3);
1492
1493 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
1494 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
1495 dst += (8 * dst_stride);
1496
1497 src2110 = src10998;
1498 src4332 = src12111110;
1499 src6554 = src14131312;
1500 src6 = src14;
1501 }
1502 if (res) {
1503 LD_SB8(src0_ptr, src_stride,
1504 src7, src8, src9, src10, src11, src12, src13, src14);
1505 src0_ptr += (8 * src_stride);
1506 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
1507 src1_ptr += (8 * src2_stride);
1508
1509 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
1510 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
1511 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1512 src76_r, src87_r, src98_r, src109_r);
1513 ILVR_B4_SB(src11, src10, src12, src11, src13, src12, src14, src13,
1514 src1110_r, src1211_r, src1312_r, src1413_r);
1515 ILVR_D4_SB(src87_r, src76_r, src109_r, src98_r, src1211_r, src1110_r,
1516 src1413_r, src1312_r,
1517 src8776, src10998, src12111110, src14131312);
1518 XORI_B4_128_SB(src8776, src10998, src12111110, src14131312);
1519
1520 DOTP_SB4_SH(src2110, src4332, src6554, src8776, filt0, filt0, filt0,
1521 filt0, dst10, dst32, dst54, dst76);
1522 DPADD_SB4_SH(src4332, src6554, src8776, src10998, filt1, filt1, filt1,
1523 filt1, dst10, dst32, dst54, dst76);
1524 DPADD_SB4_SH(src6554, src8776, src10998, src12111110, filt2, filt2,
1525 filt2, filt2, dst10, dst32, dst54, dst76);
1526 DPADD_SB4_SH(src8776, src10998, src12111110, src14131312, filt3, filt3,
1527 filt3, filt3, dst10, dst32, dst54, dst76);
1528
1529 HEVC_BIW_RND_CLIP4(dst10, dst32, dst54, dst76,
1530 in0, in1, in2, in3,
1531 weight_vec, rnd_vec, offset_vec,
1532 out0, out1, out2, out3);
1533
1534 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
1535 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
1536
1537 src2110 = src10998;
1538 src4332 = src12111110;
1539 src6554 = src14131312;
1540 src6 = src14;
1541 }
1542}
1543
1544static void hevc_vt_biwgt_8t_8w_msa(const uint8_t *src0_ptr,
1545 int32_t src_stride,
1546 const int16_t *src1_ptr,
1547 int32_t src2_stride,
1548 uint8_t *dst,
1549 int32_t dst_stride,
1550 const int8_t *filter,
1551 int32_t height,
1552 int32_t weight0,
1553 int32_t weight1,
1554 int32_t offset,
1555 int32_t rnd_val)
1556{
1557 uint32_t loop_cnt;
1558 int32_t weight;
1559 v16i8 src0, src1, src2, src3, src4, src5;
1560 v16i8 src6, src7, src8, src9, src10;
1561 v8i16 in0, in1, in2, in3;
1562 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
1563 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
1564 v8i16 tmp0, tmp1, tmp2, tmp3;
1565 v8i16 filt0, filt1, filt2, filt3;
1566 v8i16 filter_vec, out0, out1, out2, out3;
1567 v4i32 weight_vec, weight1_vec, offset_vec, rnd_vec, const_vec;
1568
1569 src0_ptr -= (3 * src_stride);
1570 offset = offset << rnd_val;
1571 weight0 = weight0 & 0x0000FFFF;
1572 weight = weight0 | (weight1 << 16);
1573
1574 const_vec = __msa_ldi_w(128);
1575 const_vec <<= 6;
1576 offset_vec = __msa_fill_w(offset);
1577 weight_vec = __msa_fill_w(weight);
1578 rnd_vec = __msa_fill_w(rnd_val + 1);
1579 weight1_vec = __msa_fill_w(weight1);
1580 offset_vec += const_vec * weight1_vec;
1581
1582 filter_vec = LD_SH(filter);
1583 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1584
1585 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
1586 src0_ptr += (7 * src_stride);
1587 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1588
1589 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1590 src10_r, src32_r, src54_r, src21_r);
1591 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1592
1593 for (loop_cnt = (height >> 2); loop_cnt--;) {
1594 LD_SB4(src0_ptr, src_stride, src7, src8, src9, src10);
1595 src0_ptr += (4 * src_stride);
1596 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
1597 src1_ptr += (4 * src2_stride);
1598
1599 XORI_B4_128_SB(src7, src8, src9, src10);
1600 ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9,
1601 src76_r, src87_r, src98_r, src109_r);
1602
1603 DOTP_SB4_SH(src10_r, src21_r, src32_r, src43_r, filt0, filt0, filt0,
1604 filt0, tmp0, tmp1, tmp2, tmp3);
1605 DPADD_SB4_SH(src32_r, src43_r, src54_r, src65_r, filt1, filt1, filt1,
1606 filt1, tmp0, tmp1, tmp2, tmp3);
1607 DPADD_SB4_SH(src54_r, src65_r, src76_r, src87_r, filt2, filt2, filt2,
1608 filt2, tmp0, tmp1, tmp2, tmp3);
1609 DPADD_SB4_SH(src76_r, src87_r, src98_r, src109_r, filt3, filt3, filt3,
1610 filt3, tmp0, tmp1, tmp2, tmp3);
1611
1612 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
1613 in0, in1, in2, in3,
1614 weight_vec, rnd_vec, offset_vec,
1615 out0, out1, out2, out3);
1616
1617 PCKEV_B2_SH(out1, out0, out3, out2, out0, out1);
1618 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
1619 dst += (4 * dst_stride);
1620
1621 src10_r = src54_r;
1622 src32_r = src76_r;
1623 src54_r = src98_r;
1624 src21_r = src65_r;
1625 src43_r = src87_r;
1626 src65_r = src109_r;
1627 src6 = src10;
1628 }
1629}
1630
1631static void hevc_vt_biwgt_8t_12w_msa(const uint8_t *src0_ptr,
1632 int32_t src_stride,
1633 const int16_t *src1_ptr,
1634 int32_t src2_stride,
1635 uint8_t *dst,
1636 int32_t dst_stride,
1637 const int8_t *filter,
1638 int32_t height,
1639 int32_t weight0,
1640 int32_t weight1,
1641 int32_t offset,
1642 int32_t rnd_val)
1643{
1644 uint32_t loop_cnt;
1645 int32_t weight;
1646 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1647 v8i16 in0, in1, in2, in3;
1648 v16i8 src10_r, src32_r, src54_r, src76_r;
1649 v16i8 src21_r, src43_r, src65_r, src87_r;
1650 v8i16 tmp0, tmp1, tmp2;
1651 v16i8 src10_l, src32_l, src54_l, src76_l;
1652 v16i8 src21_l, src43_l, src65_l, src87_l;
1653 v16i8 src2110, src4332, src6554, src8776;
1654 v8i16 filt0, filt1, filt2, filt3;
1655 v8i16 out0, out1, out2, filter_vec;
1656 v4i32 dst2_r, dst2_l;
1657 v4i32 weight_vec, weight1_vec, offset_vec, rnd_vec, const_vec;
1658
1659 src0_ptr -= (3 * src_stride);
1660 offset = offset << rnd_val;
1661 weight0 = weight0 & 0x0000FFFF;
1662 weight = weight0 | (weight1 << 16);
1663
1664 const_vec = __msa_ldi_w(128);
1665 const_vec <<= 6;
1666 offset_vec = __msa_fill_w(offset);
1667 weight_vec = __msa_fill_w(weight);
1668 rnd_vec = __msa_fill_w(rnd_val + 1);
1669 weight1_vec = __msa_fill_w(weight1);
1670 offset_vec += const_vec * weight1_vec;
1671
1672 filter_vec = LD_SH(filter);
1673 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1674
1675 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
1676 src0_ptr += (7 * src_stride);
1677 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1678
1679 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1680 src10_r, src32_r, src54_r, src21_r);
1681 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1682 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1683 src10_l, src32_l, src54_l, src21_l);
1684 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1685 ILVR_D3_SB(src21_l, src10_l, src43_l, src32_l, src65_l, src54_l,
1686 src2110, src4332, src6554);
1687
1688 for (loop_cnt = 8; loop_cnt--;) {
1689 LD_SB2(src0_ptr, src_stride, src7, src8);
1690 src0_ptr += (2 * src_stride);
1691 LD_SH2(src1_ptr, src2_stride, in0, in1);
1692 LD_SH2((src1_ptr + 8), src2_stride, in2, in3);
1693 src1_ptr += (2 * src2_stride);
1694 in2 = (v8i16) __msa_ilvr_d((v2i64) in3, (v2i64) in2);
1695 XORI_B2_128_SB(src7, src8);
1696
1697 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
1698 ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
1699 src8776 = (v16i8) __msa_ilvr_d((v2i64) src87_l, (v2i64) src76_l);
1700
1701 DOTP_SB3_SH(src10_r, src21_r, src2110, filt0, filt0, filt0,
1702 tmp0, tmp1, tmp2);
1703 DPADD_SB2_SH(src32_r, src43_r, filt1, filt1, tmp0, tmp1);
1704 tmp2 = __msa_dpadd_s_h(tmp2, src4332, (v16i8) filt1);
1705 DPADD_SB2_SH(src54_r, src65_r, filt2, filt2, tmp0, tmp1);
1706 tmp2 = __msa_dpadd_s_h(tmp2, src6554, (v16i8) filt2);
1707 DPADD_SB2_SH(src76_r, src87_r, filt3, filt3, tmp0, tmp1);
1708 tmp2 = __msa_dpadd_s_h(tmp2, src8776, (v16i8) filt3);
1709
1710 HEVC_BIW_RND_CLIP2(tmp0, tmp1, in0, in1,
1711 weight_vec, rnd_vec, offset_vec,
1712 out0, out1);
1713
1714 ILVRL_H2_SW(tmp2, in2, dst2_r, dst2_l);
1715 dst2_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_r,
1716 (v8i16) weight_vec);
1717 dst2_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst2_l,
1718 (v8i16) weight_vec);
1719 SRAR_W2_SW(dst2_r, dst2_l, rnd_vec);
1720 CLIP_SW2_0_255(dst2_r, dst2_l);
1721 out2 = __msa_pckev_h((v8i16) dst2_l, (v8i16) dst2_r);
1722 PCKEV_B2_SH(out1, out0, out2, out2, out0, out2);
1723 ST_D2(out0, 0, 1, dst, dst_stride);
1724 ST_W2(out2, 0, 1, dst + 8, dst_stride);
1725 dst += (2 * dst_stride);
1726
1727 src10_r = src32_r;
1728 src32_r = src54_r;
1729 src54_r = src76_r;
1730 src21_r = src43_r;
1731 src43_r = src65_r;
1732 src65_r = src87_r;
1733 src2110 = src4332;
1734 src4332 = src6554;
1735 src6554 = src8776;
1736 src6 = src8;
1737 }
1738}
1739
1740static void hevc_vt_biwgt_8t_16multx2mult_msa(const uint8_t *src0_ptr,
1741 int32_t src_stride,
1742 const int16_t *src1_ptr,
1743 int32_t src2_stride,
1744 uint8_t *dst,
1745 int32_t dst_stride,
1746 const int8_t *filter,
1747 int32_t height,
1748 int32_t weight0,
1749 int32_t weight1,
1750 int32_t offset,
1751 int32_t rnd_val,
1752 int32_t width)
1753{
1754 const uint8_t *src0_ptr_tmp;
1755 const int16_t *src1_ptr_tmp;
1756 uint8_t *dst_tmp;
1757 uint32_t loop_cnt, cnt;
1758 int32_t weight;
1759 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1760 v8i16 in0, in1, in2, in3;
1761 v16i8 src10_r, src32_r, src54_r, src76_r;
1762 v16i8 src21_r, src43_r, src65_r, src87_r;
1763 v16i8 src10_l, src32_l, src54_l, src76_l;
1764 v16i8 src21_l, src43_l, src65_l, src87_l;
1765 v8i16 tmp0, tmp1, tmp2, tmp3;
1766 v8i16 filt0, filt1, filt2, filt3;
1767 v8i16 filter_vec;
1768 v8i16 out0, out1, out2, out3;
1769 v4i32 weight_vec, weight1_vec, offset_vec, rnd_vec, const_vec;
1770
1771 src0_ptr -= (3 * src_stride);
1772
1773 offset = offset << rnd_val;
1774 weight0 = weight0 & 0x0000FFFF;
1775 weight = weight0 | (weight1 << 16);
1776
1777 const_vec = __msa_ldi_w(128);
1778 const_vec <<= 6;
1779 offset_vec = __msa_fill_w(offset);
1780 weight_vec = __msa_fill_w(weight);
1781 rnd_vec = __msa_fill_w(rnd_val + 1);
1782 weight1_vec = __msa_fill_w(weight1);
1783 offset_vec += const_vec * weight1_vec;
1784
1785 filter_vec = LD_SH(filter);
1786 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1787
1788 for (cnt = (width >> 4); cnt--;) {
1789 src0_ptr_tmp = src0_ptr;
1790 src1_ptr_tmp = src1_ptr;
1791 dst_tmp = dst;
1792
1793 LD_SB7(src0_ptr_tmp, src_stride,
1794 src0, src1, src2, src3, src4, src5, src6);
1795 src0_ptr_tmp += (7 * src_stride);
1796
1797 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1798 ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1799 src10_r, src32_r, src54_r, src21_r);
1800 ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1801 ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1,
1802 src10_l, src32_l, src54_l, src21_l);
1803 ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1804
1805 for (loop_cnt = (height >> 1); loop_cnt--;) {
1806 LD_SB2(src0_ptr_tmp, src_stride, src7, src8);
1807 src0_ptr_tmp += (2 * src_stride);
1808 LD_SH2(src1_ptr_tmp, src2_stride, in0, in1);
1809 LD_SH2((src1_ptr_tmp + 8), src2_stride, in2, in3);
1810 src1_ptr_tmp += (2 * src2_stride);
1811
1812 XORI_B2_128_SB(src7, src8);
1813 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
1814 ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
1815
1816 DOTP_SB4_SH(src10_r, src21_r, src10_l, src21_l, filt0, filt0,
1817 filt0, filt0, tmp0, tmp1, tmp2, tmp3);
1818 DPADD_SB4_SH(src32_r, src43_r, src32_l, src43_l, filt1, filt1,
1819 filt1, filt1, tmp0, tmp1, tmp2, tmp3);
1820 DPADD_SB4_SH(src54_r, src65_r, src54_l, src65_l, filt2, filt2,
1821 filt2, filt2, tmp0, tmp1, tmp2, tmp3);
1822 DPADD_SB4_SH(src76_r, src87_r, src76_l, src87_l, filt3, filt3,
1823 filt3, filt3, tmp0, tmp1, tmp2, tmp3);
1824
1825 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
1826 in0, in1, in2, in3,
1827 weight_vec, rnd_vec, offset_vec,
1828 out0, out1, out2, out3);
1829
1830 PCKEV_B2_SH(out2, out0, out3, out1, out0, out1);
1831 ST_SH2(out0, out1, dst_tmp, dst_stride);
1832 dst_tmp += (2 * dst_stride);
1833
1834 src10_r = src32_r;
1835 src32_r = src54_r;
1836 src54_r = src76_r;
1837 src21_r = src43_r;
1838 src43_r = src65_r;
1839 src65_r = src87_r;
1840 src10_l = src32_l;
1841 src32_l = src54_l;
1842 src54_l = src76_l;
1843 src21_l = src43_l;
1844 src43_l = src65_l;
1845 src65_l = src87_l;
1846 src6 = src8;
1847 }
1848
1849 src0_ptr += 16;
1850 src1_ptr += 16;
1851 dst += 16;
1852 }
1853}
1854
1855static void hevc_vt_biwgt_8t_16w_msa(const uint8_t *src0_ptr,
1856 int32_t src_stride,
1857 const int16_t *src1_ptr,
1858 int32_t src2_stride,
1859 uint8_t *dst,
1860 int32_t dst_stride,
1861 const int8_t *filter,
1862 int32_t height,
1863 int32_t weight0,
1864 int32_t weight1,
1865 int32_t offset,
1866 int32_t rnd_val)
1867{
1868 hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
1869 src1_ptr, src2_stride,
1870 dst, dst_stride, filter, height,
1871 weight0, weight1, offset,
1872 rnd_val, 16);
1873}
1874
1875static void hevc_vt_biwgt_8t_24w_msa(const uint8_t *src0_ptr,
1876 int32_t src_stride,
1877 const int16_t *src1_ptr,
1878 int32_t src2_stride,
1879 uint8_t *dst,
1880 int32_t dst_stride,
1881 const int8_t *filter,
1882 int32_t height,
1883 int32_t weight0,
1884 int32_t weight1,
1885 int32_t offset,
1886 int32_t rnd_val)
1887{
1888 hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
1889 src1_ptr, src2_stride,
1890 dst, dst_stride, filter, height,
1891 weight0, weight1, offset,
1892 rnd_val, 16);
1893 hevc_vt_biwgt_8t_8w_msa(src0_ptr + 16, src_stride,
1894 src1_ptr + 16, src2_stride,
1895 dst + 16, dst_stride, filter, height,
1896 weight0, weight1, offset, rnd_val);
1897}
1898
1899static void hevc_vt_biwgt_8t_32w_msa(const uint8_t *src0_ptr,
1900 int32_t src_stride,
1901 const int16_t *src1_ptr,
1902 int32_t src2_stride,
1903 uint8_t *dst,
1904 int32_t dst_stride,
1905 const int8_t *filter,
1906 int32_t height,
1907 int32_t weight0,
1908 int32_t weight1,
1909 int32_t offset,
1910 int32_t rnd_val)
1911{
1912 hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
1913 src1_ptr, src2_stride,
1914 dst, dst_stride, filter, height,
1915 weight0, weight1, offset,
1916 rnd_val, 32);
1917}
1918
1919static void hevc_vt_biwgt_8t_48w_msa(const uint8_t *src0_ptr,
1920 int32_t src_stride,
1921 const int16_t *src1_ptr,
1922 int32_t src2_stride,
1923 uint8_t *dst,
1924 int32_t dst_stride,
1925 const int8_t *filter,
1926 int32_t height,
1927 int32_t weight0,
1928 int32_t weight1,
1929 int32_t offset,
1930 int32_t rnd_val)
1931{
1932 hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
1933 src1_ptr, src2_stride,
1934 dst, dst_stride, filter, height,
1935 weight0, weight1, offset,
1936 rnd_val, 48);
1937}
1938
1939static void hevc_vt_biwgt_8t_64w_msa(const uint8_t *src0_ptr,
1940 int32_t src_stride,
1941 const int16_t *src1_ptr,
1942 int32_t src2_stride,
1943 uint8_t *dst,
1944 int32_t dst_stride,
1945 const int8_t *filter,
1946 int32_t height,
1947 int32_t weight0,
1948 int32_t weight1,
1949 int32_t offset,
1950 int32_t rnd_val)
1951{
1952 hevc_vt_biwgt_8t_16multx2mult_msa(src0_ptr, src_stride,
1953 src1_ptr, src2_stride,
1954 dst, dst_stride, filter, height,
1955 weight0, weight1, offset,
1956 rnd_val, 64);
1957}
1958
1959static void hevc_hv_biwgt_8t_4w_msa(const uint8_t *src0_ptr,
1960 int32_t src_stride,
1961 const int16_t *src1_ptr,
1962 int32_t src2_stride,
1963 uint8_t *dst,
1964 int32_t dst_stride,
1965 const int8_t *filter_x,
1966 const int8_t *filter_y,
1967 int32_t height,
1968 int32_t weight0,
1969 int32_t weight1,
1970 int32_t offset,
1971 int32_t rnd_val)
1972{
1973 uint32_t loop_cnt;
1974 uint64_t tp0, tp1;
1975 int32_t weight;
1976 v16u8 out;
1977 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1978 v8i16 in0 = { 0 }, in1 = { 0 };
1979 v8i16 filt0, filt1, filt2, filt3;
1980 v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
1981 v16i8 mask1, mask2, mask3;
1982 v8i16 filter_vec, weight_vec;
1983 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1984 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1985 v8i16 dst30, dst41, dst52, dst63, dst66, dst87;
1986 v8i16 tmp0, tmp1, tmp2, tmp3;
1987 v8i16 dst10, dst32, dst54, dst76;
1988 v8i16 dst21, dst43, dst65, dst97, dst108, dst109, dst98;
1989 v4i32 offset_vec, rnd_vec, const_vec, dst0, dst1, dst2, dst3;
1990 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
1991
1992 src0_ptr -= ((3 * src_stride) + 3);
1993
1994 filter_vec = LD_SH(filter_x);
1995 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1996
1997 filter_vec = LD_SH(filter_y);
1998 UNPCK_R_SB_SH(filter_vec, filter_vec);
1999
2000 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
2001
2002 mask1 = mask0 + 2;
2003 mask2 = mask0 + 4;
2004 mask3 = mask0 + 6;
2005
2006 offset = offset << rnd_val;
2007 weight0 = weight0 & 0x0000FFFF;
2008 weight = weight0 | (weight1 << 16);
2009
2010 const_vec = __msa_fill_w((128 * weight1));
2011 const_vec <<= 6;
2012 offset_vec = __msa_fill_w(offset);
2013 rnd_vec = __msa_fill_w(rnd_val + 1);
2014 offset_vec += const_vec;
2015 weight_vec = (v8i16) __msa_fill_w(weight);
2016
2017 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
2018 src0_ptr += (7 * src_stride);
2019
2020 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
2021
2022 VSHF_B4_SB(src0, src3, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
2023 VSHF_B4_SB(src1, src4, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
2024 VSHF_B4_SB(src2, src5, mask0, mask1, mask2, mask3,
2025 vec8, vec9, vec10, vec11);
2026 VSHF_B4_SB(src3, src6, mask0, mask1, mask2, mask3,
2027 vec12, vec13, vec14, vec15);
2028
2029 dst30 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2030 filt3);
2031 dst41 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2032 filt3);
2033 dst52 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2034 filt3);
2035 dst63 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
2036 filt3);
2037
2038 ILVRL_H2_SH(dst41, dst30, dst10, dst43);
2039 ILVRL_H2_SH(dst52, dst41, dst21, dst54);
2040 ILVRL_H2_SH(dst63, dst52, dst32, dst65);
2041
2042 dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
2043
2044 for (loop_cnt = height >> 2; loop_cnt--;) {
2045 LD_SB4(src0_ptr, src_stride, src7, src8, src9, src10);
2046 src0_ptr += (4 * src_stride);
2047 XORI_B4_128_SB(src7, src8, src9, src10);
2048
2049 LD2(src1_ptr, src2_stride, tp0, tp1);
2050 INSERT_D2_SH(tp0, tp1, in0);
2051 src1_ptr += (2 * src2_stride);
2052 LD2(src1_ptr, src2_stride, tp0, tp1);
2053 INSERT_D2_SH(tp0, tp1, in1);
2054 src1_ptr += (2 * src2_stride);
2055
2056 VSHF_B4_SB(src7, src9, mask0, mask1, mask2, mask3,
2057 vec0, vec1, vec2, vec3);
2058 VSHF_B4_SB(src8, src10, mask0, mask1, mask2, mask3,
2059 vec4, vec5, vec6, vec7);
2060 dst97 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2061 filt3);
2062 dst108 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2063 filt3);
2064
2065 dst76 = __msa_ilvr_h(dst97, dst66);
2066 ILVRL_H2_SH(dst108, dst97, dst87, dst109);
2067 dst66 = (v8i16) __msa_splati_d((v2i64) dst97, 1);
2068 dst98 = __msa_ilvr_h(dst66, dst108);
2069
2070 dst0 = HEVC_FILT_8TAP(dst10, dst32, dst54, dst76, filt_h0, filt_h1,
2071 filt_h2, filt_h3);
2072 dst1 = HEVC_FILT_8TAP(dst21, dst43, dst65, dst87, filt_h0, filt_h1,
2073 filt_h2, filt_h3);
2074 dst2 = HEVC_FILT_8TAP(dst32, dst54, dst76, dst98, filt_h0, filt_h1,
2075 filt_h2, filt_h3);
2076 dst3 = HEVC_FILT_8TAP(dst43, dst65, dst87, dst109, filt_h0, filt_h1,
2077 filt_h2, filt_h3);
2078 SRA_4V(dst0, dst1, dst2, dst3, 6);
2079 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp1, tmp3);
2080 ILVRL_H2_SH(tmp1, in0, tmp0, tmp1);
2081 ILVRL_H2_SH(tmp3, in1, tmp2, tmp3);
2082 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
2083 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
2084 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
2085 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
2086 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
2087 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
2088 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
2089 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
2090 ST_W4(out, 0, 1, 2, 3, dst, dst_stride);
2091 dst += (4 * dst_stride);
2092
2093 dst10 = dst54;
2094 dst32 = dst76;
2095 dst54 = dst98;
2096 dst21 = dst65;
2097 dst43 = dst87;
2098 dst65 = dst109;
2099 dst66 = (v8i16) __msa_splati_d((v2i64) dst108, 1);
2100 }
2101}
2102
2103static void hevc_hv_biwgt_8t_8multx2mult_msa(const uint8_t *src0_ptr,
2104 int32_t src_stride,
2105 const int16_t *src1_ptr,
2106 int32_t src2_stride,
2107 uint8_t *dst,
2108 int32_t dst_stride,
2109 const int8_t *filter_x,
2110 const int8_t *filter_y,
2111 int32_t height,
2112 int32_t weight0,
2113 int32_t weight1,
2114 int32_t offset,
2115 int32_t rnd_val,
2116 int32_t width8mult)
2117{
2118 uint32_t loop_cnt, cnt;
2119 int32_t weight;
2120 const uint8_t *src0_ptr_tmp;
2121 const int16_t *src1_ptr_tmp;
2122 uint8_t *dst_tmp;
2123 v16u8 out;
2124 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
2125 v8i16 in0, in1;
2126 v8i16 filt0, filt1, filt2, filt3;
2127 v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
2128 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
2129 v16i8 mask1, mask2, mask3;
2130 v8i16 filter_vec, weight_vec;
2131 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2132 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
2133 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
2134 v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
2135 v8i16 tmp0, tmp1, tmp2, tmp3;
2136 v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
2137 v8i16 dst10_l, dst32_l, dst54_l, dst76_l;
2138 v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
2139 v8i16 dst21_l, dst43_l, dst65_l, dst87_l;
2140 v4i32 offset_vec, rnd_vec, const_vec;
2141
2142 src0_ptr -= ((3 * src_stride) + 3);
2143
2144 offset = offset << rnd_val;
2145 weight0 = weight0 & 0x0000FFFF;
2146 weight = weight0 | (weight1 << 16);
2147
2148 const_vec = __msa_fill_w((128 * weight1));
2149 const_vec <<= 6;
2150 offset_vec = __msa_fill_w(offset);
2151 rnd_vec = __msa_fill_w(rnd_val + 1);
2152 offset_vec += const_vec;
2153 weight_vec = (v8i16) __msa_fill_w(weight);
2154
2155 filter_vec = LD_SH(filter_x);
2156 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
2157
2158 filter_vec = LD_SH(filter_y);
2159 UNPCK_R_SB_SH(filter_vec, filter_vec);
2160
2161 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
2162
2163 mask1 = mask0 + 2;
2164 mask2 = mask0 + 4;
2165 mask3 = mask0 + 6;
2166
2167 for (cnt = width8mult; cnt--;) {
2168 src0_ptr_tmp = src0_ptr;
2169 src1_ptr_tmp = src1_ptr;
2170 dst_tmp = dst;
2171
2172 LD_SB7(src0_ptr_tmp, src_stride,
2173 src0, src1, src2, src3, src4, src5, src6);
2174 src0_ptr_tmp += (7 * src_stride);
2175
2176 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
2177
2178 /* row 0 row 1 row 2 row 3 */
2179 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
2180 vec0, vec1, vec2, vec3);
2181 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
2182 vec4, vec5, vec6, vec7);
2183 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
2184 vec8, vec9, vec10, vec11);
2185 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
2186 vec12, vec13, vec14, vec15);
2187
2188 dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2189 filt3);
2190 dst1 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2191 filt3);
2192 dst2 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2193 filt3);
2194 dst3 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1,
2195 filt2, filt3);
2196
2197 /* row 4 row 5 row 6 */
2198 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
2199 vec0, vec1, vec2, vec3);
2200 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3,
2201 vec4, vec5, vec6, vec7);
2202 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3,
2203 vec8, vec9, vec10, vec11);
2204
2205 dst4 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2206 filt3);
2207 dst5 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2208 filt3);
2209 dst6 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2210 filt3);
2211
2212 for (loop_cnt = height >> 1; loop_cnt--;) {
2213 LD_SB2(src0_ptr_tmp, src_stride, src7, src8);
2214 XORI_B2_128_SB(src7, src8);
2215 src0_ptr_tmp += 2 * src_stride;
2216
2217 LD_SH2(src1_ptr_tmp, src2_stride, in0, in1);
2218 src1_ptr_tmp += (2 * src2_stride);
2219
2220 ILVR_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1, dst10_r,
2221 dst32_r, dst54_r, dst21_r);
2222 ILVL_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1, dst10_l,
2223 dst32_l, dst54_l, dst21_l);
2224 ILVR_H2_SH(dst4, dst3, dst6, dst5, dst43_r, dst65_r);
2225 ILVL_H2_SH(dst4, dst3, dst6, dst5, dst43_l, dst65_l);
2226
2227 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3,
2228 vec0, vec1, vec2, vec3);
2229 dst7 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
2230 filt2, filt3);
2231
2232 ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
2233 dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
2234 filt_h0, filt_h1, filt_h2, filt_h3);
2235 dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l,
2236 filt_h0, filt_h1, filt_h2, filt_h3);
2237
2238 dst0_r >>= 6;
2239 dst0_l >>= 6;
2240
2241 /* row 8 */
2242 VSHF_B4_SB(src8, src8, mask0, mask1, mask2, mask3,
2243 vec0, vec1, vec2, vec3);
2244 dst8 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
2245 filt2, filt3);
2246
2247 ILVRL_H2_SH(dst8, dst7, dst87_r, dst87_l);
2248 dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r,
2249 filt_h0, filt_h1, filt_h2, filt_h3);
2250 dst1_l = HEVC_FILT_8TAP(dst21_l, dst43_l, dst65_l, dst87_l,
2251 filt_h0, filt_h1, filt_h2, filt_h3);
2252
2253 dst1_r >>= 6;
2254 dst1_l >>= 6;
2255
2256 PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, tmp1, tmp3);
2257 ILVRL_H2_SH(tmp1, in0, tmp0, tmp1);
2258 ILVRL_H2_SH(tmp3, in1, tmp2, tmp3);
2259 dst0_r = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
2260 dst0_l = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
2261 dst1_r = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
2262 dst1_l = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
2263 SRAR_W4_SW(dst0_l, dst0_r, dst1_l, dst1_r, rnd_vec);
2264 CLIP_SW4_0_255(dst0_l, dst0_r, dst1_l, dst1_r);
2265 PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, tmp0, tmp1);
2266 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
2267 ST_D2(out, 0, 1, dst_tmp, dst_stride);
2268 dst_tmp += (2 * dst_stride);
2269
2270 dst0 = dst2;
2271 dst1 = dst3;
2272 dst2 = dst4;
2273 dst3 = dst5;
2274 dst4 = dst6;
2275 dst5 = dst7;
2276 dst6 = dst8;
2277 }
2278
2279 src0_ptr += 8;
2280 src1_ptr += 8;
2281 dst += 8;
2282 }
2283}
2284
2285static void hevc_hv_biwgt_8t_8w_msa(const uint8_t *src0_ptr,
2286 int32_t src_stride,
2287 const int16_t *src1_ptr,
2288 int32_t src2_stride,
2289 uint8_t *dst,
2290 int32_t dst_stride,
2291 const int8_t *filter_x,
2292 const int8_t *filter_y,
2293 int32_t height,
2294 int32_t weight0,
2295 int32_t weight1,
2296 int32_t offset,
2297 int32_t rnd_val)
2298{
2299 hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
2300 src1_ptr, src2_stride,
2301 dst, dst_stride, filter_x, filter_y,
2302 height, weight0, weight1, offset,
2303 rnd_val, 1);
2304}
2305
2306static void hevc_hv_biwgt_8t_12w_msa(const uint8_t *src0_ptr,
2307 int32_t src_stride,
2308 const int16_t *src1_ptr,
2309 int32_t src2_stride,
2310 uint8_t *dst,
2311 int32_t dst_stride,
2312 const int8_t *filter_x,
2313 const int8_t *filter_y,
2314 int32_t height,
2315 int32_t weight0,
2316 int32_t weight1,
2317 int32_t offset,
2318 int32_t rnd_val)
2319{
2320 uint32_t loop_cnt;
2321 const uint8_t *src0_ptr_tmp;
2322 uint8_t *dst_tmp;
2323 const int16_t *src1_ptr_tmp;
2324 int32_t weight;
2325 uint64_t tp0, tp1;
2326 v16u8 out;
2327 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2328 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2329 v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
2330 v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
2331 v8i16 in0 = { 0 }, in1 = { 0 };
2332 v8i16 filter_vec, weight_vec, tmp0, tmp1, tmp2, tmp3;
2333 v8i16 filt0, filt1, filt2, filt3, filt_h0, filt_h1, filt_h2, filt_h3;
2334 v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6, dsth7, dsth8;
2335 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst21_r, dst43_r, dst65_r;
2336 v8i16 dst10_l, dst32_l, dst54_l, dst76_l, dst21_l, dst43_l, dst65_l;
2337 v8i16 dst30, dst41, dst52, dst63, dst66, dst87, dst10, dst32, dst54, dst76;
2338 v8i16 dst21, dst43, dst65, dst97, dst108, dst109, dst98, dst87_r, dst87_l;
2339 v4i32 offset_vec, rnd_vec, const_vec, dst0, dst1, dst2, dst3;
2340
2341 src0_ptr -= ((3 * src_stride) + 3);
2342
2343 offset = offset << rnd_val;
2344 weight0 = weight0 & 0x0000FFFF;
2345 weight = weight0 | (weight1 << 16);
2346
2347 const_vec = __msa_fill_w((128 * weight1));
2348 const_vec <<= 6;
2349 offset_vec = __msa_fill_w(offset);
2350 rnd_vec = __msa_fill_w(rnd_val + 1);
2351 offset_vec += const_vec;
2352 weight_vec = (v8i16) __msa_fill_w(weight);
2353
2354 filter_vec = LD_SH(filter_x);
2355 SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
2356
2357 filter_vec = LD_SH(filter_y);
2358 UNPCK_R_SB_SH(filter_vec, filter_vec);
2359
2360 SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
2361
2362 mask0 = LD_SB(ff_hevc_mask_arr);
2363 mask1 = mask0 + 2;
2364 mask2 = mask0 + 4;
2365 mask3 = mask0 + 6;
2366
2367 src0_ptr_tmp = src0_ptr;
2368 src1_ptr_tmp = src1_ptr;
2369 dst_tmp = dst;
2370
2371 LD_SB7(src0_ptr_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
2372 src0_ptr_tmp += (7 * src_stride);
2373 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
2374
2375 VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
2376 VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
2377 VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
2378 vec11);
2379 VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3, vec12, vec13, vec14,
2380 vec15);
2381 dsth0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2382 filt3);
2383 dsth1 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2384 filt3);
2385 dsth2 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2386 filt3);
2387 dsth3 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1,
2388 filt2, filt3);
2389 VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
2390 VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
2391 VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
2392 vec11);
2393 dsth4 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2394 filt3);
2395 dsth5 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2396 filt3);
2397 dsth6 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2398 filt3);
2399
2400 for (loop_cnt = 8; loop_cnt--;) {
2401 LD_SB2(src0_ptr_tmp, src_stride, src7, src8);
2402 src0_ptr_tmp += (2 * src_stride);
2403 XORI_B2_128_SB(src7, src8);
2404
2405 LD_SH2(src1_ptr_tmp, src2_stride, in0, in1);
2406 src1_ptr_tmp += (2 * src2_stride);
2407
2408 ILVR_H4_SH(dsth1, dsth0, dsth3, dsth2, dsth5, dsth4, dsth2, dsth1,
2409 dst10_r, dst32_r, dst54_r, dst21_r);
2410 ILVL_H4_SH(dsth1, dsth0, dsth3, dsth2, dsth5, dsth4, dsth2, dsth1,
2411 dst10_l, dst32_l, dst54_l, dst21_l);
2412 ILVR_H2_SH(dsth4, dsth3, dsth6, dsth5, dst43_r, dst65_r);
2413 ILVL_H2_SH(dsth4, dsth3, dsth6, dsth5, dst43_l, dst65_l);
2414
2415 VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
2416 vec3);
2417 dsth7 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2418 filt3);
2419
2420 ILVRL_H2_SH(dsth7, dsth6, dst76_r, dst76_l);
2421 dst0 = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0,
2422 filt_h1, filt_h2, filt_h3);
2423 dst1 = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l, filt_h0,
2424 filt_h1, filt_h2, filt_h3);
2425 dst0 >>= 6;
2426 dst1 >>= 6;
2427
2428 VSHF_B4_SB(src8, src8, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
2429 vec3);
2430 dsth8 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2431 filt3);
2432
2433 ILVRL_H2_SH(dsth8, dsth7, dst87_r, dst87_l);
2434 dst2 = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r, filt_h0,
2435 filt_h1, filt_h2, filt_h3);
2436 dst3 = HEVC_FILT_8TAP(dst21_l, dst43_l, dst65_l, dst87_l, filt_h0,
2437 filt_h1, filt_h2, filt_h3);
2438 dst2 >>= 6;
2439 dst3 >>= 6;
2440
2441 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp1, tmp3);
2442 ILVRL_H2_SH(tmp1, in0, tmp0, tmp1);
2443 ILVRL_H2_SH(tmp3, in1, tmp2, tmp3);
2444 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
2445 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
2446 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
2447 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
2448 SRAR_W4_SW(dst1, dst0, dst3, dst2, rnd_vec);
2449 CLIP_SW4_0_255(dst1, dst0, dst3, dst2);
2450 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
2451 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
2452 ST_D2(out, 0, 1, dst_tmp, dst_stride);
2453 dst_tmp += (2 * dst_stride);
2454
2455 dsth0 = dsth2;
2456 dsth1 = dsth3;
2457 dsth2 = dsth4;
2458 dsth3 = dsth5;
2459 dsth4 = dsth6;
2460 dsth5 = dsth7;
2461 dsth6 = dsth8;
2462 }
2463
2464 src0_ptr += 8;
2465 src1_ptr += 8;
2466 dst += 8;
2467
2468 mask4 = LD_SB(ff_hevc_mask_arr + 16);
2469 mask5 = mask4 + 2;
2470 mask6 = mask4 + 4;
2471 mask7 = mask4 + 6;
2472
2473 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
2474 src0_ptr += (7 * src_stride);
2475 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
2476
2477 VSHF_B4_SB(src0, src3, mask4, mask5, mask6, mask7, vec0, vec1, vec2, vec3);
2478 VSHF_B4_SB(src1, src4, mask4, mask5, mask6, mask7, vec4, vec5, vec6, vec7);
2479 VSHF_B4_SB(src2, src5, mask4, mask5, mask6, mask7, vec8, vec9, vec10,
2480 vec11);
2481 VSHF_B4_SB(src3, src6, mask4, mask5, mask6, mask7, vec12, vec13, vec14,
2482 vec15);
2483 dst30 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2484 filt3);
2485 dst41 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2486 filt3);
2487 dst52 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
2488 filt3);
2489 dst63 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
2490 filt3);
2491 ILVRL_H2_SH(dst41, dst30, dst10, dst43);
2492 ILVRL_H2_SH(dst52, dst41, dst21, dst54);
2493 ILVRL_H2_SH(dst63, dst52, dst32, dst65);
2494
2495 dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
2496
2497 for (loop_cnt = 4; loop_cnt--;) {
2498 LD_SB4(src0_ptr, src_stride, src7, src8, src9, src10);
2499 src0_ptr += (4 * src_stride);
2500 XORI_B4_128_SB(src7, src8, src9, src10);
2501
2502 LD2(src1_ptr, src2_stride, tp0, tp1);
2503 INSERT_D2_SH(tp0, tp1, in0);
2504 src1_ptr += (2 * src2_stride);
2505 LD2(src1_ptr, src2_stride, tp0, tp1);
2506 INSERT_D2_SH(tp0, tp1, in1);
2507 src1_ptr += (2 * src2_stride);
2508
2509 VSHF_B4_SB(src7, src9, mask4, mask5, mask6, mask7, vec0, vec1, vec2,
2510 vec3);
2511 VSHF_B4_SB(src8, src10, mask4, mask5, mask6, mask7, vec4, vec5, vec6,
2512 vec7);
2513 dst97 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
2514 filt3);
2515 dst108 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
2516 filt3);
2517
2518 dst76 = __msa_ilvr_h(dst97, dst66);
2519 ILVRL_H2_SH(dst108, dst97, dst87, dst109);
2520 dst66 = (v8i16) __msa_splati_d((v2i64) dst97, 1);
2521 dst98 = __msa_ilvr_h(dst66, dst108);
2522
2523 dst0 = HEVC_FILT_8TAP(dst10, dst32, dst54, dst76, filt_h0, filt_h1,
2524 filt_h2, filt_h3);
2525 dst1 = HEVC_FILT_8TAP(dst21, dst43, dst65, dst87, filt_h0, filt_h1,
2526 filt_h2, filt_h3);
2527 dst2 = HEVC_FILT_8TAP(dst32, dst54, dst76, dst98, filt_h0, filt_h1,
2528 filt_h2, filt_h3);
2529 dst3 = HEVC_FILT_8TAP(dst43, dst65, dst87, dst109, filt_h0, filt_h1,
2530 filt_h2, filt_h3);
2531 SRA_4V(dst0, dst1, dst2, dst3, 6);
2532 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp1, tmp3);
2533 ILVRL_H2_SH(tmp1, in0, tmp0, tmp1);
2534 ILVRL_H2_SH(tmp3, in1, tmp2, tmp3);
2535 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
2536 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
2537 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
2538 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
2539 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
2540 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
2541 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
2542 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
2543 ST_W4(out, 0, 1, 2, 3, dst, dst_stride);
2544 dst += (4 * dst_stride);
2545
2546 dst10 = dst54;
2547 dst32 = dst76;
2548 dst54 = dst98;
2549 dst21 = dst65;
2550 dst43 = dst87;
2551 dst65 = dst109;
2552 dst66 = (v8i16) __msa_splati_d((v2i64) dst108, 1);
2553 }
2554}
2555
2556static void hevc_hv_biwgt_8t_16w_msa(const uint8_t *src0_ptr,
2557 int32_t src_stride,
2558 const int16_t *src1_ptr,
2559 int32_t src2_stride,
2560 uint8_t *dst,
2561 int32_t dst_stride,
2562 const int8_t *filter_x,
2563 const int8_t *filter_y,
2564 int32_t height,
2565 int32_t weight0,
2566 int32_t weight1,
2567 int32_t offset,
2568 int32_t rnd_val)
2569{
2570 hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
2571 src1_ptr, src2_stride,
2572 dst, dst_stride, filter_x, filter_y,
2573 height, weight0, weight1, offset,
2574 rnd_val, 2);
2575}
2576
2577static void hevc_hv_biwgt_8t_24w_msa(const uint8_t *src0_ptr,
2578 int32_t src_stride,
2579 const int16_t *src1_ptr,
2580 int32_t src2_stride,
2581 uint8_t *dst,
2582 int32_t dst_stride,
2583 const int8_t *filter_x,
2584 const int8_t *filter_y,
2585 int32_t height,
2586 int32_t weight0,
2587 int32_t weight1,
2588 int32_t offset,
2589 int32_t rnd_val)
2590{
2591 hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
2592 src1_ptr, src2_stride,
2593 dst, dst_stride, filter_x, filter_y,
2594 height, weight0, weight1, offset,
2595 rnd_val, 3);
2596}
2597
2598static void hevc_hv_biwgt_8t_32w_msa(const uint8_t *src0_ptr,
2599 int32_t src_stride,
2600 const int16_t *src1_ptr,
2601 int32_t src2_stride,
2602 uint8_t *dst,
2603 int32_t dst_stride,
2604 const int8_t *filter_x,
2605 const int8_t *filter_y,
2606 int32_t height,
2607 int32_t weight0,
2608 int32_t weight1,
2609 int32_t offset,
2610 int32_t rnd_val)
2611{
2612 hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
2613 src1_ptr, src2_stride,
2614 dst, dst_stride, filter_x, filter_y,
2615 height, weight0, weight1, offset,
2616 rnd_val, 4);
2617}
2618
2619static void hevc_hv_biwgt_8t_48w_msa(const uint8_t *src0_ptr,
2620 int32_t src_stride,
2621 const int16_t *src1_ptr,
2622 int32_t src2_stride,
2623 uint8_t *dst,
2624 int32_t dst_stride,
2625 const int8_t *filter_x,
2626 const int8_t *filter_y,
2627 int32_t height,
2628 int32_t weight0,
2629 int32_t weight1,
2630 int32_t offset,
2631 int32_t rnd_val)
2632{
2633 hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
2634 src1_ptr, src2_stride,
2635 dst, dst_stride, filter_x, filter_y,
2636 height, weight0, weight1, offset,
2637 rnd_val, 6);
2638}
2639
2640static void hevc_hv_biwgt_8t_64w_msa(const uint8_t *src0_ptr,
2641 int32_t src_stride,
2642 const int16_t *src1_ptr,
2643 int32_t src2_stride,
2644 uint8_t *dst,
2645 int32_t dst_stride,
2646 const int8_t *filter_x,
2647 const int8_t *filter_y,
2648 int32_t height,
2649 int32_t weight0,
2650 int32_t weight1,
2651 int32_t offset,
2652 int32_t rnd_val)
2653{
2654 hevc_hv_biwgt_8t_8multx2mult_msa(src0_ptr, src_stride,
2655 src1_ptr, src2_stride,
2656 dst, dst_stride, filter_x, filter_y,
2657 height, weight0, weight1, offset,
2658 rnd_val, 8);
2659}
2660
2661static void hevc_hz_biwgt_4t_4x2_msa(const uint8_t *src0_ptr,
2662 int32_t src_stride,
2663 const int16_t *src1_ptr,
2664 int32_t src2_stride,
2665 uint8_t *dst,
2666 int32_t dst_stride,
2667 const int8_t *filter,
2668 int32_t weight0,
2669 int32_t weight1,
2670 int32_t offset,
2671 int32_t rnd_val)
2672{
2673 int32_t weight, constant;
2674 v8i16 filt0, filt1;
2675 v16i8 src0, src1;
2676 v8i16 in0, in1;
2677 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2678 v16i8 mask1, vec0, vec1;
2679 v8i16 dst0;
2680 v4i32 dst0_r, dst0_l;
2681 v8i16 out0, filter_vec;
2682 v4i32 weight_vec, offset_vec, rnd_vec;
2683
2684 src0_ptr -= 1;
2685
2686 filter_vec = LD_SH(filter);
2687 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2688
2689 mask1 = mask0 + 2;
2690
2691 offset = offset << rnd_val;
2692 weight0 = weight0 & 0x0000FFFF;
2693 weight = weight0 | (weight1 << 16);
2694 constant = 128 * weight1;
2695 constant <<= 6;
2696 offset += constant;
2697
2698 offset_vec = __msa_fill_w(offset);
2699 weight_vec = __msa_fill_w(weight);
2700 rnd_vec = __msa_fill_w(rnd_val + 1);
2701
2702 LD_SB2(src0_ptr, src_stride, src0, src1);
2703 LD_SH2(src1_ptr, src2_stride, in0, in1);
2704 in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
2705 XORI_B2_128_SB(src0, src1);
2706
2707 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
2708 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2709
2710 ILVRL_H2_SW(dst0, in0, dst0_r, dst0_l);
2711 dst0_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_r, (v8i16) weight_vec);
2712 dst0_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst0_l, (v8i16) weight_vec);
2713 SRAR_W2_SW(dst0_r, dst0_l, rnd_vec);
2714 out0 = __msa_pckev_h((v8i16) dst0_l, (v8i16) dst0_r);
2715 CLIP_SH_0_255(out0);
2716 out0 = (v8i16) __msa_pckev_b((v16i8) out0, (v16i8) out0);
2717 ST_W2(out0, 0, 1, dst, dst_stride);
2718}
2719
2720static void hevc_hz_biwgt_4t_4x4_msa(const uint8_t *src0_ptr,
2721 int32_t src_stride,
2722 const int16_t *src1_ptr,
2723 int32_t src2_stride,
2724 uint8_t *dst,
2725 int32_t dst_stride,
2726 const int8_t *filter,
2727 int32_t weight0,
2728 int32_t weight1,
2729 int32_t offset,
2730 int32_t rnd_val)
2731{
2732 int32_t weight, constant;
2733 v8i16 filt0, filt1;
2734 v16i8 src0, src1, src2, src3;
2735 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2736 v16i8 mask1;
2737 v8i16 dst0, dst1;
2738 v16i8 vec0, vec1;
2739 v8i16 in0, in1, in2, in3;
2740 v8i16 filter_vec;
2741 v4i32 weight_vec, offset_vec, rnd_vec;
2742
2743 src0_ptr -= 1;
2744
2745 /* rearranging filter */
2746 filter_vec = LD_SH(filter);
2747 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2748
2749 mask1 = mask0 + 2;
2750
2751 offset = offset << rnd_val;
2752 weight0 = weight0 & 0x0000FFFF;
2753 weight = weight0 | (weight1 << 16);
2754 constant = 128 * weight1;
2755 constant <<= 6;
2756 offset += constant;
2757
2758 offset_vec = __msa_fill_w(offset);
2759 weight_vec = __msa_fill_w(weight);
2760 rnd_vec = __msa_fill_w(rnd_val + 1);
2761
2762 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
2763 XORI_B4_128_SB(src0, src1, src2, src3);
2764 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2765 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
2766
2767 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
2768 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2769 VSHF_B2_SB(src2, src3, src2, src3, mask0, mask1, vec0, vec1);
2770 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2771 HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
2772 weight_vec, rnd_vec, offset_vec,
2773 dst0, dst1);
2774
2775 dst0 = (v8i16) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
2776 ST_W4(dst0, 0, 1, 2, 3, dst, dst_stride);
2777}
2778
2779static void hevc_hz_biwgt_4t_4x8multiple_msa(const uint8_t *src0_ptr,
2780 int32_t src_stride,
2781 const int16_t *src1_ptr,
2782 int32_t src2_stride,
2783 uint8_t *dst,
2784 int32_t dst_stride,
2785 const int8_t *filter,
2786 int32_t height,
2787 int32_t weight0,
2788 int32_t weight1,
2789 int32_t offset,
2790 int32_t rnd_val)
2791{
2792 uint32_t loop_cnt;
2793 int32_t weight, constant;
2794 v8i16 filt0, filt1;
2795 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2796 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2797 v16i8 mask1;
2798 v16i8 vec0, vec1;
2799 v8i16 dst0, dst1, dst2, dst3;
2800 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
2801 v8i16 filter_vec;
2802 v4i32 weight_vec, offset_vec, rnd_vec;
2803
2804 src0_ptr -= 1;
2805
2806 filter_vec = LD_SH(filter);
2807 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2808
2809 offset = offset << rnd_val;
2810 weight0 = weight0 & 0x0000FFFF;
2811 weight = weight0 | (weight1 << 16);
2812 constant = 128 * weight1;
2813 constant <<= 6;
2814 offset += constant;
2815
2816 offset_vec = __msa_fill_w(offset);
2817 weight_vec = __msa_fill_w(weight);
2818 rnd_vec = __msa_fill_w(rnd_val + 1);
2819
2820 mask1 = mask0 + 2;
2821
2822 for (loop_cnt = (height >> 3); loop_cnt--;) {
2823 LD_SB8(src0_ptr, src_stride,
2824 src0, src1, src2, src3, src4, src5, src6, src7);
2825 src0_ptr += (8 * src_stride);
2826 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2827 src1_ptr += (4 * src2_stride);
2828 LD_SH4(src1_ptr, src2_stride, in4, in5, in6, in7);
2829 src1_ptr += (4 * src2_stride);
2830 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
2831 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
2832 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2833
2834 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
2835 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2836 VSHF_B2_SB(src2, src3, src2, src3, mask0, mask1, vec0, vec1);
2837 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2838 VSHF_B2_SB(src4, src5, src4, src5, mask0, mask1, vec0, vec1);
2839 dst2 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2840 VSHF_B2_SB(src6, src7, src6, src7, mask0, mask1, vec0, vec1);
2841 dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2842 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
2843 in0, in1, in2, in3,
2844 weight_vec, rnd_vec, offset_vec,
2845 dst0, dst1, dst2, dst3);
2846
2847 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2848 ST_W8(dst0, dst1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
2849 dst += (8 * dst_stride);
2850 }
2851}
2852
2853static void hevc_hz_biwgt_4t_4w_msa(const uint8_t *src0_ptr,
2854 int32_t src_stride,
2855 const int16_t *src1_ptr,
2856 int32_t src2_stride,
2857 uint8_t *dst,
2858 int32_t dst_stride,
2859 const int8_t *filter,
2860 int32_t height,
2861 int32_t weight0,
2862 int32_t weight1,
2863 int32_t offset,
2864 int32_t rnd_val)
2865{
2866 if (2 == height) {
2867 hevc_hz_biwgt_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2868 dst, dst_stride, filter,
2869 weight0, weight1, offset, rnd_val);
2870 } else if (4 == height) {
2871 hevc_hz_biwgt_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
2872 dst, dst_stride, filter,
2873 weight0, weight1, offset, rnd_val);
2874 } else if (0 == (height % 8)) {
2875 hevc_hz_biwgt_4t_4x8multiple_msa(src0_ptr, src_stride,
2876 src1_ptr, src2_stride,
2877 dst, dst_stride, filter, height,
2878 weight0, weight1, offset,
2879 rnd_val);
2880 }
2881}
2882
2883static void hevc_hz_biwgt_4t_6w_msa(const uint8_t *src0_ptr,
2884 int32_t src_stride,
2885 const int16_t *src1_ptr,
2886 int32_t src2_stride,
2887 uint8_t *dst,
2888 int32_t dst_stride,
2889 const int8_t *filter,
2890 int32_t height,
2891 int32_t weight0,
2892 int32_t weight1,
2893 int32_t offset,
2894 int32_t rnd_val)
2895{
2896 uint32_t loop_cnt;
2897 int32_t weight, constant;
2898 v8i16 filt0, filt1;
2899 v16i8 src0, src1, src2, src3;
2900 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2901 v16i8 mask1;
2902 v16i8 vec0, vec1;
2903 v8i16 in0, in1, in2, in3;
2904 v8i16 dst0, dst1, dst2, dst3;
2905 v8i16 filter_vec;
2906 v4i32 weight_vec, offset_vec, rnd_vec;
2907
2908 src0_ptr -= 1;
2909
2910 filter_vec = LD_SH(filter);
2911 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2912
2913 offset = offset << rnd_val;
2914 weight0 = weight0 & 0x0000FFFF;
2915 weight = weight0 | (weight1 << 16);
2916 constant = 128 * weight1;
2917 constant <<= 6;
2918 offset += constant;
2919
2920 offset_vec = __msa_fill_w(offset);
2921 weight_vec = __msa_fill_w(weight);
2922 rnd_vec = __msa_fill_w(rnd_val + 1);
2923
2924 mask1 = mask0 + 2;
2925
2926 for (loop_cnt = 2; loop_cnt--;) {
2927 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
2928 src0_ptr += (4 * src_stride);
2929 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
2930 src1_ptr += (4 * src2_stride);
2931 XORI_B4_128_SB(src0, src1, src2, src3);
2932
2933 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
2934 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2935 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
2936 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2937 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
2938 dst2 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2939 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
2940 dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2941
2942 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
2943 in0, in1, in2, in3,
2944 weight_vec, rnd_vec, offset_vec,
2945 dst0, dst1, dst2, dst3);
2946
2947 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
2948 ST_W2(dst0, 0, 2, dst, dst_stride);
2949 ST_H2(dst0, 2, 6, dst + 4, dst_stride);
2950 ST_W2(dst1, 0, 2, dst + 2 * dst_stride, dst_stride);
2951 ST_H2(dst1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
2952 dst += (4 * dst_stride);
2953 }
2954}
2955
2956static void hevc_hz_biwgt_4t_8x2_msa(const uint8_t *src0_ptr,
2957 int32_t src_stride,
2958 const int16_t *src1_ptr,
2959 int32_t src2_stride,
2960 uint8_t *dst,
2961 int32_t dst_stride,
2962 const int8_t *filter,
2963 int32_t weight0,
2964 int32_t weight1,
2965 int32_t offset,
2966 int32_t rnd_val)
2967{
2968 int32_t weight, constant;
2969 v8i16 filt0, filt1;
2970 v16i8 src0, src1;
2971 v8i16 in0, in1;
2972 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2973 v16i8 mask1, vec0, vec1;
2974 v8i16 dst0, dst1;
2975 v8i16 filter_vec;
2976 v4i32 weight_vec, offset_vec, rnd_vec;
2977
2978 src0_ptr -= 1;
2979
2980 filter_vec = LD_SH(filter);
2981 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2982
2983 offset = offset << rnd_val;
2984 weight0 = weight0 & 0x0000FFFF;
2985 weight = weight0 | (weight1 << 16);
2986 constant = 128 * weight1;
2987 constant <<= 6;
2988 offset += constant;
2989
2990 offset_vec = __msa_fill_w(offset);
2991 weight_vec = __msa_fill_w(weight);
2992 rnd_vec = __msa_fill_w(rnd_val + 1);
2993
2994 mask1 = mask0 + 2;
2995
2996 LD_SB2(src0_ptr, src_stride, src0, src1);
2997 LD_SH2(src1_ptr, src2_stride, in0, in1);
2998 XORI_B2_128_SB(src0, src1);
2999 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3000 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3001 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
3002 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3003 HEVC_BIW_RND_CLIP2(dst0, dst1, in0, in1,
3004 weight_vec, rnd_vec, offset_vec,
3005 dst0, dst1);
3006
3007 dst0 = (v8i16) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
3008 ST_D2(dst0, 0, 1, dst, dst_stride);
3009}
3010
3011static void hevc_hz_biwgt_4t_8x6_msa(const uint8_t *src0_ptr,
3012 int32_t src_stride,
3013 const int16_t *src1_ptr,
3014 int32_t src2_stride,
3015 uint8_t *dst,
3016 int32_t dst_stride,
3017 const int8_t *filter,
3018 int32_t weight0,
3019 int32_t weight1,
3020 int32_t offset,
3021 int32_t rnd_val)
3022{
3023 int32_t weight, constant;
3024 v8i16 filt0, filt1;
3025 v16i8 src0, src1, src2, src3, src4, src5;
3026 v8i16 in0, in1, in2, in3, in4, in5;
3027 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3028 v16i8 mask1;
3029 v16i8 vec0, vec1;
3030 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
3031 v8i16 filter_vec;
3032 v4i32 weight_vec, offset_vec, rnd_vec;
3033
3034 src0_ptr -= 1;
3035
3036 filter_vec = LD_SH(filter);
3037 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3038
3039 offset = offset << rnd_val;
3040 weight0 = weight0 & 0x0000FFFF;
3041 weight = weight0 | (weight1 << 16);
3042 constant = 128 * weight1;
3043 constant <<= 6;
3044 offset += constant;
3045
3046 offset_vec = __msa_fill_w(offset);
3047 weight_vec = __msa_fill_w(weight);
3048 rnd_vec = __msa_fill_w(rnd_val + 1);
3049
3050 mask1 = mask0 + 2;
3051
3052 LD_SB6(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5);
3053
3054 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3055 src1_ptr += (4 * src2_stride);
3056 LD_SH2(src1_ptr, src2_stride, in4, in5);
3057 XORI_B6_128_SB(src0, src1, src2, src3, src4, src5);
3058 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3059 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3060 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
3061 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3062 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
3063 dst2 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3064 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3065 dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3066 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
3067 dst4 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3068 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
3069 dst5 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3070 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
3071 in0, in1, in2, in3,
3072 weight_vec, rnd_vec, offset_vec,
3073 dst0, dst1, dst2, dst3);
3074 HEVC_BIW_RND_CLIP2(dst4, dst5, in4, in5,
3075 weight_vec, rnd_vec, offset_vec,
3076 dst4, dst5);
3077
3078 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
3079 dst3 = (v8i16) __msa_pckev_b((v16i8) dst5, (v16i8) dst4);
3080 ST_D4(dst0, dst1, 0, 1, 0, 1, dst, dst_stride);
3081 ST_D2(dst3, 0, 1, dst + 4 * dst_stride, dst_stride);
3082}
3083
3084static void hevc_hz_biwgt_4t_8x4multiple_msa(const uint8_t *src0_ptr,
3085 int32_t src_stride,
3086 const int16_t *src1_ptr,
3087 int32_t src2_stride,
3088 uint8_t *dst,
3089 int32_t dst_stride,
3090 const int8_t *filter,
3091 int32_t height,
3092 int32_t weight0,
3093 int32_t weight1,
3094 int32_t offset,
3095 int32_t rnd_val)
3096{
3097 uint32_t loop_cnt;
3098 int32_t weight, constant;
3099 v8i16 filt0, filt1;
3100 v16i8 src0, src1, src2, src3;
3101 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3102 v16i8 mask1;
3103 v16i8 vec0, vec1;
3104 v8i16 in0, in1, in2, in3;
3105 v8i16 dst0, dst1, dst2, dst3;
3106 v8i16 filter_vec;
3107 v4i32 weight_vec, offset_vec, rnd_vec;
3108
3109 src0_ptr -= 1;
3110
3111 filter_vec = LD_SH(filter);
3112 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3113
3114 offset = offset << rnd_val;
3115 weight0 = weight0 & 0x0000FFFF;
3116 weight = weight0 | (weight1 << 16);
3117 constant = 128 * weight1;
3118 constant <<= 6;
3119 offset += constant;
3120
3121 offset_vec = __msa_fill_w(offset);
3122 weight_vec = __msa_fill_w(weight);
3123 rnd_vec = __msa_fill_w(rnd_val + 1);
3124
3125 mask1 = mask0 + 2;
3126
3127 for (loop_cnt = (height >> 2); loop_cnt--;) {
3128 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
3129 src0_ptr += (4 * src_stride);
3130 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3131 src1_ptr += (4 * src2_stride);
3132 XORI_B4_128_SB(src0, src1, src2, src3);
3133
3134 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3135 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3136 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
3137 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3138 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
3139 dst2 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3140 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3141 dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3142 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
3143 in0, in1, in2, in3,
3144 weight_vec, rnd_vec, offset_vec,
3145 dst0, dst1, dst2, dst3);
3146
3147 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
3148 ST_D4(dst0, dst1, 0, 1, 0, 1, dst, dst_stride);
3149 dst += (4 * dst_stride);
3150 }
3151}
3152
3153static void hevc_hz_biwgt_4t_8w_msa(const uint8_t *src0_ptr,
3154 int32_t src_stride,
3155 const int16_t *src1_ptr,
3156 int32_t src2_stride,
3157 uint8_t *dst,
3158 int32_t dst_stride,
3159 const int8_t *filter,
3160 int32_t height,
3161 int32_t weight0,
3162 int32_t weight1,
3163 int32_t offset,
3164 int32_t rnd_val)
3165{
3166 if (2 == height) {
3167 hevc_hz_biwgt_4t_8x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
3168 dst, dst_stride, filter,
3169 weight0, weight1, offset, rnd_val);
3170 } else if (6 == height) {
3171 hevc_hz_biwgt_4t_8x6_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
3172 dst, dst_stride, filter,
3173 weight0, weight1, offset, rnd_val);
3174 } else if (0 == (height % 4)) {
3175 hevc_hz_biwgt_4t_8x4multiple_msa(src0_ptr, src_stride,
3176 src1_ptr, src2_stride,
3177 dst, dst_stride, filter, height,
3178 weight0, weight1, offset,
3179 rnd_val);
3180 }
3181}
3182
3183static void hevc_hz_biwgt_4t_12w_msa(const uint8_t *src0_ptr,
3184 int32_t src_stride,
3185 const int16_t *src1_ptr,
3186 int32_t src2_stride,
3187 uint8_t *dst,
3188 int32_t dst_stride,
3189 const int8_t *filter,
3190 int32_t height,
3191 int32_t weight0,
3192 int32_t weight1,
3193 int32_t offset,
3194 int32_t rnd_val)
3195{
3196 uint32_t loop_cnt;
3197 int32_t weight, constant;
3198 v8i16 filt0, filt1;
3199 v16i8 src0, src1, src2, src3;
3200 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
3201 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3202 v16i8 mask2 = {
3203 8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
3204 };
3205 v16i8 mask1, mask3;
3206 v16i8 vec0, vec1;
3207 v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
3208 v8i16 filter_vec;
3209 v4i32 weight_vec, offset_vec, rnd_vec;
3210
3211 src0_ptr -= 1;
3212
3213 filter_vec = LD_SH(filter);
3214 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3215
3216 offset = offset << rnd_val;
3217 weight0 = weight0 & 0x0000FFFF;
3218 weight = weight0 | (weight1 << 16);
3219 constant = 128 * weight1;
3220 constant <<= 6;
3221 offset += constant;
3222
3223 offset_vec = __msa_fill_w(offset);
3224 weight_vec = __msa_fill_w(weight);
3225 rnd_vec = __msa_fill_w(rnd_val + 1);
3226
3227 mask1 = mask0 + 2;
3228 mask3 = mask2 + 2;
3229
3230 for (loop_cnt = 4; loop_cnt--;) {
3231 LD_SB4(src0_ptr, src_stride, src0, src1, src2, src3);
3232 src0_ptr += (4 * src_stride);
3233 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3234 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
3235 src1_ptr += (4 * src2_stride);
3236 ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
3237 XORI_B4_128_SB(src0, src1, src2, src3);
3238
3239 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3240 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3241 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
3242 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3243 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
3244 dst2 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3245 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3246 dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3247 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
3248 dst4 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3249 VSHF_B2_SB(src2, src3, src2, src3, mask2, mask3, vec0, vec1);
3250 dst5 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3251
3252 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
3253 in0, in1, in2, in3,
3254 weight_vec, rnd_vec, offset_vec,
3255 dst0, dst1, dst2, dst3);
3256 HEVC_BIW_RND_CLIP2(dst4, dst5, in4, in5,
3257 weight_vec, rnd_vec, offset_vec,
3258 dst4, dst5);
3259
3260 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
3261 dst3 = (v8i16) __msa_pckev_b((v16i8) dst5, (v16i8) dst4);
3262 ST_D4(dst0, dst1, 0, 1, 0, 1, dst, dst_stride);
3263 ST_W4(dst3, 0, 1, 2, 3, dst + 8, dst_stride);
3264 dst += (4 * dst_stride);
3265 }
3266}
3267
3268static void hevc_hz_biwgt_4t_16w_msa(const uint8_t *src0_ptr,
3269 int32_t src_stride,
3270 const int16_t *src1_ptr,
3271 int32_t src2_stride,
3272 uint8_t *dst,
3273 int32_t dst_stride,
3274 const int8_t *filter,
3275 int32_t height,
3276 int32_t weight0,
3277 int32_t weight1,
3278 int32_t offset,
3279 int32_t rnd_val)
3280{
3281 uint32_t loop_cnt;
3282 int32_t weight, constant;
3283 v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
3284 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
3285 v8i16 filt0, filt1;
3286 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3287 v16i8 mask1;
3288 v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3289 v16i8 vec0, vec1;
3290 v8i16 filter_vec;
3291 v4i32 weight_vec, offset_vec, rnd_vec;
3292
3293 src0_ptr -= 1;
3294
3295 filter_vec = LD_SH(filter);
3296 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3297
3298 offset = offset << rnd_val;
3299 weight0 = weight0 & 0x0000FFFF;
3300 weight = weight0 | (weight1 << 16);
3301 constant = 128 * weight1;
3302 constant <<= 6;
3303 offset += constant;
3304
3305 offset_vec = __msa_fill_w(offset);
3306 weight_vec = __msa_fill_w(weight);
3307 rnd_vec = __msa_fill_w(rnd_val + 1);
3308
3309 mask1 = mask0 + 2;
3310
3311 for (loop_cnt = (height >> 2); loop_cnt--;) {
3312 LD_SB4(src0_ptr, src_stride, src0, src2, src4, src6);
3313 LD_SB4(src0_ptr + 8, src_stride, src1, src3, src5, src7);
3314 src0_ptr += (4 * src_stride);
3315 LD_SH4(src1_ptr, src2_stride, in0, in2, in4, in6);
3316 LD_SH4(src1_ptr + 8, src2_stride, in1, in3, in5, in7);
3317 src1_ptr += (4 * src2_stride);
3318 XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
3319
3320 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3321 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3322 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
3323 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3324 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
3325 dst2 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3326 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3327 dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3328 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec0, vec1);
3329 dst4 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3330 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec0, vec1);
3331 dst5 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3332 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec0, vec1);
3333 dst6 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3334 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
3335 dst7 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3336 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
3337 in0, in1, in2, in3,
3338 weight_vec, rnd_vec, offset_vec,
3339 dst0, dst1, dst2, dst3);
3340
3341 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
3342 ST_SH2(dst0, dst1, dst, dst_stride);
3343 dst += (2 * dst_stride);
3344
3345 HEVC_BIW_RND_CLIP4(dst4, dst5, dst6, dst7,
3346 in4, in5, in6, in7,
3347 weight_vec, rnd_vec, offset_vec,
3348 dst0, dst1, dst2, dst3);
3349
3350 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
3351 ST_SH2(dst0, dst1, dst, dst_stride);
3352 dst += (2 * dst_stride);
3353 }
3354}
3355
3356static void hevc_hz_biwgt_4t_24w_msa(const uint8_t *src0_ptr,
3357 int32_t src_stride,
3358 const int16_t *src1_ptr,
3359 int32_t src2_stride,
3360 uint8_t *dst,
3361 int32_t dst_stride,
3362 const int8_t *filter,
3363 int32_t height,
3364 int32_t weight0,
3365 int32_t weight1,
3366 int32_t offset,
3367 int32_t rnd_val)
3368{
3369 uint32_t loop_cnt;
3370 int32_t weight, constant;
3371 v16i8 src0, src1, src2, src3;
3372 v8i16 filt0, filt1;
3373 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3374 v16i8 mask1, mask2, mask3;
3375 v16i8 vec0, vec1;
3376 v8i16 dst0, dst1, dst2, dst3;
3377 v8i16 in0, in1, in2, in3, in4, in5;
3378 v8i16 filter_vec;
3379 v4i32 weight_vec, offset_vec, rnd_vec;
3380
3381 src0_ptr -= 1;
3382
3383 filter_vec = LD_SH(filter);
3384 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3385
3386 offset = offset << rnd_val;
3387 weight0 = weight0 & 0x0000FFFF;
3388 weight = weight0 | (weight1 << 16);
3389 constant = 128 * weight1;
3390 constant <<= 6;
3391 offset += constant;
3392
3393 offset_vec = __msa_fill_w(offset);
3394 weight_vec = __msa_fill_w(weight);
3395 rnd_vec = __msa_fill_w(rnd_val + 1);
3396
3397 mask1 = mask0 + 2;
3398 mask2 = mask0 + 8;
3399 mask3 = mask0 + 10;
3400
3401 for (loop_cnt = 16; loop_cnt--;) {
3402 LD_SB2(src0_ptr, src_stride, src0, src2);
3403 LD_SB2(src0_ptr + 16, src_stride, src1, src3);
3404 src0_ptr += (2 * src_stride);
3405 LD_SH2(src1_ptr, src2_stride, in0, in2);
3406 LD_SH2(src1_ptr + 8, src2_stride, in1, in3);
3407 LD_SH2(src1_ptr + 16, src2_stride, in4, in5);
3408 src1_ptr += (2 * src2_stride);
3409 XORI_B4_128_SB(src0, src1, src2, src3);
3410
3411 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3412 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3413 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
3414 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3415 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
3416 dst2 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3417 VSHF_B2_SB(src2, src3, src2, src3, mask2, mask3, vec0, vec1);
3418 dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3419 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
3420 in0, in1, in2, in3,
3421 weight_vec, rnd_vec, offset_vec,
3422 dst0, dst1, dst2, dst3);
3423
3424 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
3425 ST_SH2(dst0, dst1, dst, dst_stride);
3426
3427 /* 8 width */
3428 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
3429 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3430 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3431 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3432 HEVC_BIW_RND_CLIP2(dst0, dst1, in4, in5,
3433 weight_vec, rnd_vec, offset_vec,
3434 dst0, dst1);
3435
3436 dst0 = (v8i16) __msa_pckev_b((v16i8) dst1, (v16i8) dst0);
3437 ST_D2(dst0, 0, 1, (dst + 16), dst_stride);
3438 dst += (2 * dst_stride);
3439 }
3440}
3441
3442static void hevc_hz_biwgt_4t_32w_msa(const uint8_t *src0_ptr,
3443 int32_t src_stride,
3444 const int16_t *src1_ptr,
3445 int32_t src2_stride,
3446 uint8_t *dst,
3447 int32_t dst_stride,
3448 const int8_t *filter,
3449 int32_t height,
3450 int32_t weight0,
3451 int32_t weight1,
3452 int32_t offset,
3453 int32_t rnd_val)
3454{
3455 uint32_t loop_cnt;
3456 int32_t weight, constant;
3457 v16i8 src0, src1, src2;
3458 v8i16 filt0, filt1;
3459 v16i8 mask0 = LD_SB(&ff_hevc_mask_arr[0]);
3460 v16i8 mask1, mask2, mask3;
3461 v8i16 dst0, dst1, dst2, dst3;
3462 v16i8 vec0, vec1;
3463 v8i16 in0, in1, in2, in3;
3464 v8i16 filter_vec;
3465 v4i32 weight_vec, offset_vec, rnd_vec;
3466
3467 src0_ptr -= 1;
3468
3469 filter_vec = LD_SH(filter);
3470 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3471
3472 offset = offset << rnd_val;
3473 weight0 = weight0 & 0x0000FFFF;
3474 weight = weight0 | (weight1 << 16);
3475 constant = 128 * weight1;
3476 constant <<= 6;
3477 offset += constant;
3478
3479 offset_vec = __msa_fill_w(offset);
3480 weight_vec = __msa_fill_w(weight);
3481 rnd_vec = __msa_fill_w(rnd_val + 1);
3482
3483 mask1 = mask0 + 2;
3484 mask2 = mask0 + 8;
3485 mask3 = mask0 + 10;
3486
3487 for (loop_cnt = height; loop_cnt--;) {
3488 LD_SB2(src0_ptr, 16, src0, src1);
3489 src2 = LD_SB(src0_ptr + 24);
3490 src0_ptr += src_stride;
3491 LD_SH4(src1_ptr, 8, in0, in1, in2, in3);
3492 src1_ptr += src2_stride;
3493 XORI_B3_128_SB(src0, src1, src2);
3494
3495 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3496 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3497 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
3498 dst1 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3499 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec0, vec1);
3500 dst2 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3501 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec0, vec1);
3502 dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3503 HEVC_BIW_RND_CLIP4(dst0, dst1, dst2, dst3,
3504 in0, in1, in2, in3,
3505 weight_vec, rnd_vec, offset_vec,
3506 dst0, dst1, dst2, dst3);
3507
3508 PCKEV_B2_SH(dst1, dst0, dst3, dst2, dst0, dst1);
3509 ST_SH2(dst0, dst1, dst, 16);
3510 dst += dst_stride;
3511 }
3512}
3513
3514static void hevc_vt_biwgt_4t_4x2_msa(const uint8_t *src0_ptr,
3515 int32_t src_stride,
3516 const int16_t *src1_ptr,
3517 int32_t src2_stride,
3518 uint8_t *dst,
3519 int32_t dst_stride,
3520 const int8_t *filter,
3521 int32_t weight0,
3522 int32_t weight1,
3523 int32_t offset,
3524 int32_t rnd_val)
3525{
3526 int32_t weight, constant;
3527 v16i8 src0, src1, src2, src3, src4;
3528 v8i16 in0, in1, dst10;
3529 v16i8 src10_r, src32_r, src21_r, src43_r, src2110, src4332;
3530 v4i32 dst10_r, dst10_l;
3531 v8i16 filt0, filt1;
3532 v8i16 filter_vec, out;
3533 v4i32 weight_vec, offset_vec, rnd_vec;
3534
3535 src0_ptr -= src_stride;
3536
3537 offset = offset << rnd_val;
3538 weight0 = weight0 & 0x0000FFFF;
3539 weight = weight0 | (weight1 << 16);
3540 constant = 128 * weight1;
3541 constant <<= 6;
3542 offset += constant;
3543
3544 offset_vec = __msa_fill_w(offset);
3545 weight_vec = __msa_fill_w(weight);
3546 rnd_vec = __msa_fill_w(rnd_val + 1);
3547
3548 filter_vec = LD_SH(filter);
3549 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3550
3551 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3552 src0_ptr += (3 * src_stride);
3553 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3554 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
3555 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
3556 LD_SB2(src0_ptr, src_stride, src3, src4);
3557 src0_ptr += (2 * src_stride);
3558 LD_SH2(src1_ptr, src2_stride, in0, in1);
3559 src1_ptr += (2 * src2_stride);
3560
3561 in0 = (v8i16) __msa_ilvr_d((v2i64) in1, (v2i64) in0);
3562 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3563 src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_r, (v2i64) src32_r);
3564 src4332 = (v16i8) __msa_xori_b((v16u8) src4332, 128);
3565
3566 dst10 = HEVC_FILT_4TAP_SH(src2110, src4332, filt0, filt1);
3567
3568 ILVRL_H2_SW(dst10, in0, dst10_r, dst10_l);
3569 dst10_r = __msa_dpadd_s_w(offset_vec, (v8i16) dst10_r, (v8i16) weight_vec);
3570 dst10_l = __msa_dpadd_s_w(offset_vec, (v8i16) dst10_l, (v8i16) weight_vec);
3571 SRAR_W2_SW(dst10_r, dst10_l, rnd_vec);
3572 CLIP_SW2_0_255(dst10_r, dst10_l);
3573 out = __msa_pckev_h((v8i16) dst10_l, (v8i16) dst10_r);
3574 out = (v8i16) __msa_pckev_b((v16i8) out, (v16i8) out);
3575 ST_W2(out, 0, 1, dst, dst_stride);
3576}
3577
3578static void hevc_vt_biwgt_4t_4x4_msa(const uint8_t *src0_ptr,
3579 int32_t src_stride,
3580 const int16_t *src1_ptr,
3581 int32_t src2_stride,
3582 uint8_t *dst,
3583 int32_t dst_stride,
3584 const int8_t *filter,
3585 int32_t weight0,
3586 int32_t weight1,
3587 int32_t offset,
3588 int32_t rnd_val)
3589{
3590 int32_t weight, constant;
3591 v16i8 src0, src1, src2, src3, src4, src5, src6;
3592 v8i16 in0, in1, in2, in3;
3593 v16i8 src10_r, src32_r, src54_r, src21_r, src43_r, src65_r;
3594 v16i8 src2110, src4332, src6554;
3595 v8i16 dst10, dst32;
3596 v8i16 filt0, filt1;
3597 v8i16 filter_vec;
3598 v4i32 weight_vec, offset_vec, rnd_vec;
3599
3600 src0_ptr -= src_stride;
3601
3602 offset = offset << rnd_val;
3603 weight0 = weight0 & 0x0000FFFF;
3604 weight = weight0 | (weight1 << 16);
3605 constant = 128 * weight1;
3606 constant <<= 6;
3607 offset += constant;
3608
3609 offset_vec = __msa_fill_w(offset);
3610 weight_vec = __msa_fill_w(weight);
3611 rnd_vec = __msa_fill_w(rnd_val + 1);
3612
3613 filter_vec = LD_SH(filter);
3614 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3615
3616 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3617 src0_ptr += (3 * src_stride);
3618 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3619 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
3620 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
3621
3622 LD_SB4(src0_ptr, src_stride, src3, src4, src5, src6);
3623 src0_ptr += (4 * src_stride);
3624 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3625 src1_ptr += (4 * src2_stride);
3626 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
3627 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
3628 src32_r, src43_r, src54_r, src65_r);
3629 ILVR_D2_SB(src43_r, src32_r, src65_r, src54_r, src4332, src6554);
3630 XORI_B2_128_SB(src4332, src6554);
3631
3632 dst10 = HEVC_FILT_4TAP_SH(src2110, src4332, filt0, filt1);
3633 dst32 = HEVC_FILT_4TAP_SH(src4332, src6554, filt0, filt1);
3634
3635 HEVC_BIW_RND_CLIP2(dst10, dst32, in0, in1,
3636 weight_vec, rnd_vec, offset_vec,
3637 dst10, dst32);
3638
3639 dst10 = (v8i16) __msa_pckev_b((v16i8) dst32, (v16i8) dst10);
3640 ST_W4(dst10, 0, 1, 2, 3, dst, dst_stride);
3641 dst += (4 * dst_stride);
3642}
3643
3644static void hevc_vt_biwgt_4t_4x8multiple_msa(const uint8_t *src0_ptr,
3645 int32_t src_stride,
3646 const int16_t *src1_ptr,
3647 int32_t src2_stride,
3648 uint8_t *dst,
3649 int32_t dst_stride,
3650 const int8_t *filter,
3651 int32_t height,
3652 int32_t weight0,
3653 int32_t weight1,
3654 int32_t offset,
3655 int32_t rnd_val)
3656{
3657 uint32_t loop_cnt;
3658 int32_t weight, constant;
3659 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9;
3660 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
3661 v16i8 src10_r, src32_r, src54_r, src76_r, src98_r;
3662 v16i8 src21_r, src43_r, src65_r, src87_r, src109_r;
3663 v16i8 src2110, src4332, src6554, src8776;
3664 v8i16 dst10, dst32, dst54, dst76;
3665 v8i16 filt0, filt1;
3666 v8i16 filter_vec;
3667 v4i32 weight_vec, offset_vec, rnd_vec;
3668
3669 src0_ptr -= src_stride;
3670
3671 offset = offset << rnd_val;
3672 weight0 = weight0 & 0x0000FFFF;
3673 weight = weight0 | (weight1 << 16);
3674 constant = 128 * weight1;
3675 constant <<= 6;
3676 offset += constant;
3677
3678 offset_vec = __msa_fill_w(offset);
3679 weight_vec = __msa_fill_w(weight);
3680 rnd_vec = __msa_fill_w(rnd_val + 1);
3681
3682 filter_vec = LD_SH(filter);
3683 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3684
3685 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3686 src0_ptr += (3 * src_stride);
3687 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3688 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
3689 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
3690
3691 for (loop_cnt = (height >> 3); loop_cnt--;) {
3692 LD_SB6(src0_ptr, src_stride, src3, src4, src5, src6, src7, src8);
3693 src0_ptr += (6 * src_stride);
3694 LD_SH8(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5, in6, in7);
3695 src1_ptr += (8 * src2_stride);
3696
3697 ILVR_D2_SH(in1, in0, in3, in2, in0, in1);
3698 ILVR_D2_SH(in5, in4, in7, in6, in2, in3);
3699
3700 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
3701 src32_r, src43_r, src54_r, src65_r);
3702 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3703 ILVR_D3_SB(src43_r, src32_r, src65_r, src54_r, src87_r, src76_r,
3704 src4332, src6554, src8776);
3705 XORI_B3_128_SB(src4332, src6554, src8776);
3706
3707 dst10 = HEVC_FILT_4TAP_SH(src2110, src4332, filt0, filt1);
3708 dst32 = HEVC_FILT_4TAP_SH(src4332, src6554, filt0, filt1);
3709 dst54 = HEVC_FILT_4TAP_SH(src6554, src8776, filt0, filt1);
3710
3711 LD_SB2(src0_ptr, src_stride, src9, src2);
3712 src0_ptr += (2 * src_stride);
3713 ILVR_B2_SB(src9, src8, src2, src9, src98_r, src109_r);
3714 src2110 = (v16i8) __msa_ilvr_d((v2i64) src109_r, (v2i64) src98_r);
3715 src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
3716
3717 dst76 = HEVC_FILT_4TAP_SH(src8776, src2110, filt0, filt1);
3718 HEVC_BIW_RND_CLIP4(dst10, dst32, dst54, dst76,
3719 in0, in1, in2, in3,
3720 weight_vec, rnd_vec, offset_vec,
3721 dst10, dst32, dst54, dst76);
3722
3723 PCKEV_B2_SH(dst32, dst10, dst76, dst54, dst10, dst32);
3724 ST_W8(dst10, dst32, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
3725 dst += (8 * dst_stride);
3726 }
3727}
3728
3729static void hevc_vt_biwgt_4t_4w_msa(const uint8_t *src0_ptr,
3730 int32_t src_stride,
3731 const int16_t *src1_ptr,
3732 int32_t src2_stride,
3733 uint8_t *dst,
3734 int32_t dst_stride,
3735 const int8_t *filter,
3736 int32_t height,
3737 int32_t weight0,
3738 int32_t weight1,
3739 int32_t offset,
3740 int32_t rnd_val)
3741{
3742 if (2 == height) {
3743 hevc_vt_biwgt_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
3744 dst, dst_stride, filter,
3745 weight0, weight1, offset, rnd_val);
3746 } else if (4 == height) {
3747 hevc_vt_biwgt_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
3748 dst, dst_stride, filter,
3749 weight0, weight1, offset, rnd_val);
3750 } else if (0 == (height % 8)) {
3751 hevc_vt_biwgt_4t_4x8multiple_msa(src0_ptr, src_stride,
3752 src1_ptr, src2_stride,
3753 dst, dst_stride, filter, height,
3754 weight0, weight1, offset,
3755 rnd_val);
3756 }
3757}
3758
3759static void hevc_vt_biwgt_4t_6w_msa(const uint8_t *src0_ptr,
3760 int32_t src_stride,
3761 const int16_t *src1_ptr,
3762 int32_t src2_stride,
3763 uint8_t *dst,
3764 int32_t dst_stride,
3765 const int8_t *filter,
3766 int32_t height,
3767 int32_t weight0,
3768 int32_t weight1,
3769 int32_t offset,
3770 int32_t rnd_val)
3771{
3772 uint32_t loop_cnt;
3773 int32_t res = height & 0x03;
3774 int32_t weight, constant;
3775 v16i8 src0, src1, src2, src3, src4;
3776 v8i16 in0, in1, in2, in3;
3777 v16i8 src10_r, src32_r, src21_r, src43_r;
3778 v8i16 tmp0, tmp1, tmp2, tmp3;
3779 v8i16 filt0, filt1;
3780 v8i16 filter_vec;
3781 v4i32 weight_vec, offset_vec, rnd_vec;
3782
3783 src0_ptr -= src_stride;
3784
3785 offset = offset << rnd_val;
3786 weight0 = weight0 & 0x0000FFFF;
3787 weight = weight0 | (weight1 << 16);
3788 constant = 128 * weight1;
3789 constant <<= 6;
3790 offset += constant;
3791
3792 offset_vec = __msa_fill_w(offset);
3793 weight_vec = __msa_fill_w(weight);
3794 rnd_vec = __msa_fill_w(rnd_val + 1);
3795
3796 filter_vec = LD_SH(filter);
3797 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3798
3799 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3800 src0_ptr += (3 * src_stride);
3801 XORI_B3_128_SB(src0, src1, src2);
3802 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3803
3804 for (loop_cnt = (height >> 2); loop_cnt--;) {
3805 LD_SB2(src0_ptr, src_stride, src3, src4);
3806 src0_ptr += (2 * src_stride);
3807 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3808 src1_ptr += (4 * src2_stride);
3809 XORI_B2_128_SB(src3, src4);
3810 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3811
3812 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
3813 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
3814
3815 LD_SB2(src0_ptr, src_stride, src1, src2);
3816 src0_ptr += (2 * src_stride);
3817 XORI_B2_128_SB(src1, src2);
3818 ILVR_B2_SB(src1, src4, src2, src1, src10_r, src21_r);
3819
3820 tmp2 = HEVC_FILT_4TAP_SH(src32_r, src10_r, filt0, filt1);
3821 tmp3 = HEVC_FILT_4TAP_SH(src43_r, src21_r, filt0, filt1);
3822 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
3823 in0, in1, in2, in3,
3824 weight_vec, rnd_vec, offset_vec,
3825 tmp0, tmp1, tmp2, tmp3);
3826
3827 PCKEV_B2_SH(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
3828 ST_W2(tmp0, 0, 2, dst, dst_stride);
3829 ST_H2(tmp0, 2, 6, dst + 4, dst_stride);
3830 ST_W2(tmp1, 0, 2, dst + 2 * dst_stride, dst_stride);
3831 ST_H2(tmp1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
3832 dst += (4 * dst_stride);
3833 }
3834 if (res) {
3835 LD_SB2(src0_ptr, src_stride, src3, src4);
3836 src0_ptr += (2 * src_stride);
3837 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
3838 src1_ptr += (4 * src2_stride);
3839 XORI_B2_128_SB(src3, src4);
3840 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3841
3842 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
3843 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
3844
3845 LD_SB2(src0_ptr, src_stride, src1, src2);
3846 src0_ptr += (2 * src_stride);
3847 XORI_B2_128_SB(src1, src2);
3848 ILVR_B2_SB(src1, src4, src2, src1, src10_r, src21_r);
3849
3850 tmp2 = HEVC_FILT_4TAP_SH(src32_r, src10_r, filt0, filt1);
3851 tmp3 = HEVC_FILT_4TAP_SH(src43_r, src21_r, filt0, filt1);
3852 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
3853 in0, in1, in2, in3,
3854 weight_vec, rnd_vec, offset_vec,
3855 tmp0, tmp1, tmp2, tmp3);
3856
3857 PCKEV_B2_SH(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
3858 ST_W2(tmp0, 0, 2, dst, dst_stride);
3859 ST_H2(tmp0, 2, 6, dst + 4, dst_stride);
3860 ST_W2(tmp1, 0, 2, dst + 2 * dst_stride, dst_stride);
3861 ST_H2(tmp1, 2, 6, dst + 2 * dst_stride + 4, dst_stride);
3862 }
3863}
3864
3865static void hevc_vt_biwgt_4t_8x2_msa(const uint8_t *src0_ptr,
3866 int32_t src_stride,
3867 const int16_t *src1_ptr,
3868 int32_t src2_stride,
3869 uint8_t *dst,
3870 int32_t dst_stride,
3871 const int8_t *filter,
3872 int32_t weight0,
3873 int32_t weight1,
3874 int32_t offset,
3875 int32_t rnd_val)
3876{
3877 int32_t weight, constant;
3878 v16i8 src0, src1, src2, src3, src4;
3879 v8i16 in0, in1, tmp0, tmp1;
3880 v16i8 src10_r, src32_r, src21_r, src43_r;
3881 v8i16 filt0, filt1;
3882 v8i16 filter_vec;
3883 v4i32 weight_vec, offset_vec, rnd_vec;
3884
3885 src0_ptr -= src_stride;
3886
3887 offset = offset << rnd_val;
3888 weight0 = weight0 & 0x0000FFFF;
3889 weight = weight0 | (weight1 << 16);
3890 constant = 128 * weight1;
3891 constant <<= 6;
3892 offset += constant;
3893
3894 offset_vec = __msa_fill_w(offset);
3895 weight_vec = __msa_fill_w(weight);
3896 rnd_vec = __msa_fill_w(rnd_val + 1);
3897
3898 filter_vec = LD_SH(filter);
3899 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3900
3901 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3902 src0_ptr += (3 * src_stride);
3903 XORI_B3_128_SB(src0, src1, src2);
3904 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3905
3906 LD_SB2(src0_ptr, src_stride, src3, src4);
3907 LD_SH2(src1_ptr, src2_stride, in0, in1);
3908 XORI_B2_128_SB(src3, src4);
3909 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3910
3911 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
3912 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
3913 HEVC_BIW_RND_CLIP2(tmp0, tmp1, in0, in1,
3914 weight_vec, rnd_vec, offset_vec,
3915 tmp0, tmp1);
3916
3917 tmp0 = (v8i16) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
3918 ST_D2(tmp0, 0, 1, dst, dst_stride);
3919}
3920
3921static void hevc_vt_biwgt_4t_8x6_msa(const uint8_t *src0_ptr,
3922 int32_t src_stride,
3923 const int16_t *src1_ptr,
3924 int32_t src2_stride,
3925 uint8_t *dst,
3926 int32_t dst_stride,
3927 const int8_t *filter,
3928 int32_t weight0,
3929 int32_t weight1,
3930 int32_t offset,
3931 int32_t rnd_val)
3932{
3933 int32_t weight, constant;
3934 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3935 v8i16 in0, in1, in2, in3, in4, in5;
3936 v16i8 src10_r, src32_r, src54_r, src76_r;
3937 v16i8 src21_r, src43_r, src65_r, src87_r;
3938 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
3939 v8i16 filt0, filt1;
3940 v8i16 filter_vec;
3941 v4i32 weight_vec, offset_vec, rnd_vec;
3942
3943 src0_ptr -= src_stride;
3944
3945 offset = offset << rnd_val;
3946 weight0 = weight0 & 0x0000FFFF;
3947 weight = weight0 | (weight1 << 16);
3948 constant = 128 * weight1;
3949 constant <<= 6;
3950 offset += constant;
3951
3952 offset_vec = __msa_fill_w(offset);
3953 weight_vec = __msa_fill_w(weight);
3954 rnd_vec = __msa_fill_w(rnd_val + 1);
3955
3956 filter_vec = LD_SH(filter);
3957 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3958
3959 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
3960 src0_ptr += (3 * src_stride);
3961 XORI_B3_128_SB(src0, src1, src2);
3962 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3963
3964 LD_SB6(src0_ptr, src_stride, src3, src4, src5, src6, src7, src8);
3965 LD_SH6(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5);
3966 XORI_B6_128_SB(src3, src4, src5, src6, src7, src8);
3967 ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
3968 src32_r, src43_r, src54_r, src65_r);
3969 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3970
3971 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
3972 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
3973 tmp2 = HEVC_FILT_4TAP_SH(src32_r, src54_r, filt0, filt1);
3974 tmp3 = HEVC_FILT_4TAP_SH(src43_r, src65_r, filt0, filt1);
3975 tmp4 = HEVC_FILT_4TAP_SH(src54_r, src76_r, filt0, filt1);
3976 tmp5 = HEVC_FILT_4TAP_SH(src65_r, src87_r, filt0, filt1);
3977 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
3978 in0, in1, in2, in3,
3979 weight_vec, rnd_vec, offset_vec,
3980 tmp0, tmp1, tmp2, tmp3);
3981 HEVC_BIW_RND_CLIP2(tmp4, tmp5, in4, in5,
3982 weight_vec, rnd_vec, offset_vec,
3983 tmp4, tmp5);
3984
3985 PCKEV_B2_SH(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
3986 tmp3 = (v8i16) __msa_pckev_b((v16i8) tmp5, (v16i8) tmp4);
3987 ST_D4(tmp0, tmp1, 0, 1, 0, 1, dst, dst_stride);
3988 ST_D2(tmp3, 0, 1, dst + 4 * dst_stride, dst_stride);
3989}
3990
3991static void hevc_vt_biwgt_4t_8x4multiple_msa(const uint8_t *src0_ptr,
3992 int32_t src_stride,
3993 const int16_t *src1_ptr,
3994 int32_t src2_stride,
3995 uint8_t *dst,
3996 int32_t dst_stride,
3997 const int8_t *filter,
3998 int32_t height,
3999 int32_t weight0,
4000 int32_t weight1,
4001 int32_t offset,
4002 int32_t rnd_val)
4003{
4004 uint32_t loop_cnt;
4005 int32_t weight, constant;
4006 v16i8 src0, src1, src2, src3, src4;
4007 v8i16 in0, in1, in2, in3;
4008 v16i8 src10_r, src32_r, src21_r, src43_r;
4009 v8i16 tmp0, tmp1, tmp2, tmp3;
4010 v8i16 filt0, filt1;
4011 v8i16 filter_vec;
4012 v4i32 weight_vec, offset_vec, rnd_vec;
4013
4014 src0_ptr -= src_stride;
4015
4016 offset = offset << rnd_val;
4017 weight0 = weight0 & 0x0000FFFF;
4018 weight = weight0 | (weight1 << 16);
4019 constant = 128 * weight1;
4020 constant <<= 6;
4021 offset += constant;
4022
4023 offset_vec = __msa_fill_w(offset);
4024 weight_vec = __msa_fill_w(weight);
4025 rnd_vec = __msa_fill_w(rnd_val + 1);
4026
4027 filter_vec = LD_SH(filter);
4028 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4029
4030 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4031 src0_ptr += (3 * src_stride);
4032 XORI_B3_128_SB(src0, src1, src2);
4033 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4034
4035 for (loop_cnt = (height >> 2); loop_cnt--;) {
4036 LD_SB2(src0_ptr, src_stride, src3, src4);
4037 src0_ptr += (2 * src_stride);
4038 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
4039 src1_ptr += (4 * src2_stride);
4040 XORI_B2_128_SB(src3, src4);
4041 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4042
4043 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
4044 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
4045
4046 LD_SB2(src0_ptr, src_stride, src1, src2);
4047 src0_ptr += (2 * src_stride);
4048 XORI_B2_128_SB(src1, src2);
4049 ILVR_B2_SB(src1, src4, src2, src1, src10_r, src21_r);
4050
4051 tmp2 = HEVC_FILT_4TAP_SH(src32_r, src10_r, filt0, filt1);
4052 tmp3 = HEVC_FILT_4TAP_SH(src43_r, src21_r, filt0, filt1);
4053 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
4054 in0, in1, in2, in3,
4055 weight_vec, rnd_vec, offset_vec,
4056 tmp0, tmp1, tmp2, tmp3);
4057
4058 PCKEV_B2_SH(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
4059 ST_D4(tmp0, tmp1, 0, 1, 0, 1, dst, dst_stride);
4060 dst += (4 * dst_stride);
4061 }
4062}
4063
4064static void hevc_vt_biwgt_4t_8w_msa(const uint8_t *src0_ptr,
4065 int32_t src_stride,
4066 const int16_t *src1_ptr,
4067 int32_t src2_stride,
4068 uint8_t *dst,
4069 int32_t dst_stride,
4070 const int8_t *filter,
4071 int32_t height,
4072 int32_t weight0,
4073 int32_t weight1,
4074 int32_t offset,
4075 int32_t rnd_val)
4076{
4077 if (2 == height) {
4078 hevc_vt_biwgt_4t_8x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4079 dst, dst_stride, filter,
4080 weight0, weight1, offset, rnd_val);
4081 } else if (6 == height) {
4082 hevc_vt_biwgt_4t_8x6_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4083 dst, dst_stride, filter,
4084 weight0, weight1, offset, rnd_val);
4085 } else {
4086 hevc_vt_biwgt_4t_8x4multiple_msa(src0_ptr, src_stride,
4087 src1_ptr, src2_stride,
4088 dst, dst_stride, filter, height,
4089 weight0, weight1, offset,
4090 rnd_val);
4091 }
4092}
4093
4094static void hevc_vt_biwgt_4t_12w_msa(const uint8_t *src0_ptr,
4095 int32_t src_stride,
4096 const int16_t *src1_ptr,
4097 int32_t src2_stride,
4098 uint8_t *dst,
4099 int32_t dst_stride,
4100 const int8_t *filter,
4101 int32_t height,
4102 int32_t weight0,
4103 int32_t weight1,
4104 int32_t offset,
4105 int32_t rnd_val)
4106{
4107 uint32_t loop_cnt;
4108 int32_t weight, constant;
4109 v16i8 src0, src1, src2, src3, src4, src5;
4110 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
4111 v16i8 src10_r, src32_r, src21_r, src43_r;
4112 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
4113 v16i8 src10_l, src32_l, src54_l, src21_l, src43_l, src65_l;
4114 v16i8 src2110, src4332;
4115 v8i16 filt0, filt1;
4116 v8i16 filter_vec;
4117 v4i32 weight_vec, offset_vec, rnd_vec;
4118
4119 src0_ptr -= (1 * src_stride);
4120
4121 offset = offset << rnd_val;
4122 weight0 = weight0 & 0x0000FFFF;
4123 weight = weight0 | (weight1 << 16);
4124 constant = 128 * weight1;
4125 constant <<= 6;
4126 offset += constant;
4127
4128 offset_vec = __msa_fill_w(offset);
4129 weight_vec = __msa_fill_w(weight);
4130 rnd_vec = __msa_fill_w(rnd_val + 1);
4131
4132 filter_vec = LD_SH(filter);
4133 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4134
4135 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4136 src0_ptr += (3 * src_stride);
4137 XORI_B3_128_SB(src0, src1, src2);
4138 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4139 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
4140 src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_l, (v2i64) src10_l);
4141
4142 for (loop_cnt = (height >> 2); loop_cnt--;) {
4143 LD_SB2(src0_ptr, src_stride, src3, src4);
4144 src0_ptr += (2 * src_stride);
4145 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
4146 LD_SH4(src1_ptr + 8, src2_stride, in4, in5, in6, in7);
4147 src1_ptr += (4 * src2_stride);
4148 ILVR_D2_SH(in5, in4, in7, in6, in4, in5);
4149 XORI_B2_128_SB(src3, src4);
4150
4151 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4152 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
4153 src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_l, (v2i64) src32_l);
4154
4155 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
4156 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
4157 tmp4 = HEVC_FILT_4TAP_SH(src2110, src4332, filt0, filt1);
4158
4159 LD_SB2(src0_ptr, src_stride, src5, src2);
4160 src0_ptr += (2 * src_stride);
4161 XORI_B2_128_SB(src5, src2);
4162 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
4163 ILVL_B2_SB(src5, src4, src2, src5, src54_l, src65_l);
4164 src2110 = (v16i8) __msa_ilvr_d((v2i64) src65_l, (v2i64) src54_l);
4165
4166 tmp2 = HEVC_FILT_4TAP_SH(src32_r, src10_r, filt0, filt1);
4167 tmp3 = HEVC_FILT_4TAP_SH(src43_r, src21_r, filt0, filt1);
4168 tmp5 = HEVC_FILT_4TAP_SH(src4332, src2110, filt0, filt1);
4169 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
4170 in0, in1, in2, in3,
4171 weight_vec, rnd_vec, offset_vec,
4172 tmp0, tmp1, tmp2, tmp3);
4173 HEVC_BIW_RND_CLIP2(tmp4, tmp5, in4, in5,
4174 weight_vec, rnd_vec, offset_vec,
4175 tmp4, tmp5);
4176
4177 PCKEV_B2_SH(tmp1, tmp0, tmp3, tmp2, tmp0, tmp1);
4178 tmp2 = (v8i16) __msa_pckev_b((v16i8) tmp5, (v16i8) tmp4);
4179 ST_D4(tmp0, tmp1, 0, 1, 0, 1, dst, dst_stride);
4180 ST_W4(tmp2, 0, 1, 2, 3, dst + 8, dst_stride);
4181 dst += (4 * dst_stride);
4182 }
4183}
4184
4185static void hevc_vt_biwgt_4t_16w_msa(const uint8_t *src0_ptr,
4186 int32_t src_stride,
4187 const int16_t *src1_ptr,
4188 int32_t src2_stride,
4189 uint8_t *dst,
4190 int32_t dst_stride,
4191 const int8_t *filter,
4192 int32_t height,
4193 int32_t weight0,
4194 int32_t weight1,
4195 int32_t offset,
4196 int32_t rnd_val)
4197{
4198 uint32_t loop_cnt;
4199 int32_t weight, constant;
4200 v16i8 src0, src1, src2, src3, src4, src5;
4201 v8i16 in0, in1, in2, in3;
4202 v16i8 src10_r, src32_r, src21_r, src43_r;
4203 v16i8 src10_l, src32_l, src21_l, src43_l;
4204 v8i16 tmp0, tmp1, tmp2, tmp3;
4205 v8i16 filt0, filt1;
4206 v8i16 filter_vec;
4207 v4i32 weight_vec, offset_vec, rnd_vec;
4208
4209 src0_ptr -= src_stride;
4210
4211 offset = offset << rnd_val;
4212 weight0 = weight0 & 0x0000FFFF;
4213 weight = weight0 | (weight1 << 16);
4214 constant = 128 * weight1;
4215 constant <<= 6;
4216 offset += constant;
4217
4218 offset_vec = __msa_fill_w(offset);
4219 weight_vec = __msa_fill_w(weight);
4220 rnd_vec = __msa_fill_w(rnd_val + 1);
4221
4222 filter_vec = LD_SH(filter);
4223 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4224
4225 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4226 src0_ptr += (3 * src_stride);
4227 XORI_B3_128_SB(src0, src1, src2);
4228 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4229 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
4230
4231 for (loop_cnt = (height >> 2); loop_cnt--;) {
4232 LD_SB2(src0_ptr, src_stride, src3, src4);
4233 src0_ptr += (2 * src_stride);
4234 LD_SH2(src1_ptr, src2_stride, in0, in1);
4235 LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
4236 src1_ptr += (2 * src2_stride);
4237 XORI_B2_128_SB(src3, src4);
4238 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4239 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
4240
4241 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
4242 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
4243 tmp2 = HEVC_FILT_4TAP_SH(src10_l, src32_l, filt0, filt1);
4244 tmp3 = HEVC_FILT_4TAP_SH(src21_l, src43_l, filt0, filt1);
4245
4246 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
4247 in0, in1, in2, in3,
4248 weight_vec, rnd_vec, offset_vec,
4249 tmp0, tmp1, tmp2, tmp3);
4250 PCKEV_B2_SH(tmp2, tmp0, tmp3, tmp1, tmp0, tmp1);
4251 ST_SH2(tmp0, tmp1, dst, dst_stride);
4252 dst += (2 * dst_stride);
4253 LD_SB2(src0_ptr, src_stride, src5, src2);
4254 src0_ptr += (2 * src_stride);
4255
4256 LD_SH2(src1_ptr, src2_stride, in0, in1);
4257 LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
4258 src1_ptr += (2 * src2_stride);
4259 XORI_B2_128_SB(src5, src2);
4260 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
4261 ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
4262
4263 tmp0 = HEVC_FILT_4TAP_SH(src32_r, src10_r, filt0, filt1);
4264 tmp1 = HEVC_FILT_4TAP_SH(src43_r, src21_r, filt0, filt1);
4265 tmp2 = HEVC_FILT_4TAP_SH(src32_l, src10_l, filt0, filt1);
4266 tmp3 = HEVC_FILT_4TAP_SH(src43_l, src21_l, filt0, filt1);
4267 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp2, tmp3,
4268 in0, in1, in2, in3,
4269 weight_vec, rnd_vec, offset_vec,
4270 tmp0, tmp1, tmp2, tmp3);
4271
4272 PCKEV_B2_SH(tmp2, tmp0, tmp3, tmp1, tmp0, tmp1);
4273 ST_SH2(tmp0, tmp1, dst, dst_stride);
4274 dst += (2 * dst_stride);
4275 }
4276}
4277
4278static void hevc_vt_biwgt_4t_24w_msa(const uint8_t *src0_ptr,
4279 int32_t src_stride,
4280 const int16_t *src1_ptr,
4281 int32_t src2_stride,
4282 uint8_t *dst,
4283 int32_t dst_stride,
4284 const int8_t *filter,
4285 int32_t height,
4286 int32_t weight0,
4287 int32_t weight1,
4288 int32_t offset,
4289 int32_t rnd_val)
4290{
4291 uint32_t loop_cnt;
4292 int32_t weight, constant;
4293 v16i8 src0, src1, src2, src3, src4, src5;
4294 v16i8 src6, src7, src8, src9, src10, src11;
4295 v8i16 in0, in1, in2, in3, in4, in5;
4296 v16i8 src10_r, src32_r, src76_r, src98_r;
4297 v16i8 src10_l, src32_l, src21_l, src43_l;
4298 v16i8 src21_r, src43_r, src87_r, src109_r;
4299 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
4300 v8i16 filt0, filt1;
4301 v8i16 filter_vec;
4302 v4i32 weight_vec, offset_vec, rnd_vec;
4303
4304 src0_ptr -= src_stride;
4305
4306 offset = offset << rnd_val;
4307 weight0 = weight0 & 0x0000FFFF;
4308 weight = weight0 | (weight1 << 16);
4309 constant = 128 * weight1;
4310 constant <<= 6;
4311 offset += constant;
4312
4313 offset_vec = __msa_fill_w(offset);
4314 weight_vec = __msa_fill_w(weight);
4315 rnd_vec = __msa_fill_w(rnd_val + 1);
4316
4317 filter_vec = LD_SH(filter);
4318 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4319
4320 /* 16width */
4321 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4322 XORI_B3_128_SB(src0, src1, src2);
4323 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4324 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
4325 /* 8width */
4326 LD_SB3(src0_ptr + 16, src_stride, src6, src7, src8);
4327 src0_ptr += (3 * src_stride);
4328 XORI_B3_128_SB(src6, src7, src8);
4329 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
4330
4331 for (loop_cnt = (height >> 2); loop_cnt--;) {
4332 /* 16width */
4333 LD_SB2(src0_ptr, src_stride, src3, src4);
4334 LD_SH2(src1_ptr, src2_stride, in0, in1);
4335 LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
4336 XORI_B2_128_SB(src3, src4);
4337 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4338 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
4339
4340 /* 8width */
4341 LD_SB2(src0_ptr + 16, src_stride, src9, src10);
4342 src0_ptr += (2 * src_stride);
4343 LD_SH2(src1_ptr + 16, src2_stride, in4, in5);
4344 src1_ptr += (2 * src2_stride);
4345 XORI_B2_128_SB(src9, src10);
4346 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
4347 /* 16width */
4348 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
4349 tmp4 = HEVC_FILT_4TAP_SH(src10_l, src32_l, filt0, filt1);
4350 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
4351 tmp5 = HEVC_FILT_4TAP_SH(src21_l, src43_l, filt0, filt1);
4352 /* 8width */
4353 tmp2 = HEVC_FILT_4TAP_SH(src76_r, src98_r, filt0, filt1);
4354 tmp3 = HEVC_FILT_4TAP_SH(src87_r, src109_r, filt0, filt1);
4355 /* 16width */
4356 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp4, tmp5,
4357 in0, in1, in2, in3,
4358 weight_vec, rnd_vec, offset_vec,
4359 tmp0, tmp1, tmp4, tmp5);
4360 /* 8width */
4361 HEVC_BIW_RND_CLIP2(tmp2, tmp3, in4, in5,
4362 weight_vec, rnd_vec, offset_vec,
4363 tmp2, tmp3);
4364 /* 16width */
4365 PCKEV_B2_SH(tmp4, tmp0, tmp5, tmp1, tmp0, tmp1);
4366 /* 8width */
4367 tmp2 = (v8i16) __msa_pckev_b((v16i8) tmp3, (v16i8) tmp2);
4368 ST_SH2(tmp0, tmp1, dst, dst_stride);
4369 ST_D2(tmp2, 0, 1, dst + 16, dst_stride);
4370 dst += (2 * dst_stride);
4371
4372 /* 16width */
4373 LD_SB2(src0_ptr, src_stride, src5, src2);
4374 LD_SH2(src1_ptr, src2_stride, in0, in1);
4375 LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
4376 XORI_B2_128_SB(src5, src2);
4377 ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
4378 ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
4379 /* 8width */
4380 LD_SB2(src0_ptr + 16, src_stride, src11, src8);
4381 src0_ptr += (2 * src_stride);
4382 LD_SH2(src1_ptr + 16, src2_stride, in4, in5);
4383 src1_ptr += (2 * src2_stride);
4384 XORI_B2_128_SB(src11, src8);
4385 ILVR_B2_SB(src11, src10, src8, src11, src76_r, src87_r);
4386 /* 16width */
4387 tmp0 = HEVC_FILT_4TAP_SH(src32_r, src10_r, filt0, filt1);
4388 tmp4 = HEVC_FILT_4TAP_SH(src32_l, src10_l, filt0, filt1);
4389 tmp1 = HEVC_FILT_4TAP_SH(src43_r, src21_r, filt0, filt1);
4390 tmp5 = HEVC_FILT_4TAP_SH(src43_l, src21_l, filt0, filt1);
4391 /* 8width */
4392 tmp2 = HEVC_FILT_4TAP_SH(src98_r, src76_r, filt0, filt1);
4393 tmp3 = HEVC_FILT_4TAP_SH(src109_r, src87_r, filt0, filt1);
4394 /* 16width */
4395 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp4, tmp5,
4396 in0, in1, in2, in3,
4397 weight_vec, rnd_vec, offset_vec,
4398 tmp0, tmp1, tmp4, tmp5);
4399 /* 8width */
4400 HEVC_BIW_RND_CLIP2(tmp2, tmp3, in4, in5,
4401 weight_vec, rnd_vec, offset_vec,
4402 tmp2, tmp3);
4403 /* 16width */
4404 PCKEV_B2_SH(tmp4, tmp0, tmp5, tmp1, tmp0, tmp1);
4405
4406 /* 8width */
4407 tmp2 = (v8i16) __msa_pckev_b((v16i8) tmp3, (v16i8) tmp2);
4408 ST_SH2(tmp0, tmp1, dst, dst_stride);
4409 ST_D2(tmp2, 0, 1, dst + 16, dst_stride);
4410 dst += (2 * dst_stride);
4411 }
4412}
4413
4414static void hevc_vt_biwgt_4t_32w_msa(const uint8_t *src0_ptr,
4415 int32_t src_stride,
4416 const int16_t *src1_ptr,
4417 int32_t src2_stride,
4418 uint8_t *dst,
4419 int32_t dst_stride,
4420 const int8_t *filter,
4421 int32_t height,
4422 int32_t weight0,
4423 int32_t weight1,
4424 int32_t offset,
4425 int32_t rnd_val)
4426{
4427 uint32_t loop_cnt;
4428 uint8_t *dst_tmp = dst + 16;
4429 int32_t weight, constant;
4430 v16i8 src0, src1, src2, src3, src4, src6, src7, src8, src9, src10;
4431 v8i16 in0, in1, in2, in3, in4, in5, in6, in7;
4432 v16i8 src10_r, src32_r, src76_r, src98_r;
4433 v16i8 src21_r, src43_r, src87_r, src109_r;
4434 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4435 v16i8 src10_l, src32_l, src76_l, src98_l;
4436 v16i8 src21_l, src43_l, src87_l, src109_l;
4437 v8i16 filt0, filt1;
4438 v8i16 filter_vec;
4439 v4i32 weight_vec, offset_vec, rnd_vec;
4440
4441 src0_ptr -= src_stride;
4442
4443 offset = offset << rnd_val;
4444 weight0 = weight0 & 0x0000FFFF;
4445 weight = weight0 | (weight1 << 16);
4446 constant = 128 * weight1;
4447 constant <<= 6;
4448 offset += constant;
4449
4450 offset_vec = __msa_fill_w(offset);
4451 weight_vec = __msa_fill_w(weight);
4452 rnd_vec = __msa_fill_w(rnd_val + 1);
4453
4454 filter_vec = LD_SH(filter);
4455 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4456
4457 /* 16width */
4458 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4459 XORI_B3_128_SB(src0, src1, src2);
4460 ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
4461 ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
4462 /* next 16width */
4463 LD_SB3(src0_ptr + 16, src_stride, src6, src7, src8);
4464 src0_ptr += (3 * src_stride);
4465 XORI_B3_128_SB(src6, src7, src8);
4466 ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
4467 ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
4468
4469 for (loop_cnt = (height >> 1); loop_cnt--;) {
4470 /* 16width */
4471 LD_SB2(src0_ptr, src_stride, src3, src4);
4472 LD_SH2(src1_ptr, src2_stride, in0, in1);
4473 LD_SH2(src1_ptr + 8, src2_stride, in2, in3);
4474 XORI_B2_128_SB(src3, src4);
4475 ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
4476 ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
4477
4478 /* 16width */
4479 tmp0 = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
4480 tmp4 = HEVC_FILT_4TAP_SH(src10_l, src32_l, filt0, filt1);
4481 tmp1 = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
4482 tmp5 = HEVC_FILT_4TAP_SH(src21_l, src43_l, filt0, filt1);
4483 /* 16width */
4484 HEVC_BIW_RND_CLIP4(tmp0, tmp1, tmp4, tmp5,
4485 in0, in1, in2, in3,
4486 weight_vec, rnd_vec, offset_vec,
4487 tmp0, tmp1, tmp4, tmp5);
4488 /* 16width */
4489 PCKEV_B2_SH(tmp4, tmp0, tmp5, tmp1, tmp0, tmp1);
4490 ST_SH2(tmp0, tmp1, dst, dst_stride);
4491 dst += (2 * dst_stride);
4492
4493 src10_r = src32_r;
4494 src21_r = src43_r;
4495 src10_l = src32_l;
4496 src21_l = src43_l;
4497 src2 = src4;
4498
4499 /* next 16width */
4500 LD_SB2(src0_ptr + 16, src_stride, src9, src10);
4501 src0_ptr += (2 * src_stride);
4502 LD_SH2(src1_ptr + 16, src2_stride, in4, in5);
4503 LD_SH2(src1_ptr + 24, src2_stride, in6, in7);
4504 src1_ptr += (2 * src2_stride);
4505 XORI_B2_128_SB(src9, src10);
4506 ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
4507 ILVL_B2_SB(src9, src8, src10, src9, src98_l, src109_l);
4508 /* next 16width */
4509 tmp2 = HEVC_FILT_4TAP_SH(src76_r, src98_r, filt0, filt1);
4510 tmp6 = HEVC_FILT_4TAP_SH(src76_l, src98_l, filt0, filt1);
4511 tmp3 = HEVC_FILT_4TAP_SH(src87_r, src109_r, filt0, filt1);
4512 tmp7 = HEVC_FILT_4TAP_SH(src87_l, src109_l, filt0, filt1);
4513 /* next 16width */
4514 HEVC_BIW_RND_CLIP4(tmp2, tmp3, tmp6, tmp7,
4515 in4, in5, in6, in7,
4516 weight_vec, rnd_vec, offset_vec,
4517 tmp2, tmp3, tmp6, tmp7);
4518
4519 /* next 16width */
4520 PCKEV_B2_SH(tmp6, tmp2, tmp7, tmp3, tmp2, tmp3);
4521 ST_SH2(tmp2, tmp3, dst_tmp, dst_stride);
4522 dst_tmp += (2 * dst_stride);
4523
4524 src76_r = src98_r;
4525 src87_r = src109_r;
4526 src76_l = src98_l;
4527 src87_l = src109_l;
4528 src8 = src10;
4529 }
4530}
4531
4532static void hevc_hv_biwgt_4t_4x2_msa(const uint8_t *src0_ptr,
4533 int32_t src_stride,
4534 const int16_t *src1_ptr,
4535 int32_t src2_stride,
4536 uint8_t *dst,
4537 int32_t dst_stride,
4538 const int8_t *filter_x,
4539 const int8_t *filter_y,
4540 int32_t weight0,
4541 int32_t weight1,
4542 int32_t offset,
4543 int32_t rnd_val)
4544{
4545 uint64_t tp0, tp1;
4546 int32_t weight;
4547 v8i16 in0 = { 0 };
4548 v16u8 out;
4549 v16i8 src0, src1, src2, src3, src4;
4550 v8i16 filt0, filt1;
4551 v8i16 filt_h0, filt_h1;
4552 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
4553 v16i8 mask1;
4554 v8i16 filter_vec, tmp, weight_vec;
4555 v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
4556 v8i16 dst20, dst31, dst42, dst10, dst32, dst21, dst43, tmp0, tmp1;
4557 v4i32 dst0, dst1, offset_vec, rnd_vec, const_vec;
4558
4559 src0_ptr -= (src_stride + 1);
4560
4561 filter_vec = LD_SH(filter_x);
4562 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4563
4564 filter_vec = LD_SH(filter_y);
4565 UNPCK_R_SB_SH(filter_vec, filter_vec);
4566
4567 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
4568
4569 mask1 = mask0 + 2;
4570
4571 offset = offset << rnd_val;
4572 weight0 = weight0 & 0x0000FFFF;
4573 weight = weight0 | (weight1 << 16);
4574
4575 const_vec = __msa_fill_w((128 * weight1));
4576 const_vec <<= 6;
4577 offset_vec = __msa_fill_w(offset);
4578 weight_vec = (v8i16) __msa_fill_w(weight);
4579 rnd_vec = __msa_fill_w(rnd_val + 1);
4580 offset_vec += const_vec;
4581
4582 LD_SB5(src0_ptr, src_stride, src0, src1, src2, src3, src4);
4583 XORI_B5_128_SB(src0, src1, src2, src3, src4);
4584
4585 VSHF_B2_SB(src0, src2, src0, src2, mask0, mask1, vec0, vec1);
4586 VSHF_B2_SB(src1, src3, src1, src3, mask0, mask1, vec2, vec3);
4587 VSHF_B2_SB(src2, src4, src2, src4, mask0, mask1, vec4, vec5);
4588
4589 dst20 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
4590 dst31 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
4591 dst42 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
4592
4593 ILVRL_H2_SH(dst31, dst20, dst10, dst32);
4594 ILVRL_H2_SH(dst42, dst31, dst21, dst43);
4595
4596 dst0 = HEVC_FILT_4TAP(dst10, dst32, filt_h0, filt_h1);
4597 dst1 = HEVC_FILT_4TAP(dst21, dst43, filt_h0, filt_h1);
4598 dst0 >>= 6;
4599 dst1 >>= 6;
4600 dst0 = (v4i32) __msa_pckev_h((v8i16) dst1, (v8i16) dst0);
4601
4602 LD2(src1_ptr, src2_stride, tp0, tp1);
4603 INSERT_D2_SH(tp0, tp1, in0);
4604
4605 ILVRL_H2_SH(dst0, in0, tmp0, tmp1);
4606 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
4607 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
4608 SRAR_W2_SW(dst0, dst1, rnd_vec);
4609 CLIP_SW2_0_255(dst0, dst1);
4610 tmp = __msa_pckev_h((v8i16) dst1, (v8i16) dst0);
4611 out = (v16u8) __msa_pckev_b((v16i8) tmp, (v16i8) tmp);
4612 ST_W2(out, 0, 1, dst, dst_stride);
4613}
4614
4615static void hevc_hv_biwgt_4t_4x4_msa(const uint8_t *src0_ptr,
4616 int32_t src_stride,
4617 const int16_t *src1_ptr,
4618 int32_t src2_stride,
4619 uint8_t *dst,
4620 int32_t dst_stride,
4621 const int8_t *filter_x,
4622 const int8_t *filter_y,
4623 int32_t weight0,
4624 int32_t weight1,
4625 int32_t offset,
4626 int32_t rnd_val)
4627{
4628 uint64_t tp0, tp1;
4629 int32_t weight;
4630 v16u8 out;
4631 v8i16 in0 = { 0 }, in1 = { 0 };
4632 v16i8 src0, src1, src2, src3, src4, src5, src6;
4633 v8i16 filt0, filt1;
4634 v8i16 filt_h0, filt_h1;
4635 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
4636 v16i8 mask1;
4637 v8i16 filter_vec, weight_vec;
4638 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
4639 v8i16 tmp0, tmp1, tmp2, tmp3;
4640 v8i16 dst30, dst41, dst52, dst63;
4641 v8i16 dst10, dst32, dst54, dst21, dst43, dst65;
4642 v4i32 offset_vec, rnd_vec, const_vec;
4643 v4i32 dst0, dst1, dst2, dst3;
4644
4645 src0_ptr -= (src_stride + 1);
4646
4647 filter_vec = LD_SH(filter_x);
4648 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4649
4650 filter_vec = LD_SH(filter_y);
4651 UNPCK_R_SB_SH(filter_vec, filter_vec);
4652
4653 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
4654
4655 mask1 = mask0 + 2;
4656
4657 offset = offset << rnd_val;
4658 weight0 = weight0 & 0x0000FFFF;
4659 weight = weight0 | (weight1 << 16);
4660
4661 const_vec = __msa_fill_w((128 * weight1));
4662 const_vec <<= 6;
4663 offset_vec = __msa_fill_w(offset);
4664 weight_vec = (v8i16) __msa_fill_w(weight);
4665 rnd_vec = __msa_fill_w(rnd_val + 1);
4666 offset_vec += const_vec;
4667
4668 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
4669 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
4670
4671 VSHF_B2_SB(src0, src3, src0, src3, mask0, mask1, vec0, vec1);
4672 VSHF_B2_SB(src1, src4, src1, src4, mask0, mask1, vec2, vec3);
4673 VSHF_B2_SB(src2, src5, src2, src5, mask0, mask1, vec4, vec5);
4674 VSHF_B2_SB(src3, src6, src3, src6, mask0, mask1, vec6, vec7);
4675
4676 dst30 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
4677 dst41 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
4678 dst52 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
4679 dst63 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
4680
4681 ILVRL_H2_SH(dst41, dst30, dst10, dst43);
4682 ILVRL_H2_SH(dst52, dst41, dst21, dst54);
4683 ILVRL_H2_SH(dst63, dst52, dst32, dst65);
4684 dst0 = HEVC_FILT_4TAP(dst10, dst32, filt_h0, filt_h1);
4685 dst1 = HEVC_FILT_4TAP(dst21, dst43, filt_h0, filt_h1);
4686 dst2 = HEVC_FILT_4TAP(dst32, dst54, filt_h0, filt_h1);
4687 dst3 = HEVC_FILT_4TAP(dst43, dst65, filt_h0, filt_h1);
4688 SRA_4V(dst0, dst1, dst2, dst3, 6);
4689 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp1, tmp3);
4690
4691 LD2(src1_ptr, src2_stride, tp0, tp1);
4692 INSERT_D2_SH(tp0, tp1, in0);
4693 src1_ptr += (2 * src2_stride);
4694 LD2(src1_ptr, src2_stride, tp0, tp1);
4695 INSERT_D2_SH(tp0, tp1, in1);
4696
4697 ILVRL_H2_SH(tmp1, in0, tmp0, tmp1);
4698 ILVRL_H2_SH(tmp3, in1, tmp2, tmp3);
4699
4700 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
4701 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
4702 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
4703 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
4704 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4705 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
4706 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
4707 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
4708 ST_W4(out, 0, 1, 2, 3, dst, dst_stride);
4709}
4710
4711static void hevc_hv_biwgt_4t_4multx8mult_msa(const uint8_t *src0_ptr,
4712 int32_t src_stride,
4713 const int16_t *src1_ptr,
4714 int32_t src2_stride,
4715 uint8_t *dst,
4716 int32_t dst_stride,
4717 const int8_t *filter_x,
4718 const int8_t *filter_y,
4719 int32_t height,
4720 int32_t weight0,
4721 int32_t weight1,
4722 int32_t offset,
4723 int32_t rnd_val)
4724{
4725 uint32_t loop_cnt;
4726 uint64_t tp0, tp1;
4727 int32_t weight;
4728 v16u8 out0, out1;
4729 v8i16 in0 = { 0 }, in1 = { 0 }, in2 = { 0 }, in3 = { 0 };
4730 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
4731 v8i16 filt0, filt1;
4732 v8i16 filt_h0, filt_h1;
4733 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
4734 v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
4735 v16i8 mask1;
4736 v8i16 filter_vec, weight_vec;
4737 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
4738 v8i16 dst10, dst21, dst22, dst73, dst84, dst95, dst106;
4739 v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
4740 v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
4741 v8i16 dst98_r, dst109_r;
4742 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
4743 v4i32 offset_vec, rnd_vec, const_vec;
4744
4745 src0_ptr -= (src_stride + 1);
4746
4747 filter_vec = LD_SH(filter_x);
4748 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4749
4750 filter_vec = LD_SH(filter_y);
4751 UNPCK_R_SB_SH(filter_vec, filter_vec);
4752
4753 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
4754
4755 mask1 = mask0 + 2;
4756
4757 offset = offset << rnd_val;
4758 weight0 = weight0 & 0x0000FFFF;
4759 weight = weight0 | (weight1 << 16);
4760
4761 const_vec = __msa_fill_w((128 * weight1));
4762 const_vec <<= 6;
4763 offset_vec = __msa_fill_w(offset);
4764 weight_vec = (v8i16) __msa_fill_w(weight);
4765 rnd_vec = __msa_fill_w(rnd_val + 1);
4766 offset_vec += const_vec;
4767
4768 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4769 src0_ptr += (3 * src_stride);
4770 XORI_B3_128_SB(src0, src1, src2);
4771
4772 VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
4773 VSHF_B2_SB(src1, src2, src1, src2, mask0, mask1, vec2, vec3);
4774 dst10 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
4775 dst21 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
4776 ILVRL_H2_SH(dst21, dst10, dst10_r, dst21_r);
4777 dst22 = (v8i16) __msa_splati_d((v2i64) dst21, 1);
4778
4779 for (loop_cnt = height >> 3; loop_cnt--;) {
4780 LD_SB8(src0_ptr, src_stride,
4781 src3, src4, src5, src6, src7, src8, src9, src10);
4782 src0_ptr += (8 * src_stride);
4783 XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
4784 VSHF_B2_SB(src3, src7, src3, src7, mask0, mask1, vec0, vec1);
4785 VSHF_B2_SB(src4, src8, src4, src8, mask0, mask1, vec2, vec3);
4786 VSHF_B2_SB(src5, src9, src5, src9, mask0, mask1, vec4, vec5);
4787 VSHF_B2_SB(src6, src10, src6, src10, mask0, mask1, vec6, vec7);
4788
4789 dst73 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
4790 dst84 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
4791 dst95 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
4792 dst106 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
4793
4794 dst32_r = __msa_ilvr_h(dst73, dst22);
4795 ILVRL_H2_SH(dst84, dst73, dst43_r, dst87_r);
4796 ILVRL_H2_SH(dst95, dst84, dst54_r, dst98_r);
4797 ILVRL_H2_SH(dst106, dst95, dst65_r, dst109_r);
4798 dst22 = (v8i16) __msa_splati_d((v2i64) dst73, 1);
4799 dst76_r = __msa_ilvr_h(dst22, dst106);
4800
4801 LD2(src1_ptr, src2_stride, tp0, tp1);
4802 src1_ptr += 2 * src2_stride;
4803 INSERT_D2_SH(tp0, tp1, in0);
4804 LD2(src1_ptr, src2_stride, tp0, tp1);
4805 src1_ptr += 2 * src2_stride;
4806 INSERT_D2_SH(tp0, tp1, in1);
4807
4808 LD2(src1_ptr, src2_stride, tp0, tp1);
4809 src1_ptr += 2 * src2_stride;
4810 INSERT_D2_SH(tp0, tp1, in2);
4811 LD2(src1_ptr, src2_stride, tp0, tp1);
4812 src1_ptr += 2 * src2_stride;
4813 INSERT_D2_SH(tp0, tp1, in3);
4814
4815 dst0 = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4816 dst1 = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4817 dst2 = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
4818 dst3 = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
4819 dst4 = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
4820 dst5 = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
4821 dst6 = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
4822 dst7 = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
4823 SRA_4V(dst0, dst1, dst2, dst3, 6);
4824 SRA_4V(dst4, dst5, dst6, dst7, 6);
4825 PCKEV_H4_SW(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, dst0, dst1,
4826 dst2, dst3);
4827 ILVRL_H2_SH(dst0, in0, tmp0, tmp1);
4828 ILVRL_H2_SH(dst1, in1, tmp2, tmp3);
4829 ILVRL_H2_SH(dst2, in2, tmp4, tmp5);
4830 ILVRL_H2_SH(dst3, in3, tmp6, tmp7);
4831 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
4832 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
4833 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
4834 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
4835 dst4 = __msa_dpadd_s_w(offset_vec, tmp4, weight_vec);
4836 dst5 = __msa_dpadd_s_w(offset_vec, tmp5, weight_vec);
4837 dst6 = __msa_dpadd_s_w(offset_vec, tmp6, weight_vec);
4838 dst7 = __msa_dpadd_s_w(offset_vec, tmp7, weight_vec);
4839 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
4840 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
4841 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
4842 CLIP_SW4_0_255(dst4, dst5, dst6, dst7);
4843 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, tmp0, tmp1,
4844 tmp2, tmp3);
4845 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
4846 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
4847 dst += (8 * dst_stride);
4848
4849 dst10_r = dst98_r;
4850 dst21_r = dst109_r;
4851 dst22 = (v8i16) __msa_splati_d((v2i64) dst106, 1);
4852 }
4853}
4854
4855static void hevc_hv_biwgt_4t_4w_msa(const uint8_t *src0_ptr,
4856 int32_t src_stride,
4857 const int16_t *src1_ptr,
4858 int32_t src2_stride,
4859 uint8_t *dst,
4860 int32_t dst_stride,
4861 const int8_t *filter_x,
4862 const int8_t *filter_y,
4863 int32_t height,
4864 int32_t weight0,
4865 int32_t weight1,
4866 int32_t offset,
4867 int32_t rnd_val)
4868{
4869 if (2 == height) {
4870 hevc_hv_biwgt_4t_4x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4871 dst, dst_stride, filter_x, filter_y,
4872 weight0, weight1, offset, rnd_val);
4873 } else if (4 == height) {
4874 hevc_hv_biwgt_4t_4x4_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
4875 dst, dst_stride, filter_x, filter_y,
4876 weight0, weight1, offset, rnd_val);
4877 } else if (0 == (height % 8)) {
4878 hevc_hv_biwgt_4t_4multx8mult_msa(src0_ptr, src_stride,
4879 src1_ptr, src2_stride,
4880 dst, dst_stride, filter_x, filter_y,
4881 height, weight0, weight1,
4882 offset, rnd_val);
4883 }
4884}
4885
4886static void hevc_hv_biwgt_4t_6w_msa(const uint8_t *src0_ptr,
4887 int32_t src_stride,
4888 const int16_t *src1_ptr,
4889 int32_t src2_stride,
4890 uint8_t *dst,
4891 int32_t dst_stride,
4892 const int8_t *filter_x,
4893 const int8_t *filter_y,
4894 int32_t height,
4895 int32_t weight0,
4896 int32_t weight1,
4897 int32_t offset,
4898 int32_t rnd_val)
4899{
4900 uint32_t tpw0, tpw1, tpw2, tpw3;
4901 uint64_t tp0, tp1;
4902 int32_t weight;
4903 v16u8 out0, out1, out2;
4904 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
4905 v8i16 in0 = { 0 }, in1 = { 0 }, in2 = { 0 }, in3 = { 0 };
4906 v8i16 in4 = { 0 }, in5 = { 0 };
4907 v8i16 filt0, filt1;
4908 v8i16 filt_h0, filt_h1, filter_vec;
4909 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
4910 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
4911 v16i8 mask1;
4912 v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6, dsth7, dsth8, dsth9;
4913 v8i16 dsth10, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, weight_vec;
4914 v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst98_r, dst21_r, dst43_r;
4915 v8i16 dst65_r, dst87_r, dst109_r, dst10_l, dst32_l, dst54_l, dst76_l;
4916 v8i16 dst98_l, dst21_l, dst43_l, dst65_l, dst87_l, dst109_l;
4917 v8i16 dst1021_l, dst3243_l, dst5465_l, dst7687_l, dst98109_l;
4918 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
4919 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
4920 v4i32 dst4_r, dst5_r, dst6_r, dst7_r;
4921 v4i32 offset_vec, rnd_vec, const_vec;
4922
4923 src0_ptr -= (src_stride + 1);
4924
4925 filter_vec = LD_SH(filter_x);
4926 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
4927
4928 filter_vec = LD_SH(filter_y);
4929 UNPCK_R_SB_SH(filter_vec, filter_vec);
4930
4931 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
4932
4933 mask1 = mask0 + 2;
4934
4935 offset = offset << rnd_val;
4936 weight0 = weight0 & 0x0000FFFF;
4937 weight = weight0 | (weight1 << 16);
4938
4939 const_vec = __msa_fill_w((128 * weight1));
4940 const_vec <<= 6;
4941 offset_vec = __msa_fill_w(offset);
4942 weight_vec = (v8i16) __msa_fill_w(weight);
4943 rnd_vec = __msa_fill_w(rnd_val + 1);
4944 offset_vec += const_vec;
4945
4946 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
4947 src0_ptr += (3 * src_stride);
4948 XORI_B3_128_SB(src0, src1, src2);
4949
4950 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
4951 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
4952 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
4953 dsth0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
4954 dsth1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
4955 dsth2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
4956
4957 ILVRL_H2_SH(dsth1, dsth0, dst10_r, dst10_l);
4958 ILVRL_H2_SH(dsth2, dsth1, dst21_r, dst21_l);
4959
4960 LD_SB8(src0_ptr, src_stride, src3, src4, src5, src6, src7, src8, src9,
4961 src10);
4962 XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
4963
4964 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
4965 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
4966 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
4967 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
4968
4969 dsth3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
4970 dsth4 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
4971 dsth5 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
4972 dsth6 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
4973
4974 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
4975 VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec2, vec3);
4976 VSHF_B2_SB(src9, src9, src9, src9, mask0, mask1, vec4, vec5);
4977 VSHF_B2_SB(src10, src10, src10, src10, mask0, mask1, vec6, vec7);
4978
4979 dsth7 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
4980 dsth8 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
4981 dsth9 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
4982 dsth10 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
4983
4984 ILVRL_H2_SH(dsth3, dsth2, dst32_r, dst32_l);
4985 ILVRL_H2_SH(dsth4, dsth3, dst43_r, dst43_l);
4986 ILVRL_H2_SH(dsth5, dsth4, dst54_r, dst54_l);
4987 ILVRL_H2_SH(dsth6, dsth5, dst65_r, dst65_l);
4988 ILVRL_H2_SH(dsth7, dsth6, dst76_r, dst76_l);
4989 ILVRL_H2_SH(dsth8, dsth7, dst87_r, dst87_l);
4990 ILVRL_H2_SH(dsth9, dsth8, dst98_r, dst98_l);
4991 ILVRL_H2_SH(dsth10, dsth9, dst109_r, dst109_l);
4992 PCKEV_D2_SH(dst21_l, dst10_l, dst43_l, dst32_l, dst1021_l, dst3243_l);
4993 PCKEV_D2_SH(dst65_l, dst54_l, dst87_l, dst76_l, dst5465_l, dst7687_l);
4994 dst98109_l = (v8i16) __msa_pckev_d((v2i64) dst109_l, (v2i64) dst98_l);
4995
4996 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
4997 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
4998 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
4999 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5000 dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
5001 dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
5002 dst6_r = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
5003 dst7_r = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
5004 dst0_l = HEVC_FILT_4TAP(dst1021_l, dst3243_l, filt_h0, filt_h1);
5005 dst1_l = HEVC_FILT_4TAP(dst3243_l, dst5465_l, filt_h0, filt_h1);
5006 dst2_l = HEVC_FILT_4TAP(dst5465_l, dst7687_l, filt_h0, filt_h1);
5007 dst3_l = HEVC_FILT_4TAP(dst7687_l, dst98109_l, filt_h0, filt_h1);
5008 SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
5009 SRA_4V(dst4_r, dst5_r, dst6_r, dst7_r, 6);
5010 SRA_4V(dst0_l, dst1_l, dst2_l, dst3_l, 6);
5011 PCKEV_H2_SW(dst1_r, dst0_r, dst3_r, dst2_r, dst0, dst1);
5012 PCKEV_H2_SW(dst5_r, dst4_r, dst7_r, dst6_r, dst2, dst3);
5013
5014 LD2(src1_ptr, src2_stride, tp0, tp1);
5015 INSERT_D2_SH(tp0, tp1, in0);
5016 LD2(src1_ptr + 2 * src2_stride, src2_stride, tp0, tp1);
5017 INSERT_D2_SH(tp0, tp1, in1);
5018
5019 LD2(src1_ptr + 4 * src2_stride, src2_stride, tp0, tp1);
5020 INSERT_D2_SH(tp0, tp1, in2);
5021 LD2(src1_ptr + 6 * src2_stride, src2_stride, tp0, tp1);
5022 INSERT_D2_SH(tp0, tp1, in3);
5023
5024 ILVRL_H2_SH(dst0, in0, tmp0, tmp1);
5025 ILVRL_H2_SH(dst1, in1, tmp2, tmp3);
5026 ILVRL_H2_SH(dst2, in2, tmp4, tmp5);
5027 ILVRL_H2_SH(dst3, in3, tmp6, tmp7);
5028 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5029 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5030 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5031 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5032 dst4 = __msa_dpadd_s_w(offset_vec, tmp4, weight_vec);
5033 dst5 = __msa_dpadd_s_w(offset_vec, tmp5, weight_vec);
5034 dst6 = __msa_dpadd_s_w(offset_vec, tmp6, weight_vec);
5035 dst7 = __msa_dpadd_s_w(offset_vec, tmp7, weight_vec);
5036 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5037 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
5038 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5039 CLIP_SW4_0_255(dst4, dst5, dst6, dst7);
5040 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6, tmp0, tmp1,
5041 tmp2, tmp3);
5042 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5043 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
5044
5045 PCKEV_H2_SW(dst1_l, dst0_l, dst3_l, dst2_l, dst4, dst5);
5046
5047 LW4(src1_ptr + 4, src2_stride, tpw0, tpw1, tpw2, tpw3);
5048 src1_ptr += (4 * src2_stride);
5049 INSERT_W4_SH(tpw0, tpw1, tpw2, tpw3, in4);
5050 LW4(src1_ptr + 4, src2_stride, tpw0, tpw1, tpw2, tpw3);
5051 INSERT_W4_SH(tpw0, tpw1, tpw2, tpw3, in5);
5052
5053 ILVRL_H2_SH(dst4, in4, tmp0, tmp1);
5054 ILVRL_H2_SH(dst5, in5, tmp2, tmp3);
5055
5056 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5057 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5058 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5059 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5060 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5061 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5062 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp4, tmp5);
5063
5064 out2 = (v16u8) __msa_pckev_b((v16i8) tmp5, (v16i8) tmp4);
5065 ST_H8(out2, 0, 1, 2, 3, 4, 5, 6, 7, dst + 4, dst_stride);
5066}
5067
5068static void hevc_hv_biwgt_4t_8x2_msa(const uint8_t *src0_ptr,
5069 int32_t src_stride,
5070 const int16_t *src1_ptr,
5071 int32_t src2_stride,
5072 uint8_t *dst,
5073 int32_t dst_stride,
5074 const int8_t *filter_x,
5075 const int8_t *filter_y,
5076 int32_t weight0,
5077 int32_t weight1,
5078 int32_t offset,
5079 int32_t rnd_val)
5080{
5081 int32_t weight;
5082 v16u8 out;
5083 v16i8 src0, src1, src2, src3, src4;
5084 v8i16 filt0, filt1;
5085 v8i16 filt_h0, filt_h1;
5086 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
5087 v16i8 mask1;
5088 v8i16 filter_vec, weight_vec;
5089 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
5090 v8i16 dst0, dst1, dst2, dst3, dst4;
5091 v8i16 in0, in1;
5092 v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
5093 v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
5094 v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
5095 v8i16 tmp0, tmp1, tmp2, tmp3;
5096 v4i32 offset_vec, rnd_vec, const_vec;
5097
5098 src0_ptr -= (src_stride + 1);
5099
5100 filter_vec = LD_SH(filter_x);
5101 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
5102
5103 filter_vec = LD_SH(filter_y);
5104 UNPCK_R_SB_SH(filter_vec, filter_vec);
5105
5106 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5107
5108 mask1 = mask0 + 2;
5109
5110 offset = offset << rnd_val;
5111 weight0 = weight0 & 0x0000FFFF;
5112 weight = weight0 | (weight1 << 16);
5113
5114 const_vec = __msa_fill_w((128 * weight1));
5115 const_vec <<= 6;
5116 offset_vec = __msa_fill_w(offset);
5117 weight_vec = (v8i16) __msa_fill_w(weight);
5118 rnd_vec = __msa_fill_w(rnd_val + 1);
5119 offset_vec += const_vec;
5120
5121 LD_SB5(src0_ptr, src_stride, src0, src1, src2, src3, src4);
5122 XORI_B5_128_SB(src0, src1, src2, src3, src4);
5123
5124 LD_SH2(src1_ptr, src2_stride, in0, in1);
5125
5126 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5127 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5128 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5129 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
5130 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
5131
5132 dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5133 dst1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5134 dst2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5135 dst3 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
5136 dst4 = HEVC_FILT_4TAP_SH(vec8, vec9, filt0, filt1);
5137
5138 ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
5139 ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
5140 ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
5141 ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
5142 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5143 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5144 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5145 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5146 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5147 PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, tmp1, tmp3);
5148
5149 ILVRL_H2_SH(tmp1, in0, tmp0, tmp1);
5150 ILVRL_H2_SH(tmp3, in1, tmp2, tmp3);
5151
5152 dst0_r = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5153 dst0_l = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5154 dst1_r = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5155 dst1_l = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5156 SRAR_W4_SW(dst0_r, dst0_l, dst1_r, dst1_l, rnd_vec);
5157 CLIP_SW4_0_255(dst0_r, dst0_l, dst1_r, dst1_l);
5158 PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, tmp0, tmp1);
5159 out = (v16u8) __msa_pckev_b((v16i8) tmp1, (v16i8) tmp0);
5160 ST_D2(out, 0, 1, dst, dst_stride);
5161}
5162
5163static void hevc_hv_biwgt_4t_8multx4_msa(const uint8_t *src0_ptr,
5164 int32_t src_stride,
5165 const int16_t *src1_ptr,
5166 int32_t src2_stride,
5167 uint8_t *dst,
5168 int32_t dst_stride,
5169 const int8_t *filter_x,
5170 const int8_t *filter_y,
5171 int32_t weight0,
5172 int32_t weight1,
5173 int32_t offset,
5174 int32_t rnd_val,
5175 int32_t width8mult)
5176{
5177 int32_t weight;
5178 uint32_t cnt;
5179 v16u8 out0, out1;
5180 v16i8 src0, src1, src2, src3, src4, src5, src6, mask0, mask1;
5181 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5182 v8i16 filt0, filt1, filt_h0, filt_h1, filter_vec, weight_vec;
5183 v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6;
5184 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, in0, in1, in2, in3;
5185 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
5186 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
5187 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
5188 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
5189 v4i32 offset_vec, rnd_vec, const_vec;
5190
5191 src0_ptr -= (src_stride + 1);
5192
5193 filter_vec = LD_SH(filter_x);
5194 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
5195
5196 filter_vec = LD_SH(filter_y);
5197 UNPCK_R_SB_SH(filter_vec, filter_vec);
5198
5199 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5200
5201 mask0 = LD_SB(ff_hevc_mask_arr);
5202 mask1 = mask0 + 2;
5203
5204 offset = offset << rnd_val;
5205 weight0 = weight0 & 0x0000FFFF;
5206 weight = weight0 | (weight1 << 16);
5207
5208 const_vec = __msa_fill_w((128 * weight1));
5209 const_vec <<= 6;
5210 offset_vec = __msa_fill_w(offset);
5211 rnd_vec = __msa_fill_w(rnd_val + 1);
5212 offset_vec += const_vec;
5213 weight_vec = (v8i16) __msa_fill_w(weight);
5214
5215 for (cnt = width8mult; cnt--;) {
5216 LD_SB7(src0_ptr, src_stride, src0, src1, src2, src3, src4, src5, src6);
5217 src0_ptr += 8;
5218 XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
5219
5220 LD_SH4(src1_ptr, src2_stride, in0, in1, in2, in3);
5221 src1_ptr += 8;
5222
5223 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5224 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5225 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5226
5227 dsth0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5228 dsth1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5229 dsth2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5230
5231 ILVRL_H2_SH(dsth1, dsth0, dst10_r, dst10_l);
5232 ILVRL_H2_SH(dsth2, dsth1, dst21_r, dst21_l);
5233
5234 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
5235 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
5236 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
5237 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
5238
5239 dsth3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5240 dsth4 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5241 dsth5 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5242 dsth6 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
5243
5244 ILVRL_H2_SH(dsth3, dsth2, dst32_r, dst32_l);
5245 ILVRL_H2_SH(dsth4, dsth3, dst43_r, dst43_l);
5246 ILVRL_H2_SH(dsth5, dsth4, dst54_r, dst54_l);
5247 ILVRL_H2_SH(dsth6, dsth5, dst65_r, dst65_l);
5248
5249 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5250 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5251 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5252 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5253 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5254 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
5255 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5256 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
5257
5258 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5259 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
5260 PCKEV_H4_SW(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
5261 dst3_r, dst0, dst1, dst2, dst3);
5262
5263 ILVRL_H2_SH(dst0, in0, tmp0, tmp1);
5264 ILVRL_H2_SH(dst1, in1, tmp2, tmp3);
5265 ILVRL_H2_SH(dst2, in2, tmp4, tmp5);
5266 ILVRL_H2_SH(dst3, in3, tmp6, tmp7);
5267 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5268 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5269 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5270 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5271 dst4 = __msa_dpadd_s_w(offset_vec, tmp4, weight_vec);
5272 dst5 = __msa_dpadd_s_w(offset_vec, tmp5, weight_vec);
5273 dst6 = __msa_dpadd_s_w(offset_vec, tmp6, weight_vec);
5274 dst7 = __msa_dpadd_s_w(offset_vec, tmp7, weight_vec);
5275 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5276 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
5277 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5278 CLIP_SW4_0_255(dst4, dst5, dst6, dst7);
5279 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
5280 tmp0, tmp1, tmp2, tmp3);
5281 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5282 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
5283 dst += 8;
5284 }
5285}
5286
5287static void hevc_hv_biwgt_4t_8x6_msa(const uint8_t *src0_ptr,
5288 int32_t src_stride,
5289 const int16_t *src1_ptr,
5290 int32_t src2_stride,
5291 uint8_t *dst,
5292 int32_t dst_stride,
5293 const int8_t *filter_x,
5294 const int8_t *filter_y,
5295 int32_t weight0,
5296 int32_t weight1,
5297 int32_t offset,
5298 int32_t rnd_val)
5299{
5300 uint32_t weight;
5301 v16u8 out0, out1, out2;
5302 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
5303 v8i16 filt0, filt1;
5304 v8i16 filt_h0, filt_h1;
5305 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
5306 v16i8 mask1;
5307 v8i16 filter_vec, weight_vec;
5308 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
5309 v16i8 vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17;
5310 v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6, dsth7, dsth8;
5311 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
5312 v4i32 dst4_r, dst4_l, dst5_r, dst5_l;
5313 v8i16 dst10_r, dst32_r, dst10_l, dst32_l;
5314 v8i16 dst21_r, dst43_r, dst21_l, dst43_l;
5315 v8i16 dst54_r, dst54_l, dst65_r, dst65_l;
5316 v8i16 dst76_r, dst76_l, dst87_r, dst87_l;
5317 v8i16 in0, in1, in2, in3, in4, in5;
5318 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5319 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
5320 v4i32 offset_vec, rnd_vec, const_vec;
5321
5322 src0_ptr -= (src_stride + 1);
5323
5324 filter_vec = LD_SH(filter_x);
5325 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
5326
5327 filter_vec = LD_SH(filter_y);
5328 UNPCK_R_SB_SH(filter_vec, filter_vec);
5329
5330 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5331
5332 mask1 = mask0 + 2;
5333
5334 offset = offset << rnd_val;
5335 weight0 = weight0 & 0x0000FFFF;
5336 weight = weight0 | (weight1 << 16);
5337
5338 const_vec = __msa_fill_w((128 * weight1));
5339 const_vec <<= 6;
5340 offset_vec = __msa_fill_w(offset);
5341 weight_vec = (v8i16) __msa_fill_w(weight);
5342 rnd_vec = __msa_fill_w(rnd_val + 1);
5343 offset_vec += const_vec;
5344
5345 LD_SB5(src0_ptr, src_stride, src0, src1, src2, src3, src4);
5346 src0_ptr += (5 * src_stride);
5347 LD_SB4(src0_ptr, src_stride, src5, src6, src7, src8);
5348
5349 XORI_B5_128_SB(src0, src1, src2, src3, src4);
5350 XORI_B4_128_SB(src5, src6, src7, src8);
5351
5352 LD_SH6(src1_ptr, src2_stride, in0, in1, in2, in3, in4, in5);
5353
5354 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5355 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5356 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5357 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
5358 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
5359 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec10, vec11);
5360 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec12, vec13);
5361 VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec14, vec15);
5362 VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec16, vec17);
5363
5364 dsth0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5365 dsth1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5366 dsth2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5367 dsth3 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
5368 dsth4 = HEVC_FILT_4TAP_SH(vec8, vec9, filt0, filt1);
5369 dsth5 = HEVC_FILT_4TAP_SH(vec10, vec11, filt0, filt1);
5370 dsth6 = HEVC_FILT_4TAP_SH(vec12, vec13, filt0, filt1);
5371 dsth7 = HEVC_FILT_4TAP_SH(vec14, vec15, filt0, filt1);
5372 dsth8 = HEVC_FILT_4TAP_SH(vec16, vec17, filt0, filt1);
5373
5374 ILVRL_H2_SH(dsth1, dsth0, dst10_r, dst10_l);
5375 ILVRL_H2_SH(dsth2, dsth1, dst21_r, dst21_l);
5376 ILVRL_H2_SH(dsth3, dsth2, dst32_r, dst32_l);
5377 ILVRL_H2_SH(dsth4, dsth3, dst43_r, dst43_l);
5378 ILVRL_H2_SH(dsth5, dsth4, dst54_r, dst54_l);
5379 ILVRL_H2_SH(dsth6, dsth5, dst65_r, dst65_l);
5380 ILVRL_H2_SH(dsth7, dsth6, dst76_r, dst76_l);
5381 ILVRL_H2_SH(dsth8, dsth7, dst87_r, dst87_l);
5382
5383 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5384 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5385 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5386 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5387 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5388 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
5389 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5390 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
5391 dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
5392 dst4_l = HEVC_FILT_4TAP(dst54_l, dst76_l, filt_h0, filt_h1);
5393 dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
5394 dst5_l = HEVC_FILT_4TAP(dst65_l, dst87_l, filt_h0, filt_h1);
5395
5396 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5397 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
5398 SRA_4V(dst4_r, dst4_l, dst5_r, dst5_l, 6);
5399 PCKEV_H4_SW(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l, dst3_r,
5400 dst0, dst1, dst2, dst3);
5401
5402 ILVRL_H2_SH(dst0, in0, tmp0, tmp1);
5403 ILVRL_H2_SH(dst1, in1, tmp2, tmp3);
5404 ILVRL_H2_SH(dst2, in2, tmp4, tmp5);
5405 ILVRL_H2_SH(dst3, in3, tmp6, tmp7);
5406 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5407 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5408 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5409 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5410 dst4 = __msa_dpadd_s_w(offset_vec, tmp4, weight_vec);
5411 dst5 = __msa_dpadd_s_w(offset_vec, tmp5, weight_vec);
5412 dst6 = __msa_dpadd_s_w(offset_vec, tmp6, weight_vec);
5413 dst7 = __msa_dpadd_s_w(offset_vec, tmp7, weight_vec);
5414 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5415 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
5416 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5417 CLIP_SW4_0_255(dst4, dst5, dst6, dst7);
5418 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
5419 tmp0, tmp1, tmp2, tmp3);
5420 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5421
5422 PCKEV_H2_SW(dst4_l, dst4_r, dst5_l, dst5_r, dst0, dst1);
5423 ILVRL_H2_SH(dst0, in4, tmp0, tmp1);
5424 ILVRL_H2_SH(dst1, in5, tmp2, tmp3);
5425 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5426 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5427 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5428 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5429 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5430 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5431 PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp4, tmp5);
5432 out2 = (v16u8) __msa_pckev_b((v16i8) tmp5, (v16i8) tmp4);
5433 ST_D4(out0, out1, 0, 1, 0, 1, dst, dst_stride);
5434 ST_D2(out2, 0, 1, dst + 4 * dst_stride, dst_stride);
5435}
5436
5437static void hevc_hv_biwgt_4t_8multx4mult_msa(const uint8_t *src0_ptr,
5438 int32_t src_stride,
5439 const int16_t *src1_ptr,
5440 int32_t src2_stride,
5441 uint8_t *dst,
5442 int32_t dst_stride,
5443 const int8_t *filter_x,
5444 const int8_t *filter_y,
5445 int32_t height,
5446 int32_t weight0,
5447 int32_t weight1,
5448 int32_t offset,
5449 int32_t rnd_val,
5450 int32_t width)
5451{
5452 uint32_t loop_cnt;
5453 uint32_t cnt;
5454 int32_t weight;
5455 const uint8_t *src0_ptr_tmp;
5456 const int16_t *src1_ptr_tmp;
5457 uint8_t *dst_tmp;
5458 v16u8 out0, out1;
5459 v16i8 src0, src1, src2, src3, src4, src5, src6;
5460 v8i16 in0, in1, in2, in3;
5461 v8i16 filt0, filt1;
5462 v8i16 filt_h0, filt_h1;
5463 v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
5464 v16i8 mask1;
5465 v8i16 filter_vec;
5466 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5467 v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6;
5468 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
5469 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5470 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
5471 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
5472 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l, weight_vec;
5473 v4i32 offset_vec, rnd_vec, const_vec;
5474
5475 src0_ptr -= (src_stride + 1);
5476
5477 filter_vec = LD_SH(filter_x);
5478 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
5479
5480 filter_vec = LD_SH(filter_y);
5481 UNPCK_R_SB_SH(filter_vec, filter_vec);
5482
5483 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5484
5485 mask1 = mask0 + 2;
5486
5487 offset = offset << rnd_val;
5488 weight0 = weight0 & 0x0000FFFF;
5489 weight = weight0 | (weight1 << 16);
5490
5491 const_vec = __msa_fill_w((128 * weight1));
5492 const_vec <<= 6;
5493 offset_vec = __msa_fill_w(offset);
5494 weight_vec = (v8i16) __msa_fill_w(weight);
5495 rnd_vec = __msa_fill_w(rnd_val + 1);
5496 offset_vec += const_vec;
5497
5498 for (cnt = width >> 3; cnt--;) {
5499 src0_ptr_tmp = src0_ptr;
5500 src1_ptr_tmp = src1_ptr;
5501 dst_tmp = dst;
5502
5503 LD_SB3(src0_ptr_tmp, src_stride, src0, src1, src2);
5504 src0_ptr_tmp += (3 * src_stride);
5505 XORI_B3_128_SB(src0, src1, src2);
5506
5507 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5508 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5509 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5510 dsth0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5511 dsth1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5512 dsth2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5513
5514 ILVRL_H2_SH(dsth1, dsth0, dst10_r, dst10_l);
5515 ILVRL_H2_SH(dsth2, dsth1, dst21_r, dst21_l);
5516
5517 for (loop_cnt = height >> 2; loop_cnt--;) {
5518 LD_SB4(src0_ptr_tmp, src_stride, src3, src4, src5, src6);
5519 src0_ptr_tmp += (4 * src_stride);
5520 LD_SH4(src1_ptr_tmp, src2_stride, in0, in1, in2, in3);
5521 src1_ptr_tmp += (4 * src2_stride);
5522 XORI_B4_128_SB(src3, src4, src5, src6);
5523
5524 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
5525 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
5526 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
5527 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
5528
5529 dsth3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5530 dsth4 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5531 dsth5 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5532 dsth6 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
5533
5534 ILVRL_H2_SH(dsth3, dsth2, dst32_r, dst32_l);
5535 ILVRL_H2_SH(dsth4, dsth3, dst43_r, dst43_l);
5536 ILVRL_H2_SH(dsth5, dsth4, dst54_r, dst54_l);
5537 ILVRL_H2_SH(dsth6, dsth5, dst65_r, dst65_l);
5538
5539 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5540 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5541 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5542 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5543 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5544 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
5545 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5546 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
5547
5548 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5549 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
5550 PCKEV_H4_SW(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
5551 dst3_r, dst0, dst1, dst2, dst3);
5552 ILVRL_H2_SH(dst0, in0, tmp0, tmp1);
5553 ILVRL_H2_SH(dst1, in1, tmp2, tmp3);
5554 ILVRL_H2_SH(dst2, in2, tmp4, tmp5);
5555 ILVRL_H2_SH(dst3, in3, tmp6, tmp7);
5556 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5557 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5558 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5559 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5560 dst4 = __msa_dpadd_s_w(offset_vec, tmp4, weight_vec);
5561 dst5 = __msa_dpadd_s_w(offset_vec, tmp5, weight_vec);
5562 dst6 = __msa_dpadd_s_w(offset_vec, tmp6, weight_vec);
5563 dst7 = __msa_dpadd_s_w(offset_vec, tmp7, weight_vec);
5564 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5565 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
5566 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5567 CLIP_SW4_0_255(dst4, dst5, dst6, dst7);
5568 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
5569 tmp0, tmp1, tmp2, tmp3);
5570 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5571 ST_D4(out0, out1, 0, 1, 0, 1, dst_tmp, dst_stride);
5572 dst_tmp += (4 * dst_stride);
5573
5574 dst10_r = dst54_r;
5575 dst10_l = dst54_l;
5576 dst21_r = dst65_r;
5577 dst21_l = dst65_l;
5578 dsth2 = dsth6;
5579 }
5580
5581 src0_ptr += 8;
5582 dst += 8;
5583 src1_ptr += 8;
5584 }
5585}
5586
5587static void hevc_hv_biwgt_4t_8w_msa(const uint8_t *src0_ptr,
5588 int32_t src_stride,
5589 const int16_t *src1_ptr,
5590 int32_t src2_stride,
5591 uint8_t *dst,
5592 int32_t dst_stride,
5593 const int8_t *filter_x,
5594 const int8_t *filter_y,
5595 int32_t height,
5596 int32_t weight0,
5597 int32_t weight1,
5598 int32_t offset,
5599 int32_t rnd_val)
5600{
5601 if (2 == height) {
5602 hevc_hv_biwgt_4t_8x2_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
5603 dst, dst_stride, filter_x, filter_y,
5604 weight0, weight1, offset, rnd_val);
5605 } else if (4 == height) {
5606 hevc_hv_biwgt_4t_8multx4_msa(src0_ptr, src_stride, src1_ptr,
5607 src2_stride, dst, dst_stride, filter_x,
5608 filter_y, weight0, weight1, offset,
5609 rnd_val, 1);
5610 } else if (6 == height) {
5611 hevc_hv_biwgt_4t_8x6_msa(src0_ptr, src_stride, src1_ptr, src2_stride,
5612 dst, dst_stride, filter_x, filter_y,
5613 weight0, weight1, offset, rnd_val);
5614 } else if (0 == (height % 4)) {
5615 hevc_hv_biwgt_4t_8multx4mult_msa(src0_ptr, src_stride,
5616 src1_ptr, src2_stride,
5617 dst, dst_stride, filter_x, filter_y,
5618 height, weight0,
5619 weight1, offset, rnd_val, 8);
5620 }
5621}
5622
5623static void hevc_hv_biwgt_4t_12w_msa(const uint8_t *src0_ptr,
5624 int32_t src_stride,
5625 const int16_t *src1_ptr,
5626 int32_t src2_stride,
5627 uint8_t *dst,
5628 int32_t dst_stride,
5629 const int8_t *filter_x,
5630 const int8_t *filter_y,
5631 int32_t height,
5632 int32_t weight0,
5633 int32_t weight1,
5634 int32_t offset,
5635 int32_t rnd_val)
5636{
5637 uint32_t loop_cnt;
5638 uint64_t tp0, tp1;
5639 int32_t weight;
5640 const uint8_t *src0_ptr_tmp;
5641 const int16_t *src1_ptr_tmp;
5642 uint8_t *dst_tmp;
5643 v16u8 out0, out1;
5644 v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
5645 v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
5646 v16i8 mask0, mask1, mask2, mask3;
5647 v8i16 filt0, filt1, filt_h0, filt_h1, filter_vec;
5648 v8i16 tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7;
5649 v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6, weight_vec;
5650 v8i16 dst10, dst21, dst22, dst73, dst84, dst95, dst106;
5651 v8i16 dst76_r, dst98_r, dst87_r, dst109_r;
5652 v8i16 in0 = { 0 }, in1 = { 0 }, in2 = { 0 }, in3 = { 0 };
5653 v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
5654 v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
5655 v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
5656 v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
5657 v4i32 offset_vec, rnd_vec, const_vec;
5658
5659 src0_ptr -= (src_stride + 1);
5660
5661 filter_vec = LD_SH(filter_x);
5662 SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
5663
5664 filter_vec = LD_SH(filter_y);
5665 UNPCK_R_SB_SH(filter_vec, filter_vec);
5666
5667 SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
5668
5669 mask0 = LD_SB(ff_hevc_mask_arr);
5670 mask1 = mask0 + 2;
5671
5672 offset = offset << rnd_val;
5673 weight0 = weight0 & 0x0000FFFF;
5674 weight = weight0 | (weight1 << 16);
5675
5676 const_vec = __msa_fill_w((128 * weight1));
5677 const_vec <<= 6;
5678 offset_vec = __msa_fill_w(offset);
5679 rnd_vec = __msa_fill_w(rnd_val + 1);
5680 offset_vec += const_vec;
5681 weight_vec = (v8i16) __msa_fill_w(weight);
5682
5683 src0_ptr_tmp = src0_ptr;
5684 dst_tmp = dst;
5685 src1_ptr_tmp = src1_ptr;
5686
5687 LD_SB3(src0_ptr_tmp, src_stride, src0, src1, src2);
5688 src0_ptr_tmp += (3 * src_stride);
5689
5690 XORI_B3_128_SB(src0, src1, src2);
5691
5692 VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
5693 VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
5694 VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
5695
5696 dsth0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5697 dsth1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5698 dsth2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5699
5700 ILVRL_H2_SH(dsth1, dsth0, dst10_r, dst10_l);
5701 ILVRL_H2_SH(dsth2, dsth1, dst21_r, dst21_l);
5702
5703 for (loop_cnt = 4; loop_cnt--;) {
5704 LD_SB4(src0_ptr_tmp, src_stride, src3, src4, src5, src6);
5705 src0_ptr_tmp += (4 * src_stride);
5706 XORI_B4_128_SB(src3, src4, src5, src6);
5707
5708 LD_SH4(src1_ptr_tmp, src2_stride, in0, in1, in2, in3);
5709 src1_ptr_tmp += (4 * src2_stride);
5710
5711 VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
5712 VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
5713 VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
5714 VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
5715
5716 dsth3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5717 dsth4 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5718 dsth5 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5719 dsth6 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
5720
5721 ILVRL_H2_SH(dsth3, dsth2, dst32_r, dst32_l);
5722 ILVRL_H2_SH(dsth4, dsth3, dst43_r, dst43_l);
5723 ILVRL_H2_SH(dsth5, dsth4, dst54_r, dst54_l);
5724 ILVRL_H2_SH(dsth6, dsth5, dst65_r, dst65_l);
5725
5726 dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5727 dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
5728 dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5729 dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
5730 dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5731 dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
5732 dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5733 dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
5734
5735 SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
5736 SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
5737 PCKEV_H4_SW(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
5738 dst3_r, dst0, dst1, dst2, dst3);
5739 ILVRL_H2_SH(dst0, in0, tmp0, tmp1);
5740 ILVRL_H2_SH(dst1, in1, tmp2, tmp3);
5741 ILVRL_H2_SH(dst2, in2, tmp4, tmp5);
5742 ILVRL_H2_SH(dst3, in3, tmp6, tmp7);
5743 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5744 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5745 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5746 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5747 dst4 = __msa_dpadd_s_w(offset_vec, tmp4, weight_vec);
5748 dst5 = __msa_dpadd_s_w(offset_vec, tmp5, weight_vec);
5749 dst6 = __msa_dpadd_s_w(offset_vec, tmp6, weight_vec);
5750 dst7 = __msa_dpadd_s_w(offset_vec, tmp7, weight_vec);
5751 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5752 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
5753 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5754 CLIP_SW4_0_255(dst4, dst5, dst6, dst7);
5755 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
5756 tmp0, tmp1, tmp2, tmp3);
5757 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5758 ST_D4(out0, out1, 0, 1, 0, 1, dst_tmp, dst_stride);
5759 dst_tmp += (4 * dst_stride);
5760
5761 dst10_r = dst54_r;
5762 dst10_l = dst54_l;
5763 dst21_r = dst65_r;
5764 dst21_l = dst65_l;
5765 dsth2 = dsth6;
5766 }
5767
5768 src0_ptr += 8;
5769 dst += 8;
5770 src1_ptr += 8;
5771
5772 mask2 = LD_SB(ff_hevc_mask_arr + 16);
5773 mask3 = mask2 + 2;
5774
5775 LD_SB3(src0_ptr, src_stride, src0, src1, src2);
5776 src0_ptr += (3 * src_stride);
5777 XORI_B3_128_SB(src0, src1, src2);
5778 VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
5779 VSHF_B2_SB(src1, src2, src1, src2, mask2, mask3, vec2, vec3);
5780
5781 dst10 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5782 dst21 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5783
5784 ILVRL_H2_SH(dst21, dst10, dst10_r, dst21_r);
5785 dst22 = (v8i16) __msa_splati_d((v2i64) dst21, 1);
5786
5787 for (loop_cnt = 2; loop_cnt--;) {
5788 LD_SB8(src0_ptr, src_stride, src3, src4, src5, src6, src7, src8, src9,
5789 src10);
5790 src0_ptr += (8 * src_stride);
5791 XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
5792 VSHF_B2_SB(src3, src7, src3, src7, mask2, mask3, vec0, vec1);
5793 VSHF_B2_SB(src4, src8, src4, src8, mask2, mask3, vec2, vec3);
5794 VSHF_B2_SB(src5, src9, src5, src9, mask2, mask3, vec4, vec5);
5795 VSHF_B2_SB(src6, src10, src6, src10, mask2, mask3, vec6, vec7);
5796
5797 dst73 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
5798 dst84 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
5799 dst95 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
5800 dst106 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
5801
5802 dst32_r = __msa_ilvr_h(dst73, dst22);
5803 ILVRL_H2_SH(dst84, dst73, dst43_r, dst87_r);
5804 ILVRL_H2_SH(dst95, dst84, dst54_r, dst98_r);
5805 ILVRL_H2_SH(dst106, dst95, dst65_r, dst109_r);
5806 dst22 = (v8i16) __msa_splati_d((v2i64) dst73, 1);
5807 dst76_r = __msa_ilvr_h(dst22, dst106);
5808
5809 LD2(src1_ptr, src2_stride, tp0, tp1);
5810 src1_ptr += 2 * src2_stride;
5811 INSERT_D2_SH(tp0, tp1, in0);
5812 LD2(src1_ptr, src2_stride, tp0, tp1);
5813 src1_ptr += 2 * src2_stride;
5814 INSERT_D2_SH(tp0, tp1, in1);
5815
5816 LD2(src1_ptr, src2_stride, tp0, tp1);
5817 src1_ptr += 2 * src2_stride;
5818 INSERT_D2_SH(tp0, tp1, in2);
5819 LD2(src1_ptr, src2_stride, tp0, tp1);
5820 src1_ptr += 2 * src2_stride;
5821 INSERT_D2_SH(tp0, tp1, in3);
5822
5823 dst0 = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
5824 dst1 = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
5825 dst2 = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
5826 dst3 = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
5827 dst4 = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
5828 dst5 = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
5829 dst6 = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
5830 dst7 = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
5831
5832 SRA_4V(dst0, dst1, dst2, dst3, 6);
5833 SRA_4V(dst4, dst5, dst6, dst7, 6);
5834 PCKEV_H4_SW(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
5835 dst0, dst1, dst2, dst3);
5836 ILVRL_H2_SH(dst0, in0, tmp0, tmp1);
5837 ILVRL_H2_SH(dst1, in1, tmp2, tmp3);
5838 ILVRL_H2_SH(dst2, in2, tmp4, tmp5);
5839 ILVRL_H2_SH(dst3, in3, tmp6, tmp7);
5840 dst0 = __msa_dpadd_s_w(offset_vec, tmp0, weight_vec);
5841 dst1 = __msa_dpadd_s_w(offset_vec, tmp1, weight_vec);
5842 dst2 = __msa_dpadd_s_w(offset_vec, tmp2, weight_vec);
5843 dst3 = __msa_dpadd_s_w(offset_vec, tmp3, weight_vec);
5844 dst4 = __msa_dpadd_s_w(offset_vec, tmp4, weight_vec);
5845 dst5 = __msa_dpadd_s_w(offset_vec, tmp5, weight_vec);
5846 dst6 = __msa_dpadd_s_w(offset_vec, tmp6, weight_vec);
5847 dst7 = __msa_dpadd_s_w(offset_vec, tmp7, weight_vec);
5848 SRAR_W4_SW(dst0, dst1, dst2, dst3, rnd_vec);
5849 SRAR_W4_SW(dst4, dst5, dst6, dst7, rnd_vec);
5850 CLIP_SW4_0_255(dst0, dst1, dst2, dst3);
5851 CLIP_SW4_0_255(dst4, dst5, dst6, dst7);
5852 PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
5853 tmp0, tmp1, tmp2, tmp3);
5854 PCKEV_B2_UB(tmp1, tmp0, tmp3, tmp2, out0, out1);
5855 ST_W8(out0, out1, 0, 1, 2, 3, 0, 1, 2, 3, dst, dst_stride);
5856 dst += (8 * dst_stride);
5857
5858 dst10_r = dst98_r;
5859 dst21_r = dst109_r;
5860 dst22 = (v8i16) __msa_splati_d((v2i64) dst106, 1);
5861 }
5862}
5863
5864static void hevc_hv_biwgt_4t_16w_msa(const uint8_t *src0_ptr,
5865 int32_t src_stride,
5866 const int16_t *src1_ptr,
5867 int32_t src2_stride,
5868 uint8_t *dst,
5869 int32_t dst_stride,
5870 const int8_t *filter_x,
5871 const int8_t *filter_y,
5872 int32_t height,
5873 int32_t weight0,
5874 int32_t weight1,
5875 int32_t offset,
5876 int32_t rnd_val)
5877{
5878 if (4 == height) {
5879 hevc_hv_biwgt_4t_8multx4_msa(src0_ptr, src_stride, src1_ptr,
5880 src2_stride, dst, dst_stride, filter_x,
5881 filter_y, weight0, weight1, offset,
5882 rnd_val, 2);
5883 } else {
5884 hevc_hv_biwgt_4t_8multx4mult_msa(src0_ptr, src_stride, src1_ptr,
5885 src2_stride, dst, dst_stride,
5886 filter_x, filter_y, height, weight0,
5887 weight1, offset, rnd_val, 16);
5888 }
5889}
5890
5891static void hevc_hv_biwgt_4t_24w_msa(const uint8_t *src0_ptr,
5892 int32_t src_stride,
5893 const int16_t *src1_ptr,
5894 int32_t src2_stride,
5895 uint8_t *dst,
5896 int32_t dst_stride,
5897 const int8_t *filter_x,
5898 const int8_t *filter_y,
5899 int32_t height,
5900 int32_t weight0,
5901 int32_t weight1,
5902 int32_t offset,
5903 int32_t rnd_val)
5904{
5905 hevc_hv_biwgt_4t_8multx4mult_msa(src0_ptr, src_stride,
5906 src1_ptr, src2_stride,
5907 dst, dst_stride,
5908 filter_x, filter_y, height, weight0,
5909 weight1, offset, rnd_val, 24);
5910}
5911
5912static void hevc_hv_biwgt_4t_32w_msa(const uint8_t *src0_ptr,
5913 int32_t src_stride,
5914 const int16_t *src1_ptr,
5915 int32_t src2_stride,
5916 uint8_t *dst,
5917 int32_t dst_stride,
5918 const int8_t *filter_x,
5919 const int8_t *filter_y,
5920 int32_t height,
5921 int32_t weight0,
5922 int32_t weight1,
5923 int32_t offset,
5924 int32_t rnd_val)
5925{
5926 hevc_hv_biwgt_4t_8multx4mult_msa(src0_ptr, src_stride,
5927 src1_ptr, src2_stride,
5928 dst, dst_stride,
5929 filter_x, filter_y, height, weight0,
5930 weight1, offset, rnd_val, 32);
5931}
5932
5933#define BI_W_MC_COPY(WIDTH) \
5934void ff_hevc_put_hevc_bi_w_pel_pixels##WIDTH##_8_msa(uint8_t *dst, \
5935 ptrdiff_t dst_stride, \
5936 const uint8_t *src, \
5937 ptrdiff_t src_stride, \
5938 const int16_t *src_16bit, \
5939 int height, \
5940 int denom, \
5941 int weight0, \
5942 int weight1, \
5943 int offset, \
5944 intptr_t mx, \
5945 intptr_t my, \
5946 int width) \
5947{ \
5948 int shift = 14 + 1 - 8; \
5949 int log2Wd = denom + shift - 1; \
5950 \
5951 hevc_biwgt_copy_##WIDTH##w_msa(src, src_stride, src_16bit, MAX_PB_SIZE, \
5952 dst, dst_stride, height, \
5953 weight0, weight1, offset, log2Wd); \
5954}
5955
5956BI_W_MC_COPY(4);
5957BI_W_MC_COPY(6);
5958BI_W_MC_COPY(8);
5959BI_W_MC_COPY(12);
5960BI_W_MC_COPY(16);
5961BI_W_MC_COPY(24);
5962BI_W_MC_COPY(32);
5963BI_W_MC_COPY(48);
5964BI_W_MC_COPY(64);
5965
5966#undef BI_W_MC_COPY
5967
5968#define BI_W_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \
5969void ff_hevc_put_hevc_bi_w_##PEL##_##DIR##WIDTH##_8_msa(uint8_t *dst, \
5970 ptrdiff_t \
5971 dst_stride, \
5972 const uint8_t *src, \
5973 ptrdiff_t \
5974 src_stride, \
5975 const int16_t *src_16bit, \
5976 int height, \
5977 int denom, \
5978 int weight0, \
5979 int weight1, \
5980 int offset, \
5981 intptr_t mx, \
5982 intptr_t my, \
5983 int width) \
5984{ \
5985 const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR]; \
5986 int log2Wd = denom + 14 - 8; \
5987 \
5988 hevc_##DIR1##_biwgt_##TAP##t_##WIDTH##w_msa(src, src_stride, src_16bit, \
5989 MAX_PB_SIZE, dst, dst_stride, \
5990 filter, height, weight0, \
5991 weight1, offset, log2Wd); \
5992}
5993
5994BI_W_MC(qpel, h, 4, 8, hz, mx);
5995BI_W_MC(qpel, h, 8, 8, hz, mx);
5996BI_W_MC(qpel, h, 12, 8, hz, mx);
5997BI_W_MC(qpel, h, 16, 8, hz, mx);
5998BI_W_MC(qpel, h, 24, 8, hz, mx);
5999BI_W_MC(qpel, h, 32, 8, hz, mx);
6000BI_W_MC(qpel, h, 48, 8, hz, mx);
6001BI_W_MC(qpel, h, 64, 8, hz, mx);
6002
6003BI_W_MC(qpel, v, 4, 8, vt, my);
6004BI_W_MC(qpel, v, 8, 8, vt, my);
6005BI_W_MC(qpel, v, 12, 8, vt, my);
6006BI_W_MC(qpel, v, 16, 8, vt, my);
6007BI_W_MC(qpel, v, 24, 8, vt, my);
6008BI_W_MC(qpel, v, 32, 8, vt, my);
6009BI_W_MC(qpel, v, 48, 8, vt, my);
6010BI_W_MC(qpel, v, 64, 8, vt, my);
6011
6012BI_W_MC(epel, h, 4, 4, hz, mx);
6013BI_W_MC(epel, h, 8, 4, hz, mx);
6014BI_W_MC(epel, h, 6, 4, hz, mx);
6015BI_W_MC(epel, h, 12, 4, hz, mx);
6016BI_W_MC(epel, h, 16, 4, hz, mx);
6017BI_W_MC(epel, h, 24, 4, hz, mx);
6018BI_W_MC(epel, h, 32, 4, hz, mx);
6019
6020BI_W_MC(epel, v, 4, 4, vt, my);
6021BI_W_MC(epel, v, 8, 4, vt, my);
6022BI_W_MC(epel, v, 6, 4, vt, my);
6023BI_W_MC(epel, v, 12, 4, vt, my);
6024BI_W_MC(epel, v, 16, 4, vt, my);
6025BI_W_MC(epel, v, 24, 4, vt, my);
6026BI_W_MC(epel, v, 32, 4, vt, my);
6027
6028#undef BI_W_MC
6029
6030#define BI_W_MC_HV(PEL, WIDTH, TAP) \
6031void ff_hevc_put_hevc_bi_w_##PEL##_hv##WIDTH##_8_msa(uint8_t *dst, \
6032 ptrdiff_t dst_stride, \
6033 const uint8_t *src, \
6034 ptrdiff_t src_stride, \
6035 const int16_t *src_16bit, \
6036 int height, \
6037 int denom, \
6038 int weight0, \
6039 int weight1, \
6040 int offset, \
6041 intptr_t mx, \
6042 intptr_t my, \
6043 int width) \
6044{ \
6045 const int8_t *filter_x = ff_hevc_##PEL##_filters[mx]; \
6046 const int8_t *filter_y = ff_hevc_##PEL##_filters[my]; \
6047 int log2Wd = denom + 14 - 8; \
6048 \
6049 hevc_hv_biwgt_##TAP##t_##WIDTH##w_msa(src, src_stride, src_16bit, \
6050 MAX_PB_SIZE, dst, dst_stride, \
6051 filter_x, filter_y, height, \
6052 weight0, weight1, offset, \
6053 log2Wd); \
6054}
6055
6056BI_W_MC_HV(qpel, 4, 8);
6057BI_W_MC_HV(qpel, 8, 8);
6058BI_W_MC_HV(qpel, 12, 8);
6059BI_W_MC_HV(qpel, 16, 8);
6060BI_W_MC_HV(qpel, 24, 8);
6061BI_W_MC_HV(qpel, 32, 8);
6062BI_W_MC_HV(qpel, 48, 8);
6063BI_W_MC_HV(qpel, 64, 8);
6064
6065BI_W_MC_HV(epel, 4, 4);
6066BI_W_MC_HV(epel, 8, 4);
6067BI_W_MC_HV(epel, 6, 4);
6068BI_W_MC_HV(epel, 12, 4);
6069BI_W_MC_HV(epel, 16, 4);
6070BI_W_MC_HV(epel, 24, 4);
6071BI_W_MC_HV(epel, 32, 4);
6072
6073#undef BI_W_MC_HV
my
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t my
Definition dsp.h:57
mx
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t mx
Definition dsp.h:57
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition dsp.h:87
int32_t
int32_t
Definition audioconvert.c:56
SD
#define SD
Definition ccaption_dec.c:944
aligned
static int aligned(int val)
Definition dashdec.c:178
generic_macros_msa.h
ST_UB4
#define ST_UB4(...)
Definition generic_macros_msa.h:374
DOTP_SB3_SH
#define DOTP_SB3_SH(...)
Definition generic_macros_msa.h:776
INSERT_W4_SH
#define INSERT_W4_SH(...)
Definition generic_macros_msa.h:1155
ILVR_B2_SB
#define ILVR_B2_SB(...)
Definition generic_macros_msa.h:1338
LD_SH8
#define LD_SH8(...)
Definition generic_macros_msa.h:338
PCKEV_H2_SW
#define PCKEV_H2_SW(...)
Definition generic_macros_msa.h:1760
XORI_B8_128_SB
#define XORI_B8_128_SB(...)
Definition generic_macros_msa.h:1880
LW2
#define LW2(psrc, stride, out0, out1)
Definition generic_macros_msa.h:210
XORI_B6_128_SB
#define XORI_B6_128_SB(...)
Definition generic_macros_msa.h:1866
SPLATI_W4_SH
#define SPLATI_W4_SH(...)
Definition generic_macros_msa.h:1700
CLIP_SW2_0_255
#define CLIP_SW2_0_255(in0, in1)
Definition generic_macros_msa.h:972
LD4
#define LD4(psrc, stride, out0, out1, out2, out3)
Definition generic_macros_msa.h:228
ST_D4
#define ST_D4(in0, in1, idx0, idx1, idx2, idx3, pdst, stride)
Definition generic_macros_msa.h:499
ILVR_D3_SB
#define ILVR_D3_SB(...)
Definition generic_macros_msa.h:1452
SPLATI_H2_SH
#define SPLATI_H2_SH(...)
Definition generic_macros_msa.h:1656
ILVR_H2_SH
#define ILVR_H2_SH(...)
Definition generic_macros_msa.h:1392
XORI_B2_128_SB
#define XORI_B2_128_SB(...)
Definition generic_macros_msa.h:1835
XORI_B7_128_SB
#define XORI_B7_128_SB(...)
Definition generic_macros_msa.h:1873
SLLI_2V
#define SLLI_2V(in0, in1, shift)
Definition generic_macros_msa.h:1916
ILVR_B4_SB
#define ILVR_B4_SB(...)
Definition generic_macros_msa.h:1360
ST_SH2
#define ST_SH2(...)
Definition generic_macros_msa.h:366
SPLATI_W2_SH
#define SPLATI_W2_SH(...)
Definition generic_macros_msa.h:1692
PCKEV_B2_UB
#define PCKEV_B2_UB(...)
Definition generic_macros_msa.h:1720
VSHF_B4_SB
#define VSHF_B4_SB(...)
Definition generic_macros_msa.h:680
ILVRL_B2_SH
#define ILVRL_B2_SH(...)
Definition generic_macros_msa.h:1498
ST_SH
#define ST_SH(...)
Definition generic_macros_msa.h:43
DPADD_SB2_SH
#define DPADD_SB2_SH(...)
Definition generic_macros_msa.h:833
ILVR_B4_SH
#define ILVR_B4_SH(...)
Definition generic_macros_msa.h:1362
SRAR_W2_SW
#define SRAR_W2_SW(...)
Definition generic_macros_msa.h:2034
LD_SB7
#define LD_SB7(...)
Definition generic_macros_msa.h:327
INSERT_W4_SB
#define INSERT_W4_SB(...)
Definition generic_macros_msa.h:1154
ILVL_B4_SB
#define ILVL_B4_SB(...)
Definition generic_macros_msa.h:1274
ILVR_D4_SB
#define ILVR_D4_SB(...)
Definition generic_macros_msa.h:1460
XORI_B4_128_SB
#define XORI_B4_128_SB(...)
Definition generic_macros_msa.h:1851
LD_SB3
#define LD_SB3(...)
Definition generic_macros_msa.h:289
LD2
#define LD2(psrc, stride, out0, out1)
Definition generic_macros_msa.h:223
INSERT_D2_SH
#define INSERT_D2_SH(...)
Definition generic_macros_msa.h:1171
DOTP_SB4_SH
#define DOTP_SB4_SH(...)
Definition generic_macros_msa.h:784
ILVRL_H2_SH
#define ILVRL_H2_SH(...)
Definition generic_macros_msa.h:1508
INSERT_W2_SB
#define INSERT_W2_SB(...)
Definition generic_macros_msa.h:1144
UNPCK_R_SB_SH
#define UNPCK_R_SB_SH(in, out)
Definition generic_macros_msa.h:2156
XORI_B3_128_SB
#define XORI_B3_128_SB(...)
Definition generic_macros_msa.h:1843
PCKEV_B3_UB
#define PCKEV_B3_UB(...)
Definition generic_macros_msa.h:1729
LD_SB5
#define LD_SB5(...)
Definition generic_macros_msa.h:308
PCKEV_D2_SH
#define PCKEV_D2_SH(...)
Definition generic_macros_msa.h:1789
PCKEV_H2_SH
#define PCKEV_H2_SH(...)
Definition generic_macros_msa.h:1759
ILVR_D2_SB
#define ILVR_D2_SB(...)
Definition generic_macros_msa.h:1444
PCKEV_B2_SH
#define PCKEV_B2_SH(...)
Definition generic_macros_msa.h:1721
SRA_4V
#define SRA_4V(in0, in1, in2, in3, shift)
Definition generic_macros_msa.h:1939
ILVL_H4_SH
#define ILVL_H4_SH(...)
Definition generic_macros_msa.h:1301
LD_SB8
#define LD_SB8(...)
Definition generic_macros_msa.h:336
SPLATI_H4_SH
#define SPLATI_H4_SH(...)
Definition generic_macros_msa.h:1674
PCKEV_H4_SW
#define PCKEV_H4_SW(...)
Definition generic_macros_msa.h:1769
ST_H8
#define ST_H8(in, idx0, idx1, idx2, idx3, idx4, idx5, idx6, idx7, pdst, stride)
Definition generic_macros_msa.h:429
SLLI_4V
#define SLLI_4V(in0, in1, in2, in3, shift)
Definition generic_macros_msa.h:1921
ST_UB2
#define ST_UB2(...)
Definition generic_macros_msa.h:363
ST_W2
#define ST_W2(in, idx0, idx1, pdst, stride)
Definition generic_macros_msa.h:450
CLIP_SW4_0_255
#define CLIP_SW4_0_255(in0, in1, in2, in3)
Definition generic_macros_msa.h:978
INSERT_D2_SB
#define INSERT_D2_SB(...)
Definition generic_macros_msa.h:1170
ILVR_H4_SH
#define ILVR_H4_SH(...)
Definition generic_macros_msa.h:1408
LD_SB4
#define LD_SB4(...)
Definition generic_macros_msa.h:297
ST_W8
#define ST_W8(in0, in1, idx0, idx1, idx2, idx3, idx4, idx5, idx6, idx7, pdst, stride)
Definition generic_macros_msa.h:470
LW4
#define LW4(psrc, stride, out0, out1, out2, out3)
Definition generic_macros_msa.h:202
LD_SB6
#define LD_SB6(...)
Definition generic_macros_msa.h:316
VSHF_B2_SB
#define VSHF_B2_SB(...)
Definition generic_macros_msa.h:662
ILVL_H2_SH
#define ILVL_H2_SH(...)
Definition generic_macros_msa.h:1292
ILVR_B2_SH
#define ILVR_B2_SH(...)
Definition generic_macros_msa.h:1340
DPADD_SB4_SH
#define DPADD_SB4_SH(...)
Definition generic_macros_msa.h:841
CLIP_SH_0_255
#define CLIP_SH_0_255(in)
Definition generic_macros_msa.h:935
ILVR_D2_SH
#define ILVR_D2_SH(...)
Definition generic_macros_msa.h:1445
LD_SH4
#define LD_SH4(...)
Definition generic_macros_msa.h:299
LD_SB
#define LD_SB(...)
Definition generic_macros_msa.h:33
ILVRL_H2_SW
#define ILVRL_H2_SW(...)
Definition generic_macros_msa.h:1509
LD_SH
#define LD_SH(...)
Definition generic_macros_msa.h:35
LD_SH2
#define LD_SH2(...)
Definition generic_macros_msa.h:280
LD_SB2
#define LD_SB2(...)
Definition generic_macros_msa.h:278
XORI_B5_128_SB
#define XORI_B5_128_SB(...)
Definition generic_macros_msa.h:1859
LD_SH6
#define LD_SH6(...)
Definition generic_macros_msa.h:318
ST_H2
#define ST_H2(in, idx0, idx1, pdst, stride)
Definition generic_macros_msa.h:409
ILVL_B4_SH
#define ILVL_B4_SH(...)
Definition generic_macros_msa.h:1276
ST_D2
#define ST_D2(in, idx0, idx1, pdst, stride)
Definition generic_macros_msa.h:491
SRAR_W4_SW
#define SRAR_W4_SW(...)
Definition generic_macros_msa.h:2041
ST_UB
#define ST_UB(...)
Definition generic_macros_msa.h:40
PCKEV_H4_SH
#define PCKEV_H4_SH(...)
Definition generic_macros_msa.h:1768
ILVL_B2_SB
#define ILVL_B2_SB(...)
Definition generic_macros_msa.h:1263
ILVL_W2_SB
#define ILVL_W2_SB(...)
Definition generic_macros_msa.h:1319
src2
const pixel * src2
Definition h264pred_template.c:421
hevc_macros_msa.h
HEVC_FILT_4TAP
#define HEVC_FILT_4TAP(in0, in1, filt0, filt1)
Definition hevc_macros_msa.h:64
HEVC_FILT_8TAP
#define HEVC_FILT_8TAP(in0, in1, in2, in3, filt0, filt1, filt2, filt3)
Definition hevc_macros_msa.h:35
HEVC_FILT_4TAP_SH
#define HEVC_FILT_4TAP_SH(in0, in1, filt0, filt1)
Definition hevc_macros_msa.h:46
HEVC_FILT_8TAP_SH
#define HEVC_FILT_8TAP_SH(in0, in1, in2, in3, filt0, filt1, filt2, filt3)
Definition hevc_macros_msa.h:24
ff_hevc_mask_arr
static const uint8_t ff_hevc_mask_arr[16 *2]
Definition hevc_mc_bi_lsx.c:26
hevc_biwgt_copy_6w_msa
static void hevc_biwgt_copy_6w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:170
hevc_vt_biwgt_8t_32w_msa
static void hevc_vt_biwgt_8t_32w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1899
hevc_vt_biwgt_4t_4w_msa
static void hevc_vt_biwgt_4t_4w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3729
hevc_hz_biwgt_4t_4x2_msa
static void hevc_hz_biwgt_4t_4x2_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2661
hevc_hv_biwgt_4t_8multx4_msa
static void hevc_hv_biwgt_4t_8multx4_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val, int32_t width8mult)
Definition hevc_mc_biw_msa.c:5163
hevc_vt_biwgt_4t_4x8multiple_msa
static void hevc_vt_biwgt_4t_4x8multiple_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3644
hevc_biwgt_copy_32w_msa
static void hevc_biwgt_copy_32w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:507
hevc_hz_biwgt_4t_4x4_msa
static void hevc_hz_biwgt_4t_4x4_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2720
hevc_vt_biwgt_4t_16w_msa
static void hevc_vt_biwgt_4t_16w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4185
hevc_hz_biwgt_8t_4w_msa
static void hevc_hz_biwgt_8t_4w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:672
HEVC_BIW_RND_CLIP2_MAX_SATU
#define HEVC_BIW_RND_CLIP2_MAX_SATU(in0, in1, vec0, vec1, wgt, rnd, offset, out0, out1)
Definition hevc_mc_biw_msa.c:56
hevc_hz_biwgt_4t_4x8multiple_msa
static void hevc_hz_biwgt_4t_4x8multiple_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2779
hevc_hv_biwgt_4t_8w_msa
static void hevc_hv_biwgt_4t_8w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:5587
HEVC_BIW_RND_CLIP2
#define HEVC_BIW_RND_CLIP2(in0, in1, vec0, vec1, wgt, rnd, offset, out0, out1)
Definition hevc_mc_biw_msa.c:31
hevc_hz_biwgt_4t_24w_msa
static void hevc_hz_biwgt_4t_24w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3356
hevc_vt_biwgt_4t_8x6_msa
static void hevc_vt_biwgt_4t_8x6_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3921
hevc_hv_biwgt_8t_4w_msa
static void hevc_hv_biwgt_8t_4w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1959
hevc_hz_biwgt_4t_8x6_msa
static void hevc_hz_biwgt_4t_8x6_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3011
hevc_vt_biwgt_4t_8w_msa
static void hevc_vt_biwgt_4t_8w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4064
hevc_vt_biwgt_4t_8x4multiple_msa
static void hevc_vt_biwgt_4t_8x4multiple_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3991
hevc_hv_biwgt_4t_16w_msa
static void hevc_hv_biwgt_4t_16w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:5864
hevc_biwgt_copy_24w_msa
static void hevc_biwgt_copy_24w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:444
hevc_vt_biwgt_8t_8w_msa
static void hevc_vt_biwgt_8t_8w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1544
hevc_hv_biwgt_8t_12w_msa
static void hevc_hv_biwgt_8t_12w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2306
hevc_hz_biwgt_4t_6w_msa
static void hevc_hz_biwgt_4t_6w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2883
hevc_hv_biwgt_4t_8x6_msa
static void hevc_hv_biwgt_4t_8x6_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:5287
hevc_hz_biwgt_8t_8w_msa
static void hevc_hz_biwgt_8t_8w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:743
HEVC_BIW_RND_CLIP4_MAX_SATU
#define HEVC_BIW_RND_CLIP4_MAX_SATU(in0, in1, in2, in3, vec0, vec1, vec2, vec3, wgt, rnd, offset, out0, out1, out2, out3)
Definition hevc_mc_biw_msa.c:72
hevc_hv_biwgt_8t_32w_msa
static void hevc_hv_biwgt_8t_32w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2598
hevc_vt_biwgt_4t_12w_msa
static void hevc_vt_biwgt_4t_12w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4094
BI_W_MC
#define BI_W_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR)
Definition hevc_mc_biw_msa.c:5968
hevc_vt_biwgt_8t_4w_msa
static void hevc_vt_biwgt_8t_4w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1406
hevc_biwgt_copy_12w_msa
static void hevc_biwgt_copy_12w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:332
hevc_vt_biwgt_8t_64w_msa
static void hevc_vt_biwgt_8t_64w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1939
hevc_vt_biwgt_8t_16w_msa
static void hevc_vt_biwgt_8t_16w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1855
hevc_biwgt_copy_16w_msa
static void hevc_biwgt_copy_16w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:390
hevc_hv_biwgt_4t_4x2_msa
static void hevc_hv_biwgt_4t_4x2_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4532
hevc_biwgt_copy_64w_msa
static void hevc_biwgt_copy_64w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:618
hevc_hv_biwgt_8t_64w_msa
static void hevc_hv_biwgt_8t_64w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2640
BI_W_MC_COPY
#define BI_W_MC_COPY(WIDTH)
Definition hevc_mc_biw_msa.c:5933
hevc_hv_biwgt_4t_6w_msa
static void hevc_hv_biwgt_4t_6w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4886
hevc_vt_biwgt_4t_6w_msa
static void hevc_vt_biwgt_4t_6w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3759
hevc_hz_biwgt_4t_16w_msa
static void hevc_hz_biwgt_4t_16w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3268
hevc_hv_biwgt_8t_24w_msa
static void hevc_hv_biwgt_8t_24w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2577
hevc_hv_biwgt_4t_12w_msa
static void hevc_hv_biwgt_4t_12w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:5623
hevc_biwgt_copy_48w_msa
static void hevc_biwgt_copy_48w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:567
hevc_hz_biwgt_4t_8w_msa
static void hevc_hz_biwgt_4t_8w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3153
hevc_hv_biwgt_4t_8multx4mult_msa
static void hevc_hv_biwgt_4t_8multx4mult_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val, int32_t width)
Definition hevc_mc_biw_msa.c:5437
HEVC_BIW_RND_CLIP4
#define HEVC_BIW_RND_CLIP4(in0, in1, in2, in3, vec0, vec1, vec2, vec3, wgt, rnd, offset, out0, out1, out2, out3)
Definition hevc_mc_biw_msa.c:49
hevc_hv_biwgt_8t_8w_msa
static void hevc_hv_biwgt_8t_8w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2285
hevc_hz_biwgt_4t_12w_msa
static void hevc_hz_biwgt_4t_12w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3183
hevc_hz_biwgt_8t_16w_msa
static void hevc_hz_biwgt_8t_16w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:916
hevc_hz_biwgt_4t_8x4multiple_msa
static void hevc_hz_biwgt_4t_8x4multiple_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3084
hevc_hz_biwgt_8t_64w_msa
static void hevc_hz_biwgt_8t_64w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1308
hevc_vt_biwgt_4t_8x2_msa
static void hevc_vt_biwgt_4t_8x2_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3865
hevc_vt_biwgt_4t_24w_msa
static void hevc_vt_biwgt_4t_24w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4278
hevc_biwgt_copy_4w_msa
static void hevc_biwgt_copy_4w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:82
BI_W_MC_HV
#define BI_W_MC_HV(PEL, WIDTH, TAP)
Definition hevc_mc_biw_msa.c:6030
hevc_hz_biwgt_8t_32w_msa
static void hevc_hz_biwgt_8t_32w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1120
hevc_hv_biwgt_4t_32w_msa
static void hevc_hv_biwgt_4t_32w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:5912
hevc_hz_biwgt_8t_24w_msa
static void hevc_hz_biwgt_8t_24w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:999
hevc_vt_biwgt_4t_4x4_msa
static void hevc_vt_biwgt_4t_4x4_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3578
hevc_vt_biwgt_4t_4x2_msa
static void hevc_vt_biwgt_4t_4x2_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3514
hevc_hz_biwgt_4t_32w_msa
static void hevc_hz_biwgt_4t_32w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:3442
hevc_hv_biwgt_4t_8x2_msa
static void hevc_hv_biwgt_4t_8x2_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:5068
hevc_hz_biwgt_4t_4w_msa
static void hevc_hz_biwgt_4t_4w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2853
hevc_hv_biwgt_4t_4multx8mult_msa
static void hevc_hv_biwgt_4t_4multx8mult_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4711
hevc_vt_biwgt_8t_24w_msa
static void hevc_vt_biwgt_8t_24w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1875
hevc_hv_biwgt_4t_4w_msa
static void hevc_hv_biwgt_4t_4w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4855
hevc_hv_biwgt_8t_16w_msa
static void hevc_hv_biwgt_8t_16w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2556
hevc_vt_biwgt_8t_48w_msa
static void hevc_vt_biwgt_8t_48w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1919
hevc_hv_biwgt_8t_8multx2mult_msa
static void hevc_hv_biwgt_8t_8multx2mult_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val, int32_t width8mult)
Definition hevc_mc_biw_msa.c:2103
hevc_hz_biwgt_4t_8x2_msa
static void hevc_hz_biwgt_4t_8x2_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2956
hevc_hz_biwgt_8t_48w_msa
static void hevc_hz_biwgt_8t_48w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1205
hevc_hv_biwgt_4t_24w_msa
static void hevc_hv_biwgt_4t_24w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:5891
hevc_biwgt_copy_8w_msa
static void hevc_biwgt_copy_8w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:245
hevc_hz_biwgt_8t_12w_msa
static void hevc_hz_biwgt_8t_12w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:822
hevc_vt_biwgt_8t_12w_msa
static void hevc_vt_biwgt_8t_12w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:1631
hevc_vt_biwgt_8t_16multx2mult_msa
static void hevc_vt_biwgt_8t_16multx2mult_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val, int32_t width)
Definition hevc_mc_biw_msa.c:1740
hevc_hv_biwgt_4t_4x4_msa
static void hevc_hv_biwgt_4t_4x4_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4615
hevc_vt_biwgt_4t_32w_msa
static void hevc_vt_biwgt_4t_32w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:4414
hevc_hv_biwgt_8t_48w_msa
static void hevc_hv_biwgt_8t_48w_msa(const uint8_t *src0_ptr, int32_t src_stride, const int16_t *src1_ptr, int32_t src2_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t weight0, int32_t weight1, int32_t offset, int32_t rnd_val)
Definition hevc_mc_biw_msa.c:2619
hevcdsp_mips.h
zero
static int zero(InterplayACMContext *s, unsigned ind, unsigned col)
Definition interplayacm.c:121
offset
unsigned offset
Definition libaomenc.c:763
weight
const h264_weight_func weight
Definition h264dsp_init.c:33
tmp
static uint8_t tmp[40]
Definition aes_ctr.c:52
filter
void(* filter)(uint8_t *src, ptrdiff_t stride, int qscale)
Definition h263dsp.c:29
src1
#define src1
Definition h264pred.c:141
src0
#define src0
Definition h264pred.c:140
out
static FILE * out
Definition movenc.c:55
height
#define height
Definition dsp.h:89
width
#define width
Definition dsp.h:89
ST_W4
#define ST_W4(in, idx0, idx1, idx2, idx3, pdst, stride)
Definition vp8_lpf_lsx.c:234
h
h
Definition vp9dsp_template.c:2070
Generated on Mon Aug 17 2026 20:20:57 for FFmpeg by doxygen 1.13.2