FFmpeg
hevc_mc_uni_msa.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2015 - 2017 Manojkumar Bhosale (Manojkumar.Bhosale@imgtec.com)
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "libavutil/mips/generic_macros_msa.h"
22 #include "libavcodec/mips/hevcdsp_mips.h"
23 #include "libavcodec/mips/hevc_macros_msa.h"
24 
25 static const uint8_t ff_hevc_mask_arr[16 * 3] __attribute__((aligned(0x40))) = {
26  /* 8 width cases */
27  0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8,
28  /* 4 width cases */
29  0, 1, 1, 2, 2, 3, 3, 4, 16, 17, 17, 18, 18, 19, 19, 20,
30  /* 4 width cases */
31  8, 9, 9, 10, 10, 11, 11, 12, 24, 25, 25, 26, 26, 27, 27, 28
32 };
33 
34 #define HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, \
35  mask0, mask1, mask2, mask3, \
36  filt0, filt1, filt2, filt3, \
37  out0, out1) \
38 { \
39  v16i8 vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \
40  \
41  VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0_m, vec1_m); \
42  DOTP_SB2_SH(vec0_m, vec1_m, filt0, filt0, out0, out1); \
43  VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2_m, vec3_m); \
44  DPADD_SB2_SH(vec2_m, vec3_m, filt1, filt1, out0, out1); \
45  VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec4_m, vec5_m); \
46  DPADD_SB2_SH(vec4_m, vec5_m, filt2, filt2, out0, out1); \
47  VSHF_B2_SB(src0, src1, src2, src3, mask3, mask3, vec6_m, vec7_m); \
48  DPADD_SB2_SH(vec6_m, vec7_m, filt3, filt3, out0, out1); \
49 }
50 
51 #define HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, \
52  mask0, mask1, mask2, mask3, \
53  filt0, filt1, filt2, filt3, \
54  out0, out1, out2, out3) \
55 { \
56  v16i8 vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m; \
57  \
58  VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0_m, vec1_m); \
59  VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2_m, vec3_m); \
60  DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt0, filt0, filt0, filt0, \
61  out0, out1, out2, out3); \
62  VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0_m, vec1_m); \
63  VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2_m, vec3_m); \
64  DPADD_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt2, filt2, filt2, filt2, \
65  out0, out1, out2, out3); \
66  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4_m, vec5_m); \
67  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6_m, vec7_m); \
68  DPADD_SB4_SH(vec4_m, vec5_m, vec6_m, vec7_m, filt1, filt1, filt1, filt1, \
69  out0, out1, out2, out3); \
70  VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec4_m, vec5_m); \
71  VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec6_m, vec7_m); \
72  DPADD_SB4_SH(vec4_m, vec5_m, vec6_m, vec7_m, filt3, filt3, filt3, filt3, \
73  out0, out1, out2, out3); \
74 }
75 
76 #define HORIZ_4TAP_4WID_4VECS_FILT(src0, src1, src2, src3, \
77  mask0, mask1, filt0, filt1, \
78  out0, out1) \
79 { \
80  v16i8 vec0_m, vec1_m, vec2_m, vec3_m; \
81  \
82  VSHF_B2_SB(src0, src1, src2, src3, mask0, mask0, vec0_m, vec1_m); \
83  DOTP_SB2_SH(vec0_m, vec1_m, filt0, filt0, out0, out1); \
84  VSHF_B2_SB(src0, src1, src2, src3, mask1, mask1, vec2_m, vec3_m); \
85  DPADD_SB2_SH(vec2_m, vec3_m, filt1, filt1, out0, out1); \
86 }
87 
88 #define HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, \
89  mask0, mask1, filt0, filt1, \
90  out0, out1, out2, out3) \
91 { \
92  v16i8 vec0_m, vec1_m, vec2_m, vec3_m; \
93  \
94  VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0_m, vec1_m); \
95  VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2_m, vec3_m); \
96  DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt0, filt0, filt0, filt0, \
97  out0, out1, out2, out3); \
98  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0_m, vec1_m); \
99  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2_m, vec3_m); \
100  DPADD_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt1, filt1, filt1, filt1, \
101  out0, out1, out2, out3); \
102 }
103 
104 static void copy_width8_msa(uint8_t *src, int32_t src_stride,
105  uint8_t *dst, int32_t dst_stride,
106  int32_t height)
107 {
108  int32_t cnt;
109  uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
110 
111  if (2 == height) {
112  LD2(src, src_stride, out0, out1);
113  SD(out0, dst);
114  dst += dst_stride;
115  SD(out1, dst);
116  } else if (6 == height) {
117  LD4(src, src_stride, out0, out1, out2, out3);
118  src += (4 * src_stride);
119  SD4(out0, out1, out2, out3, dst, dst_stride);
120  dst += (4 * dst_stride);
121  LD2(src, src_stride, out0, out1);
122  SD(out0, dst);
123  dst += dst_stride;
124  SD(out1, dst);
125  } else if (0 == (height % 8)) {
126  for (cnt = (height >> 3); cnt--;) {
127  LD4(src, src_stride, out0, out1, out2, out3);
128  src += (4 * src_stride);
129  LD4(src, src_stride, out4, out5, out6, out7);
130  src += (4 * src_stride);
131  SD4(out0, out1, out2, out3, dst, dst_stride);
132  dst += (4 * dst_stride);
133  SD4(out4, out5, out6, out7, dst, dst_stride);
134  dst += (4 * dst_stride);
135  }
136  } else if (0 == (height % 4)) {
137  for (cnt = (height >> 2); cnt--;) {
138  LD4(src, src_stride, out0, out1, out2, out3);
139  src += (4 * src_stride);
140  SD4(out0, out1, out2, out3, dst, dst_stride);
141  dst += (4 * dst_stride);
142  }
143  }
144 }
145 
146 static void copy_width12_msa(uint8_t *src, int32_t src_stride,
147  uint8_t *dst, int32_t dst_stride,
148  int32_t height)
149 {
150  v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
151 
152  LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
153  src += (8 * src_stride);
154  ST12x8_UB(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride);
155  dst += (8 * dst_stride);
156  LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
157  ST12x8_UB(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride);
158 }
159 
160 static void copy_width16_msa(uint8_t *src, int32_t src_stride,
161  uint8_t *dst, int32_t dst_stride,
162  int32_t height)
163 {
164  int32_t cnt;
165  v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
166 
167  if (12 == height) {
168  LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
169  src += (8 * src_stride);
170  ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride);
171  dst += (8 * dst_stride);
172  LD_UB4(src, src_stride, src0, src1, src2, src3);
173  src += (4 * src_stride);
174  ST_UB4(src0, src1, src2, src3, dst, dst_stride);
175  dst += (4 * dst_stride);
176  } else if (0 == (height % 8)) {
177  for (cnt = (height >> 3); cnt--;) {
178  LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6,
179  src7);
180  src += (8 * src_stride);
181  ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7, dst,
182  dst_stride);
183  dst += (8 * dst_stride);
184  }
185  } else if (0 == (height % 4)) {
186  for (cnt = (height >> 2); cnt--;) {
187  LD_UB4(src, src_stride, src0, src1, src2, src3);
188  src += (4 * src_stride);
189 
190  ST_UB4(src0, src1, src2, src3, dst, dst_stride);
191  dst += (4 * dst_stride);
192  }
193  }
194 }
195 
196 static void copy_width24_msa(uint8_t *src, int32_t src_stride,
197  uint8_t *dst, int32_t dst_stride,
198  int32_t height)
199 {
200  int32_t cnt;
201  v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
202  uint64_t out0, out1, out2, out3, out4, out5, out6, out7;
203 
204  for (cnt = 4; cnt--;) {
205  LD_UB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
206  LD4(src + 16, src_stride, out0, out1, out2, out3);
207  src += (4 * src_stride);
208  LD4(src + 16, src_stride, out4, out5, out6, out7);
209  src += (4 * src_stride);
210 
211  ST_UB8(src0, src1, src2, src3, src4, src5, src6, src7, dst, dst_stride);
212  SD4(out0, out1, out2, out3, dst + 16, dst_stride);
213  dst += (4 * dst_stride);
214  SD4(out4, out5, out6, out7, dst + 16, dst_stride);
215  dst += (4 * dst_stride);
216  }
217 }
218 
219 static void copy_width32_msa(uint8_t *src, int32_t src_stride,
220  uint8_t *dst, int32_t dst_stride,
221  int32_t height)
222 {
223  int32_t cnt;
224  v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
225 
226  for (cnt = (height >> 2); cnt--;) {
227  LD_UB4(src, src_stride, src0, src1, src2, src3);
228  LD_UB4(src + 16, src_stride, src4, src5, src6, src7);
229  src += (4 * src_stride);
230  ST_UB4(src0, src1, src2, src3, dst, dst_stride);
231  ST_UB4(src4, src5, src6, src7, dst + 16, dst_stride);
232  dst += (4 * dst_stride);
233  }
234 }
235 
236 static void copy_width48_msa(uint8_t *src, int32_t src_stride,
237  uint8_t *dst, int32_t dst_stride,
238  int32_t height)
239 {
240  int32_t cnt;
241  v16u8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
242  v16u8 src11;
243 
244  for (cnt = (height >> 2); cnt--;) {
245  LD_UB4(src, src_stride, src0, src1, src2, src3);
246  LD_UB4(src + 16, src_stride, src4, src5, src6, src7);
247  LD_UB4(src + 32, src_stride, src8, src9, src10, src11);
248  src += (4 * src_stride);
249 
250  ST_UB4(src0, src1, src2, src3, dst, dst_stride);
251  ST_UB4(src4, src5, src6, src7, dst + 16, dst_stride);
252  ST_UB4(src8, src9, src10, src11, dst + 32, dst_stride);
253  dst += (4 * dst_stride);
254  }
255 }
256 
257 static void copy_width64_msa(uint8_t *src, int32_t src_stride,
258  uint8_t *dst, int32_t dst_stride,
259  int32_t height)
260 {
261  int32_t cnt;
262  v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
263  v16u8 src8, src9, src10, src11, src12, src13, src14, src15;
264 
265  for (cnt = (height >> 2); cnt--;) {
266  LD_UB4(src, 16, src0, src1, src2, src3);
267  src += src_stride;
268  LD_UB4(src, 16, src4, src5, src6, src7);
269  src += src_stride;
270  LD_UB4(src, 16, src8, src9, src10, src11);
271  src += src_stride;
272  LD_UB4(src, 16, src12, src13, src14, src15);
273  src += src_stride;
274 
275  ST_UB4(src0, src1, src2, src3, dst, 16);
276  dst += dst_stride;
277  ST_UB4(src4, src5, src6, src7, dst, 16);
278  dst += dst_stride;
279  ST_UB4(src8, src9, src10, src11, dst, 16);
280  dst += dst_stride;
281  ST_UB4(src12, src13, src14, src15, dst, 16);
282  dst += dst_stride;
283  }
284 }
285 
286 static void common_hz_8t_4x4_msa(uint8_t *src, int32_t src_stride,
287  uint8_t *dst, int32_t dst_stride,
288  const int8_t *filter)
289 {
290  v16u8 mask0, mask1, mask2, mask3, out;
291  v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3;
292  v8i16 filt, out0, out1;
293 
294  mask0 = LD_UB(&ff_hevc_mask_arr[16]);
295  src -= 3;
296 
297  /* rearranging filter */
298  filt = LD_SH(filter);
299  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
300 
301  mask1 = mask0 + 2;
302  mask2 = mask0 + 4;
303  mask3 = mask0 + 6;
304 
305  LD_SB4(src, src_stride, src0, src1, src2, src3);
306  XORI_B4_128_SB(src0, src1, src2, src3);
307  HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
308  mask3, filt0, filt1, filt2, filt3, out0, out1);
309  SRARI_H2_SH(out0, out1, 6);
310  SAT_SH2_SH(out0, out1, 7);
311  out = PCKEV_XORI128_UB(out0, out1);
312  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
313 }
314 
315 static void common_hz_8t_4x8_msa(uint8_t *src, int32_t src_stride,
316  uint8_t *dst, int32_t dst_stride,
317  const int8_t *filter)
318 {
319  v16i8 filt0, filt1, filt2, filt3;
320  v16i8 src0, src1, src2, src3;
321  v16u8 mask0, mask1, mask2, mask3, out;
322  v8i16 filt, out0, out1, out2, out3;
323 
324  mask0 = LD_UB(&ff_hevc_mask_arr[16]);
325  src -= 3;
326 
327  /* rearranging filter */
328  filt = LD_SH(filter);
329  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
330 
331  mask1 = mask0 + 2;
332  mask2 = mask0 + 4;
333  mask3 = mask0 + 6;
334 
335  LD_SB4(src, src_stride, src0, src1, src2, src3);
336  XORI_B4_128_SB(src0, src1, src2, src3);
337  src += (4 * src_stride);
338  HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
339  mask3, filt0, filt1, filt2, filt3, out0, out1);
340  LD_SB4(src, src_stride, src0, src1, src2, src3);
341  XORI_B4_128_SB(src0, src1, src2, src3);
342  HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
343  mask3, filt0, filt1, filt2, filt3, out2, out3);
344  SRARI_H4_SH(out0, out1, out2, out3, 6);
345  SAT_SH4_SH(out0, out1, out2, out3, 7);
346  out = PCKEV_XORI128_UB(out0, out1);
347  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
348  dst += (4 * dst_stride);
349  out = PCKEV_XORI128_UB(out2, out3);
350  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
351 }
352 
353 static void common_hz_8t_4x16_msa(uint8_t *src, int32_t src_stride,
354  uint8_t *dst, int32_t dst_stride,
355  const int8_t *filter)
356 {
357  v16u8 mask0, mask1, mask2, mask3, out;
358  v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3;
359  v8i16 filt, out0, out1, out2, out3;
360 
361  mask0 = LD_UB(&ff_hevc_mask_arr[16]);
362  src -= 3;
363 
364  /* rearranging filter */
365  filt = LD_SH(filter);
366  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
367 
368  mask1 = mask0 + 2;
369  mask2 = mask0 + 4;
370  mask3 = mask0 + 6;
371 
372  LD_SB4(src, src_stride, src0, src1, src2, src3);
373  XORI_B4_128_SB(src0, src1, src2, src3);
374  src += (4 * src_stride);
375  HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
376  mask3, filt0, filt1, filt2, filt3, out0, out1);
377  LD_SB4(src, src_stride, src0, src1, src2, src3);
378  XORI_B4_128_SB(src0, src1, src2, src3);
379  src += (4 * src_stride);
380  HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
381  mask3, filt0, filt1, filt2, filt3, out2, out3);
382  SRARI_H4_SH(out0, out1, out2, out3, 6);
383  SAT_SH4_SH(out0, out1, out2, out3, 7);
384  out = PCKEV_XORI128_UB(out0, out1);
385  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
386  dst += (4 * dst_stride);
387  out = PCKEV_XORI128_UB(out2, out3);
388  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
389  dst += (4 * dst_stride);
390 
391  LD_SB4(src, src_stride, src0, src1, src2, src3);
392  XORI_B4_128_SB(src0, src1, src2, src3);
393  src += (4 * src_stride);
394  HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
395  mask3, filt0, filt1, filt2, filt3, out0, out1);
396  LD_SB4(src, src_stride, src0, src1, src2, src3);
397  XORI_B4_128_SB(src0, src1, src2, src3);
398  src += (4 * src_stride);
399  HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
400  mask3, filt0, filt1, filt2, filt3, out2, out3);
401 
402  SRARI_H4_SH(out0, out1, out2, out3, 6);
403  SAT_SH4_SH(out0, out1, out2, out3, 7);
404  out = PCKEV_XORI128_UB(out0, out1);
405  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
406  dst += (4 * dst_stride);
407  out = PCKEV_XORI128_UB(out2, out3);
408  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
409 }
410 
411 static void common_hz_8t_4w_msa(uint8_t *src, int32_t src_stride,
412  uint8_t *dst, int32_t dst_stride,
413  const int8_t *filter, int32_t height)
414 {
415  if (4 == height) {
416  common_hz_8t_4x4_msa(src, src_stride, dst, dst_stride, filter);
417  } else if (8 == height) {
418  common_hz_8t_4x8_msa(src, src_stride, dst, dst_stride, filter);
419  } else if (16 == height) {
420  common_hz_8t_4x16_msa(src, src_stride, dst, dst_stride, filter);
421  }
422 }
423 
424 static void common_hz_8t_8w_msa(uint8_t *src, int32_t src_stride,
425  uint8_t *dst, int32_t dst_stride,
426  const int8_t *filter, int32_t height)
427 {
428  uint32_t loop_cnt;
429  v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3;
430  v16u8 mask0, mask1, mask2, mask3, tmp0, tmp1;
431  v16i8 vec0_m, vec1_m, vec2_m, vec3_m, vec4_m, vec5_m, vec6_m, vec7_m;
432  v8i16 filt, out0, out1, out2, out3;
433 
434  mask0 = LD_UB(&ff_hevc_mask_arr[0]);
435  src -= 3;
436 
437  /* rearranging filter */
438  filt = LD_SH(filter);
439  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
440 
441  mask1 = mask0 + 2;
442  mask2 = mask0 + 4;
443  mask3 = mask0 + 6;
444 
445  for (loop_cnt = (height >> 2); loop_cnt--;) {
446  LD_SB4(src, src_stride, src0, src1, src2, src3);
447  XORI_B4_128_SB(src0, src1, src2, src3);
448  src += (4 * src_stride);
449 
450  VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0_m, vec1_m);
451  VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2_m, vec3_m);
452  DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt0, filt0, filt0, filt0,
453  out0, out1, out2, out3);
454  VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0_m, vec1_m);
455  VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2_m, vec3_m);
456  DPADD_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt2, filt2, filt2, filt2,
457  out0, out1, out2, out3);
458  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4_m, vec5_m);
459  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6_m, vec7_m);
460  DPADD_SB4_SH(vec4_m, vec5_m, vec6_m, vec7_m, filt1, filt1, filt1, filt1,
461  out0, out1, out2, out3);
462  VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec4_m, vec5_m);
463  VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec6_m, vec7_m);
464  DPADD_SB4_SH(vec4_m, vec5_m, vec6_m, vec7_m, filt3, filt3, filt3, filt3,
465  out0, out1, out2, out3);
466 
467  SRARI_H4_SH(out0, out1, out2, out3, 6);
468  SAT_SH4_SH(out0, out1, out2, out3, 7);
469  tmp0 = PCKEV_XORI128_UB(out0, out1);
470  tmp1 = PCKEV_XORI128_UB(out2, out3);
471  ST8x4_UB(tmp0, tmp1, dst, dst_stride);
472  dst += (4 * dst_stride);
473  }
474 }
475 
476 static void common_hz_8t_12w_msa(uint8_t *src, int32_t src_stride,
477  uint8_t *dst, int32_t dst_stride,
478  const int8_t *filter, int32_t height)
479 {
480  uint32_t loop_cnt;
481  v16u8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask00;
482  v16u8 tmp0, tmp1, tmp2;
483  v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
484  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
485  v16i8 filt0, filt1, filt2, filt3;
486  v8i16 filt, out0, out1, out2, out3, out4, out5;
487 
488  mask00 = LD_UB(&ff_hevc_mask_arr[0]);
489  mask0 = LD_UB(&ff_hevc_mask_arr[16]);
490 
491  src = src - 3;
492 
493  /* rearranging filter */
494  filt = LD_SH(filter);
495  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
496 
497  mask1 = mask00 + 2;
498  mask2 = mask00 + 4;
499  mask3 = mask00 + 6;
500  mask4 = mask0 + 2;
501  mask5 = mask0 + 4;
502  mask6 = mask0 + 6;
503 
504  for (loop_cnt = 4; loop_cnt--;) {
505  /* 8 width */
506  LD_SB4(src, src_stride, src0, src1, src2, src3);
507  /* 4 width */
508  LD_SB4(src + 8, src_stride, src4, src5, src6, src7);
509 
510  XORI_B4_128_SB(src0, src1, src2, src3);
511  XORI_B4_128_SB(src4, src5, src6, src7);
512  src += (4 * src_stride);
513 
514  VSHF_B2_SB(src0, src0, src1, src1, mask00, mask00, vec0, vec1);
515  VSHF_B2_SB(src2, src2, src3, src3, mask00, mask00, vec2, vec3);
516  DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, out0,
517  out1, out2, out3);
518  VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0, vec1);
519  VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2, vec3);
520  DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, out0,
521  out1, out2, out3);
522  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
523  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
524  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1, out0,
525  out1, out2, out3);
526  VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec4, vec5);
527  VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec6, vec7);
528  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt3, filt3, filt3, filt3, out0,
529  out1, out2, out3);
530 
531  /* 4 width */
532  VSHF_B2_SB(src4, src5, src6, src7, mask0, mask0, vec0, vec1);
533  DOTP_SB2_SH(vec0, vec1, filt0, filt0, out4, out5);
534  VSHF_B2_SB(src4, src5, src6, src7, mask4, mask4, vec2, vec3);
535  DPADD_SB2_SH(vec2, vec3, filt1, filt1, out4, out5);
536  VSHF_B2_SB(src4, src5, src6, src7, mask5, mask5, vec4, vec5);
537  DPADD_SB2_SH(vec4, vec5, filt2, filt2, out4, out5);
538  VSHF_B2_SB(src4, src5, src6, src7, mask6, mask6, vec6, vec7);
539  DPADD_SB2_SH(vec6, vec7, filt3, filt3, out4, out5);
540 
541  SRARI_H4_SH(out0, out1, out2, out3, 6);
542  SRARI_H2_SH(out4, out5, 6);
543  SAT_SH4_SH(out0, out1, out2, out3, 7);
544  SAT_SH2_SH(out4, out5, 7);
545  tmp0 = PCKEV_XORI128_UB(out0, out1);
546  tmp1 = PCKEV_XORI128_UB(out2, out3);
547  tmp2 = PCKEV_XORI128_UB(out4, out5);
548 
549  ST8x4_UB(tmp0, tmp1, dst, dst_stride);
550  ST4x4_UB(tmp2, tmp2, 0, 1, 2, 3, dst + 8, dst_stride);
551  dst += (4 * dst_stride);
552  }
553 }
554 
555 static void common_hz_8t_16w_msa(uint8_t *src, int32_t src_stride,
556  uint8_t *dst, int32_t dst_stride,
557  const int8_t *filter, int32_t height)
558 {
559  uint32_t loop_cnt;
560  v16u8 mask0, mask1, mask2, mask3, out;
561  v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
562  v16i8 filt0, filt1, filt2, filt3;
563  v8i16 filt, out0, out1, out2, out3;
564 
565  mask0 = LD_UB(&ff_hevc_mask_arr[0]);
566  src -= 3;
567 
568  /* rearranging filter */
569  filt = LD_SH(filter);
570  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
571 
572  mask1 = mask0 + 2;
573  mask2 = mask0 + 4;
574  mask3 = mask0 + 6;
575 
576  for (loop_cnt = (height >> 2); loop_cnt--;) {
577  LD_SB2(src, src_stride, src0, src2);
578  LD_SB2(src + 8, src_stride, src1, src3);
579  src += (2 * src_stride);
580 
581  LD_SB2(src, src_stride, src4, src6);
582  LD_SB2(src + 8, src_stride, src5, src7);
583  src += (2 * src_stride);
584 
585  XORI_B4_128_SB(src0, src1, src2, src3);
586  XORI_B4_128_SB(src4, src5, src6, src7);
587  HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
588  mask3, filt0, filt1, filt2, filt3, out0,
589  out1, out2, out3);
590  SRARI_H4_SH(out0, out1, out2, out3, 6);
591  SAT_SH4_SH(out0, out1, out2, out3, 7);
592  out = PCKEV_XORI128_UB(out0, out1);
593  ST_UB(out, dst);
594  dst += dst_stride;
595  out = PCKEV_XORI128_UB(out2, out3);
596  ST_UB(out, dst);
597  dst += dst_stride;
598 
599  HORIZ_8TAP_8WID_4VECS_FILT(src4, src5, src6, src7, mask0, mask1, mask2,
600  mask3, filt0, filt1, filt2, filt3, out0,
601  out1, out2, out3);
602  SRARI_H4_SH(out0, out1, out2, out3, 6);
603  SAT_SH4_SH(out0, out1, out2, out3, 7);
604  out = PCKEV_XORI128_UB(out0, out1);
605  ST_UB(out, dst);
606  dst += dst_stride;
607  out = PCKEV_XORI128_UB(out2, out3);
608  ST_UB(out, dst);
609  dst += dst_stride;
610  }
611 }
612 
613 static void common_hz_8t_24w_msa(uint8_t *src, int32_t src_stride,
614  uint8_t *dst, int32_t dst_stride,
615  const int8_t *filter, int32_t height)
616 {
617  uint32_t loop_cnt;
618  v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3;
619  v16u8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7, out;
620  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9, vec10;
621  v16i8 vec11;
622  v8i16 out0, out1, out2, out3, out8, out9, filt;
623 
624  mask0 = LD_UB(&ff_hevc_mask_arr[0]);
625  src -= 3;
626 
627  /* rearranging filter */
628  filt = LD_SH(filter);
629  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
630 
631  mask1 = mask0 + 2;
632  mask2 = mask0 + 4;
633  mask3 = mask0 + 6;
634  mask4 = mask0 + 8;
635  mask5 = mask0 + 10;
636  mask6 = mask0 + 12;
637  mask7 = mask0 + 14;
638 
639  for (loop_cnt = 16; loop_cnt--;) {
640  LD_SB2(src, src_stride, src0, src2);
641  LD_SB2(src + 16, src_stride, src1, src3);
642  XORI_B4_128_SB(src0, src1, src2, src3);
643  src += (2 * src_stride);
644  VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec8);
645  VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec9);
646  VSHF_B2_SB(src0, src1, src2, src3, mask4, mask4, vec1, vec3);
647  DOTP_SB4_SH(vec0, vec8, vec2, vec9, filt0, filt0, filt0, filt0, out0,
648  out8, out2, out9);
649  DOTP_SB2_SH(vec1, vec3, filt0, filt0, out1, out3);
650  VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0, vec8);
651  VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2, vec9);
652  VSHF_B2_SB(src0, src1, src2, src3, mask6, mask6, vec1, vec3);
653  DPADD_SB4_SH(vec0, vec8, vec2, vec9, filt2, filt2, filt2, filt2,
654  out0, out8, out2, out9);
655  DPADD_SB2_SH(vec1, vec3, filt2, filt2, out1, out3);
656  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec10);
657  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec11);
658  VSHF_B2_SB(src0, src1, src2, src3, mask5, mask5, vec5, vec7);
659  DPADD_SB4_SH(vec4, vec10, vec6, vec11, filt1, filt1, filt1, filt1,
660  out0, out8, out2, out9);
661  DPADD_SB2_SH(vec5, vec7, filt1, filt1, out1, out3);
662  VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec4, vec10);
663  VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec6, vec11);
664  VSHF_B2_SB(src0, src1, src2, src3, mask7, mask7, vec5, vec7);
665  DPADD_SB4_SH(vec4, vec10, vec6, vec11, filt3, filt3, filt3, filt3,
666  out0, out8, out2, out9);
667  DPADD_SB2_SH(vec5, vec7, filt3, filt3, out1, out3);
668  SRARI_H4_SH(out0, out8, out2, out9, 6);
669  SRARI_H2_SH(out1, out3, 6);
670  SAT_SH4_SH(out0, out8, out2, out9, 7);
671  SAT_SH2_SH(out1, out3, 7);
672  out = PCKEV_XORI128_UB(out8, out9);
673  ST8x2_UB(out, dst + 16, dst_stride);
674  out = PCKEV_XORI128_UB(out0, out1);
675  ST_UB(out, dst);
676  dst += dst_stride;
677  out = PCKEV_XORI128_UB(out2, out3);
678  ST_UB(out, dst);
679  dst += dst_stride;
680  }
681 }
682 
683 static void common_hz_8t_32w_msa(uint8_t *src, int32_t src_stride,
684  uint8_t *dst, int32_t dst_stride,
685  const int8_t *filter, int32_t height)
686 {
687  uint32_t loop_cnt;
688  v16u8 mask0, mask1, mask2, mask3, out;
689  v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
690  v16i8 filt0, filt1, filt2, filt3;
691  v8i16 filt, out0, out1, out2, out3;
692 
693  mask0 = LD_UB(&ff_hevc_mask_arr[0]);
694  src -= 3;
695 
696  /* rearranging filter */
697  filt = LD_SH(filter);
698  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
699 
700  mask1 = mask0 + 2;
701  mask2 = mask0 + 4;
702  mask3 = mask0 + 6;
703 
704  for (loop_cnt = (height >> 1); loop_cnt--;) {
705  src0 = LD_SB(src);
706  src1 = LD_SB(src + 8);
707  src2 = LD_SB(src + 16);
708  src3 = LD_SB(src + 24);
709  src += src_stride;
710  XORI_B4_128_SB(src0, src1, src2, src3);
711 
712  src4 = LD_SB(src);
713  src5 = LD_SB(src + 8);
714  src6 = LD_SB(src + 16);
715  src7 = LD_SB(src + 24);
716  src += src_stride;
717  XORI_B4_128_SB(src4, src5, src6, src7);
718 
719  HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, mask2,
720  mask3, filt0, filt1, filt2, filt3, out0,
721  out1, out2, out3);
722  SRARI_H4_SH(out0, out1, out2, out3, 6);
723  SAT_SH4_SH(out0, out1, out2, out3, 7);
724 
725  out = PCKEV_XORI128_UB(out0, out1);
726  ST_UB(out, dst);
727  out = PCKEV_XORI128_UB(out2, out3);
728  ST_UB(out, dst + 16);
729  dst += dst_stride;
730 
731  HORIZ_8TAP_8WID_4VECS_FILT(src4, src5, src6, src7, mask0, mask1, mask2,
732  mask3, filt0, filt1, filt2, filt3, out0,
733  out1, out2, out3);
734  SRARI_H4_SH(out0, out1, out2, out3, 6);
735  SAT_SH4_SH(out0, out1, out2, out3, 7);
736  out = PCKEV_XORI128_UB(out0, out1);
737  ST_UB(out, dst);
738  out = PCKEV_XORI128_UB(out2, out3);
739  ST_UB(out, dst + 16);
740  dst += dst_stride;
741  }
742 }
743 
744 static void common_hz_8t_48w_msa(uint8_t *src, int32_t src_stride,
745  uint8_t *dst, int32_t dst_stride,
746  const int8_t *filter, int32_t height)
747 {
748  uint32_t loop_cnt;
749  v16i8 src0, src1, src2, src3, filt0, filt1, filt2, filt3, vec0, vec1, vec2;
750  v16i8 src4;
751  v16u8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7, out;
752  v8i16 filt, out0, out1, out2, out3;
753 
754  mask0 = LD_UB(&ff_hevc_mask_arr[0]);
755  src -= 3;
756 
757  /* rearranging filter */
758  filt = LD_SH(filter);
759  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
760 
761  mask1 = mask0 + 2;
762  mask2 = mask0 + 4;
763  mask3 = mask0 + 6;
764  mask4 = mask0 + 8;
765  mask5 = mask0 + 10;
766  mask6 = mask0 + 12;
767  mask7 = mask0 + 14;
768 
769  for (loop_cnt = 64; loop_cnt--;) {
770  src0 = LD_SB(src);
771  src1 = LD_SB(src + 8);
772  src2 = LD_SB(src + 16);
773  src3 = LD_SB(src + 32);
774  src4 = LD_SB(src + 40);
775  src += src_stride;
776 
777  XORI_B4_128_SB(src0, src1, src2, src3);
778  src4 = (v16i8) __msa_xori_b((v16u8) src4, 128);
779 
780  VSHF_B3_SB(src0, src0, src1, src1, src2, src2, mask0, mask0, mask0,
781  vec0, vec1, vec2);
782  DOTP_SB3_SH(vec0, vec1, vec2, filt0, filt0, filt0, out0, out1, out2);
783  VSHF_B3_SB(src0, src0, src1, src1, src2, src2, mask1, mask1, mask1,
784  vec0, vec1, vec2);
785  DPADD_SB2_SH(vec0, vec1, filt1, filt1, out0, out1);
786  out2 = __msa_dpadd_s_h(out2, vec2, filt1);
787  VSHF_B3_SB(src0, src0, src1, src1, src2, src2, mask2, mask2, mask2,
788  vec0, vec1, vec2);
789  DPADD_SB2_SH(vec0, vec1, filt2, filt2, out0, out1);
790  out2 = __msa_dpadd_s_h(out2, vec2, filt2);
791 
792  VSHF_B3_SB(src0, src0, src1, src1, src2, src2, mask3, mask3, mask3,
793  vec0, vec1, vec2);
794  DPADD_SB2_SH(vec0, vec1, filt3, filt3, out0, out1);
795  out2 = __msa_dpadd_s_h(out2, vec2, filt3);
796 
797  SRARI_H2_SH(out0, out1, 6);
798  out3 = __msa_srari_h(out2, 6);
799  SAT_SH3_SH(out0, out1, out3, 7);
800  out = PCKEV_XORI128_UB(out0, out1);
801  ST_UB(out, dst);
802 
803  VSHF_B3_SB(src2, src3, src3, src3, src4, src4, mask4, mask0, mask0,
804  vec0, vec1, vec2);
805  DOTP_SB3_SH(vec0, vec1, vec2, filt0, filt0, filt0, out0, out1, out2);
806  VSHF_B3_SB(src2, src3, src3, src3, src4, src4, mask5, mask1, mask1,
807  vec0, vec1, vec2);
808  DPADD_SB2_SH(vec0, vec1, filt1, filt1, out0, out1);
809  out2 = __msa_dpadd_s_h(out2, vec2, filt1);
810  VSHF_B3_SB(src2, src3, src3, src3, src4, src4, mask6, mask2, mask2,
811  vec0, vec1, vec2);
812  DPADD_SB2_SH(vec0, vec1, filt2, filt2, out0, out1);
813  out2 = __msa_dpadd_s_h(out2, vec2, filt2);
814  VSHF_B3_SB(src2, src3, src3, src3, src4, src4, mask7, mask3, mask3,
815  vec0, vec1, vec2);
816  DPADD_SB2_SH(vec0, vec1, filt3, filt3, out0, out1);
817  out2 = __msa_dpadd_s_h(out2, vec2, filt3);
818 
819  SRARI_H2_SH(out0, out1, 6);
820  out2 = __msa_srari_h(out2, 6);
821  SAT_SH3_SH(out0, out1, out2, 7);
822  out = PCKEV_XORI128_UB(out3, out0);
823  ST_UB(out, dst + 16);
824  out = PCKEV_XORI128_UB(out1, out2);
825  ST_UB(out, dst + 32);
826  dst += dst_stride;
827  }
828 }
829 
830 static void common_hz_8t_64w_msa(uint8_t *src, int32_t src_stride,
831  uint8_t *dst, int32_t dst_stride,
832  const int8_t *filter, int32_t height)
833 {
834  int32_t loop_cnt;
835  v16u8 mask0, mask1, mask2, mask3, out;
836  v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
837  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
838  v16i8 filt0, filt1, filt2, filt3;
839  v8i16 res0, res1, res2, res3, filt;
840 
841  mask0 = LD_UB(&ff_hevc_mask_arr[0]);
842  src -= 3;
843 
844  /* rearranging filter */
845  filt = LD_SH(filter);
846  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
847 
848  mask1 = mask0 + 2;
849  mask2 = mask0 + 4;
850  mask3 = mask0 + 6;
851 
852  for (loop_cnt = height; loop_cnt--;) {
853  LD_SB8(src, 8, src0, src1, src2, src3, src4, src5, src6, src7);
854  src += src_stride;
855 
856  XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
857 
858  VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
859  VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2, vec3);
860  DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, res0,
861  res1, res2, res3);
862  VSHF_B2_SB(src0, src0, src1, src1, mask2, mask2, vec0, vec1);
863  VSHF_B2_SB(src2, src2, src3, src3, mask2, mask2, vec2, vec3);
864  DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, res0,
865  res1, res2, res3);
866  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec4, vec5);
867  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec6, vec7);
868  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1, res0,
869  res1, res2, res3);
870  VSHF_B2_SB(src0, src0, src1, src1, mask3, mask3, vec4, vec5);
871  VSHF_B2_SB(src2, src2, src3, src3, mask3, mask3, vec6, vec7);
872  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt3, filt3, filt3, filt3, res0,
873  res1, res2, res3);
874 
875  SRARI_H4_SH(res0, res1, res2, res3, 6);
876  SAT_SH4_SH(res0, res1, res2, res3, 7);
877  out = PCKEV_XORI128_UB(res0, res1);
878  ST_UB(out, dst);
879  out = PCKEV_XORI128_UB(res2, res3);
880  ST_UB(out, dst + 16);
881 
882  VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0, vec1);
883  VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2, vec3);
884  DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0, res0,
885  res1, res2, res3);
886  VSHF_B2_SB(src4, src4, src5, src5, mask2, mask2, vec0, vec1);
887  VSHF_B2_SB(src6, src6, src7, src7, mask2, mask2, vec2, vec3);
888  DPADD_SB4_SH(vec0, vec1, vec2, vec3, filt2, filt2, filt2, filt2, res0,
889  res1, res2, res3);
890  VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec4, vec5);
891  VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec6, vec7);
892  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1, res0,
893  res1, res2, res3);
894  VSHF_B2_SB(src4, src4, src5, src5, mask3, mask3, vec4, vec5);
895  VSHF_B2_SB(src6, src6, src7, src7, mask3, mask3, vec6, vec7);
896  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt3, filt3, filt3, filt3, res0,
897  res1, res2, res3);
898 
899  SRARI_H4_SH(res0, res1, res2, res3, 6);
900  SAT_SH4_SH(res0, res1, res2, res3, 7);
901  out = PCKEV_XORI128_UB(res0, res1);
902  ST_UB(out, dst + 32);
903  out = PCKEV_XORI128_UB(res2, res3);
904  ST_UB(out, dst + 48);
905  dst += dst_stride;
906  }
907 }
908 
909 static void common_vt_8t_4w_msa(uint8_t *src, int32_t src_stride,
910  uint8_t *dst, int32_t dst_stride,
911  const int8_t *filter, int32_t height)
912 {
913  uint32_t loop_cnt;
914  v16u8 out0, out1;
915  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
916  v16i8 src11, src12, src13, src14;
917  v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r;
918  v16i8 src65_r, src87_r, src109_r, src2110, src4332, src6554, src8776;
919  v16i8 src1110_r, src1211_r, src1312_r, src1413_r, src12111110, src14131312;
920  v16i8 src10998, filt0, filt1, filt2, filt3;
921  v8i16 filt, out10, out32, out54, out76;
922 
923  src -= (3 * src_stride);
924 
925  filt = LD_SH(filter);
926  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
927 
928  LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
929  src += (7 * src_stride);
930 
931  ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r,
932  src54_r, src21_r);
933  ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
934  ILVR_D3_SB(src21_r, src10_r, src43_r, src32_r, src65_r, src54_r, src2110,
935  src4332, src6554);
936  XORI_B3_128_SB(src2110, src4332, src6554);
937 
938  for (loop_cnt = (height >> 3); loop_cnt--;) {
939  LD_SB4(src, src_stride, src7, src8, src9, src10);
940  src += (4 * src_stride);
941  LD_SB4(src, src_stride, src11, src12, src13, src14);
942  src += (4 * src_stride);
943 
944  ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r,
945  src87_r, src98_r, src109_r);
946  ILVR_B4_SB(src11, src10, src12, src11, src13, src12, src14, src13,
947  src1110_r, src1211_r, src1312_r, src1413_r);
948  ILVR_D2_SB(src87_r, src76_r, src109_r, src98_r, src8776, src10998);
949  ILVR_D2_SB(src1211_r, src1110_r, src1413_r, src1312_r,
950  src12111110, src14131312);
951  XORI_B2_128_SB(src8776, src10998);
952  XORI_B2_128_SB(src12111110, src14131312);
953 
954  DOTP_SB2_SH(src2110, src4332, filt0, filt0, out10, out32);
955  DOTP_SB2_SH(src6554, src8776, filt0, filt0, out54, out76);
956  DPADD_SB2_SH(src4332, src6554, filt1, filt1, out10, out32);
957  DPADD_SB2_SH(src8776, src10998, filt1, filt1, out54, out76);
958  DPADD_SB2_SH(src6554, src8776, filt2, filt2, out10, out32);
959  DPADD_SB2_SH(src10998, src12111110, filt2, filt2, out54, out76);
960  DPADD_SB2_SH(src8776, src10998, filt3, filt3, out10, out32);
961  DPADD_SB2_SH(src12111110, src14131312, filt3, filt3, out54, out76);
962  SRARI_H2_SH(out10, out32, 6);
963  SRARI_H2_SH(out54, out76, 6);
964  SAT_SH2_SH(out10, out32, 7);
965  SAT_SH2_SH(out54, out76, 7);
966  out0 = PCKEV_XORI128_UB(out10, out32);
967  out1 = PCKEV_XORI128_UB(out54, out76);
968  ST4x4_UB(out0, out0, 0, 1, 2, 3, dst, dst_stride);
969  dst += (4 * dst_stride);
970  ST4x4_UB(out1, out1, 0, 1, 2, 3, dst, dst_stride);
971  dst += (4 * dst_stride);
972 
973  src2110 = src10998;
974  src4332 = src12111110;
975  src6554 = src14131312;
976  src6 = src14;
977  }
978 }
979 
980 static void common_vt_8t_8w_msa(uint8_t *src, int32_t src_stride,
981  uint8_t *dst, int32_t dst_stride,
982  const int8_t *filter, int32_t height)
983 {
984  uint32_t loop_cnt;
985  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
986  v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r;
987  v16i8 src65_r, src87_r, src109_r, filt0, filt1, filt2, filt3;
988  v16u8 tmp0, tmp1;
989  v8i16 filt, out0_r, out1_r, out2_r, out3_r;
990 
991  src -= (3 * src_stride);
992 
993  filt = LD_SH(filter);
994  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
995 
996  LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
997  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
998  src += (7 * src_stride);
999  ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r,
1000  src54_r, src21_r);
1001  ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1002 
1003  for (loop_cnt = (height >> 2); loop_cnt--;) {
1004  LD_SB4(src, src_stride, src7, src8, src9, src10);
1005  XORI_B4_128_SB(src7, src8, src9, src10);
1006  src += (4 * src_stride);
1007 
1008  ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r,
1009  src87_r, src98_r, src109_r);
1010  DOTP_SB4_SH(src10_r, src21_r, src32_r, src43_r, filt0, filt0, filt0,
1011  filt0, out0_r, out1_r, out2_r, out3_r);
1012  DPADD_SB4_SH(src32_r, src43_r, src54_r, src65_r, filt1, filt1, filt1,
1013  filt1, out0_r, out1_r, out2_r, out3_r);
1014  DPADD_SB4_SH(src54_r, src65_r, src76_r, src87_r, filt2, filt2, filt2,
1015  filt2, out0_r, out1_r, out2_r, out3_r);
1016  DPADD_SB4_SH(src76_r, src87_r, src98_r, src109_r, filt3, filt3, filt3,
1017  filt3, out0_r, out1_r, out2_r, out3_r);
1018  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
1019  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
1020  tmp0 = PCKEV_XORI128_UB(out0_r, out1_r);
1021  tmp1 = PCKEV_XORI128_UB(out2_r, out3_r);
1022  ST8x4_UB(tmp0, tmp1, dst, dst_stride);
1023  dst += (4 * dst_stride);
1024 
1025  src10_r = src54_r;
1026  src32_r = src76_r;
1027  src54_r = src98_r;
1028  src21_r = src65_r;
1029  src43_r = src87_r;
1030  src65_r = src109_r;
1031  src6 = src10;
1032  }
1033 }
1034 
1035 static void common_vt_8t_12w_msa(uint8_t *src, int32_t src_stride,
1036  uint8_t *dst, int32_t dst_stride,
1037  const int8_t *filter, int32_t height)
1038 {
1039  uint32_t loop_cnt;
1040  uint32_t out2, out3;
1041  uint64_t out0, out1;
1042  v16u8 tmp0, tmp1, tmp2, tmp3;
1043  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1044  v16i8 filt0, filt1, filt2, filt3;
1045  v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r;
1046  v16i8 src65_r, src87_r, src109_r, src10_l, src32_l, src54_l, src76_l;
1047  v16i8 src98_l, src21_l, src43_l, src65_l, src87_l, src109_l;
1048  v8i16 filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
1049 
1050  src -= (3 * src_stride);
1051 
1052  filt = LD_SH(filter);
1053  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1054 
1055  LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1056  src += (7 * src_stride);
1057 
1058  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1059 
1060  ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r,
1061  src54_r, src21_r);
1062  ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1063  ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_l, src32_l,
1064  src54_l, src21_l);
1065  ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1066 
1067  for (loop_cnt = 4; loop_cnt--;) {
1068  LD_SB4(src, src_stride, src7, src8, src9, src10);
1069  XORI_B4_128_SB(src7, src8, src9, src10);
1070  src += (4 * src_stride);
1071 
1072  ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r,
1073  src87_r, src98_r, src109_r);
1074  ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_l,
1075  src87_l, src98_l, src109_l);
1076  out0_r = HEVC_FILT_8TAP_SH(src10_r, src32_r, src54_r, src76_r, filt0,
1077  filt1, filt2, filt3);
1078  out1_r = HEVC_FILT_8TAP_SH(src21_r, src43_r, src65_r, src87_r, filt0,
1079  filt1, filt2, filt3);
1080  out2_r = HEVC_FILT_8TAP_SH(src32_r, src54_r, src76_r, src98_r, filt0,
1081  filt1, filt2, filt3);
1082  out3_r = HEVC_FILT_8TAP_SH(src43_r, src65_r, src87_r, src109_r, filt0,
1083  filt1, filt2, filt3);
1084  out0_l = HEVC_FILT_8TAP_SH(src10_l, src32_l, src54_l, src76_l, filt0,
1085  filt1, filt2, filt3);
1086  out1_l = HEVC_FILT_8TAP_SH(src21_l, src43_l, src65_l, src87_l, filt0,
1087  filt1, filt2, filt3);
1088  out2_l = HEVC_FILT_8TAP_SH(src32_l, src54_l, src76_l, src98_l, filt0,
1089  filt1, filt2, filt3);
1090  out3_l = HEVC_FILT_8TAP_SH(src43_l, src65_l, src87_l, src109_l, filt0,
1091  filt1, filt2, filt3);
1092  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
1093  SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 6);
1094  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
1095  SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
1096  PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
1097  out3_r, tmp0, tmp1, tmp2, tmp3);
1098  XORI_B4_128_UB(tmp0, tmp1, tmp2, tmp3);
1099 
1100  out0 = __msa_copy_u_d((v2i64) tmp0, 0);
1101  out1 = __msa_copy_u_d((v2i64) tmp1, 0);
1102  out2 = __msa_copy_u_w((v4i32) tmp0, 2);
1103  out3 = __msa_copy_u_w((v4i32) tmp1, 2);
1104  SD(out0, dst);
1105  SW(out2, (dst + 8));
1106  dst += dst_stride;
1107  SD(out1, dst);
1108  SW(out3, (dst + 8));
1109  dst += dst_stride;
1110  out0 = __msa_copy_u_d((v2i64) tmp2, 0);
1111  out1 = __msa_copy_u_d((v2i64) tmp3, 0);
1112  out2 = __msa_copy_u_w((v4i32) tmp2, 2);
1113  out3 = __msa_copy_u_w((v4i32) tmp3, 2);
1114  SD(out0, dst);
1115  SW(out2, (dst + 8));
1116  dst += dst_stride;
1117  SD(out1, dst);
1118  SW(out3, (dst + 8));
1119  dst += dst_stride;
1120 
1121  src10_r = src54_r;
1122  src32_r = src76_r;
1123  src54_r = src98_r;
1124  src21_r = src65_r;
1125  src43_r = src87_r;
1126  src65_r = src109_r;
1127  src10_l = src54_l;
1128  src32_l = src76_l;
1129  src54_l = src98_l;
1130  src21_l = src65_l;
1131  src43_l = src87_l;
1132  src65_l = src109_l;
1133  src6 = src10;
1134  }
1135 }
1136 
1137 static void common_vt_8t_16w_msa(uint8_t *src, int32_t src_stride,
1138  uint8_t *dst, int32_t dst_stride,
1139  const int8_t *filter, int32_t height)
1140 {
1141  uint32_t loop_cnt;
1142  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1143  v16i8 filt0, filt1, filt2, filt3;
1144  v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r;
1145  v16i8 src65_r, src87_r, src109_r, src10_l, src32_l, src54_l, src76_l;
1146  v16i8 src98_l, src21_l, src43_l, src65_l, src87_l, src109_l;
1147  v16u8 tmp0, tmp1, tmp2, tmp3;
1148  v8i16 filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
1149 
1150  src -= (3 * src_stride);
1151 
1152  filt = LD_SH(filter);
1153  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1154 
1155  LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1156  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1157  src += (7 * src_stride);
1158  ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r, src32_r,
1159  src54_r, src21_r);
1160  ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1161  ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_l, src32_l,
1162  src54_l, src21_l);
1163  ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1164 
1165  for (loop_cnt = (height >> 2); loop_cnt--;) {
1166  LD_SB4(src, src_stride, src7, src8, src9, src10);
1167  XORI_B4_128_SB(src7, src8, src9, src10);
1168  src += (4 * src_stride);
1169 
1170  ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r,
1171  src87_r, src98_r, src109_r);
1172  ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_l,
1173  src87_l, src98_l, src109_l);
1174  out0_r = HEVC_FILT_8TAP_SH(src10_r, src32_r, src54_r, src76_r, filt0,
1175  filt1, filt2, filt3);
1176  out1_r = HEVC_FILT_8TAP_SH(src21_r, src43_r, src65_r, src87_r, filt0,
1177  filt1, filt2, filt3);
1178  out2_r = HEVC_FILT_8TAP_SH(src32_r, src54_r, src76_r, src98_r, filt0,
1179  filt1, filt2, filt3);
1180  out3_r = HEVC_FILT_8TAP_SH(src43_r, src65_r, src87_r, src109_r, filt0,
1181  filt1, filt2, filt3);
1182  out0_l = HEVC_FILT_8TAP_SH(src10_l, src32_l, src54_l, src76_l, filt0,
1183  filt1, filt2, filt3);
1184  out1_l = HEVC_FILT_8TAP_SH(src21_l, src43_l, src65_l, src87_l, filt0,
1185  filt1, filt2, filt3);
1186  out2_l = HEVC_FILT_8TAP_SH(src32_l, src54_l, src76_l, src98_l, filt0,
1187  filt1, filt2, filt3);
1188  out3_l = HEVC_FILT_8TAP_SH(src43_l, src65_l, src87_l, src109_l, filt0,
1189  filt1, filt2, filt3);
1190  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
1191  SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 6);
1192  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
1193  SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
1194  PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
1195  out3_r, tmp0, tmp1, tmp2, tmp3);
1196  XORI_B4_128_UB(tmp0, tmp1, tmp2, tmp3);
1197  ST_UB4(tmp0, tmp1, tmp2, tmp3, dst, dst_stride);
1198  dst += (4 * dst_stride);
1199 
1200  src10_r = src54_r;
1201  src32_r = src76_r;
1202  src54_r = src98_r;
1203  src21_r = src65_r;
1204  src43_r = src87_r;
1205  src65_r = src109_r;
1206  src10_l = src54_l;
1207  src32_l = src76_l;
1208  src54_l = src98_l;
1209  src21_l = src65_l;
1210  src43_l = src87_l;
1211  src65_l = src109_l;
1212  src6 = src10;
1213  }
1214 }
1215 
1216 static void common_vt_8t_16w_mult_msa(uint8_t *src, int32_t src_stride,
1217  uint8_t *dst, int32_t dst_stride,
1218  const int8_t *filter, int32_t height,
1219  int32_t width)
1220 {
1221  uint8_t *src_tmp;
1222  uint8_t *dst_tmp;
1223  uint32_t loop_cnt, cnt;
1224  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1225  v16i8 filt0, filt1, filt2, filt3;
1226  v16i8 src10_r, src32_r, src54_r, src76_r, src98_r, src21_r, src43_r;
1227  v16i8 src65_r, src87_r, src109_r, src10_l, src32_l, src54_l, src76_l;
1228  v16i8 src98_l, src21_l, src43_l, src65_l, src87_l, src109_l;
1229  v16u8 tmp0, tmp1, tmp2, tmp3;
1230  v8i16 filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
1231 
1232  src -= (3 * src_stride);
1233 
1234  filt = LD_SH(filter);
1235  SPLATI_H4_SB(filt, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1236 
1237  for (cnt = (width >> 4); cnt--;) {
1238  src_tmp = src;
1239  dst_tmp = dst;
1240 
1241  LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
1242  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1243  src_tmp += (7 * src_stride);
1244  ILVR_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_r,
1245  src32_r, src54_r, src21_r);
1246  ILVR_B2_SB(src4, src3, src6, src5, src43_r, src65_r);
1247  ILVL_B4_SB(src1, src0, src3, src2, src5, src4, src2, src1, src10_l,
1248  src32_l, src54_l, src21_l);
1249  ILVL_B2_SB(src4, src3, src6, src5, src43_l, src65_l);
1250 
1251  for (loop_cnt = (height >> 2); loop_cnt--;) {
1252  LD_SB4(src_tmp, src_stride, src7, src8, src9, src10);
1253  XORI_B4_128_SB(src7, src8, src9, src10);
1254  src_tmp += (4 * src_stride);
1255  ILVR_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_r,
1256  src87_r, src98_r, src109_r);
1257  ILVL_B4_SB(src7, src6, src8, src7, src9, src8, src10, src9, src76_l,
1258  src87_l, src98_l, src109_l);
1259  out0_r = HEVC_FILT_8TAP_SH(src10_r, src32_r, src54_r, src76_r,
1260  filt0, filt1, filt2, filt3);
1261  out1_r = HEVC_FILT_8TAP_SH(src21_r, src43_r, src65_r, src87_r,
1262  filt0, filt1, filt2, filt3);
1263  out2_r = HEVC_FILT_8TAP_SH(src32_r, src54_r, src76_r, src98_r,
1264  filt0, filt1, filt2, filt3);
1265  out3_r = HEVC_FILT_8TAP_SH(src43_r, src65_r, src87_r, src109_r,
1266  filt0, filt1, filt2, filt3);
1267  out0_l = HEVC_FILT_8TAP_SH(src10_l, src32_l, src54_l, src76_l,
1268  filt0, filt1, filt2, filt3);
1269  out1_l = HEVC_FILT_8TAP_SH(src21_l, src43_l, src65_l, src87_l,
1270  filt0, filt1, filt2, filt3);
1271  out2_l = HEVC_FILT_8TAP_SH(src32_l, src54_l, src76_l, src98_l,
1272  filt0, filt1, filt2, filt3);
1273  out3_l = HEVC_FILT_8TAP_SH(src43_l, src65_l, src87_l, src109_l,
1274  filt0, filt1, filt2, filt3);
1275  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
1276  SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 6);
1277  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
1278  SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
1279  PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
1280  out3_r, tmp0, tmp1, tmp2, tmp3);
1281  XORI_B4_128_UB(tmp0, tmp1, tmp2, tmp3);
1282  ST_UB4(tmp0, tmp1, tmp2, tmp3, dst_tmp, dst_stride);
1283  dst_tmp += (4 * dst_stride);
1284 
1285  src10_r = src54_r;
1286  src32_r = src76_r;
1287  src54_r = src98_r;
1288  src21_r = src65_r;
1289  src43_r = src87_r;
1290  src65_r = src109_r;
1291  src10_l = src54_l;
1292  src32_l = src76_l;
1293  src54_l = src98_l;
1294  src21_l = src65_l;
1295  src43_l = src87_l;
1296  src65_l = src109_l;
1297  src6 = src10;
1298  }
1299 
1300  src += 16;
1301  dst += 16;
1302  }
1303 }
1304 
1305 static void common_vt_8t_24w_msa(uint8_t *src, int32_t src_stride,
1306  uint8_t *dst, int32_t dst_stride,
1307  const int8_t *filter, int32_t height)
1308 {
1309  common_vt_8t_16w_mult_msa(src, src_stride, dst, dst_stride, filter, height,
1310  16);
1311 
1312  common_vt_8t_8w_msa(src + 16, src_stride, dst + 16, dst_stride, filter,
1313  height);
1314 }
1315 
1316 static void common_vt_8t_32w_msa(uint8_t *src, int32_t src_stride,
1317  uint8_t *dst, int32_t dst_stride,
1318  const int8_t *filter, int32_t height)
1319 {
1320  common_vt_8t_16w_mult_msa(src, src_stride, dst, dst_stride, filter, height,
1321  32);
1322 }
1323 
1324 static void common_vt_8t_48w_msa(uint8_t *src, int32_t src_stride,
1325  uint8_t *dst, int32_t dst_stride,
1326  const int8_t *filter, int32_t height)
1327 {
1328  common_vt_8t_16w_mult_msa(src, src_stride, dst, dst_stride, filter, height,
1329  48);
1330 }
1331 
1332 static void common_vt_8t_64w_msa(uint8_t *src, int32_t src_stride,
1333  uint8_t *dst, int32_t dst_stride,
1334  const int8_t *filter, int32_t height)
1335 {
1336  common_vt_8t_16w_mult_msa(src, src_stride, dst, dst_stride, filter, height,
1337  64);
1338 }
1339 
1340 static void hevc_hv_uni_8t_4w_msa(uint8_t *src,
1341  int32_t src_stride,
1342  uint8_t *dst,
1343  int32_t dst_stride,
1344  const int8_t *filter_x,
1345  const int8_t *filter_y,
1346  int32_t height)
1347 {
1348  uint32_t loop_cnt;
1349  v16u8 out0, out1;
1350  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1351  v16i8 src9, src10, src11, src12, src13, src14;
1352  v8i16 filt0, filt1, filt2, filt3;
1353  v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
1354  v16i8 mask1, mask2, mask3;
1355  v8i16 filter_vec;
1356  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1357  v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1358  v8i16 dst30, dst41, dst52, dst63, dst66, dst117, dst128, dst139, dst1410;
1359  v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst98_r, dst1110_r, dst1312_r;
1360  v8i16 dst21_r, dst43_r, dst65_r, dst87_r, dst109_r, dst1211_r, dst1413_r;
1361  v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r, dst6_r, dst7_r;
1362  v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
1363 
1364  src -= ((3 * src_stride) + 3);
1365  filter_vec = LD_SH(filter_x);
1366  SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1367 
1368  filter_vec = LD_SH(filter_y);
1369  UNPCK_R_SB_SH(filter_vec, filter_vec);
1370 
1371  SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1372 
1373  mask1 = mask0 + 2;
1374  mask2 = mask0 + 4;
1375  mask3 = mask0 + 6;
1376 
1377  LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1378  src += (7 * src_stride);
1379  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1380 
1381  VSHF_B4_SB(src0, src3, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1382  VSHF_B4_SB(src1, src4, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
1383  VSHF_B4_SB(src2, src5, mask0, mask1, mask2, mask3,
1384  vec8, vec9, vec10, vec11);
1385  VSHF_B4_SB(src3, src6, mask0, mask1, mask2, mask3,
1386  vec12, vec13, vec14, vec15);
1387 
1388  dst30 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1389  filt3);
1390  dst41 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1391  filt3);
1392  dst52 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1393  filt3);
1394  dst63 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1395  filt3);
1396 
1397  ILVRL_H2_SH(dst41, dst30, dst10_r, dst43_r);
1398  ILVRL_H2_SH(dst52, dst41, dst21_r, dst54_r);
1399  ILVRL_H2_SH(dst63, dst52, dst32_r, dst65_r);
1400 
1401  dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
1402 
1403  for (loop_cnt = height >> 3; loop_cnt--;) {
1404  LD_SB8(src, src_stride, src7, src8, src9, src10, src11, src12, src13,
1405  src14);
1406  src += (8 * src_stride);
1407  XORI_B8_128_SB(src7, src8, src9, src10, src11, src12, src13, src14);
1408 
1409  VSHF_B4_SB(src7, src11, mask0, mask1, mask2, mask3,
1410  vec0, vec1, vec2, vec3);
1411  VSHF_B4_SB(src8, src12, mask0, mask1, mask2, mask3,
1412  vec4, vec5, vec6, vec7);
1413  VSHF_B4_SB(src9, src13, mask0, mask1, mask2, mask3,
1414  vec8, vec9, vec10, vec11);
1415  VSHF_B4_SB(src10, src14, mask0, mask1, mask2, mask3,
1416  vec12, vec13, vec14, vec15);
1417 
1418  dst117 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1419  filt3);
1420  dst128 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1421  filt3);
1422  dst139 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1,
1423  filt2, filt3);
1424  dst1410 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1,
1425  filt2, filt3);
1426 
1427  dst76_r = __msa_ilvr_h(dst117, dst66);
1428  ILVRL_H2_SH(dst128, dst117, dst87_r, dst1211_r);
1429  ILVRL_H2_SH(dst139, dst128, dst98_r, dst1312_r);
1430  ILVRL_H2_SH(dst1410, dst139, dst109_r, dst1413_r);
1431  dst117 = (v8i16) __msa_splati_d((v2i64) dst117, 1);
1432  dst1110_r = __msa_ilvr_h(dst117, dst1410);
1433 
1434  dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0,
1435  filt_h1, filt_h2, filt_h3);
1436  dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r, filt_h0,
1437  filt_h1, filt_h2, filt_h3);
1438  dst2_r = HEVC_FILT_8TAP(dst32_r, dst54_r, dst76_r, dst98_r, filt_h0,
1439  filt_h1, filt_h2, filt_h3);
1440  dst3_r = HEVC_FILT_8TAP(dst43_r, dst65_r, dst87_r, dst109_r, filt_h0,
1441  filt_h1, filt_h2, filt_h3);
1442  dst4_r = HEVC_FILT_8TAP(dst54_r, dst76_r, dst98_r, dst1110_r, filt_h0,
1443  filt_h1, filt_h2, filt_h3);
1444  dst5_r = HEVC_FILT_8TAP(dst65_r, dst87_r, dst109_r, dst1211_r, filt_h0,
1445  filt_h1, filt_h2, filt_h3);
1446  dst6_r = HEVC_FILT_8TAP(dst76_r, dst98_r, dst1110_r, dst1312_r, filt_h0,
1447  filt_h1, filt_h2, filt_h3);
1448  dst7_r = HEVC_FILT_8TAP(dst87_r, dst109_r, dst1211_r, dst1413_r,
1449  filt_h0, filt_h1, filt_h2, filt_h3);
1450 
1451  SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
1452  SRA_4V(dst4_r, dst5_r, dst6_r, dst7_r, 6);
1453  SRARI_W4_SW(dst0_r, dst1_r, dst2_r, dst3_r, 6);
1454  SRARI_W4_SW(dst4_r, dst5_r, dst6_r, dst7_r, 6);
1455  SAT_SW4_SW(dst0_r, dst1_r, dst2_r, dst3_r, 7);
1456  SAT_SW4_SW(dst4_r, dst5_r, dst6_r, dst7_r, 7);
1457  PCKEV_H2_SW(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
1458  PCKEV_H2_SW(dst5_r, dst4_r, dst7_r, dst6_r, dst4_r, dst5_r);
1459  out0 = PCKEV_XORI128_UB(dst0_r, dst1_r);
1460  out1 = PCKEV_XORI128_UB(dst4_r, dst5_r);
1461  ST4x4_UB(out0, out0, 0, 1, 2, 3, dst, dst_stride);
1462  dst += (4 * dst_stride);
1463  ST4x4_UB(out1, out1, 0, 1, 2, 3, dst, dst_stride);
1464  dst += (4 * dst_stride);
1465 
1466  dst10_r = dst98_r;
1467  dst32_r = dst1110_r;
1468  dst54_r = dst1312_r;
1469  dst21_r = dst109_r;
1470  dst43_r = dst1211_r;
1471  dst65_r = dst1413_r;
1472  dst66 = (v8i16) __msa_splati_d((v2i64) dst1410, 1);
1473  }
1474 }
1475 
1476 static void hevc_hv_uni_8t_8multx2mult_msa(uint8_t *src,
1477  int32_t src_stride,
1478  uint8_t *dst,
1479  int32_t dst_stride,
1480  const int8_t *filter_x,
1481  const int8_t *filter_y,
1482  int32_t height, int32_t width)
1483 {
1484  uint32_t loop_cnt, cnt;
1485  uint8_t *src_tmp;
1486  uint8_t *dst_tmp;
1487  v16u8 out;
1488  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
1489  v8i16 filt0, filt1, filt2, filt3;
1490  v8i16 filt_h0, filt_h1, filt_h2, filt_h3;
1491  v16i8 mask1, mask2, mask3;
1492  v8i16 filter_vec;
1493  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1494  v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1495  v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
1496  v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
1497  v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
1498  v8i16 dst10_l, dst32_l, dst54_l, dst76_l;
1499  v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
1500  v8i16 dst21_l, dst43_l, dst65_l, dst87_l;
1501  v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
1502 
1503  src -= ((3 * src_stride) + 3);
1504 
1505  filter_vec = LD_SH(filter_x);
1506  SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1507 
1508  filter_vec = LD_SH(filter_y);
1509  UNPCK_R_SB_SH(filter_vec, filter_vec);
1510 
1511  SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1512 
1513  mask1 = mask0 + 2;
1514  mask2 = mask0 + 4;
1515  mask3 = mask0 + 6;
1516 
1517  for (cnt = width >> 3; cnt--;) {
1518  src_tmp = src;
1519  dst_tmp = dst;
1520 
1521  LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
1522  src_tmp += (7 * src_stride);
1523  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1524 
1525  /* row 0 row 1 row 2 row 3 */
1526  VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3,
1527  vec0, vec1, vec2, vec3);
1528  VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3,
1529  vec4, vec5, vec6, vec7);
1530  VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3,
1531  vec8, vec9, vec10, vec11);
1532  VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3,
1533  vec12, vec13, vec14, vec15);
1534  dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1535  filt3);
1536  dst1 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1537  filt3);
1538  dst2 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1539  filt3);
1540  dst3 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1,
1541  filt2, filt3);
1542 
1543  VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3,
1544  vec0, vec1, vec2, vec3);
1545  VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3,
1546  vec4, vec5, vec6, vec7);
1547  VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3,
1548  vec8, vec9, vec10, vec11);
1549  dst4 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1550  filt3);
1551  dst5 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1552  filt3);
1553  dst6 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1554  filt3);
1555 
1556  for (loop_cnt = height >> 1; loop_cnt--;) {
1557  LD_SB2(src_tmp, src_stride, src7, src8);
1558  XORI_B2_128_SB(src7, src8);
1559  src_tmp += 2 * src_stride;
1560 
1561  ILVR_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1,
1562  dst10_r, dst32_r, dst54_r, dst21_r);
1563  ILVL_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1,
1564  dst10_l, dst32_l, dst54_l, dst21_l);
1565  ILVR_H2_SH(dst4, dst3, dst6, dst5, dst43_r, dst65_r);
1566  ILVL_H2_SH(dst4, dst3, dst6, dst5, dst43_l, dst65_l);
1567 
1568  VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3,
1569  vec0, vec1, vec2, vec3);
1570  dst7 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
1571  filt2, filt3);
1572 
1573  ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
1574  dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
1575  filt_h0, filt_h1, filt_h2, filt_h3);
1576  dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l,
1577  filt_h0, filt_h1, filt_h2, filt_h3);
1578  dst0_r >>= 6;
1579  dst0_l >>= 6;
1580 
1581  VSHF_B4_SB(src8, src8, mask0, mask1, mask2, mask3,
1582  vec0, vec1, vec2, vec3);
1583  dst8 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1,
1584  filt2, filt3);
1585 
1586  ILVRL_H2_SH(dst8, dst7, dst87_r, dst87_l);
1587  dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r,
1588  filt_h0, filt_h1, filt_h2, filt_h3);
1589  dst1_l = HEVC_FILT_8TAP(dst21_l, dst43_l, dst65_l, dst87_l,
1590  filt_h0, filt_h1, filt_h2, filt_h3);
1591  dst1_r >>= 6;
1592  dst1_l >>= 6;
1593  SRARI_W4_SW(dst0_r, dst0_l, dst1_r, dst1_l, 6);
1594  SAT_SW4_SW(dst0_r, dst0_l, dst1_r, dst1_l, 7);
1595 
1596  PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst0, dst1);
1597  out = PCKEV_XORI128_UB(dst0, dst1);
1598  ST8x2_UB(out, dst_tmp, dst_stride);
1599  dst_tmp += (2 * dst_stride);
1600 
1601  dst0 = dst2;
1602  dst1 = dst3;
1603  dst2 = dst4;
1604  dst3 = dst5;
1605  dst4 = dst6;
1606  dst5 = dst7;
1607  dst6 = dst8;
1608  }
1609 
1610  src += 8;
1611  dst += 8;
1612  }
1613 }
1614 
1615 static void hevc_hv_uni_8t_8w_msa(uint8_t *src,
1616  int32_t src_stride,
1617  uint8_t *dst,
1618  int32_t dst_stride,
1619  const int8_t *filter_x,
1620  const int8_t *filter_y,
1621  int32_t height)
1622 {
1623  hevc_hv_uni_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
1624  filter_x, filter_y, height, 8);
1625 }
1626 
1627 static void hevc_hv_uni_8t_12w_msa(uint8_t *src,
1628  int32_t src_stride,
1629  uint8_t *dst,
1630  int32_t dst_stride,
1631  const int8_t *filter_x,
1632  const int8_t *filter_y,
1633  int32_t height)
1634 {
1635  uint32_t loop_cnt;
1636  uint8_t *src_tmp, *dst_tmp;
1637  v16u8 out0, out1;
1638  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
1639  v16i8 src11, src12, src13, src14;
1640  v16i8 mask0, mask1, mask2, mask3, mask4, mask5, mask6, mask7;
1641  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
1642  v16i8 vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15;
1643  v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
1644  v8i16 dst30, dst41, dst52, dst63, dst66, dst117, dst128, dst139, dst1410;
1645  v8i16 filt0, filt1, filt2, filt3, filt_h0, filt_h1, filt_h2, filt_h3;
1646  v8i16 dst10_r, dst32_r, dst54_r, dst76_r, dst21_r, dst43_r, dst65_r;
1647  v8i16 dst10_l, dst32_l, dst54_l, dst76_l, dst21_l, dst43_l, dst65_l;
1648  v8i16 dst87_r, dst98_r, dst1110_r, dst1312_r, dst109_r, dst1211_r;
1649  v8i16 dst1413_r, dst87_l, filter_vec;
1650  v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r, dst6_r, dst7_r;
1651  v4i32 dst0_l, dst1_l;
1652 
1653  src -= ((3 * src_stride) + 3);
1654 
1655  filter_vec = LD_SH(filter_x);
1656  SPLATI_H4_SH(filter_vec, 0, 1, 2, 3, filt0, filt1, filt2, filt3);
1657 
1658  filter_vec = LD_SH(filter_y);
1659  UNPCK_R_SB_SH(filter_vec, filter_vec);
1660 
1661  SPLATI_W4_SH(filter_vec, filt_h0, filt_h1, filt_h2, filt_h3);
1662 
1663  mask0 = LD_SB(ff_hevc_mask_arr);
1664  mask1 = mask0 + 2;
1665  mask2 = mask0 + 4;
1666  mask3 = mask0 + 6;
1667 
1668  src_tmp = src;
1669  dst_tmp = dst;
1670 
1671  LD_SB7(src_tmp, src_stride, src0, src1, src2, src3, src4, src5, src6);
1672  src_tmp += (7 * src_stride);
1673  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1674 
1675  /* row 0 row 1 row 2 row 3 */
1676  VSHF_B4_SB(src0, src0, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1677  VSHF_B4_SB(src1, src1, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
1678  VSHF_B4_SB(src2, src2, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
1679  vec11);
1680  VSHF_B4_SB(src3, src3, mask0, mask1, mask2, mask3, vec12, vec13, vec14,
1681  vec15);
1682  dst0 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1683  filt3);
1684  dst1 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1685  filt3);
1686  dst2 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1687  filt3);
1688  dst3 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1,
1689  filt2, filt3);
1690 
1691  VSHF_B4_SB(src4, src4, mask0, mask1, mask2, mask3, vec0, vec1, vec2, vec3);
1692  VSHF_B4_SB(src5, src5, mask0, mask1, mask2, mask3, vec4, vec5, vec6, vec7);
1693  VSHF_B4_SB(src6, src6, mask0, mask1, mask2, mask3, vec8, vec9, vec10,
1694  vec11);
1695  dst4 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1696  filt3);
1697  dst5 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1698  filt3);
1699  dst6 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1700  filt3);
1701 
1702  for (loop_cnt = 8; loop_cnt--;) {
1703  LD_SB2(src_tmp, src_stride, src7, src8);
1704  XORI_B2_128_SB(src7, src8);
1705  src_tmp += 2 * src_stride;
1706 
1707  ILVR_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1, dst10_r,
1708  dst32_r, dst54_r, dst21_r);
1709  ILVL_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst2, dst1, dst10_l,
1710  dst32_l, dst54_l, dst21_l);
1711  ILVR_H2_SH(dst4, dst3, dst6, dst5, dst43_r, dst65_r);
1712  ILVL_H2_SH(dst4, dst3, dst6, dst5, dst43_l, dst65_l);
1713 
1714  VSHF_B4_SB(src7, src7, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
1715  vec3);
1716  dst7 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1717  filt3);
1718 
1719  ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
1720  dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r,
1721  filt_h0, filt_h1, filt_h2, filt_h3);
1722  dst0_l = HEVC_FILT_8TAP(dst10_l, dst32_l, dst54_l, dst76_l,
1723  filt_h0, filt_h1, filt_h2, filt_h3);
1724  dst0_r >>= 6;
1725  dst0_l >>= 6;
1726 
1727  VSHF_B4_SB(src8, src8, mask0, mask1, mask2, mask3, vec0, vec1, vec2,
1728  vec3);
1729  dst8 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1730  filt3);
1731 
1732  ILVRL_H2_SH(dst8, dst7, dst87_r, dst87_l);
1733  dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r,
1734  filt_h0, filt_h1, filt_h2, filt_h3);
1735  dst1_l = HEVC_FILT_8TAP(dst21_l, dst43_l, dst65_l, dst87_l,
1736  filt_h0, filt_h1, filt_h2, filt_h3);
1737  dst1_r >>= 6;
1738  dst1_l >>= 6;
1739  SRARI_W4_SW(dst0_r, dst0_l, dst1_r, dst1_l, 6);
1740  SAT_SW4_SW(dst0_r, dst0_l, dst1_r, dst1_l, 7);
1741 
1742  PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst0, dst1);
1743  out0 = PCKEV_XORI128_UB(dst0, dst1);
1744  ST8x2_UB(out0, dst_tmp, dst_stride);
1745  dst_tmp += (2 * dst_stride);
1746 
1747  dst0 = dst2;
1748  dst1 = dst3;
1749  dst2 = dst4;
1750  dst3 = dst5;
1751  dst4 = dst6;
1752  dst5 = dst7;
1753  dst6 = dst8;
1754  }
1755 
1756  src += 8;
1757  dst += 8;
1758 
1759  mask4 = LD_SB(ff_hevc_mask_arr + 16);
1760  mask5 = mask4 + 2;
1761  mask6 = mask4 + 4;
1762  mask7 = mask4 + 6;
1763 
1764  LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
1765  src += (7 * src_stride);
1766  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
1767 
1768  VSHF_B4_SB(src0, src3, mask4, mask5, mask6, mask7, vec0, vec1, vec2, vec3);
1769  VSHF_B4_SB(src1, src4, mask4, mask5, mask6, mask7, vec4, vec5, vec6, vec7);
1770  VSHF_B4_SB(src2, src5, mask4, mask5, mask6, mask7, vec8, vec9, vec10,
1771  vec11);
1772  VSHF_B4_SB(src3, src6, mask4, mask5, mask6, mask7, vec12, vec13, vec14,
1773  vec15);
1774 
1775  dst30 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1776  filt3);
1777  dst41 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1778  filt3);
1779  dst52 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1, filt2,
1780  filt3);
1781  dst63 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1, filt2,
1782  filt3);
1783 
1784  ILVRL_H2_SH(dst41, dst30, dst10_r, dst43_r);
1785  ILVRL_H2_SH(dst52, dst41, dst21_r, dst54_r);
1786  ILVRL_H2_SH(dst63, dst52, dst32_r, dst65_r);
1787 
1788  dst66 = (v8i16) __msa_splati_d((v2i64) dst63, 1);
1789 
1790  for (loop_cnt = 2; loop_cnt--;) {
1791  LD_SB8(src, src_stride, src7, src8, src9, src10, src11, src12, src13,
1792  src14);
1793  src += (8 * src_stride);
1794  XORI_B8_128_SB(src7, src8, src9, src10, src11, src12, src13, src14);
1795 
1796  VSHF_B4_SB(src7, src11, mask4, mask5, mask6, mask7, vec0, vec1, vec2,
1797  vec3);
1798  VSHF_B4_SB(src8, src12, mask4, mask5, mask6, mask7, vec4, vec5, vec6,
1799  vec7);
1800  VSHF_B4_SB(src9, src13, mask4, mask5, mask6, mask7, vec8, vec9, vec10,
1801  vec11);
1802  VSHF_B4_SB(src10, src14, mask4, mask5, mask6, mask7, vec12, vec13,
1803  vec14, vec15);
1804 
1805  dst117 = HEVC_FILT_8TAP_SH(vec0, vec1, vec2, vec3, filt0, filt1, filt2,
1806  filt3);
1807  dst128 = HEVC_FILT_8TAP_SH(vec4, vec5, vec6, vec7, filt0, filt1, filt2,
1808  filt3);
1809  dst139 = HEVC_FILT_8TAP_SH(vec8, vec9, vec10, vec11, filt0, filt1,
1810  filt2, filt3);
1811  dst1410 = HEVC_FILT_8TAP_SH(vec12, vec13, vec14, vec15, filt0, filt1,
1812  filt2, filt3);
1813 
1814  dst76_r = __msa_ilvr_h(dst117, dst66);
1815  ILVRL_H2_SH(dst128, dst117, dst87_r, dst1211_r);
1816  ILVRL_H2_SH(dst139, dst128, dst98_r, dst1312_r);
1817  ILVRL_H2_SH(dst1410, dst139, dst109_r, dst1413_r);
1818  dst117 = (v8i16) __msa_splati_d((v2i64) dst117, 1);
1819  dst1110_r = __msa_ilvr_h(dst117, dst1410);
1820 
1821  dst0_r = HEVC_FILT_8TAP(dst10_r, dst32_r, dst54_r, dst76_r, filt_h0,
1822  filt_h1, filt_h2, filt_h3);
1823  dst1_r = HEVC_FILT_8TAP(dst21_r, dst43_r, dst65_r, dst87_r, filt_h0,
1824  filt_h1, filt_h2, filt_h3);
1825  dst2_r = HEVC_FILT_8TAP(dst32_r, dst54_r, dst76_r, dst98_r, filt_h0,
1826  filt_h1, filt_h2, filt_h3);
1827  dst3_r = HEVC_FILT_8TAP(dst43_r, dst65_r, dst87_r, dst109_r, filt_h0,
1828  filt_h1, filt_h2, filt_h3);
1829  dst4_r = HEVC_FILT_8TAP(dst54_r, dst76_r, dst98_r, dst1110_r, filt_h0,
1830  filt_h1, filt_h2, filt_h3);
1831  dst5_r = HEVC_FILT_8TAP(dst65_r, dst87_r, dst109_r, dst1211_r, filt_h0,
1832  filt_h1, filt_h2, filt_h3);
1833  dst6_r = HEVC_FILT_8TAP(dst76_r, dst98_r, dst1110_r, dst1312_r, filt_h0,
1834  filt_h1, filt_h2, filt_h3);
1835  dst7_r = HEVC_FILT_8TAP(dst87_r, dst109_r, dst1211_r, dst1413_r,
1836  filt_h0, filt_h1, filt_h2, filt_h3);
1837 
1838  SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
1839  SRA_4V(dst4_r, dst5_r, dst6_r, dst7_r, 6);
1840  SRARI_W4_SW(dst0_r, dst1_r, dst2_r, dst3_r, 6);
1841  SRARI_W4_SW(dst4_r, dst5_r, dst6_r, dst7_r, 6);
1842  SAT_SW4_SW(dst0_r, dst1_r, dst2_r, dst3_r, 7);
1843  SAT_SW4_SW(dst4_r, dst5_r, dst6_r, dst7_r, 7);
1844  PCKEV_H2_SW(dst1_r, dst0_r, dst3_r, dst2_r, dst0_r, dst1_r);
1845  PCKEV_H2_SW(dst5_r, dst4_r, dst7_r, dst6_r, dst4_r, dst5_r);
1846  out0 = PCKEV_XORI128_UB(dst0_r, dst1_r);
1847  out1 = PCKEV_XORI128_UB(dst4_r, dst5_r);
1848  ST4x4_UB(out0, out0, 0, 1, 2, 3, dst, dst_stride);
1849  dst += (4 * dst_stride);
1850  ST4x4_UB(out1, out1, 0, 1, 2, 3, dst, dst_stride);
1851  dst += (4 * dst_stride);
1852 
1853  dst10_r = dst98_r;
1854  dst32_r = dst1110_r;
1855  dst54_r = dst1312_r;
1856  dst21_r = dst109_r;
1857  dst43_r = dst1211_r;
1858  dst65_r = dst1413_r;
1859  dst66 = (v8i16) __msa_splati_d((v2i64) dst1410, 1);
1860  }
1861 }
1862 
1863 static void hevc_hv_uni_8t_16w_msa(uint8_t *src,
1864  int32_t src_stride,
1865  uint8_t *dst,
1866  int32_t dst_stride,
1867  const int8_t *filter_x,
1868  const int8_t *filter_y,
1869  int32_t height)
1870 {
1871  hevc_hv_uni_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
1872  filter_x, filter_y, height, 16);
1873 }
1874 
1875 static void hevc_hv_uni_8t_24w_msa(uint8_t *src,
1876  int32_t src_stride,
1877  uint8_t *dst,
1878  int32_t dst_stride,
1879  const int8_t *filter_x,
1880  const int8_t *filter_y,
1881  int32_t height)
1882 {
1883  hevc_hv_uni_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
1884  filter_x, filter_y, height, 24);
1885 }
1886 
1887 static void hevc_hv_uni_8t_32w_msa(uint8_t *src,
1888  int32_t src_stride,
1889  uint8_t *dst,
1890  int32_t dst_stride,
1891  const int8_t *filter_x,
1892  const int8_t *filter_y,
1893  int32_t height)
1894 {
1895  hevc_hv_uni_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
1896  filter_x, filter_y, height, 32);
1897 }
1898 
1899 static void hevc_hv_uni_8t_48w_msa(uint8_t *src,
1900  int32_t src_stride,
1901  uint8_t *dst,
1902  int32_t dst_stride,
1903  const int8_t *filter_x,
1904  const int8_t *filter_y,
1905  int32_t height)
1906 {
1907  hevc_hv_uni_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
1908  filter_x, filter_y, height, 48);
1909 }
1910 
1911 static void hevc_hv_uni_8t_64w_msa(uint8_t *src,
1912  int32_t src_stride,
1913  uint8_t *dst,
1914  int32_t dst_stride,
1915  const int8_t *filter_x,
1916  const int8_t *filter_y,
1917  int32_t height)
1918 {
1919  hevc_hv_uni_8t_8multx2mult_msa(src, src_stride, dst, dst_stride,
1920  filter_x, filter_y, height, 64);
1921 }
1922 
1923 static void common_hz_4t_4x2_msa(uint8_t *src, int32_t src_stride,
1924  uint8_t *dst, int32_t dst_stride,
1925  const int8_t *filter)
1926 {
1927  v16i8 filt0, filt1, src0, src1, mask0, mask1, vec0, vec1;
1928  v16u8 out;
1929  v8i16 filt, res0;
1930 
1931  mask0 = LD_SB(&ff_hevc_mask_arr[16]);
1932  src -= 1;
1933 
1934  /* rearranging filter */
1935  filt = LD_SH(filter);
1936  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
1937 
1938  mask1 = mask0 + 2;
1939 
1940  LD_SB2(src, src_stride, src0, src1);
1941  XORI_B2_128_SB(src0, src1);
1942  VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
1943  res0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
1944  res0 = __msa_srari_h(res0, 6);
1945  res0 = __msa_sat_s_h(res0, 7);
1946  out = PCKEV_XORI128_UB(res0, res0);
1947  ST4x2_UB(out, dst, dst_stride);
1948 }
1949 
1950 static void common_hz_4t_4x4_msa(uint8_t *src, int32_t src_stride,
1951  uint8_t *dst, int32_t dst_stride,
1952  const int8_t *filter)
1953 {
1954  v16i8 src0, src1, src2, src3, filt0, filt1, mask0, mask1;
1955  v8i16 filt, out0, out1;
1956  v16u8 out;
1957 
1958  mask0 = LD_SB(&ff_hevc_mask_arr[16]);
1959  src -= 1;
1960 
1961  /* rearranging filter */
1962  filt = LD_SH(filter);
1963  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
1964 
1965  mask1 = mask0 + 2;
1966 
1967  LD_SB4(src, src_stride, src0, src1, src2, src3);
1968  XORI_B4_128_SB(src0, src1, src2, src3);
1969  HORIZ_4TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1,
1970  filt0, filt1, out0, out1);
1971  SRARI_H2_SH(out0, out1, 6);
1972  SAT_SH2_SH(out0, out1, 7);
1973  out = PCKEV_XORI128_UB(out0, out1);
1974  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
1975 }
1976 
1977 static void common_hz_4t_4x8_msa(uint8_t *src, int32_t src_stride,
1978  uint8_t *dst, int32_t dst_stride,
1979  const int8_t *filter)
1980 {
1981  v16i8 src0, src1, src2, src3, filt0, filt1, mask0, mask1;
1982  v16u8 out;
1983  v8i16 filt, out0, out1, out2, out3;
1984 
1985  mask0 = LD_SB(&ff_hevc_mask_arr[16]);
1986  src -= 1;
1987 
1988  /* rearranging filter */
1989  filt = LD_SH(filter);
1990  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
1991 
1992  mask1 = mask0 + 2;
1993 
1994  LD_SB4(src, src_stride, src0, src1, src2, src3);
1995  src += (4 * src_stride);
1996 
1997  XORI_B4_128_SB(src0, src1, src2, src3);
1998  HORIZ_4TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1,
1999  filt0, filt1, out0, out1);
2000  LD_SB4(src, src_stride, src0, src1, src2, src3);
2001  XORI_B4_128_SB(src0, src1, src2, src3);
2002  HORIZ_4TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1,
2003  filt0, filt1, out2, out3);
2004  SRARI_H4_SH(out0, out1, out2, out3, 6);
2005  SAT_SH4_SH(out0, out1, out2, out3, 7);
2006  out = PCKEV_XORI128_UB(out0, out1);
2007  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
2008  dst += (4 * dst_stride);
2009  out = PCKEV_XORI128_UB(out2, out3);
2010  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
2011 }
2012 
2013 static void common_hz_4t_4x16_msa(uint8_t *src, int32_t src_stride,
2014  uint8_t *dst, int32_t dst_stride,
2015  const int8_t *filter)
2016 {
2017  v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2018  v16i8 filt0, filt1, mask0, mask1;
2019  v16u8 out;
2020  v8i16 filt, out0, out1, out2, out3;
2021 
2022  mask0 = LD_SB(&ff_hevc_mask_arr[16]);
2023  src -= 1;
2024 
2025  /* rearranging filter */
2026  filt = LD_SH(filter);
2027  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2028 
2029  mask1 = mask0 + 2;
2030 
2031  LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
2032  src += (8 * src_stride);
2033  XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2034  HORIZ_4TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1,
2035  filt0, filt1, out0, out1);
2036  HORIZ_4TAP_4WID_4VECS_FILT(src4, src5, src6, src7, mask0, mask1,
2037  filt0, filt1, out2, out3);
2038  SRARI_H4_SH(out0, out1, out2, out3, 6);
2039  SAT_SH4_SH(out0, out1, out2, out3, 7);
2040  out = PCKEV_XORI128_UB(out0, out1);
2041  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
2042  dst += (4 * dst_stride);
2043  out = PCKEV_XORI128_UB(out2, out3);
2044  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
2045  dst += (4 * dst_stride);
2046 
2047  LD_SB8(src, src_stride, src0, src1, src2, src3, src4, src5, src6, src7);
2048  src += (8 * src_stride);
2049  XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2050  HORIZ_4TAP_4WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1,
2051  filt0, filt1, out0, out1);
2052  HORIZ_4TAP_4WID_4VECS_FILT(src4, src5, src6, src7, mask0, mask1,
2053  filt0, filt1, out2, out3);
2054  SRARI_H4_SH(out0, out1, out2, out3, 6);
2055  SAT_SH4_SH(out0, out1, out2, out3, 7);
2056  out = PCKEV_XORI128_UB(out0, out1);
2057  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
2058  dst += (4 * dst_stride);
2059  out = PCKEV_XORI128_UB(out2, out3);
2060  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
2061 }
2062 
2063 static void common_hz_4t_4w_msa(uint8_t *src, int32_t src_stride,
2064  uint8_t *dst, int32_t dst_stride,
2065  const int8_t *filter, int32_t height)
2066 {
2067  if (2 == height) {
2068  common_hz_4t_4x2_msa(src, src_stride, dst, dst_stride, filter);
2069  } else if (4 == height) {
2070  common_hz_4t_4x4_msa(src, src_stride, dst, dst_stride, filter);
2071  } else if (8 == height) {
2072  common_hz_4t_4x8_msa(src, src_stride, dst, dst_stride, filter);
2073  } else if (16 == height) {
2074  common_hz_4t_4x16_msa(src, src_stride, dst, dst_stride, filter);
2075  }
2076 }
2077 
2078 static void common_hz_4t_6w_msa(uint8_t *src, int32_t src_stride,
2079  uint8_t *dst, int32_t dst_stride,
2080  const int8_t *filter, int32_t height)
2081 {
2082  v16i8 src0, src1, src2, src3, filt0, filt1, mask0, mask1;
2083  v16u8 out4, out5;
2084  v8i16 filt, out0, out1, out2, out3;
2085 
2086  mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2087  src -= 1;
2088 
2089  /* rearranging filter */
2090  filt = LD_SH(filter);
2091  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2092 
2093  mask1 = mask0 + 2;
2094 
2095  LD_SB4(src, src_stride, src0, src1, src2, src3);
2096  src += (4 * src_stride);
2097 
2098  XORI_B4_128_SB(src0, src1, src2, src3);
2099  HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, filt0,
2100  filt1, out0, out1, out2, out3);
2101  SRARI_H4_SH(out0, out1, out2, out3, 6);
2102  SAT_SH4_SH(out0, out1, out2, out3, 7);
2103  out4 = PCKEV_XORI128_UB(out0, out1);
2104  out5 = PCKEV_XORI128_UB(out2, out3);
2105  ST6x4_UB(out4, out5, dst, dst_stride);
2106  dst += (4 * dst_stride);
2107 
2108  LD_SB4(src, src_stride, src0, src1, src2, src3);
2109  src += (4 * src_stride);
2110 
2111  XORI_B4_128_SB(src0, src1, src2, src3);
2112  HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, filt0,
2113  filt1, out0, out1, out2, out3);
2114  SRARI_H4_SH(out0, out1, out2, out3, 6);
2115  SAT_SH4_SH(out0, out1, out2, out3, 7);
2116  out4 = PCKEV_XORI128_UB(out0, out1);
2117  out5 = PCKEV_XORI128_UB(out2, out3);
2118  ST6x4_UB(out4, out5, dst, dst_stride);
2119  dst += (4 * dst_stride);
2120 }
2121 
2122 static void common_hz_4t_8x2mult_msa(uint8_t *src, int32_t src_stride,
2123  uint8_t *dst, int32_t dst_stride,
2124  const int8_t *filter, int32_t height)
2125 {
2126  uint32_t loop_cnt;
2127  v16i8 src0, src1, filt0, filt1, mask0, mask1;
2128  v16u8 out;
2129  v8i16 filt, vec0, vec1, vec2, vec3;
2130 
2131  mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2132  src -= 1;
2133 
2134  filt = LD_SH(filter);
2135  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2136 
2137  mask1 = mask0 + 2;
2138 
2139  for (loop_cnt = (height >> 1); loop_cnt--;) {
2140  LD_SB2(src, src_stride, src0, src1);
2141  src += (2 * src_stride);
2142 
2143  XORI_B2_128_SB(src0, src1);
2144  VSHF_B2_SH(src0, src0, src1, src1, mask0, mask0, vec0, vec1);
2145  DOTP_SB2_SH(vec0, vec1, filt0, filt0, vec0, vec1);
2146  VSHF_B2_SH(src0, src0, src1, src1, mask1, mask1, vec2, vec3);
2147  DPADD_SB2_SH(vec2, vec3, filt1, filt1, vec0, vec1);
2148  SRARI_H2_SH(vec0, vec1, 6);
2149  SAT_SH2_SH(vec0, vec1, 7);
2150  out = PCKEV_XORI128_UB(vec0, vec1);
2151  ST8x2_UB(out, dst, dst_stride);
2152  dst += (2 * dst_stride);
2153  }
2154 }
2155 
2156 static void common_hz_4t_8x4mult_msa(uint8_t *src, int32_t src_stride,
2157  uint8_t *dst, int32_t dst_stride,
2158  const int8_t *filter, int32_t height)
2159 {
2160  uint32_t loop_cnt;
2161  v16i8 src0, src1, src2, src3, filt0, filt1, mask0, mask1;
2162  v16u8 tmp0, tmp1;
2163  v8i16 filt, out0, out1, out2, out3;
2164 
2165  mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2166  src -= 1;
2167 
2168  /* rearranging filter */
2169  filt = LD_SH(filter);
2170  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2171 
2172  mask1 = mask0 + 2;
2173 
2174  for (loop_cnt = (height >> 2); loop_cnt--;) {
2175  LD_SB4(src, src_stride, src0, src1, src2, src3);
2176  src += (4 * src_stride);
2177 
2178  XORI_B4_128_SB(src0, src1, src2, src3);
2179  HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3, mask0, mask1, filt0,
2180  filt1, out0, out1, out2, out3);
2181  SRARI_H4_SH(out0, out1, out2, out3, 6);
2182  SAT_SH4_SH(out0, out1, out2, out3, 7);
2183  tmp0 = PCKEV_XORI128_UB(out0, out1);
2184  tmp1 = PCKEV_XORI128_UB(out2, out3);
2185  ST8x4_UB(tmp0, tmp1, dst, dst_stride);
2186  dst += (4 * dst_stride);
2187  }
2188 }
2189 
2190 static void common_hz_4t_8w_msa(uint8_t *src, int32_t src_stride,
2191  uint8_t *dst, int32_t dst_stride,
2192  const int8_t *filter, int32_t height)
2193 {
2194  if ((2 == height) || (6 == height)) {
2195  common_hz_4t_8x2mult_msa(src, src_stride, dst, dst_stride, filter,
2196  height);
2197  } else {
2198  common_hz_4t_8x4mult_msa(src, src_stride, dst, dst_stride, filter,
2199  height);
2200  }
2201 }
2202 
2203 static void common_hz_4t_12w_msa(uint8_t *src, int32_t src_stride,
2204  uint8_t *dst, int32_t dst_stride,
2205  const int8_t *filter, int32_t height)
2206 {
2207  uint32_t loop_cnt;
2208  v16i8 src0, src1, src2, src3, filt0, filt1, mask0, mask1, mask2, mask3;
2209  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
2210  v16i8 vec10, vec11;
2211  v16u8 tmp0, tmp1;
2212  v8i16 filt, out0, out1, out2, out3, out4, out5;
2213 
2214  mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2215  mask2 = LD_SB(&ff_hevc_mask_arr[32]);
2216 
2217  src -= 1;
2218 
2219  /* rearranging filter */
2220  filt = LD_SH(filter);
2221  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2222 
2223  mask1 = mask0 + 2;
2224  mask3 = mask2 + 2;
2225 
2226  for (loop_cnt = 4; loop_cnt--;) {
2227  LD_SB4(src, src_stride, src0, src1, src2, src3);
2228  src += (4 * src_stride);
2229 
2230  XORI_B4_128_SB(src0, src1, src2, src3);
2231  VSHF_B2_SB(src0, src1, src2, src3, mask2, mask2, vec0, vec1);
2232  DOTP_SB2_SH(vec0, vec1, filt0, filt0, out0, out1);
2233  VSHF_B2_SB(src0, src1, src2, src3, mask3, mask3, vec2, vec3);
2234  DPADD_SB2_SH(vec2, vec3, filt1, filt1, out0, out1);
2235  SRARI_H2_SH(out0, out1, 6);
2236  SAT_SH2_SH(out0, out1, 7);
2237  tmp0 = PCKEV_XORI128_UB(out0, out1);
2238  ST4x4_UB(tmp0, tmp0, 0, 1, 2, 3, dst + 8, dst_stride);
2239 
2240  VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec4, vec5);
2241  VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec6, vec7);
2242  DOTP_SB4_SH(vec4, vec5, vec6, vec7, filt0, filt0, filt0, filt0,
2243  out2, out3, out4, out5);
2244  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec8, vec9);
2245  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec10, vec11);
2246  DPADD_SB4_SH(vec8, vec9, vec10, vec11, filt1, filt1, filt1, filt1,
2247  out2, out3, out4, out5);
2248  SRARI_H4_SH(out2, out3, out4, out5, 6);
2249  SAT_SH4_SH(out2, out3, out4, out5, 7);
2250  tmp0 = PCKEV_XORI128_UB(out2, out3);
2251  tmp1 = PCKEV_XORI128_UB(out4, out5);
2252  ST8x4_UB(tmp0, tmp1, dst, dst_stride);
2253  dst += (4 * dst_stride);
2254  }
2255 }
2256 
2257 static void common_hz_4t_16w_msa(uint8_t *src, int32_t src_stride,
2258  uint8_t *dst, int32_t dst_stride,
2259  const int8_t *filter, int32_t height)
2260 {
2261  uint32_t loop_cnt;
2262  v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2263  v16i8 filt0, filt1, mask0, mask1;
2264  v16i8 vec0_m, vec1_m, vec2_m, vec3_m;
2265  v8i16 filt, out0, out1, out2, out3, out4, out5, out6, out7;
2266  v16u8 out;
2267 
2268  mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2269  src -= 1;
2270 
2271  /* rearranging filter */
2272  filt = LD_SH(filter);
2273  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2274 
2275  mask1 = mask0 + 2;
2276 
2277  for (loop_cnt = (height >> 2); loop_cnt--;) {
2278  LD_SB4(src, src_stride, src0, src2, src4, src6);
2279  LD_SB4(src + 8, src_stride, src1, src3, src5, src7);
2280  src += (4 * src_stride);
2281 
2282  XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2283 
2284  VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0_m, vec1_m);
2285  VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2_m, vec3_m);
2286  DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt0, filt0, filt0, filt0,
2287  out0, out1, out2, out3);
2288  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0_m, vec1_m);
2289  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2_m, vec3_m);
2290  DPADD_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt1, filt1, filt1, filt1,
2291  out0, out1, out2, out3);
2292  SRARI_H4_SH(out0, out1, out2, out3, 6);
2293  SAT_SH4_SH(out0, out1, out2, out3, 7);
2294  out = PCKEV_XORI128_UB(out0, out1);
2295  ST_UB(out, dst);
2296  dst += dst_stride;
2297  out = PCKEV_XORI128_UB(out2, out3);
2298  ST_UB(out, dst);
2299  dst += dst_stride;
2300 
2301  VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0_m, vec1_m);
2302  VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2_m, vec3_m);
2303  DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt0, filt0, filt0, filt0,
2304  out4, out5, out6, out7);
2305  VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec0_m, vec1_m);
2306  VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec2_m, vec3_m);
2307  DPADD_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt1, filt1, filt1, filt1,
2308  out4, out5, out6, out7);
2309  SRARI_H4_SH(out4, out5, out6, out7, 6);
2310  SAT_SH4_SH(out4, out5, out6, out7, 7);
2311  out = PCKEV_XORI128_UB(out4, out5);
2312  ST_UB(out, dst);
2313  dst += dst_stride;
2314  out = PCKEV_XORI128_UB(out6, out7);
2315  ST_UB(out, dst);
2316  dst += dst_stride;
2317  }
2318 }
2319 
2320 static void common_hz_4t_24w_msa(uint8_t *src, int32_t src_stride,
2321  uint8_t *dst, int32_t dst_stride,
2322  const int8_t *filter, int32_t height)
2323 {
2324  uint8_t *dst1 = dst + 16;
2325  uint32_t loop_cnt;
2326  v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2327  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
2328  v16i8 filt0, filt1, mask0, mask1, mask00, mask11;
2329  v8i16 filt, out0, out1, out2, out3;
2330  v16u8 tmp0, tmp1;
2331 
2332  mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2333  src -= 1;
2334 
2335  /* rearranging filter */
2336  filt = LD_SH(filter);
2337  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2338 
2339  mask1 = mask0 + 2;
2340  mask00 = mask0 + 8;
2341  mask11 = mask0 + 10;
2342 
2343  for (loop_cnt = 8; loop_cnt--;) {
2344  LD_SB4(src, src_stride, src0, src2, src4, src6);
2345  LD_SB4(src + 16, src_stride, src1, src3, src5, src7);
2346  src += (4 * src_stride);
2347 
2348  XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2349  VSHF_B2_SB(src0, src0, src0, src1, mask0, mask00, vec0, vec1);
2350  VSHF_B2_SB(src2, src2, src2, src3, mask0, mask00, vec2, vec3);
2351  VSHF_B2_SB(src0, src0, src0, src1, mask1, mask11, vec4, vec5);
2352  VSHF_B2_SB(src2, src2, src2, src3, mask1, mask11, vec6, vec7);
2353  DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0,
2354  out0, out1, out2, out3);
2355  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1,
2356  out0, out1, out2, out3);
2357  SRARI_H4_SH(out0, out1, out2, out3, 6);
2358  SAT_SH4_SH(out0, out1, out2, out3, 7);
2359  tmp0 = PCKEV_XORI128_UB(out0, out1);
2360  ST_UB(tmp0, dst);
2361  dst += dst_stride;
2362  tmp0 = PCKEV_XORI128_UB(out2, out3);
2363  ST_UB(tmp0, dst);
2364  dst += dst_stride;
2365 
2366  VSHF_B2_SB(src4, src4, src4, src5, mask0, mask00, vec0, vec1);
2367  VSHF_B2_SB(src6, src6, src6, src7, mask0, mask00, vec2, vec3);
2368  VSHF_B2_SB(src4, src4, src4, src5, mask1, mask11, vec4, vec5);
2369  VSHF_B2_SB(src6, src6, src6, src7, mask1, mask11, vec6, vec7);
2370  DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0,
2371  out0, out1, out2, out3);
2372  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1,
2373  out0, out1, out2, out3);
2374  SRARI_H4_SH(out0, out1, out2, out3, 6);
2375  SAT_SH4_SH(out0, out1, out2, out3, 7);
2376  tmp0 = PCKEV_XORI128_UB(out0, out1);
2377  ST_UB(tmp0, dst);
2378  dst += dst_stride;
2379  tmp0 = PCKEV_XORI128_UB(out2, out3);
2380  ST_UB(tmp0, dst);
2381  dst += dst_stride;
2382 
2383  /* 8 width */
2384  VSHF_B2_SB(src1, src1, src3, src3, mask0, mask0, vec0, vec1);
2385  VSHF_B2_SB(src5, src5, src7, src7, mask0, mask0, vec2, vec3);
2386  VSHF_B2_SB(src1, src1, src3, src3, mask1, mask1, vec4, vec5);
2387  VSHF_B2_SB(src5, src5, src7, src7, mask1, mask1, vec6, vec7);
2388 
2389  DOTP_SB4_SH(vec0, vec1, vec2, vec3, filt0, filt0, filt0, filt0,
2390  out0, out1, out2, out3);
2391  DPADD_SB4_SH(vec4, vec5, vec6, vec7, filt1, filt1, filt1, filt1,
2392  out0, out1, out2, out3);
2393 
2394  SRARI_H4_SH(out0, out1, out2, out3, 6);
2395  SAT_SH4_SH(out0, out1, out2, out3, 7);
2396  tmp0 = PCKEV_XORI128_UB(out0, out1);
2397  tmp1 = PCKEV_XORI128_UB(out2, out3);
2398  ST8x4_UB(tmp0, tmp1, dst1, dst_stride);
2399  dst1 += (4 * dst_stride);
2400  }
2401 }
2402 
2403 static void common_hz_4t_32w_msa(uint8_t *src, int32_t src_stride,
2404  uint8_t *dst, int32_t dst_stride,
2405  const int8_t *filter, int32_t height)
2406 {
2407  uint32_t loop_cnt;
2408  v16i8 src0, src1, src2, src3, src4, src5, src6, src7;
2409  v16i8 filt0, filt1, mask0, mask1;
2410  v16u8 out;
2411  v16i8 vec0_m, vec1_m, vec2_m, vec3_m;
2412  v8i16 filt, out0, out1, out2, out3, out4, out5, out6, out7;
2413 
2414  mask0 = LD_SB(&ff_hevc_mask_arr[0]);
2415  src -= 1;
2416 
2417  /* rearranging filter */
2418  filt = LD_SH(filter);
2419  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2420 
2421  mask1 = mask0 + 2;
2422 
2423  for (loop_cnt = (height >> 1); loop_cnt--;) {
2424  src0 = LD_SB(src);
2425  src1 = LD_SB(src + 8);
2426  src2 = LD_SB(src + 16);
2427  src3 = LD_SB(src + 24);
2428  src += src_stride;
2429  src4 = LD_SB(src);
2430  src5 = LD_SB(src + 8);
2431  src6 = LD_SB(src + 16);
2432  src7 = LD_SB(src + 24);
2433  src += src_stride;
2434 
2435  XORI_B8_128_SB(src0, src1, src2, src3, src4, src5, src6, src7);
2436 
2437  VSHF_B2_SB(src0, src0, src1, src1, mask0, mask0, vec0_m, vec1_m);
2438  VSHF_B2_SB(src2, src2, src3, src3, mask0, mask0, vec2_m, vec3_m);
2439  DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt0, filt0, filt0, filt0,
2440  out0, out1, out2, out3);
2441  VSHF_B2_SB(src0, src0, src1, src1, mask1, mask1, vec0_m, vec1_m);
2442  VSHF_B2_SB(src2, src2, src3, src3, mask1, mask1, vec2_m, vec3_m);
2443  DPADD_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt1, filt1, filt1, filt1,
2444  out0, out1, out2, out3);
2445 
2446  VSHF_B2_SB(src4, src4, src5, src5, mask0, mask0, vec0_m, vec1_m);
2447  VSHF_B2_SB(src6, src6, src7, src7, mask0, mask0, vec2_m, vec3_m);
2448  DOTP_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt0, filt0, filt0, filt0,
2449  out4, out5, out6, out7);
2450  VSHF_B2_SB(src4, src4, src5, src5, mask1, mask1, vec0_m, vec1_m);
2451  VSHF_B2_SB(src6, src6, src7, src7, mask1, mask1, vec2_m, vec3_m);
2452  DPADD_SB4_SH(vec0_m, vec1_m, vec2_m, vec3_m, filt1, filt1, filt1, filt1,
2453  out4, out5, out6, out7);
2454  SRARI_H4_SH(out0, out1, out2, out3, 6);
2455  SRARI_H4_SH(out4, out5, out6, out7, 6);
2456  SAT_SH4_SH(out0, out1, out2, out3, 7);
2457  SAT_SH4_SH(out4, out5, out6, out7, 7);
2458  out = PCKEV_XORI128_UB(out0, out1);
2459  ST_UB(out, dst);
2460  out = PCKEV_XORI128_UB(out2, out3);
2461  ST_UB(out, dst + 16);
2462  dst += dst_stride;
2463  out = PCKEV_XORI128_UB(out4, out5);
2464  ST_UB(out, dst);
2465  out = PCKEV_XORI128_UB(out6, out7);
2466  ST_UB(out, dst + 16);
2467  dst += dst_stride;
2468  }
2469 }
2470 
2471 static void common_vt_4t_4x2_msa(uint8_t *src, int32_t src_stride,
2472  uint8_t *dst, int32_t dst_stride,
2473  const int8_t *filter)
2474 {
2475  v16i8 src0, src1, src2, src3, src4, src10_r, src32_r, src21_r, src43_r;
2476  v16i8 src2110, src4332, filt0, filt1;
2477  v16u8 out;
2478  v8i16 filt, out10;
2479 
2480  src -= src_stride;
2481 
2482  filt = LD_SH(filter);
2483  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2484 
2485  LD_SB3(src, src_stride, src0, src1, src2);
2486  src += (3 * src_stride);
2487 
2488  ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2489  src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
2490  src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
2491  LD_SB2(src, src_stride, src3, src4);
2492  ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2493  src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_r, (v2i64) src32_r);
2494  src4332 = (v16i8) __msa_xori_b((v16u8) src4332, 128);
2495  out10 = HEVC_FILT_4TAP_SH(src2110, src4332, filt0, filt1);
2496  out10 = __msa_srari_h(out10, 6);
2497  out10 = __msa_sat_s_h(out10, 7);
2498  out = PCKEV_XORI128_UB(out10, out10);
2499  ST4x2_UB(out, dst, dst_stride);
2500 }
2501 
2502 static void common_vt_4t_4x4multiple_msa(uint8_t *src, int32_t src_stride,
2503  uint8_t *dst, int32_t dst_stride,
2504  const int8_t *filter, int32_t height)
2505 {
2506  uint32_t loop_cnt;
2507  v16i8 src0, src1, src2, src3, src4, src5;
2508  v16i8 src10_r, src32_r, src54_r, src21_r, src43_r, src65_r;
2509  v16i8 src2110, src4332, filt0, filt1;
2510  v8i16 filt, out10, out32;
2511  v16u8 out;
2512 
2513  src -= src_stride;
2514 
2515  filt = LD_SH(filter);
2516  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2517 
2518  LD_SB3(src, src_stride, src0, src1, src2);
2519  src += (3 * src_stride);
2520 
2521  ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2522 
2523  src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_r, (v2i64) src10_r);
2524  src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
2525 
2526  for (loop_cnt = (height >> 2); loop_cnt--;) {
2527  LD_SB3(src, src_stride, src3, src4, src5);
2528  src += (3 * src_stride);
2529  ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2530  src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_r, (v2i64) src32_r);
2531  src4332 = (v16i8) __msa_xori_b((v16u8) src4332, 128);
2532  out10 = HEVC_FILT_4TAP_SH(src2110, src4332, filt0, filt1);
2533 
2534  src2 = LD_SB(src);
2535  src += (src_stride);
2536  ILVR_B2_SB(src5, src4, src2, src5, src54_r, src65_r);
2537  src2110 = (v16i8) __msa_ilvr_d((v2i64) src65_r, (v2i64) src54_r);
2538  src2110 = (v16i8) __msa_xori_b((v16u8) src2110, 128);
2539  out32 = HEVC_FILT_4TAP_SH(src4332, src2110, filt0, filt1);
2540  SRARI_H2_SH(out10, out32, 6);
2541  SAT_SH2_SH(out10, out32, 7);
2542  out = PCKEV_XORI128_UB(out10, out32);
2543  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
2544  dst += (4 * dst_stride);
2545  }
2546 }
2547 
2548 static void common_vt_4t_4w_msa(uint8_t *src, int32_t src_stride,
2549  uint8_t *dst, int32_t dst_stride,
2550  const int8_t *filter, int32_t height)
2551 {
2552  if (2 == height) {
2553  common_vt_4t_4x2_msa(src, src_stride, dst, dst_stride, filter);
2554  } else {
2555  common_vt_4t_4x4multiple_msa(src, src_stride, dst, dst_stride, filter,
2556  height);
2557  }
2558 }
2559 
2560 static void common_vt_4t_6w_msa(uint8_t *src, int32_t src_stride,
2561  uint8_t *dst, int32_t dst_stride,
2562  const int8_t *filter, int32_t height)
2563 {
2564  v16u8 out0, out1;
2565  v16i8 src0, src1, src2, src3, src4, src5, src6;
2566  v16i8 src10_r, src32_r, src21_r, src43_r, src54_r, src65_r;
2567  v8i16 dst0_r, dst1_r, dst2_r, dst3_r, filt0, filt1, filter_vec;
2568 
2569  src -= src_stride;
2570 
2571  filter_vec = LD_SH(filter);
2572  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2573 
2574  LD_SB3(src, src_stride, src0, src1, src2);
2575  src += (3 * src_stride);
2576  XORI_B3_128_SB(src0, src1, src2);
2577  ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2578 
2579  LD_SB2(src, src_stride, src3, src4);
2580  src += (2 * src_stride);
2581  XORI_B2_128_SB(src3, src4);
2582  ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2583 
2584  dst0_r = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
2585  dst1_r = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
2586 
2587  LD_SB2(src, src_stride, src5, src6);
2588  src += (2 * src_stride);
2589  XORI_B2_128_SB(src5, src6);
2590  ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
2591 
2592  dst2_r = HEVC_FILT_4TAP_SH(src32_r, src54_r, filt0, filt1);
2593  dst3_r = HEVC_FILT_4TAP_SH(src43_r, src65_r, filt0, filt1);
2594 
2595  SRARI_H4_SH(dst0_r, dst1_r, dst2_r, dst3_r, 6);
2596  SAT_SH4_SH(dst0_r, dst1_r, dst2_r, dst3_r, 7);
2597  out0 = PCKEV_XORI128_UB(dst0_r, dst1_r);
2598  out1 = PCKEV_XORI128_UB(dst2_r, dst3_r);
2599  ST6x4_UB(out0, out1, dst, dst_stride);
2600  dst += (4 * dst_stride);
2601 
2602  LD_SB2(src, src_stride, src3, src4);
2603  src += (2 * src_stride);
2604  XORI_B2_128_SB(src3, src4);
2605  ILVR_B2_SB(src3, src6, src4, src3, src32_r, src43_r);
2606 
2607  dst0_r = HEVC_FILT_4TAP_SH(src54_r, src32_r, filt0, filt1);
2608  dst1_r = HEVC_FILT_4TAP_SH(src65_r, src43_r, filt0, filt1);
2609 
2610  LD_SB2(src, src_stride, src5, src6);
2611  src += (2 * src_stride);
2612  XORI_B2_128_SB(src5, src6);
2613  ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
2614 
2615  dst2_r = HEVC_FILT_4TAP_SH(src32_r, src54_r, filt0, filt1);
2616  dst3_r = HEVC_FILT_4TAP_SH(src43_r, src65_r, filt0, filt1);
2617 
2618  SRARI_H4_SH(dst0_r, dst1_r, dst2_r, dst3_r, 6);
2619  SAT_SH4_SH(dst0_r, dst1_r, dst2_r, dst3_r, 7);
2620  out0 = PCKEV_XORI128_UB(dst0_r, dst1_r);
2621  out1 = PCKEV_XORI128_UB(dst2_r, dst3_r);
2622  ST6x4_UB(out0, out1, dst, dst_stride);
2623 }
2624 
2625 static void common_vt_4t_8x2_msa(uint8_t *src, int32_t src_stride,
2626  uint8_t *dst, int32_t dst_stride,
2627  const int8_t *filter)
2628 {
2629  v16i8 src0, src1, src2, src3, src4;
2630  v8i16 src01, src12, src23, src34, tmp0, tmp1, filt, filt0, filt1;
2631  v16u8 out;
2632 
2633  src -= src_stride;
2634 
2635  /* rearranging filter_y */
2636  filt = LD_SH(filter);
2637  SPLATI_H2_SH(filt, 0, 1, filt0, filt1);
2638 
2639  LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
2640  XORI_B5_128_SB(src0, src1, src2, src3, src4);
2641  ILVR_B2_SH(src1, src0, src3, src2, src01, src23);
2642  tmp0 = HEVC_FILT_4TAP_SH(src01, src23, filt0, filt1);
2643  ILVR_B2_SH(src2, src1, src4, src3, src12, src34);
2644  tmp1 = HEVC_FILT_4TAP_SH(src12, src34, filt0, filt1);
2645  SRARI_H2_SH(tmp0, tmp1, 6);
2646  SAT_SH2_SH(tmp0, tmp1, 7);
2647  out = PCKEV_XORI128_UB(tmp0, tmp1);
2648  ST8x2_UB(out, dst, dst_stride);
2649 }
2650 
2651 static void common_vt_4t_8x6_msa(uint8_t *src, int32_t src_stride,
2652  uint8_t *dst, int32_t dst_stride,
2653  const int8_t *filter)
2654 {
2655  uint32_t loop_cnt;
2656  uint64_t out0, out1, out2;
2657  v16i8 src0, src1, src2, src3, src4, src5;
2658  v8i16 vec0, vec1, vec2, vec3, vec4, tmp0, tmp1, tmp2;
2659  v8i16 filt, filt0, filt1;
2660 
2661  src -= src_stride;
2662 
2663  /* rearranging filter_y */
2664  filt = LD_SH(filter);
2665  SPLATI_H2_SH(filt, 0, 1, filt0, filt1);
2666 
2667  LD_SB3(src, src_stride, src0, src1, src2);
2668  src += (3 * src_stride);
2669 
2670  XORI_B3_128_SB(src0, src1, src2);
2671  ILVR_B2_SH(src1, src0, src2, src1, vec0, vec2);
2672 
2673  for (loop_cnt = 2; loop_cnt--;) {
2674  LD_SB3(src, src_stride, src3, src4, src5);
2675  src += (3 * src_stride);
2676 
2677  XORI_B3_128_SB(src3, src4, src5);
2678  ILVR_B3_SH(src3, src2, src4, src3, src5, src4, vec1, vec3, vec4);
2679  tmp0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
2680  tmp1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
2681  tmp2 = HEVC_FILT_4TAP_SH(vec1, vec4, filt0, filt1);
2682  SRARI_H2_SH(tmp0, tmp1, 6);
2683  tmp2 = __msa_srari_h(tmp2, 6);
2684  SAT_SH3_SH(tmp0, tmp1, tmp2, 7);
2685  PCKEV_B2_SH(tmp1, tmp0, tmp2, tmp2, tmp0, tmp2);
2686  XORI_B2_128_SH(tmp0, tmp2);
2687 
2688  out0 = __msa_copy_u_d((v2i64) tmp0, 0);
2689  out1 = __msa_copy_u_d((v2i64) tmp0, 1);
2690  out2 = __msa_copy_u_d((v2i64) tmp2, 0);
2691  SD(out0, dst);
2692  dst += dst_stride;
2693  SD(out1, dst);
2694  dst += dst_stride;
2695  SD(out2, dst);
2696  dst += dst_stride;
2697 
2698  src2 = src5;
2699  vec0 = vec3;
2700  vec2 = vec4;
2701  }
2702 }
2703 
2704 static void common_vt_4t_8x4mult_msa(uint8_t *src, int32_t src_stride,
2705  uint8_t *dst, int32_t dst_stride,
2706  const int8_t *filter, int32_t height)
2707 {
2708  uint32_t loop_cnt;
2709  v16i8 src0, src1, src2, src7, src8, src9, src10;
2710  v16i8 src10_r, src72_r, src98_r, src21_r, src87_r, src109_r, filt0, filt1;
2711  v16u8 tmp0, tmp1;
2712  v8i16 filt, out0_r, out1_r, out2_r, out3_r;
2713 
2714  src -= src_stride;
2715 
2716  filt = LD_SH(filter);
2717  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2718 
2719  LD_SB3(src, src_stride, src0, src1, src2);
2720  src += (3 * src_stride);
2721 
2722  XORI_B3_128_SB(src0, src1, src2);
2723  ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2724 
2725  for (loop_cnt = (height >> 2); loop_cnt--;) {
2726  LD_SB4(src, src_stride, src7, src8, src9, src10);
2727  src += (4 * src_stride);
2728 
2729  XORI_B4_128_SB(src7, src8, src9, src10);
2730  ILVR_B4_SB(src7, src2, src8, src7, src9, src8, src10, src9,
2731  src72_r, src87_r, src98_r, src109_r);
2732  out0_r = HEVC_FILT_4TAP_SH(src10_r, src72_r, filt0, filt1);
2733  out1_r = HEVC_FILT_4TAP_SH(src21_r, src87_r, filt0, filt1);
2734  out2_r = HEVC_FILT_4TAP_SH(src72_r, src98_r, filt0, filt1);
2735  out3_r = HEVC_FILT_4TAP_SH(src87_r, src109_r, filt0, filt1);
2736  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
2737  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2738  tmp0 = PCKEV_XORI128_UB(out0_r, out1_r);
2739  tmp1 = PCKEV_XORI128_UB(out2_r, out3_r);
2740  ST8x4_UB(tmp0, tmp1, dst, dst_stride);
2741  dst += (4 * dst_stride);
2742 
2743  src10_r = src98_r;
2744  src21_r = src109_r;
2745  src2 = src10;
2746  }
2747 }
2748 
2749 static void common_vt_4t_8w_msa(uint8_t *src, int32_t src_stride,
2750  uint8_t *dst, int32_t dst_stride,
2751  const int8_t *filter, int32_t height)
2752 {
2753  if (2 == height) {
2754  common_vt_4t_8x2_msa(src, src_stride, dst, dst_stride, filter);
2755  } else if (6 == height) {
2756  common_vt_4t_8x6_msa(src, src_stride, dst, dst_stride, filter);
2757  } else {
2758  common_vt_4t_8x4mult_msa(src, src_stride, dst, dst_stride,
2759  filter, height);
2760  }
2761 }
2762 
2763 static void common_vt_4t_12w_msa(uint8_t *src, int32_t src_stride,
2764  uint8_t *dst, int32_t dst_stride,
2765  const int8_t *filter, int32_t height)
2766 {
2767  uint32_t loop_cnt;
2768  v16i8 src0, src1, src2, src3, src4, src5, src6;
2769  v16u8 out0, out1;
2770  v16i8 src10_r, src32_r, src21_r, src43_r, src54_r, src65_r;
2771  v16i8 src10_l, src32_l, src54_l, src21_l, src43_l, src65_l;
2772  v16i8 src2110, src4332, src6554;
2773  v8i16 dst0_r, dst1_r, dst2_r, dst3_r, dst0_l, dst1_l, filt0, filt1;
2774  v8i16 filter_vec;
2775 
2776  src -= (1 * src_stride);
2777 
2778  filter_vec = LD_SH(filter);
2779  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
2780 
2781  LD_SB3(src, src_stride, src0, src1, src2);
2782  src += (3 * src_stride);
2783 
2784  XORI_B3_128_SB(src0, src1, src2);
2785  ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2786  ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
2787  src2110 = (v16i8) __msa_ilvr_d((v2i64) src21_l, (v2i64) src10_l);
2788 
2789  for (loop_cnt = 4; loop_cnt--;) {
2790  LD_SB4(src, src_stride, src3, src4, src5, src6);
2791  src += (4 * src_stride);
2792 
2793  XORI_B4_128_SB(src3, src4, src5, src6);
2794  ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2795  ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
2796  src4332 = (v16i8) __msa_ilvr_d((v2i64) src43_l, (v2i64) src32_l);
2797  ILVR_B2_SB(src5, src4, src6, src5, src54_r, src65_r);
2798  ILVL_B2_SB(src5, src4, src6, src5, src54_l, src65_l);
2799  src6554 = (v16i8) __msa_ilvr_d((v2i64) src65_l, (v2i64) src54_l);
2800 
2801  dst0_r = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
2802  dst1_r = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
2803  dst0_l = HEVC_FILT_4TAP_SH(src2110, src4332, filt0, filt1);
2804  dst2_r = HEVC_FILT_4TAP_SH(src32_r, src54_r, filt0, filt1);
2805  dst3_r = HEVC_FILT_4TAP_SH(src43_r, src65_r, filt0, filt1);
2806  dst1_l = HEVC_FILT_4TAP_SH(src4332, src6554, filt0, filt1);
2807 
2808  SRARI_H4_SH(dst0_r, dst1_r, dst2_r, dst3_r, 6);
2809  SRARI_H2_SH(dst0_l, dst1_l, 6);
2810  SAT_SH4_SH(dst0_r, dst1_r, dst2_r, dst3_r, 7);
2811  SAT_SH2_SH(dst0_l, dst1_l, 7);
2812  out0 = PCKEV_XORI128_UB(dst0_r, dst1_r);
2813  out1 = PCKEV_XORI128_UB(dst2_r, dst3_r);
2814  ST8x4_UB(out0, out1, dst, dst_stride);
2815  out0 = PCKEV_XORI128_UB(dst0_l, dst1_l);
2816  ST4x4_UB(out0, out0, 0, 1, 2, 3, dst + 8, dst_stride);
2817  dst += (4 * dst_stride);
2818 
2819  src2 = src6;
2820  src10_r = src54_r;
2821  src21_r = src65_r;
2822  src2110 = src6554;
2823  }
2824 }
2825 
2826 static void common_vt_4t_16w_msa(uint8_t *src, int32_t src_stride,
2827  uint8_t *dst, int32_t dst_stride,
2828  const int8_t *filter, int32_t height)
2829 {
2830  uint32_t loop_cnt;
2831  v16i8 src0, src1, src2, src3, src4, src5, src6;
2832  v16i8 src10_r, src32_r, src54_r, src21_r, src43_r, src65_r, src10_l;
2833  v16i8 src32_l, src54_l, src21_l, src43_l, src65_l, filt0, filt1;
2834  v16u8 tmp0, tmp1, tmp2, tmp3;
2835  v8i16 filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
2836 
2837  src -= src_stride;
2838 
2839  filt = LD_SH(filter);
2840  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2841 
2842  LD_SB3(src, src_stride, src0, src1, src2);
2843  src += (3 * src_stride);
2844 
2845  XORI_B3_128_SB(src0, src1, src2);
2846  ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2847  ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
2848 
2849  for (loop_cnt = (height >> 2); loop_cnt--;) {
2850  LD_SB4(src, src_stride, src3, src4, src5, src6);
2851  src += (4 * src_stride);
2852 
2853  XORI_B4_128_SB(src3, src4, src5, src6);
2854  ILVR_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
2855  src32_r, src43_r, src54_r, src65_r);
2856  ILVL_B4_SB(src3, src2, src4, src3, src5, src4, src6, src5,
2857  src32_l, src43_l, src54_l, src65_l);
2858  out0_r = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
2859  out1_r = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
2860  out2_r = HEVC_FILT_4TAP_SH(src32_r, src54_r, filt0, filt1);
2861  out3_r = HEVC_FILT_4TAP_SH(src43_r, src65_r, filt0, filt1);
2862  out0_l = HEVC_FILT_4TAP_SH(src10_l, src32_l, filt0, filt1);
2863  out1_l = HEVC_FILT_4TAP_SH(src21_l, src43_l, filt0, filt1);
2864  out2_l = HEVC_FILT_4TAP_SH(src32_l, src54_l, filt0, filt1);
2865  out3_l = HEVC_FILT_4TAP_SH(src43_l, src65_l, filt0, filt1);
2866  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
2867  SRARI_H4_SH(out0_l, out1_l, out2_l, out3_l, 6);
2868  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2869  SAT_SH4_SH(out0_l, out1_l, out2_l, out3_l, 7);
2870  PCKEV_B4_UB(out0_l, out0_r, out1_l, out1_r, out2_l, out2_r, out3_l,
2871  out3_r, tmp0, tmp1, tmp2, tmp3);
2872  XORI_B4_128_UB(tmp0, tmp1, tmp2, tmp3);
2873  ST_UB4(tmp0, tmp1, tmp2, tmp3, dst, dst_stride);
2874  dst += (4 * dst_stride);
2875 
2876  src10_r = src54_r;
2877  src21_r = src65_r;
2878  src10_l = src54_l;
2879  src21_l = src65_l;
2880  src2 = src6;
2881  }
2882 }
2883 
2884 static void common_vt_4t_24w_msa(uint8_t *src, int32_t src_stride,
2885  uint8_t *dst, int32_t dst_stride,
2886  const int8_t *filter, int32_t height)
2887 {
2888  uint32_t loop_cnt;
2889  uint64_t out0, out1;
2890  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
2891  v16i8 src11, filt0, filt1;
2892  v16i8 src10_r, src32_r, src76_r, src98_r, src21_r, src43_r, src87_r;
2893  v16i8 src109_r, src10_l, src32_l, src21_l, src43_l;
2894  v16u8 out;
2895  v8i16 filt, out0_r, out1_r, out2_r, out3_r, out0_l, out1_l;
2896 
2897  src -= src_stride;
2898 
2899  filt = LD_SH(filter);
2900  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
2901 
2902  /* 16 width */
2903  LD_SB3(src, src_stride, src0, src1, src2);
2904  XORI_B3_128_SB(src0, src1, src2);
2905  ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
2906  ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
2907 
2908  /* 8 width */
2909  LD_SB3(src + 16, src_stride, src6, src7, src8);
2910  src += (3 * src_stride);
2911  XORI_B3_128_SB(src6, src7, src8);
2912  ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
2913 
2914  for (loop_cnt = 8; loop_cnt--;) {
2915  /* 16 width */
2916  LD_SB2(src, src_stride, src3, src4);
2917  XORI_B2_128_SB(src3, src4);
2918  ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
2919  ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
2920 
2921  /* 8 width */
2922  LD_SB2(src + 16, src_stride, src9, src10);
2923  src += (2 * src_stride);
2924  XORI_B2_128_SB(src9, src10);
2925  ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
2926 
2927  /* 16 width */
2928  out0_r = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
2929  out0_l = HEVC_FILT_4TAP_SH(src10_l, src32_l, filt0, filt1);
2930  out1_r = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
2931  out1_l = HEVC_FILT_4TAP_SH(src21_l, src43_l, filt0, filt1);
2932 
2933  /* 8 width */
2934  out2_r = HEVC_FILT_4TAP_SH(src76_r, src98_r, filt0, filt1);
2935  out3_r = HEVC_FILT_4TAP_SH(src87_r, src109_r, filt0, filt1);
2936 
2937  /* 16 + 8 width */
2938  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
2939  SRARI_H2_SH(out0_l, out1_l, 6);
2940  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2941  SAT_SH2_SH(out0_l, out1_l, 7);
2942  out = PCKEV_XORI128_UB(out0_r, out0_l);
2943  ST_UB(out, dst);
2944  PCKEV_B2_SH(out2_r, out2_r, out3_r, out3_r, out2_r, out3_r);
2945  XORI_B2_128_SH(out2_r, out3_r);
2946  out0 = __msa_copy_u_d((v2i64) out2_r, 0);
2947  out1 = __msa_copy_u_d((v2i64) out3_r, 0);
2948  SD(out0, dst + 16);
2949  dst += dst_stride;
2950  out = PCKEV_XORI128_UB(out1_r, out1_l);
2951  ST_UB(out, dst);
2952  SD(out1, dst + 16);
2953  dst += dst_stride;
2954 
2955  /* 16 width */
2956  LD_SB2(src, src_stride, src5, src2);
2957  XORI_B2_128_SB(src5, src2);
2958  ILVR_B2_SB(src5, src4, src2, src5, src10_r, src21_r);
2959  ILVL_B2_SB(src5, src4, src2, src5, src10_l, src21_l);
2960 
2961  /* 8 width */
2962  LD_SB2(src + 16, src_stride, src11, src8);
2963  src += (2 * src_stride);
2964  XORI_B2_128_SB(src11, src8);
2965  ILVR_B2_SB(src11, src10, src8, src11, src76_r, src87_r);
2966 
2967  /* 16 width */
2968  out0_r = HEVC_FILT_4TAP_SH(src32_r, src10_r, filt0, filt1);
2969  out0_l = HEVC_FILT_4TAP_SH(src32_l, src10_l, filt0, filt1);
2970  out1_r = HEVC_FILT_4TAP_SH(src43_r, src21_r, filt0, filt1);
2971  out1_l = HEVC_FILT_4TAP_SH(src43_l, src21_l, filt0, filt1);
2972 
2973  /* 8 width */
2974  out2_r = HEVC_FILT_4TAP_SH(src98_r, src76_r, filt0, filt1);
2975  out3_r = HEVC_FILT_4TAP_SH(src109_r, src87_r, filt0, filt1);
2976 
2977  /* 16 + 8 width */
2978  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
2979  SRARI_H2_SH(out0_l, out1_l, 6);
2980  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
2981  SAT_SH2_SH(out0_l, out1_l, 7);
2982  out = PCKEV_XORI128_UB(out0_r, out0_l);
2983  ST_UB(out, dst);
2984  out = PCKEV_XORI128_UB(out2_r, out2_r);
2985  ST8x1_UB(out, dst + 16);
2986  dst += dst_stride;
2987  out = PCKEV_XORI128_UB(out1_r, out1_l);
2988  ST_UB(out, dst);
2989  out = PCKEV_XORI128_UB(out3_r, out3_r);
2990  ST8x1_UB(out, dst + 16);
2991  dst += dst_stride;
2992  }
2993 }
2994 
2995 static void common_vt_4t_32w_msa(uint8_t *src, int32_t src_stride,
2996  uint8_t *dst, int32_t dst_stride,
2997  const int8_t *filter, int32_t height)
2998 {
2999  uint32_t loop_cnt;
3000  v16i8 src0, src1, src2, src3, src4, src6, src7, src8, src9, src10;
3001  v16i8 src10_r, src32_r, src76_r, src98_r;
3002  v16i8 src21_r, src43_r, src87_r, src109_r;
3003  v8i16 out0_r, out1_r, out2_r, out3_r, out0_l, out1_l, out2_l, out3_l;
3004  v16i8 src10_l, src32_l, src76_l, src98_l;
3005  v16i8 src21_l, src43_l, src87_l, src109_l;
3006  v8i16 filt;
3007  v16i8 filt0, filt1;
3008  v16u8 out;
3009 
3010  src -= src_stride;
3011 
3012  filt = LD_SH(filter);
3013  SPLATI_H2_SB(filt, 0, 1, filt0, filt1);
3014 
3015  /* 16 width */
3016  LD_SB3(src, src_stride, src0, src1, src2);
3017  XORI_B3_128_SB(src0, src1, src2);
3018 
3019  ILVR_B2_SB(src1, src0, src2, src1, src10_r, src21_r);
3020  ILVL_B2_SB(src1, src0, src2, src1, src10_l, src21_l);
3021 
3022  /* next 16 width */
3023  LD_SB3(src + 16, src_stride, src6, src7, src8);
3024  src += (3 * src_stride);
3025 
3026  XORI_B3_128_SB(src6, src7, src8);
3027  ILVR_B2_SB(src7, src6, src8, src7, src76_r, src87_r);
3028  ILVL_B2_SB(src7, src6, src8, src7, src76_l, src87_l);
3029 
3030  for (loop_cnt = (height >> 1); loop_cnt--;) {
3031  /* 16 width */
3032  LD_SB2(src, src_stride, src3, src4);
3033  XORI_B2_128_SB(src3, src4);
3034  ILVR_B2_SB(src3, src2, src4, src3, src32_r, src43_r);
3035  ILVL_B2_SB(src3, src2, src4, src3, src32_l, src43_l);
3036 
3037  /* 16 width */
3038  out0_r = HEVC_FILT_4TAP_SH(src10_r, src32_r, filt0, filt1);
3039  out0_l = HEVC_FILT_4TAP_SH(src10_l, src32_l, filt0, filt1);
3040  out1_r = HEVC_FILT_4TAP_SH(src21_r, src43_r, filt0, filt1);
3041  out1_l = HEVC_FILT_4TAP_SH(src21_l, src43_l, filt0, filt1);
3042 
3043  /* 16 width */
3044  SRARI_H4_SH(out0_r, out1_r, out0_l, out1_l, 6);
3045  SAT_SH4_SH(out0_r, out1_r, out0_l, out1_l, 7);
3046  out = PCKEV_XORI128_UB(out0_r, out0_l);
3047  ST_UB(out, dst);
3048  out = PCKEV_XORI128_UB(out1_r, out1_l);
3049  ST_UB(out, dst + dst_stride);
3050 
3051  src10_r = src32_r;
3052  src21_r = src43_r;
3053  src10_l = src32_l;
3054  src21_l = src43_l;
3055  src2 = src4;
3056 
3057  /* next 16 width */
3058  LD_SB2(src + 16, src_stride, src9, src10);
3059  src += (2 * src_stride);
3060  XORI_B2_128_SB(src9, src10);
3061  ILVR_B2_SB(src9, src8, src10, src9, src98_r, src109_r);
3062  ILVL_B2_SB(src9, src8, src10, src9, src98_l, src109_l);
3063 
3064  /* next 16 width */
3065  out2_r = HEVC_FILT_4TAP_SH(src76_r, src98_r, filt0, filt1);
3066  out2_l = HEVC_FILT_4TAP_SH(src76_l, src98_l, filt0, filt1);
3067  out3_r = HEVC_FILT_4TAP_SH(src87_r, src109_r, filt0, filt1);
3068  out3_l = HEVC_FILT_4TAP_SH(src87_l, src109_l, filt0, filt1);
3069 
3070  /* next 16 width */
3071  SRARI_H4_SH(out2_r, out3_r, out2_l, out3_l, 6);
3072  SAT_SH4_SH(out2_r, out3_r, out2_l, out3_l, 7);
3073  out = PCKEV_XORI128_UB(out2_r, out2_l);
3074  ST_UB(out, dst + 16);
3075  out = PCKEV_XORI128_UB(out3_r, out3_l);
3076  ST_UB(out, dst + 16 + dst_stride);
3077 
3078  dst += 2 * dst_stride;
3079 
3080  src76_r = src98_r;
3081  src87_r = src109_r;
3082  src76_l = src98_l;
3083  src87_l = src109_l;
3084  src8 = src10;
3085  }
3086 }
3087 
3088 static void hevc_hv_uni_4t_4x2_msa(uint8_t *src,
3089  int32_t src_stride,
3090  uint8_t *dst,
3091  int32_t dst_stride,
3092  const int8_t *filter_x,
3093  const int8_t *filter_y)
3094 {
3095  v16u8 out;
3096  v16i8 src0, src1, src2, src3, src4;
3097  v8i16 filt0, filt1;
3098  v8i16 filt_h0, filt_h1;
3099  v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
3100  v16i8 mask1;
3101  v8i16 filter_vec, tmp;
3102  v16i8 vec0, vec1, vec2, vec3, vec4, vec5;
3103  v8i16 dst20, dst31, dst42, dst10, dst32, dst21, dst43;
3104  v4i32 dst0, dst1;
3105 
3106  src -= (src_stride + 1);
3107 
3108  filter_vec = LD_SH(filter_x);
3109  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3110 
3111  filter_vec = LD_SH(filter_y);
3112  UNPCK_R_SB_SH(filter_vec, filter_vec);
3113 
3114  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3115 
3116  mask1 = mask0 + 2;
3117 
3118  LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
3119  XORI_B5_128_SB(src0, src1, src2, src3, src4);
3120 
3121  VSHF_B2_SB(src0, src2, src0, src2, mask0, mask1, vec0, vec1);
3122  VSHF_B2_SB(src1, src3, src1, src3, mask0, mask1, vec2, vec3);
3123  VSHF_B2_SB(src2, src4, src2, src4, mask0, mask1, vec4, vec5);
3124 
3125  dst20 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3126  dst31 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3127  dst42 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3128 
3129  ILVRL_H2_SH(dst31, dst20, dst10, dst32);
3130  ILVRL_H2_SH(dst42, dst31, dst21, dst43);
3131 
3132  dst0 = HEVC_FILT_4TAP(dst10, dst32, filt_h0, filt_h1);
3133  dst1 = HEVC_FILT_4TAP(dst21, dst43, filt_h0, filt_h1);
3134  dst0 >>= 6;
3135  dst1 >>= 6;
3136  tmp = __msa_pckev_h((v8i16) dst1, (v8i16) dst0);
3137  tmp = __msa_srari_h(tmp, 6);
3138  tmp = __msa_sat_s_h(tmp, 7);
3139  out = PCKEV_XORI128_UB(tmp, tmp);
3140  ST4x2_UB(out, dst, dst_stride);
3141 }
3142 
3143 static void hevc_hv_uni_4t_4x4_msa(uint8_t *src,
3144  int32_t src_stride,
3145  uint8_t *dst,
3146  int32_t dst_stride,
3147  const int8_t *filter_x,
3148  const int8_t *filter_y)
3149 {
3150  v16u8 out;
3151  v16i8 src0, src1, src2, src3, src4, src5, src6;
3152  v8i16 filt0, filt1;
3153  v8i16 filt_h0, filt_h1;
3154  v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
3155  v16i8 mask1;
3156  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3157  v8i16 filter_vec, tmp0, tmp1;
3158  v8i16 dst30, dst41, dst52, dst63;
3159  v8i16 dst10, dst32, dst54, dst21, dst43, dst65;
3160  v4i32 dst0, dst1, dst2, dst3;
3161 
3162  src -= (src_stride + 1);
3163 
3164  filter_vec = LD_SH(filter_x);
3165  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3166 
3167  filter_vec = LD_SH(filter_y);
3168  UNPCK_R_SB_SH(filter_vec, filter_vec);
3169 
3170  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3171 
3172  mask1 = mask0 + 2;
3173 
3174  LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
3175  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
3176 
3177  VSHF_B2_SB(src0, src3, src0, src3, mask0, mask1, vec0, vec1);
3178  VSHF_B2_SB(src1, src4, src1, src4, mask0, mask1, vec2, vec3);
3179  VSHF_B2_SB(src2, src5, src2, src5, mask0, mask1, vec4, vec5);
3180  VSHF_B2_SB(src3, src6, src3, src6, mask0, mask1, vec6, vec7);
3181 
3182  dst30 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3183  dst41 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3184  dst52 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3185  dst63 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3186 
3187  ILVRL_H2_SH(dst41, dst30, dst10, dst43);
3188  ILVRL_H2_SH(dst52, dst41, dst21, dst54);
3189  ILVRL_H2_SH(dst63, dst52, dst32, dst65);
3190  dst0 = HEVC_FILT_4TAP(dst10, dst32, filt_h0, filt_h1);
3191  dst1 = HEVC_FILT_4TAP(dst21, dst43, filt_h0, filt_h1);
3192  dst2 = HEVC_FILT_4TAP(dst32, dst54, filt_h0, filt_h1);
3193  dst3 = HEVC_FILT_4TAP(dst43, dst65, filt_h0, filt_h1);
3194  SRA_4V(dst0, dst1, dst2, dst3, 6);
3195  PCKEV_H2_SH(dst1, dst0, dst3, dst2, tmp0, tmp1);
3196  SRARI_H2_SH(tmp0, tmp1, 6);
3197  SAT_SH2_SH(tmp0, tmp1, 7);
3198  out = PCKEV_XORI128_UB(tmp0, tmp1);
3199  ST4x4_UB(out, out, 0, 1, 2, 3, dst, dst_stride);
3200 }
3201 
3202 static void hevc_hv_uni_4t_4multx8mult_msa(uint8_t *src,
3203  int32_t src_stride,
3204  uint8_t *dst,
3205  int32_t dst_stride,
3206  const int8_t *filter_x,
3207  const int8_t *filter_y,
3208  int32_t height)
3209 {
3210  uint32_t loop_cnt;
3211  v16u8 out0, out1;
3212  v16i8 src0, src1, src2, src3, src4, src5;
3213  v16i8 src6, src7, src8, src9, src10;
3214  v8i16 filt0, filt1;
3215  v8i16 filt_h0, filt_h1;
3216  v16i8 mask0 = LD_SB(ff_hevc_mask_arr + 16);
3217  v16i8 mask1;
3218  v8i16 filter_vec, tmp0, tmp1, tmp2, tmp3;
3219  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3220  v8i16 dst10, dst21, dst22, dst73, dst84, dst95, dst106;
3221  v4i32 dst0_r, dst1_r, dst2_r, dst3_r, dst4_r, dst5_r, dst6_r, dst7_r;
3222  v8i16 dst10_r, dst32_r, dst54_r, dst76_r;
3223  v8i16 dst21_r, dst43_r, dst65_r, dst87_r;
3224  v8i16 dst98_r, dst109_r;
3225 
3226  src -= (src_stride + 1);
3227 
3228  filter_vec = LD_SH(filter_x);
3229  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3230 
3231  filter_vec = LD_SH(filter_y);
3232  UNPCK_R_SB_SH(filter_vec, filter_vec);
3233 
3234  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3235 
3236  mask1 = mask0 + 2;
3237 
3238  LD_SB3(src, src_stride, src0, src1, src2);
3239  src += (3 * src_stride);
3240 
3241  XORI_B3_128_SB(src0, src1, src2);
3242 
3243  VSHF_B2_SB(src0, src1, src0, src1, mask0, mask1, vec0, vec1);
3244  VSHF_B2_SB(src1, src2, src1, src2, mask0, mask1, vec2, vec3);
3245  dst10 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3246  dst21 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3247  ILVRL_H2_SH(dst21, dst10, dst10_r, dst21_r);
3248  dst22 = (v8i16) __msa_splati_d((v2i64) dst21, 1);
3249 
3250  for (loop_cnt = height >> 3; loop_cnt--;) {
3251  LD_SB8(src, src_stride,
3252  src3, src4, src5, src6, src7, src8, src9, src10);
3253  src += (8 * src_stride);
3254 
3255  XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
3256 
3257  VSHF_B2_SB(src3, src7, src3, src7, mask0, mask1, vec0, vec1);
3258  VSHF_B2_SB(src4, src8, src4, src8, mask0, mask1, vec2, vec3);
3259  VSHF_B2_SB(src5, src9, src5, src9, mask0, mask1, vec4, vec5);
3260  VSHF_B2_SB(src6, src10, src6, src10, mask0, mask1, vec6, vec7);
3261 
3262  dst73 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3263  dst84 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3264  dst95 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3265  dst106 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3266 
3267  dst32_r = __msa_ilvr_h(dst73, dst22);
3268  ILVRL_H2_SH(dst84, dst73, dst43_r, dst87_r);
3269  ILVRL_H2_SH(dst95, dst84, dst54_r, dst98_r);
3270  ILVRL_H2_SH(dst106, dst95, dst65_r, dst109_r);
3271  dst22 = (v8i16) __msa_splati_d((v2i64) dst73, 1);
3272  dst76_r = __msa_ilvr_h(dst22, dst106);
3273 
3274  dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3275  dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3276  dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3277  dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3278  dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
3279  dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
3280  dst6_r = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
3281  dst7_r = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
3282  SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
3283  SRA_4V(dst4_r, dst5_r, dst6_r, dst7_r, 6);
3284  PCKEV_H4_SH(dst1_r, dst0_r, dst3_r, dst2_r,
3285  dst5_r, dst4_r, dst7_r, dst6_r,
3286  tmp0, tmp1, tmp2, tmp3);
3287  SRARI_H4_SH(tmp0, tmp1, tmp2, tmp3, 6);
3288  SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
3289  out0 = PCKEV_XORI128_UB(tmp0, tmp1);
3290  out1 = PCKEV_XORI128_UB(tmp2, tmp3);
3291  ST4x8_UB(out0, out1, dst, dst_stride);
3292  dst += (8 * dst_stride);
3293 
3294  dst10_r = dst98_r;
3295  dst21_r = dst109_r;
3296  dst22 = (v8i16) __msa_splati_d((v2i64) dst106, 1);
3297  }
3298 }
3299 
3300 static void hevc_hv_uni_4t_4w_msa(uint8_t *src,
3301  int32_t src_stride,
3302  uint8_t *dst,
3303  int32_t dst_stride,
3304  const int8_t *filter_x,
3305  const int8_t *filter_y,
3306  int32_t height)
3307 {
3308  if (2 == height) {
3309  hevc_hv_uni_4t_4x2_msa(src, src_stride, dst, dst_stride,
3310  filter_x, filter_y);
3311  } else if (4 == height) {
3312  hevc_hv_uni_4t_4x4_msa(src, src_stride, dst, dst_stride,
3313  filter_x, filter_y);
3314  } else if (0 == (height % 8)) {
3315  hevc_hv_uni_4t_4multx8mult_msa(src, src_stride, dst, dst_stride,
3316  filter_x, filter_y, height);
3317  }
3318 }
3319 
3320 static void hevc_hv_uni_4t_6w_msa(uint8_t *src,
3321  int32_t src_stride,
3322  uint8_t *dst,
3323  int32_t dst_stride,
3324  const int8_t *filter_x,
3325  const int8_t *filter_y,
3326  int32_t height)
3327 {
3328  v16u8 out0, out1, out2;
3329  v16i8 src0, src1, src2, src3, src4, src5, src6;
3330  v16i8 src7, src8, src9, src10;
3331  v8i16 filt0, filt1;
3332  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3333  v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3334  v16i8 mask1;
3335  v8i16 filt_h0, filt_h1, filter_vec;
3336  v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6, dsth7, dsth8, dsth9;
3337  v8i16 dsth10, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5;
3338  v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
3339  v4i32 dst4_r, dst5_r, dst6_r, dst7_r;
3340  v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
3341  v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
3342  v8i16 dst54_r, dst76_r, dst98_r, dst65_r, dst87_r, dst109_r;
3343  v8i16 dst98_l, dst65_l, dst54_l, dst76_l, dst87_l, dst109_l;
3344  v8i16 dst1021_l, dst3243_l, dst5465_l, dst7687_l, dst98109_l;
3345 
3346  src -= (src_stride + 1);
3347 
3348  filter_vec = LD_SH(filter_x);
3349  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3350 
3351  filter_vec = LD_SH(filter_y);
3352  UNPCK_R_SB_SH(filter_vec, filter_vec);
3353 
3354  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3355 
3356  mask1 = mask0 + 2;
3357 
3358  LD_SB3(src, src_stride, src0, src1, src2);
3359  src += (3 * src_stride);
3360 
3361  XORI_B3_128_SB(src0, src1, src2);
3362 
3363  VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3364  VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3365  VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3366 
3367  dsth0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3368  dsth1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3369  dsth2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3370 
3371  ILVRL_H2_SH(dsth1, dsth0, dst10_r, dst10_l);
3372  ILVRL_H2_SH(dsth2, dsth1, dst21_r, dst21_l);
3373 
3374  LD_SB8(src, src_stride, src3, src4, src5, src6, src7, src8, src9, src10);
3375  XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
3376 
3377  VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3378  VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
3379  VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
3380  VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
3381 
3382  dsth3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3383  dsth4 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3384  dsth5 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3385  dsth6 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3386 
3387  VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec0, vec1);
3388  VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec2, vec3);
3389  VSHF_B2_SB(src9, src9, src9, src9, mask0, mask1, vec4, vec5);
3390  VSHF_B2_SB(src10, src10, src10, src10, mask0, mask1, vec6, vec7);
3391 
3392  dsth7 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3393  dsth8 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3394  dsth9 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3395  dsth10 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3396 
3397  ILVRL_H2_SH(dsth3, dsth2, dst32_r, dst32_l);
3398  ILVRL_H2_SH(dsth4, dsth3, dst43_r, dst43_l);
3399  ILVRL_H2_SH(dsth5, dsth4, dst54_r, dst54_l);
3400  ILVRL_H2_SH(dsth6, dsth5, dst65_r, dst65_l);
3401  ILVRL_H2_SH(dsth7, dsth6, dst76_r, dst76_l);
3402  ILVRL_H2_SH(dsth8, dsth7, dst87_r, dst87_l);
3403  ILVRL_H2_SH(dsth9, dsth8, dst98_r, dst98_l);
3404  ILVRL_H2_SH(dsth10, dsth9, dst109_r, dst109_l);
3405 
3406  PCKEV_D2_SH(dst21_l, dst10_l, dst43_l, dst32_l, dst1021_l, dst3243_l);
3407  PCKEV_D2_SH(dst65_l, dst54_l, dst87_l, dst76_l, dst5465_l, dst7687_l);
3408  dst98109_l = (v8i16) __msa_pckev_d((v2i64) dst109_l, (v2i64) dst98_l);
3409 
3410  dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3411  dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3412  dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3413  dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3414  dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
3415  dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
3416  dst6_r = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
3417  dst7_r = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
3418  dst0_l = HEVC_FILT_4TAP(dst1021_l, dst3243_l, filt_h0, filt_h1);
3419  dst1_l = HEVC_FILT_4TAP(dst3243_l, dst5465_l, filt_h0, filt_h1);
3420  dst2_l = HEVC_FILT_4TAP(dst5465_l, dst7687_l, filt_h0, filt_h1);
3421  dst3_l = HEVC_FILT_4TAP(dst7687_l, dst98109_l, filt_h0, filt_h1);
3422  SRA_4V(dst0_r, dst1_r, dst2_r, dst3_r, 6);
3423  SRA_4V(dst4_r, dst5_r, dst6_r, dst7_r, 6);
3424  SRA_4V(dst0_l, dst1_l, dst2_l, dst3_l, 6);
3425  PCKEV_H2_SH(dst1_r, dst0_r, dst3_r, dst2_r, tmp0, tmp1);
3426  PCKEV_H2_SH(dst5_r, dst4_r, dst7_r, dst6_r, tmp2, tmp3);
3427  PCKEV_H2_SH(dst1_l, dst0_l, dst3_l, dst2_l, tmp4, tmp5);
3428  SRARI_H4_SH(tmp0, tmp1, tmp2, tmp3, 6);
3429  SRARI_H2_SH(tmp4, tmp5, 6);
3430  SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3,7);
3431  SAT_SH2_SH(tmp4, tmp5,7);
3432  out0 = PCKEV_XORI128_UB(tmp0, tmp1);
3433  out1 = PCKEV_XORI128_UB(tmp2, tmp3);
3434  out2 = PCKEV_XORI128_UB(tmp4, tmp5);
3435  ST4x8_UB(out0, out1, dst, dst_stride);
3436  ST2x4_UB(out2, 0, dst + 4, dst_stride);
3437  dst += 4 * dst_stride;
3438  ST2x4_UB(out2, 4, dst + 4, dst_stride);
3439 }
3440 
3441 static void hevc_hv_uni_4t_8x2_msa(uint8_t *src,
3442  int32_t src_stride,
3443  uint8_t *dst,
3444  int32_t dst_stride,
3445  const int8_t *filter_x,
3446  const int8_t *filter_y)
3447 {
3448  v16u8 out;
3449  v16i8 src0, src1, src2, src3, src4;
3450  v8i16 filt0, filt1;
3451  v8i16 filt_h0, filt_h1, filter_vec;
3452  v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3453  v16i8 mask1;
3454  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
3455  v8i16 dst0, dst1, dst2, dst3, dst4;
3456  v4i32 dst0_r, dst0_l, dst1_r, dst1_l;
3457  v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
3458  v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
3459  v8i16 out0_r, out1_r;
3460 
3461  src -= (src_stride + 1);
3462 
3463  filter_vec = LD_SH(filter_x);
3464  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3465 
3466  filter_vec = LD_SH(filter_y);
3467  UNPCK_R_SB_SH(filter_vec, filter_vec);
3468 
3469  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3470 
3471  mask1 = mask0 + 2;
3472 
3473  LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
3474  XORI_B5_128_SB(src0, src1, src2, src3, src4);
3475 
3476  VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3477  VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3478  VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3479  VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3480  VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
3481 
3482  dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3483  dst1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3484  dst2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3485  dst3 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3486  dst4 = HEVC_FILT_4TAP_SH(vec8, vec9, filt0, filt1);
3487  ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
3488  ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
3489  ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
3490  ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
3491  dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3492  dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
3493  dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3494  dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
3495  SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
3496  PCKEV_H2_SH(dst0_l, dst0_r, dst1_l, dst1_r, out0_r, out1_r);
3497  SRARI_H2_SH(out0_r, out1_r, 6);
3498  SAT_SH2_SH(out0_r, out1_r, 7);
3499  out = PCKEV_XORI128_UB(out0_r, out1_r);
3500  ST8x2_UB(out, dst, dst_stride);
3501 }
3502 
3503 static void hevc_hv_uni_4t_8multx4_msa(uint8_t *src,
3504  int32_t src_stride,
3505  uint8_t *dst,
3506  int32_t dst_stride,
3507  const int8_t *filter_x,
3508  const int8_t *filter_y,
3509  int32_t width8mult)
3510 {
3511  uint32_t cnt;
3512  v16u8 out0, out1;
3513  v16i8 src0, src1, src2, src3, src4, src5, src6, mask0, mask1;
3514  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3515  v8i16 filt0, filt1, filt_h0, filt_h1, filter_vec;
3516  v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, tmp0, tmp1, tmp2, tmp3;
3517  v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
3518  v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
3519  v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
3520 
3521  src -= (src_stride + 1);
3522 
3523  filter_vec = LD_SH(filter_x);
3524  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3525 
3526  filter_vec = LD_SH(filter_y);
3527  UNPCK_R_SB_SH(filter_vec, filter_vec);
3528 
3529  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3530 
3531  mask0 = LD_SB(ff_hevc_mask_arr);
3532  mask1 = mask0 + 2;
3533 
3534  for (cnt = width8mult; cnt--;) {
3535  LD_SB7(src, src_stride, src0, src1, src2, src3, src4, src5, src6);
3536  src += 8;
3537  XORI_B7_128_SB(src0, src1, src2, src3, src4, src5, src6);
3538 
3539  VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3540  VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3541  VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3542 
3543  dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3544  dst1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3545  dst2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3546 
3547  ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
3548  ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
3549 
3550  VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3551  VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
3552  VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
3553  VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
3554 
3555  dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3556  dst4 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3557  dst5 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3558  dst6 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3559 
3560  ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
3561  ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
3562  ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
3563  ILVRL_H2_SH(dst6, dst5, dst65_r, dst65_l);
3564 
3565  dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3566  dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
3567  dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3568  dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
3569  dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3570  dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
3571  dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3572  dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
3573 
3574  SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
3575  SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
3576 
3577  PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
3578  dst3_r, tmp0, tmp1, tmp2, tmp3);
3579  SRARI_H4_SH(tmp0, tmp1, tmp2, tmp3, 6);
3580  SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
3581  out0 = PCKEV_XORI128_UB(tmp0, tmp1);
3582  out1 = PCKEV_XORI128_UB(tmp2, tmp3);
3583  ST8x4_UB(out0, out1, dst, dst_stride);
3584  dst += 8;
3585  }
3586 }
3587 
3588 static void hevc_hv_uni_4t_8x6_msa(uint8_t *src,
3589  int32_t src_stride,
3590  uint8_t *dst,
3591  int32_t dst_stride,
3592  const int8_t *filter_x,
3593  const int8_t *filter_y)
3594 {
3595  v16u8 out0, out1, out2;
3596  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8;
3597  v8i16 filt0, filt1;
3598  v8i16 filt_h0, filt_h1, filter_vec;
3599  v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3600  v16i8 mask1;
3601  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7, vec8, vec9;
3602  v16i8 vec10, vec11, vec12, vec13, vec14, vec15, vec16, vec17;
3603  v8i16 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7, dst8;
3604  v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
3605  v4i32 dst4_r, dst4_l, dst5_r, dst5_l;
3606  v8i16 dst10_r, dst32_r, dst10_l, dst32_l;
3607  v8i16 dst21_r, dst43_r, dst21_l, dst43_l;
3608  v8i16 dst54_r, dst54_l, dst65_r, dst65_l;
3609  v8i16 dst76_r, dst76_l, dst87_r, dst87_l;
3610  v8i16 out0_r, out1_r, out2_r, out3_r, out4_r, out5_r;
3611 
3612  src -= (src_stride + 1);
3613 
3614  filter_vec = LD_SH(filter_x);
3615  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3616 
3617  filter_vec = LD_SH(filter_y);
3618  UNPCK_R_SB_SH(filter_vec, filter_vec);
3619 
3620  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3621 
3622  mask1 = mask0 + 2;
3623 
3624  LD_SB5(src, src_stride, src0, src1, src2, src3, src4);
3625  src += (5 * src_stride);
3626  LD_SB4(src, src_stride, src5, src6, src7, src8);
3627 
3628  XORI_B5_128_SB(src0, src1, src2, src3, src4);
3629  XORI_B4_128_SB(src5, src6, src7, src8);
3630 
3631  VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3632  VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3633  VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3634  VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec6, vec7);
3635  VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec8, vec9);
3636  VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec10, vec11);
3637  VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec12, vec13);
3638  VSHF_B2_SB(src7, src7, src7, src7, mask0, mask1, vec14, vec15);
3639  VSHF_B2_SB(src8, src8, src8, src8, mask0, mask1, vec16, vec17);
3640 
3641  dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3642  dst1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3643  dst2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3644  dst3 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3645  dst4 = HEVC_FILT_4TAP_SH(vec8, vec9, filt0, filt1);
3646  dst5 = HEVC_FILT_4TAP_SH(vec10, vec11, filt0, filt1);
3647  dst6 = HEVC_FILT_4TAP_SH(vec12, vec13, filt0, filt1);
3648  dst7 = HEVC_FILT_4TAP_SH(vec14, vec15, filt0, filt1);
3649  dst8 = HEVC_FILT_4TAP_SH(vec16, vec17, filt0, filt1);
3650 
3651  ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
3652  ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
3653  ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
3654  ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
3655  ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
3656  ILVRL_H2_SH(dst6, dst5, dst65_r, dst65_l);
3657  ILVRL_H2_SH(dst7, dst6, dst76_r, dst76_l);
3658  ILVRL_H2_SH(dst8, dst7, dst87_r, dst87_l);
3659 
3660  dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3661  dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
3662  dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3663  dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
3664  dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3665  dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
3666  dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3667  dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
3668  dst4_r = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
3669  dst4_l = HEVC_FILT_4TAP(dst54_l, dst76_l, filt_h0, filt_h1);
3670  dst5_r = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
3671  dst5_l = HEVC_FILT_4TAP(dst65_l, dst87_l, filt_h0, filt_h1);
3672 
3673  SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
3674  SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
3675  SRA_4V(dst4_r, dst4_l, dst5_r, dst5_l, 6);
3676  PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r,
3677  dst2_l, dst2_r, dst3_l, dst3_r, out0_r, out1_r, out2_r, out3_r);
3678  PCKEV_H2_SH(dst4_l, dst4_r, dst5_l, dst5_r, out4_r, out5_r);
3679  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
3680  SRARI_H2_SH(out4_r, out5_r, 6);
3681  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
3682  SAT_SH2_SH(out4_r, out5_r, 7);
3683  out0 = PCKEV_XORI128_UB(out0_r, out1_r);
3684  out1 = PCKEV_XORI128_UB(out2_r, out3_r);
3685  out2 = PCKEV_XORI128_UB(out4_r, out5_r);
3686 
3687  ST8x4_UB(out0, out1, dst, dst_stride);
3688  dst += (4 * dst_stride);
3689  ST8x2_UB(out2, dst, dst_stride);
3690 }
3691 
3692 static void hevc_hv_uni_4t_8multx4mult_msa(uint8_t *src,
3693  int32_t src_stride,
3694  uint8_t *dst,
3695  int32_t dst_stride,
3696  const int8_t *filter_x,
3697  const int8_t *filter_y,
3698  int32_t height,
3699  int32_t width8mult)
3700 {
3701  uint32_t loop_cnt, cnt;
3702  uint8_t *src_tmp;
3703  uint8_t *dst_tmp;
3704  v16u8 out0, out1;
3705  v16i8 src0, src1, src2, src3, src4, src5, src6;
3706  v8i16 filt0, filt1;
3707  v8i16 filt_h0, filt_h1, filter_vec;
3708  v16i8 mask0 = LD_SB(ff_hevc_mask_arr);
3709  v16i8 mask1;
3710  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3711  v8i16 dst0, dst1, dst2, dst3, dst4, dst5;
3712  v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
3713  v8i16 dst10_r, dst32_r, dst21_r, dst43_r;
3714  v8i16 dst10_l, dst32_l, dst21_l, dst43_l;
3715  v8i16 dst54_r, dst54_l, dst65_r, dst65_l, dst6;
3716  v8i16 out0_r, out1_r, out2_r, out3_r;
3717 
3718  src -= (src_stride + 1);
3719 
3720  filter_vec = LD_SH(filter_x);
3721  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3722 
3723  filter_vec = LD_SH(filter_y);
3724  UNPCK_R_SB_SH(filter_vec, filter_vec);
3725 
3726  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3727 
3728  mask1 = mask0 + 2;
3729 
3730  for (cnt = width8mult; cnt--;) {
3731  src_tmp = src;
3732  dst_tmp = dst;
3733 
3734  LD_SB3(src_tmp, src_stride, src0, src1, src2);
3735  src_tmp += (3 * src_stride);
3736 
3737  XORI_B3_128_SB(src0, src1, src2);
3738 
3739  VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3740  VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3741  VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3742 
3743  dst0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3744  dst1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3745  dst2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3746 
3747  ILVRL_H2_SH(dst1, dst0, dst10_r, dst10_l);
3748  ILVRL_H2_SH(dst2, dst1, dst21_r, dst21_l);
3749 
3750  for (loop_cnt = (height >> 2); loop_cnt--;) {
3751  LD_SB4(src_tmp, src_stride, src3, src4, src5, src6);
3752  src_tmp += (4 * src_stride);
3753 
3754  XORI_B4_128_SB(src3, src4, src5, src6);
3755 
3756  VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3757  VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
3758  VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
3759  VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
3760 
3761  dst3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3762  dst4 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3763  dst5 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3764  dst6 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3765 
3766  ILVRL_H2_SH(dst3, dst2, dst32_r, dst32_l);
3767  ILVRL_H2_SH(dst4, dst3, dst43_r, dst43_l);
3768  ILVRL_H2_SH(dst5, dst4, dst54_r, dst54_l);
3769  ILVRL_H2_SH(dst6, dst5, dst65_r, dst65_l);
3770 
3771  dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3772  dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
3773  dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3774  dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
3775  dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3776  dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
3777  dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3778  dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
3779 
3780  SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
3781  SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
3782 
3783  PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r,
3784  dst2_l, dst2_r, dst3_l, dst3_r,
3785  out0_r, out1_r, out2_r, out3_r);
3786 
3787  SRARI_H4_SH(out0_r, out1_r, out2_r, out3_r, 6);
3788  SAT_SH4_SH(out0_r, out1_r, out2_r, out3_r, 7);
3789  out0 = PCKEV_XORI128_UB(out0_r, out1_r);
3790  out1 = PCKEV_XORI128_UB(out2_r, out3_r);
3791  ST8x4_UB(out0, out1, dst_tmp, dst_stride);
3792  dst_tmp += (4 * dst_stride);
3793 
3794  dst10_r = dst54_r;
3795  dst10_l = dst54_l;
3796  dst21_r = dst65_r;
3797  dst21_l = dst65_l;
3798  dst2 = dst6;
3799  }
3800 
3801  src += 8;
3802  dst += 8;
3803  }
3804 }
3805 
3806 static void hevc_hv_uni_4t_8w_msa(uint8_t *src,
3807  int32_t src_stride,
3808  uint8_t *dst,
3809  int32_t dst_stride,
3810  const int8_t *filter_x,
3811  const int8_t *filter_y,
3812  int32_t height)
3813 {
3814  if (2 == height) {
3815  hevc_hv_uni_4t_8x2_msa(src, src_stride, dst, dst_stride,
3816  filter_x, filter_y);
3817  } else if (4 == height) {
3818  hevc_hv_uni_4t_8multx4_msa(src, src_stride, dst, dst_stride,
3819  filter_x, filter_y, 1);
3820  } else if (6 == height) {
3821  hevc_hv_uni_4t_8x6_msa(src, src_stride, dst, dst_stride,
3822  filter_x, filter_y);
3823  } else if (0 == (height % 4)) {
3824  hevc_hv_uni_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
3825  filter_x, filter_y, height, 1);
3826  }
3827 }
3828 
3829 static void hevc_hv_uni_4t_12w_msa(uint8_t *src,
3830  int32_t src_stride,
3831  uint8_t *dst,
3832  int32_t dst_stride,
3833  const int8_t *filter_x,
3834  const int8_t *filter_y,
3835  int32_t height)
3836 {
3837  uint32_t loop_cnt;
3838  uint8_t *src_tmp, *dst_tmp;
3839  v16u8 out0, out1;
3840  v16i8 src0, src1, src2, src3, src4, src5, src6, src7, src8, src9, src10;
3841  v16i8 vec0, vec1, vec2, vec3, vec4, vec5, vec6, vec7;
3842  v16i8 mask0, mask1, mask2, mask3;
3843  v8i16 filt0, filt1, filt_h0, filt_h1, filter_vec, tmp0, tmp1, tmp2, tmp3;
3844  v8i16 dsth0, dsth1, dsth2, dsth3, dsth4, dsth5, dsth6;
3845  v8i16 dst10, dst21, dst22, dst73, dst84, dst95, dst106;
3846  v8i16 dst76_r, dst98_r, dst87_r, dst109_r;
3847  v8i16 dst10_r, dst32_r, dst54_r, dst21_r, dst43_r, dst65_r;
3848  v8i16 dst10_l, dst32_l, dst54_l, dst21_l, dst43_l, dst65_l;
3849  v4i32 dst0_r, dst0_l, dst1_r, dst1_l, dst2_r, dst2_l, dst3_r, dst3_l;
3850  v4i32 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
3851 
3852  src -= (src_stride + 1);
3853 
3854  filter_vec = LD_SH(filter_x);
3855  SPLATI_H2_SH(filter_vec, 0, 1, filt0, filt1);
3856 
3857  filter_vec = LD_SH(filter_y);
3858  UNPCK_R_SB_SH(filter_vec, filter_vec);
3859 
3860  SPLATI_W2_SH(filter_vec, 0, filt_h0, filt_h1);
3861 
3862  mask0 = LD_SB(ff_hevc_mask_arr);
3863  mask1 = mask0 + 2;
3864 
3865  src_tmp = src;
3866  dst_tmp = dst;
3867 
3868  LD_SB3(src_tmp, src_stride, src0, src1, src2);
3869  src_tmp += (3 * src_stride);
3870 
3871  XORI_B3_128_SB(src0, src1, src2);
3872 
3873  VSHF_B2_SB(src0, src0, src0, src0, mask0, mask1, vec0, vec1);
3874  VSHF_B2_SB(src1, src1, src1, src1, mask0, mask1, vec2, vec3);
3875  VSHF_B2_SB(src2, src2, src2, src2, mask0, mask1, vec4, vec5);
3876 
3877  dsth0 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3878  dsth1 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3879  dsth2 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3880 
3881  ILVRL_H2_SH(dsth1, dsth0, dst10_r, dst10_l);
3882  ILVRL_H2_SH(dsth2, dsth1, dst21_r, dst21_l);
3883 
3884  for (loop_cnt = 4; loop_cnt--;) {
3885  LD_SB4(src_tmp, src_stride, src3, src4, src5, src6);
3886  src_tmp += (4 * src_stride);
3887  XORI_B4_128_SB(src3, src4, src5, src6);
3888 
3889  VSHF_B2_SB(src3, src3, src3, src3, mask0, mask1, vec0, vec1);
3890  VSHF_B2_SB(src4, src4, src4, src4, mask0, mask1, vec2, vec3);
3891  VSHF_B2_SB(src5, src5, src5, src5, mask0, mask1, vec4, vec5);
3892  VSHF_B2_SB(src6, src6, src6, src6, mask0, mask1, vec6, vec7);
3893 
3894  dsth3 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3895  dsth4 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3896  dsth5 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3897  dsth6 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3898 
3899  ILVRL_H2_SH(dsth3, dsth2, dst32_r, dst32_l);
3900  ILVRL_H2_SH(dsth4, dsth3, dst43_r, dst43_l);
3901  ILVRL_H2_SH(dsth5, dsth4, dst54_r, dst54_l);
3902  ILVRL_H2_SH(dsth6, dsth5, dst65_r, dst65_l);
3903 
3904  dst0_r = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3905  dst0_l = HEVC_FILT_4TAP(dst10_l, dst32_l, filt_h0, filt_h1);
3906  dst1_r = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3907  dst1_l = HEVC_FILT_4TAP(dst21_l, dst43_l, filt_h0, filt_h1);
3908  dst2_r = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3909  dst2_l = HEVC_FILT_4TAP(dst32_l, dst54_l, filt_h0, filt_h1);
3910  dst3_r = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3911  dst3_l = HEVC_FILT_4TAP(dst43_l, dst65_l, filt_h0, filt_h1);
3912 
3913  SRA_4V(dst0_r, dst0_l, dst1_r, dst1_l, 6);
3914  SRA_4V(dst2_r, dst2_l, dst3_r, dst3_l, 6);
3915 
3916  PCKEV_H4_SH(dst0_l, dst0_r, dst1_l, dst1_r, dst2_l, dst2_r, dst3_l,
3917  dst3_r, tmp0, tmp1, tmp2, tmp3);
3918  SRARI_H4_SH(tmp0, tmp1, tmp2, tmp3, 6);
3919  SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
3920  out0 = PCKEV_XORI128_UB(tmp0, tmp1);
3921  out1 = PCKEV_XORI128_UB(tmp2, tmp3);
3922  ST8x4_UB(out0, out1, dst_tmp, dst_stride);
3923  dst_tmp += (4 * dst_stride);
3924 
3925  dst10_r = dst54_r;
3926  dst10_l = dst54_l;
3927  dst21_r = dst65_r;
3928  dst21_l = dst65_l;
3929  dsth2 = dsth6;
3930  }
3931 
3932  src += 8;
3933  dst += 8;
3934 
3935  mask2 = LD_SB(ff_hevc_mask_arr + 16);
3936  mask3 = mask2 + 2;
3937 
3938  LD_SB3(src, src_stride, src0, src1, src2);
3939  src += (3 * src_stride);
3940  XORI_B3_128_SB(src0, src1, src2);
3941  VSHF_B2_SB(src0, src1, src0, src1, mask2, mask3, vec0, vec1);
3942  VSHF_B2_SB(src1, src2, src1, src2, mask2, mask3, vec2, vec3);
3943 
3944  dst10 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3945  dst21 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3946 
3947  ILVRL_H2_SH(dst21, dst10, dst10_r, dst21_r);
3948  dst22 = (v8i16) __msa_splati_d((v2i64) dst21, 1);
3949 
3950  for (loop_cnt = 2; loop_cnt--;) {
3951  LD_SB8(src, src_stride,
3952  src3, src4, src5, src6, src7, src8, src9, src10);
3953  src += (8 * src_stride);
3954  XORI_B8_128_SB(src3, src4, src5, src6, src7, src8, src9, src10);
3955  VSHF_B2_SB(src3, src7, src3, src7, mask2, mask3, vec0, vec1);
3956  VSHF_B2_SB(src4, src8, src4, src8, mask2, mask3, vec2, vec3);
3957  VSHF_B2_SB(src5, src9, src5, src9, mask2, mask3, vec4, vec5);
3958  VSHF_B2_SB(src6, src10, src6, src10, mask2, mask3, vec6, vec7);
3959 
3960  dst73 = HEVC_FILT_4TAP_SH(vec0, vec1, filt0, filt1);
3961  dst84 = HEVC_FILT_4TAP_SH(vec2, vec3, filt0, filt1);
3962  dst95 = HEVC_FILT_4TAP_SH(vec4, vec5, filt0, filt1);
3963  dst106 = HEVC_FILT_4TAP_SH(vec6, vec7, filt0, filt1);
3964 
3965  dst32_r = __msa_ilvr_h(dst73, dst22);
3966  ILVRL_H2_SH(dst84, dst73, dst43_r, dst87_r);
3967  ILVRL_H2_SH(dst95, dst84, dst54_r, dst98_r);
3968  ILVRL_H2_SH(dst106, dst95, dst65_r, dst109_r);
3969  dst22 = (v8i16) __msa_splati_d((v2i64) dst73, 1);
3970  dst76_r = __msa_ilvr_h(dst22, dst106);
3971 
3972  dst0 = HEVC_FILT_4TAP(dst10_r, dst32_r, filt_h0, filt_h1);
3973  dst1 = HEVC_FILT_4TAP(dst21_r, dst43_r, filt_h0, filt_h1);
3974  dst2 = HEVC_FILT_4TAP(dst32_r, dst54_r, filt_h0, filt_h1);
3975  dst3 = HEVC_FILT_4TAP(dst43_r, dst65_r, filt_h0, filt_h1);
3976  dst4 = HEVC_FILT_4TAP(dst54_r, dst76_r, filt_h0, filt_h1);
3977  dst5 = HEVC_FILT_4TAP(dst65_r, dst87_r, filt_h0, filt_h1);
3978  dst6 = HEVC_FILT_4TAP(dst76_r, dst98_r, filt_h0, filt_h1);
3979  dst7 = HEVC_FILT_4TAP(dst87_r, dst109_r, filt_h0, filt_h1);
3980  SRA_4V(dst0, dst1, dst2, dst3, 6);
3981  SRA_4V(dst4, dst5, dst6, dst7, 6);
3982  PCKEV_H4_SH(dst1, dst0, dst3, dst2, dst5, dst4, dst7, dst6,
3983  tmp0, tmp1, tmp2, tmp3);
3984  SRARI_H4_SH(tmp0, tmp1, tmp2, tmp3, 6);
3985  SAT_SH4_SH(tmp0, tmp1, tmp2, tmp3, 7);
3986  out0 = PCKEV_XORI128_UB(tmp0, tmp1);
3987  out1 = PCKEV_XORI128_UB(tmp2, tmp3);
3988  ST4x8_UB(out0, out1, dst, dst_stride);
3989  dst += (8 * dst_stride);
3990 
3991  dst10_r = dst98_r;
3992  dst21_r = dst109_r;
3993  dst22 = (v8i16) __msa_splati_d((v2i64) dst106, 1);
3994  }
3995 }
3996 
3997 static void hevc_hv_uni_4t_16w_msa(uint8_t *src,
3998  int32_t src_stride,
3999  uint8_t *dst,
4000  int32_t dst_stride,
4001  const int8_t *filter_x,
4002  const int8_t *filter_y,
4003  int32_t height)
4004 {
4005  if (4 == height) {
4006  hevc_hv_uni_4t_8multx4_msa(src, src_stride, dst, dst_stride, filter_x,
4007  filter_y, 2);
4008  } else {
4009  hevc_hv_uni_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
4010  filter_x, filter_y, height, 2);
4011  }
4012 }
4013 
4014 static void hevc_hv_uni_4t_24w_msa(uint8_t *src,
4015  int32_t src_stride,
4016  uint8_t *dst,
4017  int32_t dst_stride,
4018  const int8_t *filter_x,
4019  const int8_t *filter_y,
4020  int32_t height)
4021 {
4022  hevc_hv_uni_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
4023  filter_x, filter_y, height, 3);
4024 }
4025 
4026 static void hevc_hv_uni_4t_32w_msa(uint8_t *src,
4027  int32_t src_stride,
4028  uint8_t *dst,
4029  int32_t dst_stride,
4030  const int8_t *filter_x,
4031  const int8_t *filter_y,
4032  int32_t height)
4033 {
4034  hevc_hv_uni_4t_8multx4mult_msa(src, src_stride, dst, dst_stride,
4035  filter_x, filter_y, height, 4);
4036 }
4037 
4038 #define UNI_MC_COPY(WIDTH) \
4039 void ff_hevc_put_hevc_uni_pel_pixels##WIDTH##_8_msa(uint8_t *dst, \
4040  ptrdiff_t dst_stride, \
4041  uint8_t *src, \
4042  ptrdiff_t src_stride, \
4043  int height, \
4044  intptr_t mx, \
4045  intptr_t my, \
4046  int width) \
4047 { \
4048  copy_width##WIDTH##_msa(src, src_stride, dst, dst_stride, height); \
4049 }
4050 
4051 UNI_MC_COPY(8);
4052 UNI_MC_COPY(12);
4053 UNI_MC_COPY(16);
4054 UNI_MC_COPY(24);
4055 UNI_MC_COPY(32);
4056 UNI_MC_COPY(48);
4057 UNI_MC_COPY(64);
4058 
4059 #undef UNI_MC_COPY
4060 
4061 #define UNI_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR) \
4062 void ff_hevc_put_hevc_uni_##PEL##_##DIR##WIDTH##_8_msa(uint8_t *dst, \
4063  ptrdiff_t dst_stride, \
4064  uint8_t *src, \
4065  ptrdiff_t src_stride, \
4066  int height, \
4067  intptr_t mx, \
4068  intptr_t my, \
4069  int width) \
4070 { \
4071  const int8_t *filter = ff_hevc_##PEL##_filters[FILT_DIR - 1]; \
4072  \
4073  common_##DIR1##_##TAP##t_##WIDTH##w_msa(src, src_stride, dst, dst_stride, \
4074  filter, height); \
4075 }
4076 
4077 UNI_MC(qpel, h, 4, 8, hz, mx);
4078 UNI_MC(qpel, h, 8, 8, hz, mx);
4079 UNI_MC(qpel, h, 12, 8, hz, mx);
4080 UNI_MC(qpel, h, 16, 8, hz, mx);
4081 UNI_MC(qpel, h, 24, 8, hz, mx);
4082 UNI_MC(qpel, h, 32, 8, hz, mx);
4083 UNI_MC(qpel, h, 48, 8, hz, mx);
4084 UNI_MC(qpel, h, 64, 8, hz, mx);
4085 
4086 UNI_MC(qpel, v, 4, 8, vt, my);
4087 UNI_MC(qpel, v, 8, 8, vt, my);
4088 UNI_MC(qpel, v, 12, 8, vt, my);
4089 UNI_MC(qpel, v, 16, 8, vt, my);
4090 UNI_MC(qpel, v, 24, 8, vt, my);
4091 UNI_MC(qpel, v, 32, 8, vt, my);
4092 UNI_MC(qpel, v, 48, 8, vt, my);
4093 UNI_MC(qpel, v, 64, 8, vt, my);
4094 
4095 UNI_MC(epel, h, 4, 4, hz, mx);
4096 UNI_MC(epel, h, 6, 4, hz, mx);
4097 UNI_MC(epel, h, 8, 4, hz, mx);
4098 UNI_MC(epel, h, 12, 4, hz, mx);
4099 UNI_MC(epel, h, 16, 4, hz, mx);
4100 UNI_MC(epel, h, 24, 4, hz, mx);
4101 UNI_MC(epel, h, 32, 4, hz, mx);
4102 
4103 UNI_MC(epel, v, 4, 4, vt, my);
4104 UNI_MC(epel, v, 6, 4, vt, my);
4105 UNI_MC(epel, v, 8, 4, vt, my);
4106 UNI_MC(epel, v, 12, 4, vt, my);
4107 UNI_MC(epel, v, 16, 4, vt, my);
4108 UNI_MC(epel, v, 24, 4, vt, my);
4109 UNI_MC(epel, v, 32, 4, vt, my);
4110 
4111 #undef UNI_MC
4112 
4113 #define UNI_MC_HV(PEL, WIDTH, TAP) \
4114 void ff_hevc_put_hevc_uni_##PEL##_hv##WIDTH##_8_msa(uint8_t *dst, \
4115  ptrdiff_t dst_stride, \
4116  uint8_t *src, \
4117  ptrdiff_t src_stride, \
4118  int height, \
4119  intptr_t mx, \
4120  intptr_t my, \
4121  int width) \
4122 { \
4123  const int8_t *filter_x = ff_hevc_##PEL##_filters[mx - 1]; \
4124  const int8_t *filter_y = ff_hevc_##PEL##_filters[my - 1]; \
4125  \
4126  hevc_hv_uni_##TAP##t_##WIDTH##w_msa(src, src_stride, dst, dst_stride, \
4127  filter_x, filter_y, height); \
4128 }
4129 
4130 UNI_MC_HV(qpel, 4, 8);
4131 UNI_MC_HV(qpel, 8, 8);
4132 UNI_MC_HV(qpel, 12, 8);
4133 UNI_MC_HV(qpel, 16, 8);
4134 UNI_MC_HV(qpel, 24, 8);
4135 UNI_MC_HV(qpel, 32, 8);
4136 UNI_MC_HV(qpel, 48, 8);
4137 UNI_MC_HV(qpel, 64, 8);
4138 
4139 UNI_MC_HV(epel, 4, 4);
4140 UNI_MC_HV(epel, 6, 4);
4141 UNI_MC_HV(epel, 8, 4);
4142 UNI_MC_HV(epel, 12, 4);
4143 UNI_MC_HV(epel, 16, 4);
4144 UNI_MC_HV(epel, 24, 4);
4145 UNI_MC_HV(epel, 32, 4);
4146 
4147 #undef UNI_MC_HV
VSHF_B4_SB
#define VSHF_B4_SB(...)
Definition: generic_macros_msa.h:778
XORI_B5_128_SB
#define XORI_B5_128_SB(...)
Definition: generic_macros_msa.h:1977
XORI_B8_128_SB
#define XORI_B8_128_SB(...)
Definition: generic_macros_msa.h:1998
hevc_hv_uni_4t_12w_msa
static void hevc_hv_uni_4t_12w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:3829
common_hz_4t_8x4mult_msa
static void common_hz_4t_8x4mult_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2156
h
h
Definition: vp9dsp_template.c:2038
hevc_hv_uni_8t_16w_msa
static void hevc_hv_uni_8t_16w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:1863
hevc_hv_uni_4t_4x4_msa
static void hevc_hv_uni_4t_4x4_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition: hevc_mc_uni_msa.c:3143
SRARI_W4_SW
#define SRARI_W4_SW(...)
Definition: generic_macros_msa.h:2210
common_vt_4t_8x6_msa
static void common_vt_4t_8x6_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:2651
hevc_hv_uni_4t_6w_msa
static void hevc_hv_uni_4t_6w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:3320
ILVR_H4_SH
#define ILVR_H4_SH(...)
Definition: generic_macros_msa.h:1526
PCKEV_B2_SH
#define PCKEV_B2_SH(...)
Definition: generic_macros_msa.h:1839
copy_width24_msa
static void copy_width24_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height)
Definition: hevc_mc_uni_msa.c:196
common_vt_4t_4x2_msa
static void common_vt_4t_4x2_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:2471
XORI_B2_128_SB
#define XORI_B2_128_SB(...)
Definition: generic_macros_msa.h:1953
hevc_hv_uni_4t_8x2_msa
static void hevc_hv_uni_4t_8x2_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition: hevc_mc_uni_msa.c:3441
PCKEV_XORI128_UB
#define PCKEV_XORI128_UB(in0, in1)
Definition: generic_macros_msa.h:2872
hevc_hv_uni_8t_8multx2mult_msa
static void hevc_hv_uni_8t_8multx2mult_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t width)
Definition: hevc_mc_uni_msa.c:1476
SD
#define SD
Definition: ccaption_dec.c:819
LD_SB
#define LD_SB(...)
Definition: generic_macros_msa.h:32
XORI_B3_128_SB
#define XORI_B3_128_SB(...)
Definition: generic_macros_msa.h:1961
hevc_hv_uni_4t_32w_msa
static void hevc_hv_uni_4t_32w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:4026
common_hz_8t_16w_msa
static void common_hz_8t_16w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:555
UNPCK_R_SB_SH
#define UNPCK_R_SB_SH(in, out)
Definition: generic_macros_msa.h:2274
LD_UB4
#define LD_UB4(...)
Definition: generic_macros_msa.h:293
DPADD_SB4_SH
#define DPADD_SB4_SH(...)
Definition: generic_macros_msa.h:939
ILVR_B2_SB
#define ILVR_B2_SB(...)
Definition: generic_macros_msa.h:1457
SPLATI_H2_SH
#define SPLATI_H2_SH(...)
Definition: generic_macros_msa.h:1774
hevc_hv_uni_8t_48w_msa
static void hevc_hv_uni_8t_48w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:1899
src
#define src
Definition: vp8dsp.c:254
copy_width48_msa
static void copy_width48_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height)
Definition: hevc_mc_uni_msa.c:236
ILVL_H2_SH
#define ILVL_H2_SH(...)
Definition: generic_macros_msa.h:1411
common_hz_8t_32w_msa
static void common_hz_8t_32w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:683
common_hz_8t_24w_msa
static void common_hz_8t_24w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:613
LD_SB2
#define LD_SB2(...)
Definition: generic_macros_msa.h:275
ST4x4_UB
#define ST4x4_UB(in0, in1, idx0, idx1, idx2, idx3, pdst, stride)
Definition: generic_macros_msa.h:463
ILVL_H4_SH
#define ILVL_H4_SH(...)
Definition: generic_macros_msa.h:1420
common_hz_4t_4x2_msa
static void common_hz_4t_4x2_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:1923
common_hz_8t_4w_msa
static void common_hz_8t_4w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:411
common_hz_4t_32w_msa
static void common_hz_4t_32w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2403
XORI_B2_128_SH
#define XORI_B2_128_SH(...)
Definition: generic_macros_msa.h:1954
VSHF_B2_SB
#define VSHF_B2_SB(...)
Definition: generic_macros_msa.h:760
SRA_4V
#define SRA_4V(in0, in1, in2, in3, shift)
Definition: generic_macros_msa.h:2057
XORI_B4_128_UB
#define XORI_B4_128_UB(...)
Definition: generic_macros_msa.h:1968
HORIZ_4TAP_8WID_4VECS_FILT
#define HORIZ_4TAP_8WID_4VECS_FILT(src0, src1, src2, src3,mask0, mask1, filt0, filt1,out0, out1, out2, out3)
Definition: hevc_mc_uni_msa.c:88
ILVR_D2_SB
#define ILVR_D2_SB(...)
Definition: generic_macros_msa.h:1562
uint8_t
uint8_t
Definition: audio_convert.c:194
LD4
#define LD4(psrc, stride, out0, out1, out2, out3)
Definition: generic_macros_msa.h:225
SPLATI_W2_SH
#define SPLATI_W2_SH(...)
Definition: generic_macros_msa.h:1810
SRARI_H4_SH
#define SRARI_H4_SH(...)
Definition: generic_macros_msa.h:2185
SPLATI_H4_SH
#define SPLATI_H4_SH(...)
Definition: generic_macros_msa.h:1792
hevc_hv_uni_4t_8w_msa
static void hevc_hv_uni_4t_8w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:3806
ILVL_B2_SB
#define ILVL_B2_SB(...)
Definition: generic_macros_msa.h:1382
height
#define height
LD_SH
#define LD_SH(...)
Definition: generic_macros_msa.h:34
hevc_hv_uni_8t_32w_msa
static void hevc_hv_uni_8t_32w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:1887
ILVRL_H2_SH
#define ILVRL_H2_SH(...)
Definition: generic_macros_msa.h:1626
ILVR_D3_SB
#define ILVR_D3_SB(...)
Definition: generic_macros_msa.h:1570
LD_SB8
#define LD_SB8(...)
Definition: generic_macros_msa.h:332
common_hz_8t_8w_msa
static void common_hz_8t_8w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:424
HORIZ_8TAP_4WID_4VECS_FILT
#define HORIZ_8TAP_4WID_4VECS_FILT(src0, src1, src2, src3,mask0, mask1, mask2, mask3,filt0, filt1, filt2, filt3,out0, out1)
Definition: hevc_mc_uni_msa.c:34
UNI_MC
#define UNI_MC(PEL, DIR, WIDTH, TAP, DIR1, FILT_DIR)
Definition: hevc_mc_uni_msa.c:4061
aligned
static int aligned(int val)
Definition: dashdec.c:178
common_vt_4t_4w_msa
static void common_vt_4t_4w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2548
hevc_hv_uni_8t_64w_msa
static void hevc_hv_uni_8t_64w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:1911
hevc_macros_msa.h
filter
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce then the filter should push the output frames on the output link immediately As an exception to the previous rule if the input frame is enough to produce several output frames then the filter needs output only at least one per link The additional frames can be left buffered in the filter
Definition: filter_design.txt:228
ST8x2_UB
#define ST8x2_UB(in, pdst, stride)
Definition: generic_macros_msa.h:544
VSHF_B2_SH
#define VSHF_B2_SH(...)
Definition: generic_macros_msa.h:762
SPLATI_H2_SB
#define SPLATI_H2_SB(...)
Definition: generic_macros_msa.h:1773
hevc_hv_uni_4t_24w_msa
static void hevc_hv_uni_4t_24w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:4014
hevc_hv_uni_4t_4w_msa
static void hevc_hv_uni_4t_4w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:3300
generic_macros_msa.h
XORI_B7_128_SB
#define XORI_B7_128_SB(...)
Definition: generic_macros_msa.h:1991
hevc_hv_uni_4t_4x2_msa
static void hevc_hv_uni_4t_4x2_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition: hevc_mc_uni_msa.c:3088
common_vt_4t_12w_msa
static void common_vt_4t_12w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2763
ILVR_B2_SH
#define ILVR_B2_SH(...)
Definition: generic_macros_msa.h:1459
hevcdsp_mips.h
XORI_B4_128_SB
#define XORI_B4_128_SB(...)
Definition: generic_macros_msa.h:1969
hevc_hv_uni_8t_8w_msa
static void hevc_hv_uni_8t_8w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:1615
SPLATI_W4_SH
#define SPLATI_W4_SH(...)
Definition: generic_macros_msa.h:1818
DPADD_SB2_SH
#define DPADD_SB2_SH(...)
Definition: generic_macros_msa.h:931
SRARI_H2_SH
#define SRARI_H2_SH(...)
Definition: generic_macros_msa.h:2177
common_hz_8t_4x8_msa
static void common_hz_8t_4x8_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:315
common_hz_8t_12w_msa
static void common_hz_8t_12w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:476
common_vt_8t_32w_msa
static void common_vt_8t_32w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:1316
LD_UB8
#define LD_UB8(...)
Definition: generic_macros_msa.h:331
width
#define width
PCKEV_D2_SH
#define PCKEV_D2_SH(...)
Definition: generic_macros_msa.h:1907
UNI_MC_HV
#define UNI_MC_HV(PEL, WIDTH, TAP)
Definition: hevc_mc_uni_msa.c:4113
SAT_SW4_SW
#define SAT_SW4_SW(...)
Definition: generic_macros_msa.h:1757
PCKEV_H2_SW
#define PCKEV_H2_SW(...)
Definition: generic_macros_msa.h:1878
common_hz_4t_6w_msa
static void common_hz_4t_6w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2078
int32_t
int32_t
Definition: audio_convert.c:194
common_vt_4t_8x4mult_msa
static void common_vt_4t_8x4mult_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2704
copy_width8_msa
static void copy_width8_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height)
Definition: hevc_mc_uni_msa.c:104
PCKEV_H2_SH
#define PCKEV_H2_SH(...)
Definition: generic_macros_msa.h:1877
LD_SB3
#define LD_SB3(...)
Definition: generic_macros_msa.h:286
common_hz_4t_24w_msa
static void common_hz_4t_24w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2320
copy_width12_msa
static void copy_width12_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height)
Definition: hevc_mc_uni_msa.c:146
common_hz_8t_48w_msa
static void common_hz_8t_48w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:744
common_vt_4t_16w_msa
static void common_vt_4t_16w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2826
ST_UB
#define ST_UB(...)
Definition: generic_macros_msa.h:39
hevc_hv_uni_4t_4multx8mult_msa
static void hevc_hv_uni_4t_4multx8mult_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:3202
SAT_SH4_SH
#define SAT_SH4_SH(...)
Definition: generic_macros_msa.h:1733
SPLATI_H4_SB
#define SPLATI_H4_SB(...)
Definition: generic_macros_msa.h:1791
LD_SB4
#define LD_SB4(...)
Definition: generic_macros_msa.h:294
ST2x4_UB
#define ST2x4_UB(in, stidx, pdst, stride)
Definition: generic_macros_msa.h:415
PCKEV_B4_UB
#define PCKEV_B4_UB(...)
Definition: generic_macros_msa.h:1857
HEVC_FILT_8TAP
#define HEVC_FILT_8TAP(in0, in1, in2, in3,filt0, filt1, filt2, filt3)
Definition: hevc_macros_msa.h:35
ST_UB8
#define ST_UB8(...)
Definition: generic_macros_msa.h:399
ST_UB4
#define ST_UB4(...)
Definition: generic_macros_msa.h:382
common_vt_8t_4w_msa
static void common_vt_8t_4w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:909
src1
#define src1
Definition: h264pred.c:139
common_hz_4t_4x4_msa
static void common_hz_4t_4x4_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:1950
ILVL_B4_SB
#define ILVL_B4_SB(...)
Definition: generic_macros_msa.h:1393
common_hz_4t_4x16_msa
static void common_hz_4t_4x16_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:2013
HORIZ_8TAP_8WID_4VECS_FILT
#define HORIZ_8TAP_8WID_4VECS_FILT(src0, src1, src2, src3,mask0, mask1, mask2, mask3,filt0, filt1, filt2, filt3,out0, out1, out2, out3)
Definition: hevc_mc_uni_msa.c:51
SAT_SH2_SH
#define SAT_SH2_SH(...)
Definition: generic_macros_msa.h:1719
common_vt_4t_8x2_msa
static void common_vt_4t_8x2_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:2625
copy_width64_msa
static void copy_width64_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height)
Definition: hevc_mc_uni_msa.c:257
hevc_hv_uni_4t_8multx4mult_msa
static void hevc_hv_uni_4t_8multx4mult_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height, int32_t width8mult)
Definition: hevc_mc_uni_msa.c:3692
hevc_hv_uni_8t_4w_msa
static void hevc_hv_uni_8t_4w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:1340
hevc_hv_uni_4t_16w_msa
static void hevc_hv_uni_4t_16w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:3997
hevc_hv_uni_8t_12w_msa
static void hevc_hv_uni_8t_12w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:1627
DOTP_SB4_SH
#define DOTP_SB4_SH(...)
Definition: generic_macros_msa.h:882
common_vt_4t_6w_msa
static void common_vt_4t_6w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2560
HEVC_FILT_8TAP_SH
#define HEVC_FILT_8TAP_SH(in0, in1, in2, in3,filt0, filt1, filt2, filt3)
Definition: hevc_macros_msa.h:24
common_vt_4t_8w_msa
static void common_vt_4t_8w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2749
HORIZ_4TAP_4WID_4VECS_FILT
#define HORIZ_4TAP_4WID_4VECS_FILT(src0, src1, src2, src3,mask0, mask1, filt0, filt1,out0, out1)
Definition: hevc_mc_uni_msa.c:76
SAT_SH3_SH
#define SAT_SH3_SH(...)
Definition: generic_macros_msa.h:1726
common_vt_8t_48w_msa
static void common_vt_8t_48w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:1324
src0
#define src0
Definition: h264pred.c:138
common_hz_4t_16w_msa
static void common_hz_4t_16w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2257
copy_width16_msa
static void copy_width16_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height)
Definition: hevc_mc_uni_msa.c:160
SD4
#define SD4(in0, in1, in2, in3, pdst, stride)
Definition: generic_macros_msa.h:253
ST12x8_UB
#define ST12x8_UB(in0, in1, in2, in3, in4, in5, in6, in7, pdst, stride)
Definition: generic_macros_msa.h:606
common_vt_4t_4x4multiple_msa
static void common_vt_4t_4x4multiple_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2502
SW
#define SW(val, pdst)
Definition: generic_macros_msa.h:164
filt
static const int8_t filt[NUMTAPS]
Definition: af_earwax.c:39
hevc_hv_uni_4t_8x6_msa
static void hevc_hv_uni_4t_8x6_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y)
Definition: hevc_mc_uni_msa.c:3588
ST4x8_UB
#define ST4x8_UB(in0, in1, pdst, stride)
Definition: generic_macros_msa.h:475
ST6x4_UB
#define ST6x4_UB(in0, in1, pdst, stride)
Definition: generic_macros_msa.h:496
common_vt_8t_16w_msa
static void common_vt_8t_16w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:1137
LD_SB7
#define LD_SB7(...)
Definition: generic_macros_msa.h:323
common_hz_4t_8w_msa
static void common_hz_4t_8w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2190
LD_SB5
#define LD_SB5(...)
Definition: generic_macros_msa.h:304
HEVC_FILT_4TAP
#define HEVC_FILT_4TAP(in0, in1, filt0, filt1)
Definition: hevc_macros_msa.h:55
common_hz_4t_12w_msa
static void common_hz_4t_12w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2203
common_vt_4t_24w_msa
static void common_vt_4t_24w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2884
common_hz_8t_4x16_msa
static void common_hz_8t_4x16_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:353
ST8x4_UB
#define ST8x4_UB(in0, in1, pdst, stride)
Definition: generic_macros_msa.h:568
ILVR_H2_SH
#define ILVR_H2_SH(...)
Definition: generic_macros_msa.h:1510
ff_hevc_mask_arr
static const uint8_t ff_hevc_mask_arr[16 *3]
Definition: hevc_mc_uni_msa.c:25
UNI_MC_COPY
#define UNI_MC_COPY(WIDTH)
Definition: hevc_mc_uni_msa.c:4038
hevc_hv_uni_4t_8multx4_msa
static void hevc_hv_uni_4t_8multx4_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t width8mult)
Definition: hevc_mc_uni_msa.c:3503
LD_UB
#define LD_UB(...)
Definition: generic_macros_msa.h:31
common_hz_4t_4w_msa
static void common_hz_4t_4w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2063
common_hz_8t_64w_msa
static void common_hz_8t_64w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:830
common_hz_8t_4x4_msa
static void common_hz_8t_4x4_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:286
ILVR_B4_SB
#define ILVR_B4_SB(...)
Definition: generic_macros_msa.h:1479
hevc_hv_uni_8t_24w_msa
static void hevc_hv_uni_8t_24w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter_x, const int8_t *filter_y, int32_t height)
Definition: hevc_mc_uni_msa.c:1875
out
FILE * out
Definition: movenc.c:54
ILVR_B3_SH
#define ILVR_B3_SH(...)
Definition: generic_macros_msa.h:1470
common_vt_8t_12w_msa
static void common_vt_8t_12w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:1035
ST8x1_UB
#define ST8x1_UB(in, pdst)
Definition: generic_macros_msa.h:530
common_vt_8t_8w_msa
static void common_vt_8t_8w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:980
PCKEV_H4_SH
#define PCKEV_H4_SH(...)
Definition: generic_macros_msa.h:1886
common_vt_4t_32w_msa
static void common_vt_4t_32w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2995
ST4x2_UB
#define ST4x2_UB(in, pdst, stride)
Definition: generic_macros_msa.h:439
LD2
#define LD2(psrc, stride, out0, out1)
Definition: generic_macros_msa.h:220
common_hz_4t_8x2mult_msa
static void common_hz_4t_8x2mult_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:2122
common_vt_8t_24w_msa
static void common_vt_8t_24w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:1305
common_hz_4t_4x8_msa
static void common_hz_4t_4x8_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter)
Definition: hevc_mc_uni_msa.c:1977
common_vt_8t_16w_mult_msa
static void common_vt_8t_16w_mult_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height, int32_t width)
Definition: hevc_mc_uni_msa.c:1216
HEVC_FILT_4TAP_SH
#define HEVC_FILT_4TAP_SH(in0, in1, filt0, filt1)
Definition: hevc_macros_msa.h:46
copy_width32_msa
static void copy_width32_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, int32_t height)
Definition: hevc_mc_uni_msa.c:219
common_vt_8t_64w_msa
static void common_vt_8t_64w_msa(uint8_t *src, int32_t src_stride, uint8_t *dst, int32_t dst_stride, const int8_t *filter, int32_t height)
Definition: hevc_mc_uni_msa.c:1332
DOTP_SB2_SH
#define DOTP_SB2_SH(...)
Definition: generic_macros_msa.h:866
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:26
VSHF_B3_SB
#define VSHF_B3_SB(...)
Definition: generic_macros_msa.h:770
DOTP_SB3_SH
#define DOTP_SB3_SH(...)
Definition: generic_macros_msa.h:874
Generated on Sun Mar 17 2019 19:21:10 for FFmpeg by   doxygen 1.8.11