FFmpeg
Loading...
Searching...
No Matches
hevc_lpf_sao_msa.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2015 -2017 Manojkumar Bhosale (Manojkumar.Bhosale@imgtec.com)
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21#include "libavutil/mips/generic_macros_msa.h"
22#include "libavcodec/mips/hevcdsp_mips.h"
23
24static void hevc_loopfilter_luma_hor_msa(uint8_t *src, int32_t stride,
25 int32_t beta, const int32_t *tc,
26 const uint8_t *p_is_pcm, const uint8_t *q_is_pcm)
27{
28 uint8_t *p3 = src - (stride << 2);
29 uint8_t *p2 = src - ((stride << 1) + stride);
30 uint8_t *p1 = src - (stride << 1);
31 uint8_t *p0 = src - stride;
32 uint8_t *q0 = src;
33 uint8_t *q1 = src + stride;
34 uint8_t *q2 = src + (stride << 1);
35 uint8_t *q3 = src + (stride << 1) + stride;
36 uint8_t flag0, flag1;
37 int32_t dp00, dq00, dp30, dq30, d00, d30;
38 int32_t d0030, d0434;
39 int32_t dp04, dq04, dp34, dq34, d04, d34;
40 int32_t tc0, p_is_pcm0, q_is_pcm0, beta30, beta20, tc250;
41 int32_t tc4, p_is_pcm4, q_is_pcm4, tc254, tmp;
42 uint64_t dst_val0, dst_val1;
43 v16u8 dst0, dst1, dst2, dst3, dst4, dst5;
44 v2i64 cmp0, cmp1, cmp2, p_is_pcm_vec, q_is_pcm_vec;
45 v2i64 cmp3;
46 v8u16 temp0, temp1;
47 v8i16 temp2;
48 v8i16 tc_pos, tc_neg;
49 v8i16 diff0, diff1, delta0, delta1, delta2, abs_delta0;
50 v16i8 zero = { 0 };
51 v8u16 p3_src, p2_src, p1_src, p0_src, q0_src, q1_src, q2_src, q3_src;
52
53 dp00 = abs(p2[0] - (p1[0] << 1) + p0[0]);
54 dq00 = abs(q2[0] - (q1[0] << 1) + q0[0]);
55 dp30 = abs(p2[3] - (p1[3] << 1) + p0[3]);
56 dq30 = abs(q2[3] - (q1[3] << 1) + q0[3]);
57 d00 = dp00 + dq00;
58 d30 = dp30 + dq30;
59 dp04 = abs(p2[4] - (p1[4] << 1) + p0[4]);
60 dq04 = abs(q2[4] - (q1[4] << 1) + q0[4]);
61 dp34 = abs(p2[7] - (p1[7] << 1) + p0[7]);
62 dq34 = abs(q2[7] - (q1[7] << 1) + q0[7]);
63 d04 = dp04 + dq04;
64 d34 = dp34 + dq34;
65
66 p_is_pcm0 = p_is_pcm[0];
67 p_is_pcm4 = p_is_pcm[1];
68 q_is_pcm0 = q_is_pcm[0];
69 q_is_pcm4 = q_is_pcm[1];
70
71 cmp0 = __msa_fill_d(p_is_pcm0);
72 cmp1 = __msa_fill_d(p_is_pcm4);
73 p_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
74 p_is_pcm_vec = __msa_ceqi_d(p_is_pcm_vec, 0);
75
76 d0030 = (d00 + d30) >= beta;
77 d0434 = (d04 + d34) >= beta;
78
79 cmp0 = (v2i64) __msa_fill_w(d0030);
80 cmp1 = (v2i64) __msa_fill_w(d0434);
81 cmp3 = (v2i64) __msa_ilvev_w((v4i32) cmp1, (v4i32) cmp0);
82 cmp3 = (v2i64) __msa_ceqi_w((v4i32) cmp3, 0);
83
84 if ((!p_is_pcm0 || !p_is_pcm4 || !q_is_pcm0 || !q_is_pcm4) &&
85 (!d0030 || !d0434)) {
86 p3_src = LD_UH(p3);
87 p2_src = LD_UH(p2);
88 p1_src = LD_UH(p1);
89 p0_src = LD_UH(p0);
90
91 cmp0 = __msa_fill_d(q_is_pcm0);
92 cmp1 = __msa_fill_d(q_is_pcm4);
93 q_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
94 q_is_pcm_vec = __msa_ceqi_d(q_is_pcm_vec, 0);
95
96 tc0 = tc[0];
97 beta30 = beta >> 3;
98 beta20 = beta >> 2;
99 tc250 = ((tc0 * 5 + 1) >> 1);
100 tc4 = tc[1];
101 tc254 = ((tc4 * 5 + 1) >> 1);
102
103 cmp0 = (v2i64) __msa_fill_h(tc0);
104 cmp1 = (v2i64) __msa_fill_h(tc4);
105
106 ILVR_B4_UH(zero, p3_src, zero, p2_src, zero, p1_src, zero, p0_src,
107 p3_src, p2_src, p1_src, p0_src);
108 q0_src = LD_UH(q0);
109 q1_src = LD_UH(q1);
110 q2_src = LD_UH(q2);
111 q3_src = LD_UH(q3);
112
113 flag0 = abs(p3[0] - p0[0]) + abs(q3[0] - q0[0]) < beta30 &&
114 abs(p0[0] - q0[0]) < tc250;
115 flag0 = flag0 && (abs(p3[3] - p0[3]) + abs(q3[3] - q0[3]) < beta30 &&
116 abs(p0[3] - q0[3]) < tc250 && (d00 << 1) < beta20 &&
117 (d30 << 1) < beta20);
118
119 tc_pos = (v8i16) __msa_ilvev_d(cmp1, cmp0);
120 ILVR_B4_UH(zero, q0_src, zero, q1_src, zero, q2_src, zero, q3_src,
121 q0_src, q1_src, q2_src, q3_src);
122 flag1 = abs(p3[4] - p0[4]) + abs(q3[4] - q0[4]) < beta30 &&
123 abs(p0[4] - q0[4]) < tc254;
124 flag1 = flag1 && (abs(p3[7] - p0[7]) + abs(q3[7] - q0[7]) < beta30 &&
125 abs(p0[7] - q0[7]) < tc254 && (d04 << 1) < beta20 &&
126 (d34 << 1) < beta20);
127
128 cmp0 = (v2i64) __msa_fill_w(flag0);
129 cmp1 = (v2i64) __msa_fill_w(flag1);
130 cmp2 = (v2i64) __msa_ilvev_w((v4i32) cmp1, (v4i32) cmp0);
131 cmp2 = (v2i64) __msa_ceqi_w((v4i32) cmp2, 0);
132
133 if (flag0 && flag1) { /* strong only */
134 /* strong filter */
135 tc_pos <<= 1;
136 tc_neg = -tc_pos;
137
138 /* p part */
139 temp0 = (p1_src + p0_src + q0_src);
140 temp1 = ((p3_src + p2_src) << 1) + p2_src + temp0;
141 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
142 temp2 = (v8i16) (temp1 - p2_src);
143 CLIP_SH(temp2, tc_neg, tc_pos);
144 dst0 = (v16u8) (temp2 + (v8i16) p2_src);
145
146 temp1 = temp0 + p2_src;
147 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 2);
148 temp2 = (v8i16) (temp1 - p1_src);
149 CLIP_SH(temp2, tc_neg, tc_pos);
150 dst1 = (v16u8) (temp2 + (v8i16) p1_src);
151
152 temp1 = (temp0 << 1) + p2_src + q1_src;
153 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
154 temp2 = (v8i16) (temp1 - p0_src);
155 CLIP_SH(temp2, tc_neg, tc_pos);
156 dst2 = (v16u8) (temp2 + (v8i16) p0_src);
157
158 dst0 = __msa_bmz_v(dst0, (v16u8) p2_src, (v16u8) p_is_pcm_vec);
159 dst1 = __msa_bmz_v(dst1, (v16u8) p1_src, (v16u8) p_is_pcm_vec);
160 dst2 = __msa_bmz_v(dst2, (v16u8) p0_src, (v16u8) p_is_pcm_vec);
161
162 /* q part */
163 temp0 = (q1_src + p0_src + q0_src);
164
165 temp1 = ((q3_src + q2_src) << 1) + q2_src + temp0;
166 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
167 temp2 = (v8i16) (temp1 - q2_src);
168 CLIP_SH(temp2, tc_neg, tc_pos);
169 dst5 = (v16u8) (temp2 + (v8i16) q2_src);
170
171 temp1 = temp0 + q2_src;
172 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 2);
173 temp2 = (v8i16) (temp1 - q1_src);
174 CLIP_SH(temp2, tc_neg, tc_pos);
175 dst4 = (v16u8) (temp2 + (v8i16) q1_src);
176
177 temp1 = (temp0 << 1) + p1_src + q2_src;
178 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
179 temp2 = (v8i16) (temp1 - q0_src);
180 CLIP_SH(temp2, tc_neg, tc_pos);
181 dst3 = (v16u8) (temp2 + (v8i16) q0_src);
182
183 dst3 = __msa_bmz_v(dst3, (v16u8) q0_src, (v16u8) q_is_pcm_vec);
184 dst4 = __msa_bmz_v(dst4, (v16u8) q1_src, (v16u8) q_is_pcm_vec);
185 dst5 = __msa_bmz_v(dst5, (v16u8) q2_src, (v16u8) q_is_pcm_vec);
186
187 /* pack results to 8 bit */
188 PCKEV_B2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
189 dst2 = (v16u8) __msa_pckev_b((v16i8) dst5, (v16i8) dst4);
190
191 /* pack src to 8 bit */
192 PCKEV_B2_UB(p1_src, p2_src, q0_src, p0_src, dst3, dst4);
193 dst5 = (v16u8) __msa_pckev_b((v16i8) q2_src, (v16i8) q1_src);
194
195 dst0 = __msa_bmz_v(dst0, dst3, (v16u8) cmp3);
196 dst1 = __msa_bmz_v(dst1, dst4, (v16u8) cmp3);
197 dst2 = __msa_bmz_v(dst2, dst5, (v16u8) cmp3);
198
199 dst_val0 = __msa_copy_u_d((v2i64) dst2, 0);
200 dst_val1 = __msa_copy_u_d((v2i64) dst2, 1);
201
202 ST_D4(dst0, dst1, 0, 1, 0, 1, p2, stride);
203 SD(dst_val0, p2 + 4 * stride);
204 SD(dst_val1, p2 + 5 * stride);
205 /* strong filter ends */
206 } else if (flag0 == flag1) { /* weak only */
207 /* weak filter */
208 tc_neg = -tc_pos;
209
210 diff0 = (v8i16) (q0_src - p0_src);
211 diff1 = (v8i16) (q1_src - p1_src);
212 diff0 = (diff0 << 3) + diff0;
213 diff1 = (diff1 << 1) + diff1;
214 delta0 = diff0 - diff1;
215 delta0 = __msa_srari_h(delta0, 4);
216
217 temp1 = (v8u16) ((tc_pos << 3) + (tc_pos << 1));
218 abs_delta0 = __msa_add_a_h(delta0, (v8i16) zero);
219 abs_delta0 = (v8u16) abs_delta0 < temp1;
220
221 CLIP_SH(delta0, tc_neg, tc_pos);
222
223 temp2 = (v8i16) (delta0 + p0_src);
224 CLIP_SH_0_255(temp2);
225 temp0 = (v8u16) __msa_bmz_v((v16u8) temp2, (v16u8) p0_src,
226 (v16u8) p_is_pcm_vec);
227
228 temp2 = (v8i16) (q0_src - delta0);
229 CLIP_SH_0_255(temp2);
230 temp2 = (v8i16) __msa_bmz_v((v16u8) temp2, (v16u8) q0_src,
231 (v16u8) q_is_pcm_vec);
232
233 p_is_pcm_vec = ~p_is_pcm_vec;
234 q_is_pcm_vec = ~q_is_pcm_vec;
235 tmp = (beta + (beta >> 1)) >> 3;
236 cmp0 = __msa_fill_d(dp00 + dp30 < tmp);
237 cmp1 = __msa_fill_d(dp04 + dp34 < tmp);
238 cmp0 = __msa_ilvev_d(cmp1, cmp0);
239 cmp0 = __msa_ceqi_d(cmp0, 0);
240 p_is_pcm_vec = p_is_pcm_vec | cmp0;
241
242 cmp0 = __msa_fill_d(dq00 + dq30 < tmp);
243 cmp1 = __msa_fill_d(dq04 + dq34 < tmp);
244 cmp0 = __msa_ilvev_d(cmp1, cmp0);
245 cmp0 = __msa_ceqi_d(cmp0, 0);
246 q_is_pcm_vec = q_is_pcm_vec | cmp0;
247
248 tc_pos >>= 1;
249 tc_neg = -tc_pos;
250
251 delta1 = (v8i16) __msa_aver_u_h(p2_src, p0_src);
252 delta1 -= (v8i16) p1_src;
253 delta1 += delta0;
254 delta1 >>= 1;
255 CLIP_SH(delta1, tc_neg, tc_pos);
256 delta1 = (v8i16) p1_src + (v8i16) delta1;
257 CLIP_SH_0_255(delta1);
258 delta1 = (v8i16) __msa_bmnz_v((v16u8) delta1, (v16u8) p1_src,
259 (v16u8) p_is_pcm_vec);
260
261 delta2 = (v8i16) __msa_aver_u_h(q0_src, q2_src);
262 delta2 = delta2 - (v8i16) q1_src;
263 delta2 = delta2 - delta0;
264 delta2 = delta2 >> 1;
265 CLIP_SH(delta2, tc_neg, tc_pos);
266 delta2 = (v8i16) q1_src + (v8i16) delta2;
267 CLIP_SH_0_255(delta2);
268 delta2 = (v8i16) __msa_bmnz_v((v16u8) delta2, (v16u8) q1_src,
269 (v16u8) q_is_pcm_vec);
270
271 dst1 = (v16u8) __msa_bmz_v((v16u8) delta1, (v16u8) p1_src,
272 (v16u8) abs_delta0);
273 dst2 = (v16u8) __msa_bmz_v((v16u8) temp0, (v16u8) p0_src,
274 (v16u8) abs_delta0);
275 dst3 = (v16u8) __msa_bmz_v((v16u8) temp2, (v16u8) q0_src,
276 (v16u8) abs_delta0);
277 dst4 = (v16u8) __msa_bmz_v((v16u8) delta2, (v16u8) q1_src,
278 (v16u8) abs_delta0);
279 /* pack results to 8 bit */
280 PCKEV_B2_UB(dst2, dst1, dst4, dst3, dst0, dst1);
281
282 /* pack src to 8 bit */
283 PCKEV_B2_UB(p0_src, p1_src, q1_src, q0_src, dst2, dst3);
284
285 dst0 = __msa_bmz_v(dst0, dst2, (v16u8) cmp3);
286 dst1 = __msa_bmz_v(dst1, dst3, (v16u8) cmp3);
287
288 p2 += stride;
289 ST_D4(dst0, dst1, 0, 1, 0, 1, p2, stride);
290 /* weak filter ends */
291 } else { /* strong + weak */
292 /* strong filter */
293 tc_pos <<= 1;
294 tc_neg = -tc_pos;
295
296 /* p part */
297 temp0 = (p1_src + p0_src + q0_src);
298 temp1 = ((p3_src + p2_src) << 1) + p2_src + temp0;
299 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
300 temp2 = (v8i16) (temp1 - p2_src);
301 CLIP_SH(temp2, tc_neg, tc_pos);
302 dst0 = (v16u8) (temp2 + (v8i16) p2_src);
303
304 temp1 = temp0 + p2_src;
305 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 2);
306 temp2 = (v8i16) (temp1 - p1_src);
307 CLIP_SH(temp2, tc_neg, tc_pos);
308 dst1 = (v16u8) (temp2 + (v8i16) p1_src);
309
310 temp1 = (temp0 << 1) + p2_src + q1_src;
311 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
312 temp2 = (v8i16) (temp1 - p0_src);
313 CLIP_SH(temp2, tc_neg, tc_pos);
314 dst2 = (v16u8) (temp2 + (v8i16) p0_src);
315
316 dst0 = __msa_bmz_v(dst0, (v16u8) p2_src, (v16u8) p_is_pcm_vec);
317 dst1 = __msa_bmz_v(dst1, (v16u8) p1_src, (v16u8) p_is_pcm_vec);
318 dst2 = __msa_bmz_v(dst2, (v16u8) p0_src, (v16u8) p_is_pcm_vec);
319
320 /* q part */
321 temp0 = (q1_src + p0_src + q0_src);
322
323 temp1 = ((q3_src + q2_src) << 1) + q2_src + temp0;
324 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
325 temp2 = (v8i16) (temp1 - q2_src);
326 CLIP_SH(temp2, tc_neg, tc_pos);
327 dst5 = (v16u8) (temp2 + (v8i16) q2_src);
328
329 temp1 = temp0 + q2_src;
330 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 2);
331 temp2 = (v8i16) (temp1 - q1_src);
332 CLIP_SH(temp2, tc_neg, tc_pos);
333 dst4 = (v16u8) (temp2 + (v8i16) q1_src);
334
335 temp1 = (temp0 << 1) + p1_src + q2_src;
336 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
337 temp2 = (v8i16) (temp1 - q0_src);
338 CLIP_SH(temp2, tc_neg, tc_pos);
339 dst3 = (v16u8) (temp2 + (v8i16) q0_src);
340
341 dst3 = __msa_bmz_v(dst3, (v16u8) q0_src, (v16u8) q_is_pcm_vec);
342 dst4 = __msa_bmz_v(dst4, (v16u8) q1_src, (v16u8) q_is_pcm_vec);
343 dst5 = __msa_bmz_v(dst5, (v16u8) q2_src, (v16u8) q_is_pcm_vec);
344
345 /* pack strong results to 8 bit */
346 PCKEV_B2_UB(dst1, dst0, dst3, dst2, dst0, dst1);
347 dst2 = (v16u8) __msa_pckev_b((v16i8) dst5, (v16i8) dst4);
348 /* strong filter ends */
349
350 /* weak filter */
351 tc_pos >>= 1;
352 tc_neg = -tc_pos;
353
354 diff0 = (v8i16) (q0_src - p0_src);
355 diff1 = (v8i16) (q1_src - p1_src);
356 diff0 = (diff0 << 3) + diff0;
357 diff1 = (diff1 << 1) + diff1;
358 delta0 = diff0 - diff1;
359 delta0 = __msa_srari_h(delta0, 4);
360
361 temp1 = (v8u16) ((tc_pos << 3) + (tc_pos << 1));
362 abs_delta0 = __msa_add_a_h(delta0, (v8i16) zero);
363 abs_delta0 = (v8u16) abs_delta0 < temp1;
364
365 CLIP_SH(delta0, tc_neg, tc_pos);
366
367 temp2 = (v8i16) (delta0 + p0_src);
368 CLIP_SH_0_255(temp2);
369 temp0 = (v8u16) __msa_bmz_v((v16u8) temp2, (v16u8) p0_src,
370 (v16u8) p_is_pcm_vec);
371
372 temp2 = (v8i16) (q0_src - delta0);
373 CLIP_SH_0_255(temp2);
374 temp2 = (v8i16) __msa_bmz_v((v16u8) temp2, (v16u8) q0_src,
375 (v16u8) q_is_pcm_vec);
376
377 p_is_pcm_vec = ~p_is_pcm_vec;
378 q_is_pcm_vec = ~q_is_pcm_vec;
379 tmp = (beta + (beta >> 1)) >> 3;
380 cmp0 = __msa_fill_d(dp00 + dp30 < tmp);
381 cmp1 = __msa_fill_d(dp04 + dp34 < tmp);
382 cmp0 = __msa_ilvev_d(cmp1, cmp0);
383 p_is_pcm_vec = p_is_pcm_vec | __msa_ceqi_d(cmp0, 0);
384
385 cmp0 = __msa_fill_d(dq00 + dq30 < tmp);
386 cmp1 = __msa_fill_d(dq04 + dq34 < tmp);
387 cmp0 = __msa_ilvev_d(cmp1, cmp0);
388 q_is_pcm_vec = q_is_pcm_vec | __msa_ceqi_d(cmp0, 0);
389
390 tc_pos >>= 1;
391 tc_neg = -tc_pos;
392
393 delta1 = (v8i16) __msa_aver_u_h(p2_src, p0_src);
394 delta1 -= (v8i16) p1_src;
395 delta1 += delta0;
396 delta1 >>= 1;
397 CLIP_SH(delta1, tc_neg, tc_pos);
398 delta1 = (v8i16) p1_src + (v8i16) delta1;
399 CLIP_SH_0_255(delta1);
400 delta1 = (v8i16) __msa_bmnz_v((v16u8) delta1, (v16u8) p1_src,
401 (v16u8) p_is_pcm_vec);
402
403 delta2 = (v8i16) __msa_aver_u_h(q0_src, q2_src);
404 delta2 = delta2 - (v8i16) q1_src;
405 delta2 = delta2 - delta0;
406 delta2 = delta2 >> 1;
407 CLIP_SH(delta2, tc_neg, tc_pos);
408 delta2 = (v8i16) q1_src + (v8i16) delta2;
409 CLIP_SH_0_255(delta2);
410 delta2 = (v8i16) __msa_bmnz_v((v16u8) delta2, (v16u8) q1_src,
411 (v16u8) q_is_pcm_vec);
412
413 delta1 = (v8i16) __msa_bmz_v((v16u8) delta1, (v16u8) p1_src,
414 (v16u8) abs_delta0);
415 temp0 = (v8u16) __msa_bmz_v((v16u8) temp0, (v16u8) p0_src,
416 (v16u8) abs_delta0);
417 temp2 = (v8i16) __msa_bmz_v((v16u8) temp2, (v16u8) q0_src,
418 (v16u8) abs_delta0);
419 delta2 = (v8i16) __msa_bmz_v((v16u8) delta2, (v16u8) q1_src,
420 (v16u8) abs_delta0);
421 /* weak filter ends */
422
423 /* pack weak results to 8 bit */
424 PCKEV_B2_UB(delta1, p2_src, temp2, temp0, dst3, dst4);
425 dst5 = (v16u8) __msa_pckev_b((v16i8) q2_src, (v16i8) delta2);
426
427 /* select between weak or strong */
428 dst0 = __msa_bmnz_v(dst0, dst3, (v16u8) cmp2);
429 dst1 = __msa_bmnz_v(dst1, dst4, (v16u8) cmp2);
430 dst2 = __msa_bmnz_v(dst2, dst5, (v16u8) cmp2);
431
432 /* pack src to 8 bit */
433 PCKEV_B2_UB(p1_src, p2_src, q0_src, p0_src, dst3, dst4);
434 dst5 = (v16u8) __msa_pckev_b((v16i8) q2_src, (v16i8) q1_src);
435
436 dst0 = __msa_bmz_v(dst0, dst3, (v16u8) cmp3);
437 dst1 = __msa_bmz_v(dst1, dst4, (v16u8) cmp3);
438 dst2 = __msa_bmz_v(dst2, dst5, (v16u8) cmp3);
439
440 dst_val0 = __msa_copy_u_d((v2i64) dst2, 0);
441 dst_val1 = __msa_copy_u_d((v2i64) dst2, 1);
442
443 ST_D4(dst0, dst1, 0, 1, 0, 1, p2, stride);
444 SD(dst_val0, p2 + 4 * stride);
445 SD(dst_val1, p2 + 5 * stride);
446 }
447 }
448}
449
450static void hevc_loopfilter_luma_ver_msa(uint8_t *src, int32_t stride,
451 int32_t beta, const int32_t *tc,
452 const uint8_t *p_is_pcm, const uint8_t *q_is_pcm)
453{
454 uint8_t *p3 = src;
455 uint8_t *p2 = src + 3 * stride;
456 uint8_t *p1 = src + (stride << 2);
457 uint8_t *p0 = src + 7 * stride;
458 uint8_t flag0, flag1;
459 uint16_t tmp0, tmp1;
460 uint32_t tmp2, tmp3;
461 int32_t dp00, dq00, dp30, dq30, d00, d30;
462 int32_t d0030, d0434;
463 int32_t dp04, dq04, dp34, dq34, d04, d34;
464 int32_t tc0, p_is_pcm0, q_is_pcm0, beta30, beta20, tc250;
465 int32_t tc4, p_is_pcm4, q_is_pcm4, tc254, tmp;
466 v16u8 dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7;
467 v2i64 cmp0, cmp1, cmp2, p_is_pcm_vec, q_is_pcm_vec;
468 v2i64 cmp3;
469 v8u16 temp0, temp1;
470 v8i16 temp2;
471 v8i16 tc_pos, tc_neg;
472 v8i16 diff0, diff1, delta0, delta1, delta2, abs_delta0;
473 v16i8 zero = { 0 };
474 v8u16 p3_src, p2_src, p1_src, p0_src, q0_src, q1_src, q2_src, q3_src;
475
476 dp00 = abs(p3[-3] - (p3[-2] << 1) + p3[-1]);
477 dq00 = abs(p3[2] - (p3[1] << 1) + p3[0]);
478 dp30 = abs(p2[-3] - (p2[-2] << 1) + p2[-1]);
479 dq30 = abs(p2[2] - (p2[1] << 1) + p2[0]);
480 d00 = dp00 + dq00;
481 d30 = dp30 + dq30;
482 p_is_pcm0 = p_is_pcm[0];
483 q_is_pcm0 = q_is_pcm[0];
484
485 dp04 = abs(p1[-3] - (p1[-2] << 1) + p1[-1]);
486 dq04 = abs(p1[2] - (p1[1] << 1) + p1[0]);
487 dp34 = abs(p0[-3] - (p0[-2] << 1) + p0[-1]);
488 dq34 = abs(p0[2] - (p0[1] << 1) + p0[0]);
489 d04 = dp04 + dq04;
490 d34 = dp34 + dq34;
491 p_is_pcm4 = p_is_pcm[1];
492 q_is_pcm4 = q_is_pcm[1];
493
494 cmp0 = __msa_fill_d(p_is_pcm0);
495 cmp1 = __msa_fill_d(p_is_pcm4);
496 p_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
497 p_is_pcm_vec = __msa_ceqi_d(p_is_pcm_vec, 0);
498
499 d0030 = (d00 + d30) >= beta;
500 d0434 = (d04 + d34) >= beta;
501
502 cmp0 = __msa_fill_d(d0030);
503 cmp1 = __msa_fill_d(d0434);
504 cmp3 = __msa_ilvev_d(cmp1, cmp0);
505 cmp3 = (v2i64) __msa_ceqi_d(cmp3, 0);
506
507 if ((!p_is_pcm0 || !p_is_pcm4 || !q_is_pcm0 || !q_is_pcm4) &&
508 (!d0030 || !d0434)) {
509 src -= 4;
510 LD_UH8(src, stride, p3_src, p2_src, p1_src, p0_src, q0_src, q1_src,
511 q2_src, q3_src);
512
513 cmp0 = __msa_fill_d(q_is_pcm0);
514 cmp1 = __msa_fill_d(q_is_pcm4);
515 q_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
516 q_is_pcm_vec = __msa_ceqi_d(q_is_pcm_vec, 0);
517
518 tc0 = tc[0];
519 beta30 = beta >> 3;
520 beta20 = beta >> 2;
521 tc250 = ((tc0 * 5 + 1) >> 1);
522
523 tc4 = tc[1];
524 tc254 = ((tc4 * 5 + 1) >> 1);
525 cmp0 = (v2i64) __msa_fill_h(tc0 << 1);
526 cmp1 = (v2i64) __msa_fill_h(tc4 << 1);
527 tc_pos = (v8i16) __msa_ilvev_d(cmp1, cmp0);
528
529 TRANSPOSE8x8_UB_UH(p3_src, p2_src, p1_src, p0_src, q0_src, q1_src,
530 q2_src, q3_src, p3_src, p2_src, p1_src, p0_src,
531 q0_src, q1_src, q2_src, q3_src);
532
533 flag0 = abs(p3[-4] - p3[-1]) + abs(p3[3] - p3[0]) < beta30 &&
534 abs(p3[-1] - p3[0]) < tc250;
535 flag0 = flag0 && (abs(p2[-4] - p2[-1]) + abs(p2[3] - p2[0]) < beta30 &&
536 abs(p2[-1] - p2[0]) < tc250 && (d00 << 1) < beta20 &&
537 (d30 << 1) < beta20);
538 cmp0 = __msa_fill_d(flag0);
539 ILVR_B4_UH(zero, p3_src, zero, p2_src, zero, p1_src, zero, p0_src,
540 p3_src, p2_src, p1_src, p0_src);
541
542 flag1 = abs(p1[-4] - p1[-1]) + abs(p1[3] - p1[0]) < beta30 &&
543 abs(p1[-1] - p1[0]) < tc254;
544 flag1 = flag1 && (abs(p0[-4] - p0[-1]) + abs(p0[3] - p0[0]) < beta30 &&
545 abs(p0[-1] - p0[0]) < tc254 && (d04 << 1) < beta20 &&
546 (d34 << 1) < beta20);
547 ILVR_B4_UH(zero, q0_src, zero, q1_src, zero, q2_src, zero, q3_src,
548 q0_src, q1_src, q2_src, q3_src);
549
550 cmp1 = __msa_fill_d(flag1);
551 cmp2 = __msa_ilvev_d(cmp1, cmp0);
552 cmp2 = __msa_ceqi_d(cmp2, 0);
553
554 if (flag0 && flag1) { /* strong only */
555 /* strong filter */
556 tc_neg = -tc_pos;
557
558 /* p part */
559 temp0 = (p1_src + p0_src + q0_src);
560
561 temp1 = ((p3_src + p2_src) << 1) + p2_src + temp0;
562 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
563 temp2 = (v8i16) (temp1 - p2_src);
564 CLIP_SH(temp2, tc_neg, tc_pos);
565 dst0 = (v16u8) (temp2 + (v8i16) p2_src);
566
567 temp1 = temp0 + p2_src;
568 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 2);
569 temp2 = (v8i16) (temp1 - p1_src);
570 CLIP_SH(temp2, tc_neg, tc_pos);
571 dst1 = (v16u8) (temp2 + (v8i16) p1_src);
572
573 temp1 = (temp0 << 1) + p2_src + q1_src;
574 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
575 temp2 = (v8i16) (temp1 - p0_src);
576 CLIP_SH(temp2, tc_neg, tc_pos);
577 dst2 = (v16u8) (temp2 + (v8i16) p0_src);
578
579 dst0 = __msa_bmz_v(dst0, (v16u8) p2_src, (v16u8) p_is_pcm_vec);
580 dst1 = __msa_bmz_v(dst1, (v16u8) p1_src, (v16u8) p_is_pcm_vec);
581 dst2 = __msa_bmz_v(dst2, (v16u8) p0_src, (v16u8) p_is_pcm_vec);
582
583 /* q part */
584 temp0 = (q1_src + p0_src + q0_src);
585 temp1 = ((q3_src + q2_src) << 1) + q2_src + temp0;
586 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
587 temp2 = (v8i16) (temp1 - q2_src);
588 CLIP_SH(temp2, tc_neg, tc_pos);
589 dst5 = (v16u8) (temp2 + (v8i16) q2_src);
590
591 temp1 = temp0 + q2_src;
592 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 2);
593 temp2 = (v8i16) (temp1 - q1_src);
594 CLIP_SH(temp2, tc_neg, tc_pos);
595 dst4 = (v16u8) (temp2 + (v8i16) q1_src);
596
597 temp1 = (temp0 << 1) + p1_src + q2_src;
598 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
599 temp2 = (v8i16) (temp1 - q0_src);
600 CLIP_SH(temp2, tc_neg, tc_pos);
601 dst3 = (v16u8) (temp2 + (v8i16) q0_src);
602
603 dst3 = __msa_bmz_v(dst3, (v16u8) q0_src, (v16u8) q_is_pcm_vec);
604 dst4 = __msa_bmz_v(dst4, (v16u8) q1_src, (v16u8) q_is_pcm_vec);
605 dst5 = __msa_bmz_v(dst5, (v16u8) q2_src, (v16u8) q_is_pcm_vec);
606 /* strong filter ends */
607 } else if (flag0 == flag1) { /* weak only */
608 /* weak filter */
609 tc_pos >>= 1;
610 tc_neg = -tc_pos;
611
612 diff0 = (v8i16) (q0_src - p0_src);
613 diff1 = (v8i16) (q1_src - p1_src);
614 diff0 = (diff0 << 3) + diff0;
615 diff1 = (diff1 << 1) + diff1;
616 delta0 = diff0 - diff1;
617 delta0 = __msa_srari_h(delta0, 4);
618
619 temp1 = (v8u16) ((tc_pos << 3) + (tc_pos << 1));
620 abs_delta0 = __msa_add_a_h(delta0, (v8i16) zero);
621 abs_delta0 = (v8u16) abs_delta0 < temp1;
622
623 CLIP_SH(delta0, tc_neg, tc_pos);
624 temp2 = (v8i16) (delta0 + p0_src);
625 CLIP_SH_0_255(temp2);
626 temp0 = (v8u16) __msa_bmz_v((v16u8) temp2, (v16u8) p0_src,
627 (v16u8) p_is_pcm_vec);
628
629 temp2 = (v8i16) (q0_src - delta0);
630 CLIP_SH_0_255(temp2);
631 temp2 = (v8i16) __msa_bmz_v((v16u8) temp2, (v16u8) q0_src,
632 (v16u8) q_is_pcm_vec);
633
634 tmp = ((beta + (beta >> 1)) >> 3);
635 cmp0 = __msa_fill_d(!p_is_pcm0 && ((dp00 + dp30) < tmp));
636 cmp1 = __msa_fill_d(!p_is_pcm4 && ((dp04 + dp34) < tmp));
637 p_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
638 p_is_pcm_vec = __msa_ceqi_d(p_is_pcm_vec, 0);
639
640 cmp0 = (v2i64) __msa_fill_h((!q_is_pcm0) && (dq00 + dq30 < tmp));
641 cmp1 = (v2i64) __msa_fill_h((!q_is_pcm4) && (dq04 + dq34 < tmp));
642 q_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
643 q_is_pcm_vec = __msa_ceqi_d(q_is_pcm_vec, 0);
644
645 tc_pos >>= 1;
646 tc_neg = -tc_pos;
647
648 delta1 = (v8i16) __msa_aver_u_h(p2_src, p0_src);
649 delta1 -= (v8i16) p1_src;
650 delta1 += delta0;
651 delta1 >>= 1;
652 CLIP_SH(delta1, tc_neg, tc_pos);
653 delta1 = (v8i16) p1_src + (v8i16) delta1;
654 CLIP_SH_0_255(delta1);
655 delta1 = (v8i16) __msa_bmnz_v((v16u8) delta1, (v16u8) p1_src,
656 (v16u8) p_is_pcm_vec);
657
658 delta2 = (v8i16) __msa_aver_u_h(q0_src, q2_src);
659 delta2 = delta2 - (v8i16) q1_src;
660 delta2 = delta2 - delta0;
661 delta2 = delta2 >> 1;
662 CLIP_SH(delta2, tc_neg, tc_pos);
663 delta2 = (v8i16) q1_src + (v8i16) delta2;
664 CLIP_SH_0_255(delta2);
665 delta2 = (v8i16) __msa_bmnz_v((v16u8) delta2, (v16u8) q1_src,
666 (v16u8) q_is_pcm_vec);
667
668 dst0 = __msa_bmz_v((v16u8) delta1, (v16u8) p1_src,
669 (v16u8) abs_delta0);
670 dst1 = __msa_bmz_v((v16u8) temp0, (v16u8) p0_src,
671 (v16u8) abs_delta0);
672 dst2 = __msa_bmz_v((v16u8) temp2, (v16u8) q0_src,
673 (v16u8) abs_delta0);
674 dst3 = __msa_bmz_v((v16u8) delta2, (v16u8) q1_src,
675 (v16u8) abs_delta0);
676 /* weak filter ends */
677
678 dst0 = __msa_bmz_v(dst0, (v16u8) p1_src, (v16u8) cmp3);
679 dst1 = __msa_bmz_v(dst1, (v16u8) p0_src, (v16u8) cmp3);
680 dst2 = __msa_bmz_v(dst2, (v16u8) q0_src, (v16u8) cmp3);
681 dst3 = __msa_bmz_v(dst3, (v16u8) q1_src, (v16u8) cmp3);
682
683 PCKEV_B2_UB(dst2, dst0, dst3, dst1, dst0, dst1);
684
685 /* transpose */
686 ILVRL_B2_UB(dst1, dst0, dst4, dst5);
687 ILVRL_H2_UB(dst5, dst4, dst0, dst1);
688
689 src += 2;
690
691 tmp2 = __msa_copy_u_w((v4i32) dst0, 0);
692 tmp3 = __msa_copy_u_w((v4i32) dst0, 1);
693 SW(tmp2, src);
694 src += stride;
695 SW(tmp3, src);
696 src += stride;
697
698 tmp2 = __msa_copy_u_w((v4i32) dst0, 2);
699 tmp3 = __msa_copy_u_w((v4i32) dst0, 3);
700 SW(tmp2, src);
701 src += stride;
702 SW(tmp3, src);
703 src += stride;
704
705 tmp2 = __msa_copy_u_w((v4i32) dst1, 0);
706 tmp3 = __msa_copy_u_w((v4i32) dst1, 1);
707 SW(tmp2, src);
708 src += stride;
709 SW(tmp3, src);
710 src += stride;
711
712 tmp2 = __msa_copy_u_w((v4i32) dst1, 2);
713 tmp3 = __msa_copy_u_w((v4i32) dst1, 3);
714 SW(tmp2, src);
715 src += stride;
716 SW(tmp3, src);
717
718 return;
719 } else { /* strong + weak */
720 /* strong filter */
721 tc_neg = -tc_pos;
722
723 /* p part */
724 temp0 = (p1_src + p0_src + q0_src);
725
726 temp1 = ((p3_src + p2_src) << 1) + p2_src + temp0;
727 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
728 temp2 = (v8i16) (temp1 - p2_src);
729 CLIP_SH(temp2, tc_neg, tc_pos);
730 dst0 = (v16u8) (temp2 + (v8i16) p2_src);
731
732 temp1 = temp0 + p2_src;
733 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 2);
734 temp2 = (v8i16) (temp1 - p1_src);
735 CLIP_SH(temp2, tc_neg, tc_pos);
736 dst1 = (v16u8) (temp2 + (v8i16) p1_src);
737
738 temp1 = (temp0 << 1) + p2_src + q1_src;
739 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
740 temp2 = (v8i16) (temp1 - p0_src);
741 CLIP_SH(temp2, tc_neg, tc_pos);
742 dst2 = (v16u8) (temp2 + (v8i16) p0_src);
743
744 dst0 = __msa_bmz_v(dst0, (v16u8) p2_src, (v16u8) p_is_pcm_vec);
745 dst1 = __msa_bmz_v(dst1, (v16u8) p1_src, (v16u8) p_is_pcm_vec);
746 dst2 = __msa_bmz_v(dst2, (v16u8) p0_src, (v16u8) p_is_pcm_vec);
747
748 /* q part */
749 temp0 = (q1_src + p0_src + q0_src);
750 temp1 = ((q3_src + q2_src) << 1) + q2_src + temp0;
751 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
752 temp2 = (v8i16) (temp1 - q2_src);
753 CLIP_SH(temp2, tc_neg, tc_pos);
754 dst5 = (v16u8) (temp2 + (v8i16) q2_src);
755
756 temp1 = temp0 + q2_src;
757 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 2);
758 temp2 = (v8i16) (temp1 - q1_src);
759 CLIP_SH(temp2, tc_neg, tc_pos);
760 dst4 = (v16u8) (temp2 + (v8i16) q1_src);
761
762 temp1 = (temp0 << 1) + p1_src + q2_src;
763 temp1 = (v8u16) __msa_srari_h((v8i16) temp1, 3);
764 temp2 = (v8i16) (temp1 - q0_src);
765 CLIP_SH(temp2, tc_neg, tc_pos);
766 dst3 = (v16u8) (temp2 + (v8i16) q0_src);
767
768 dst3 = __msa_bmz_v(dst3, (v16u8) q0_src, (v16u8) q_is_pcm_vec);
769 dst4 = __msa_bmz_v(dst4, (v16u8) q1_src, (v16u8) q_is_pcm_vec);
770 dst5 = __msa_bmz_v(dst5, (v16u8) q2_src, (v16u8) q_is_pcm_vec);
771 /* strong filter ends */
772
773 /* weak filter */
774 tc_pos >>= 1;
775 tc_neg = -tc_pos;
776
777 diff0 = (v8i16) (q0_src - p0_src);
778 diff1 = (v8i16) (q1_src - p1_src);
779 diff0 = (diff0 << 3) + diff0;
780 diff1 = (diff1 << 1) + diff1;
781 delta0 = diff0 - diff1;
782 delta0 = __msa_srari_h(delta0, 4);
783
784 temp1 = (v8u16) ((tc_pos << 3) + (tc_pos << 1));
785 abs_delta0 = __msa_add_a_h(delta0, (v8i16) zero);
786 abs_delta0 = (v8u16) abs_delta0 < temp1;
787
788 CLIP_SH(delta0, tc_neg, tc_pos);
789
790 temp2 = (v8i16) (delta0 + p0_src);
791 CLIP_SH_0_255(temp2);
792 temp0 = (v8u16) __msa_bmz_v((v16u8) temp2, (v16u8) p0_src,
793 (v16u8) p_is_pcm_vec);
794
795 temp2 = (v8i16) (q0_src - delta0);
796 CLIP_SH_0_255(temp2);
797 temp2 = (v8i16) __msa_bmz_v((v16u8) temp2, (v16u8) q0_src,
798 (v16u8) q_is_pcm_vec);
799
800 tmp = (beta + (beta >> 1)) >> 3;
801 cmp0 = __msa_fill_d(!p_is_pcm0 && ((dp00 + dp30) < tmp));
802 cmp1 = __msa_fill_d(!p_is_pcm4 && ((dp04 + dp34) < tmp));
803 p_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
804 p_is_pcm_vec = __msa_ceqi_d(p_is_pcm_vec, 0);
805
806 cmp0 = (v2i64) __msa_fill_h((!q_is_pcm0) && (dq00 + dq30 < tmp));
807 cmp1 = (v2i64) __msa_fill_h((!q_is_pcm4) && (dq04 + dq34 < tmp));
808 q_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
809 q_is_pcm_vec = __msa_ceqi_d(q_is_pcm_vec, 0);
810
811 tc_pos >>= 1;
812 tc_neg = -tc_pos;
813
814 delta1 = (v8i16) __msa_aver_u_h(p2_src, p0_src);
815 delta1 -= (v8i16) p1_src;
816 delta1 += delta0;
817 delta1 >>= 1;
818 CLIP_SH(delta1, tc_neg, tc_pos);
819 delta1 = (v8i16) p1_src + (v8i16) delta1;
820 CLIP_SH_0_255(delta1);
821 delta1 = (v8i16) __msa_bmnz_v((v16u8) delta1, (v16u8) p1_src,
822 (v16u8) p_is_pcm_vec);
823
824 delta2 = (v8i16) __msa_aver_u_h(q0_src, q2_src);
825 delta2 = delta2 - (v8i16) q1_src;
826 delta2 = delta2 - delta0;
827 delta2 = delta2 >> 1;
828 CLIP_SH(delta2, tc_neg, tc_pos);
829 delta2 = (v8i16) q1_src + (v8i16) delta2;
830 CLIP_SH_0_255(delta2);
831 delta2 = (v8i16) __msa_bmnz_v((v16u8) delta2, (v16u8) q1_src,
832 (v16u8) q_is_pcm_vec);
833 delta1 = (v8i16) __msa_bmz_v((v16u8) delta1, (v16u8) p1_src,
834 (v16u8) abs_delta0);
835 temp0 = (v8u16) __msa_bmz_v((v16u8) temp0, (v16u8) p0_src,
836 (v16u8) abs_delta0);
837 temp2 = (v8i16) __msa_bmz_v((v16u8) temp2, (v16u8) q0_src,
838 (v16u8) abs_delta0);
839 delta2 = (v8i16) __msa_bmz_v((v16u8) delta2, (v16u8) q1_src,
840 (v16u8) abs_delta0);
841 /* weak filter ends*/
842
843 /* select between weak or strong */
844 dst2 = __msa_bmnz_v(dst2, (v16u8) temp0, (v16u8) cmp2);
845 dst3 = __msa_bmnz_v(dst3, (v16u8) temp2, (v16u8) cmp2);
846 dst1 = __msa_bmnz_v(dst1, (v16u8) delta1, (v16u8) cmp2);
847 dst4 = __msa_bmnz_v(dst4, (v16u8) delta2, (v16u8) cmp2);
848 dst0 = __msa_bmnz_v(dst0, (v16u8) p2_src, (v16u8) cmp2);
849 dst5 = __msa_bmnz_v(dst5, (v16u8) q2_src, (v16u8) cmp2);
850 }
851
852 dst0 = __msa_bmz_v(dst0, (v16u8) p2_src, (v16u8) cmp3);
853 dst1 = __msa_bmz_v(dst1, (v16u8) p1_src, (v16u8) cmp3);
854 dst2 = __msa_bmz_v(dst2, (v16u8) p0_src, (v16u8) cmp3);
855 dst3 = __msa_bmz_v(dst3, (v16u8) q0_src, (v16u8) cmp3);
856 dst4 = __msa_bmz_v(dst4, (v16u8) q1_src, (v16u8) cmp3);
857 dst5 = __msa_bmz_v(dst5, (v16u8) q2_src, (v16u8) cmp3);
858
859 /* pack results to 8 bit */
860 PCKEV_B4_UB(dst2, dst0, dst3, dst1, dst4, dst4, dst5, dst5, dst0, dst1,
861 dst2, dst3);
862
863 /* transpose */
864 ILVRL_B2_UB(dst1, dst0, dst4, dst5);
865 ILVRL_B2_UB(dst3, dst2, dst6, dst7);
866 ILVRL_H2_UB(dst5, dst4, dst0, dst1);
867 ILVRL_H2_UB(dst7, dst6, dst2, dst3);
868
869 src += 1;
870
871 tmp2 = __msa_copy_u_w((v4i32) dst0, 0);
872 tmp3 = __msa_copy_u_w((v4i32) dst0, 1);
873 tmp0 = __msa_copy_u_h((v8i16) dst2, 0);
874 tmp1 = __msa_copy_u_h((v8i16) dst2, 2);
875 SW(tmp2, src);
876 SH(tmp0, src + 4);
877 src += stride;
878 SW(tmp3, src);
879 SH(tmp1, src + 4);
880 src += stride;
881
882 tmp2 = __msa_copy_u_w((v4i32) dst0, 2);
883 tmp3 = __msa_copy_u_w((v4i32) dst0, 3);
884 tmp0 = __msa_copy_u_h((v8i16) dst2, 4);
885 tmp1 = __msa_copy_u_h((v8i16) dst2, 6);
886 SW(tmp2, src);
887 SH(tmp0, src + 4);
888 src += stride;
889 SW(tmp3, src);
890 SH(tmp1, src + 4);
891 src += stride;
892
893 tmp2 = __msa_copy_u_w((v4i32) dst1, 0);
894 tmp3 = __msa_copy_u_w((v4i32) dst1, 1);
895 tmp0 = __msa_copy_u_h((v8i16) dst3, 0);
896 tmp1 = __msa_copy_u_h((v8i16) dst3, 2);
897 SW(tmp2, src);
898 SH(tmp0, src + 4);
899 src += stride;
900 SW(tmp3, src);
901 SH(tmp1, src + 4);
902 src += stride;
903
904 tmp2 = __msa_copy_u_w((v4i32) dst1, 2);
905 tmp3 = __msa_copy_u_w((v4i32) dst1, 3);
906 tmp0 = __msa_copy_u_h((v8i16) dst3, 4);
907 tmp1 = __msa_copy_u_h((v8i16) dst3, 6);
908 SW(tmp2, src);
909 SH(tmp0, src + 4);
910 src += stride;
911 SW(tmp3, src);
912 SH(tmp1, src + 4);
913 }
914}
915
916static void hevc_loopfilter_chroma_hor_msa(uint8_t *src, int32_t stride,
917 const int32_t *tc, const uint8_t *p_is_pcm,
918 const uint8_t *q_is_pcm)
919{
920 uint8_t *p1_ptr = src - (stride << 1);
921 uint8_t *p0_ptr = src - stride;
922 uint8_t *q0_ptr = src;
923 uint8_t *q1_ptr = src + stride;
924 v2i64 cmp0, cmp1, p_is_pcm_vec, q_is_pcm_vec;
925 v8u16 p1, p0, q0, q1;
926 v8i16 tc_pos, tc_neg;
927 v16i8 zero = { 0 };
928 v8i16 temp0, temp1, delta;
929
930 if (!(tc[0] <= 0) || !(tc[1] <= 0)) {
931 cmp0 = (v2i64) __msa_fill_h(tc[0]);
932 cmp1 = (v2i64) __msa_fill_h(tc[1]);
933 tc_pos = (v8i16) __msa_ilvev_d(cmp1, cmp0);
934 tc_neg = -tc_pos;
935
936 cmp0 = __msa_fill_d(p_is_pcm[0]);
937 cmp1 = __msa_fill_d(p_is_pcm[1]);
938 p_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
939 p_is_pcm_vec = __msa_ceqi_d(p_is_pcm_vec, 0);
940
941 cmp0 = __msa_fill_d(q_is_pcm[0]);
942 cmp1 = __msa_fill_d(q_is_pcm[1]);
943 q_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
944 q_is_pcm_vec = __msa_ceqi_d(q_is_pcm_vec, 0);
945
946 p1 = LD_UH(p1_ptr);
947 p0 = LD_UH(p0_ptr);
948 q0 = LD_UH(q0_ptr);
949 q1 = LD_UH(q1_ptr);
950
951 ILVR_B4_UH(zero, p1, zero, p0, zero, q0, zero, q1, p1, p0, q0, q1);
952
953 temp0 = (v8i16) (q0 - p0);
954 temp1 = (v8i16) (p1 - q1);
955 temp0 <<= 2;
956 temp0 += temp1;
957 delta = __msa_srari_h((v8i16) temp0, 3);
958 CLIP_SH(delta, tc_neg, tc_pos);
959
960 temp0 = (v8i16) ((v8i16) p0 + delta);
961 CLIP_SH_0_255(temp0);
962 temp0 = (v8i16) __msa_bmz_v((v16u8) temp0, (v16u8) p0,
963 (v16u8) p_is_pcm_vec);
964
965 temp1 = (v8i16) ((v8i16) q0 - delta);
966 CLIP_SH_0_255(temp1);
967 temp1 = (v8i16) __msa_bmz_v((v16u8) temp1, (v16u8) q0,
968 (v16u8) q_is_pcm_vec);
969
970 tc_pos = (v8i16) __msa_clei_s_d((v2i64) tc_pos, 0);
971 temp0 = (v8i16) __msa_bmnz_v((v16u8) temp0, (v16u8) p0, (v16u8) tc_pos);
972 temp1 = (v8i16) __msa_bmnz_v((v16u8) temp1, (v16u8) q0, (v16u8) tc_pos);
973
974 temp0 = (v8i16) __msa_pckev_b((v16i8) temp1, (v16i8) temp0);
975 ST_D2(temp0, 0, 1, p0_ptr, stride);
976 }
977}
978
979static void hevc_loopfilter_chroma_ver_msa(uint8_t *src, int32_t stride,
980 const int32_t *tc, const uint8_t *p_is_pcm,
981 const uint8_t *q_is_pcm)
982{
983 v2i64 cmp0, cmp1, p_is_pcm_vec, q_is_pcm_vec;
984 v16u8 src0, src1, src2, src3, src4, src5, src6, src7;
985 v8u16 p1, p0, q0, q1;
986 v8i16 tc_pos, tc_neg;
987 v16i8 zero = { 0 };
988 v8i16 temp0, temp1, delta;
989
990 if (!(tc[0] <= 0) || !(tc[1] <= 0)) {
991 cmp0 = (v2i64) __msa_fill_h(tc[0]);
992 cmp1 = (v2i64) __msa_fill_h(tc[1]);
993 tc_pos = (v8i16) __msa_ilvev_d(cmp1, cmp0);
994 tc_neg = -tc_pos;
995
996 cmp0 = __msa_fill_d(p_is_pcm[0]);
997 cmp1 = __msa_fill_d(p_is_pcm[1]);
998 p_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
999 p_is_pcm_vec = __msa_ceqi_d(p_is_pcm_vec, 0);
1000
1001 cmp0 = __msa_fill_d(q_is_pcm[0]);
1002 cmp1 = __msa_fill_d(q_is_pcm[1]);
1003 q_is_pcm_vec = __msa_ilvev_d(cmp1, cmp0);
1004 q_is_pcm_vec = __msa_ceqi_d(q_is_pcm_vec, 0);
1005
1006 src -= 2;
1007 LD_UB8(src, stride, src0, src1, src2, src3, src4, src5, src6, src7);
1008 TRANSPOSE8x4_UB_UH(src0, src1, src2, src3, src4, src5, src6, src7,
1009 p1, p0, q0, q1);
1010 ILVR_B4_UH(zero, p1, zero, p0, zero, q0, zero, q1, p1, p0, q0, q1);
1011
1012 temp0 = (v8i16) (q0 - p0);
1013 temp1 = (v8i16) (p1 - q1);
1014 temp0 <<= 2;
1015 temp0 += temp1;
1016 delta = __msa_srari_h((v8i16) temp0, 3);
1017 CLIP_SH(delta, tc_neg, tc_pos);
1018
1019 temp0 = (v8i16) ((v8i16) p0 + delta);
1020 CLIP_SH_0_255(temp0);
1021 temp0 = (v8i16) __msa_bmz_v((v16u8) temp0, (v16u8) p0,
1022 (v16u8) p_is_pcm_vec);
1023
1024 temp1 = (v8i16) ((v8i16) q0 - delta);
1025 CLIP_SH_0_255(temp1);
1026 temp1 = (v8i16) __msa_bmz_v((v16u8) temp1, (v16u8) q0,
1027 (v16u8) q_is_pcm_vec);
1028
1029 tc_pos = (v8i16) __msa_clei_s_d((v2i64) tc_pos, 0);
1030 temp0 = (v8i16) __msa_bmnz_v((v16u8) temp0, (v16u8) p0, (v16u8) tc_pos);
1031 temp1 = (v8i16) __msa_bmnz_v((v16u8) temp1, (v16u8) q0, (v16u8) tc_pos);
1032
1033 temp0 = (v8i16) __msa_ilvev_b((v16i8) temp1, (v16i8) temp0);
1034
1035 src += 1;
1036 ST_H8(temp0, 0, 1, 2, 3, 4, 5, 6, 7, src, stride);
1037 }
1038}
1039
1040static void hevc_sao_band_filter_4width_msa(uint8_t *dst, int32_t dst_stride,
1041 const uint8_t *src, int32_t src_stride,
1042 int32_t sao_left_class,
1043 const int16_t *sao_offset_val,
1044 int32_t height)
1045{
1046 v16u8 src0, src1, src2, src3;
1047 v16i8 src0_r, src1_r;
1048 v16i8 offset, offset_val, mask;
1049 v16i8 dst0, offset0, offset1;
1050 v16i8 zero = { 0 };
1051
1052 offset_val = LD_SB(sao_offset_val + 1);
1053 offset_val = (v16i8) __msa_pckev_d((v2i64) offset_val, (v2i64) offset_val);
1054
1055 offset_val = __msa_pckev_b(offset_val, offset_val);
1056 offset1 = (v16i8) __msa_insve_w((v4i32) zero, 3, (v4i32) offset_val);
1057 offset0 = __msa_sld_b(offset1, zero, 28 - ((sao_left_class) & 31));
1058 offset1 = __msa_sld_b(zero, offset1, 28 - ((sao_left_class) & 31));
1059
1060 /* load in advance. */
1061 LD_UB4(src, src_stride, src0, src1, src2, src3);
1062
1063 if (!((sao_left_class > 12) & (sao_left_class < 29))) {
1064 SWAP(offset0, offset1);
1065 }
1066
1067 for (height -= 4; height; height -= 4) {
1068 src += (4 * src_stride);
1069
1070 ILVEV_D2_SB(src0, src1, src2, src3, src0_r, src1_r);
1071
1072 src0_r = (v16i8) __msa_pckev_w((v4i32) src1_r, (v4i32) src0_r);
1073 mask = __msa_srli_b(src0_r, 3);
1074 offset = __msa_vshf_b(mask, offset1, offset0);
1075
1076 src0_r = (v16i8) __msa_xori_b((v16u8) src0_r, 128);
1077 dst0 = __msa_adds_s_b(src0_r, offset);
1078 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1079
1080 /* load in advance. */
1081 LD_UB4(src, src_stride, src0, src1, src2, src3);
1082
1083 /* store results */
1084 ST_W4(dst0, 0, 1, 2, 3, dst, dst_stride);
1085 dst += (4 * dst_stride);
1086 }
1087
1088 ILVEV_D2_SB(src0, src1, src2, src3, src0_r, src1_r);
1089
1090 src0_r = (v16i8) __msa_pckev_w((v4i32) src1_r, (v4i32) src0_r);
1091 mask = __msa_srli_b(src0_r, 3);
1092 offset = __msa_vshf_b(mask, offset1, offset0);
1093
1094 src0_r = (v16i8) __msa_xori_b((v16u8) src0_r, 128);
1095 dst0 = __msa_adds_s_b(src0_r, offset);
1096 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1097
1098 /* store results */
1099 ST_W4(dst0, 0, 1, 2, 3, dst, dst_stride);
1100}
1101
1102static void hevc_sao_band_filter_8width_msa(uint8_t *dst, int32_t dst_stride,
1103 const uint8_t *src, int32_t src_stride,
1104 int32_t sao_left_class,
1105 const int16_t *sao_offset_val,
1106 int32_t height)
1107{
1108 v16u8 src0, src1, src2, src3;
1109 v16i8 src0_r, src1_r, mask0, mask1;
1110 v16i8 offset_mask0, offset_mask1, offset_val;
1111 v16i8 offset0, offset1, dst0, dst1;
1112 v16i8 zero = { 0 };
1113
1114 offset_val = LD_SB(sao_offset_val + 1);
1115 offset_val = (v16i8) __msa_pckev_d((v2i64) offset_val, (v2i64) offset_val);
1116 offset_val = __msa_pckev_b(offset_val, offset_val);
1117 offset1 = (v16i8) __msa_insve_w((v4i32) zero, 3, (v4i32) offset_val);
1118 offset0 = __msa_sld_b(offset1, zero, 28 - ((sao_left_class) & 31));
1119 offset1 = __msa_sld_b(zero, offset1, 28 - ((sao_left_class) & 31));
1120
1121 /* load in advance. */
1122 LD_UB4(src, src_stride, src0, src1, src2, src3);
1123
1124 if (!((sao_left_class > 12) & (sao_left_class < 29))) {
1125 SWAP(offset0, offset1);
1126 }
1127
1128 for (height -= 4; height; height -= 4) {
1129 src += src_stride << 2;
1130
1131 ILVR_D2_SB(src1, src0, src3, src2, src0_r, src1_r);
1132
1133 mask0 = __msa_srli_b(src0_r, 3);
1134 mask1 = __msa_srli_b(src1_r, 3);
1135
1136 offset_mask0 = __msa_vshf_b(mask0, offset1, offset0);
1137 offset_mask1 = __msa_vshf_b(mask1, offset1, offset0);
1138
1139 /* load in advance. */
1140 LD_UB4(src, src_stride, src0, src1, src2, src3);
1141
1142 XORI_B2_128_SB(src0_r, src1_r);
1143
1144 dst0 = __msa_adds_s_b(src0_r, offset_mask0);
1145 dst1 = __msa_adds_s_b(src1_r, offset_mask1);
1146
1147 XORI_B2_128_SB(dst0, dst1);
1148
1149 /* store results */
1150 ST_D4(dst0, dst1, 0, 1, 0, 1, dst, dst_stride);
1151 dst += dst_stride << 2;
1152 }
1153
1154 ILVR_D2_SB(src1, src0, src3, src2, src0_r, src1_r);
1155
1156 mask0 = __msa_srli_b(src0_r, 3);
1157 mask1 = __msa_srli_b(src1_r, 3);
1158
1159 offset_mask0 = __msa_vshf_b(mask0, offset1, offset0);
1160 offset_mask1 = __msa_vshf_b(mask1, offset1, offset0);
1161
1162 XORI_B2_128_SB(src0_r, src1_r);
1163
1164 dst0 = __msa_adds_s_b(src0_r, offset_mask0);
1165 dst1 = __msa_adds_s_b(src1_r, offset_mask1);
1166
1167 XORI_B2_128_SB(dst0, dst1);
1168
1169 /* store results */
1170 ST_D4(dst0, dst1, 0, 1, 0, 1, dst, dst_stride);
1171}
1172
1173static void hevc_sao_band_filter_16multiple_msa(uint8_t *dst,
1174 int32_t dst_stride,
1175 const uint8_t *src,
1176 int32_t src_stride,
1177 int32_t sao_left_class,
1178 const int16_t *sao_offset_val,
1179 int32_t width, int32_t height)
1180{
1181 int32_t w_cnt;
1182 v16u8 src0, src1, src2, src3;
1183 v16i8 out0, out1, out2, out3;
1184 v16i8 mask0, mask1, mask2, mask3;
1185 v16i8 tmp0, tmp1, tmp2, tmp3, offset_val;
1186 v16i8 offset0, offset1;
1187 v16i8 zero = { 0 };
1188
1189 offset_val = LD_SB(sao_offset_val + 1);
1190 offset_val = (v16i8) __msa_pckev_d((v2i64) offset_val, (v2i64) offset_val);
1191 offset_val = __msa_pckev_b(offset_val, offset_val);
1192 offset1 = (v16i8) __msa_insve_w((v4i32) zero, 3, (v4i32) offset_val);
1193 offset0 = __msa_sld_b(offset1, zero, 28 - ((sao_left_class) & 31));
1194 offset1 = __msa_sld_b(zero, offset1, 28 - ((sao_left_class) & 31));
1195
1196 if (!((sao_left_class > 12) & (sao_left_class < 29))) {
1197 SWAP(offset0, offset1);
1198 }
1199
1200 while (height > 0) {
1201 /* load in advance */
1202 LD_UB4(src, src_stride, src0, src1, src2, src3);
1203
1204 for (w_cnt = 16; w_cnt < width; w_cnt += 16) {
1205 mask0 = __msa_srli_b((v16i8) src0, 3);
1206 mask1 = __msa_srli_b((v16i8) src1, 3);
1207 mask2 = __msa_srli_b((v16i8) src2, 3);
1208 mask3 = __msa_srli_b((v16i8) src3, 3);
1209
1210 VSHF_B2_SB(offset0, offset1, offset0, offset1, mask0, mask1,
1211 tmp0, tmp1);
1212 VSHF_B2_SB(offset0, offset1, offset0, offset1, mask2, mask3,
1213 tmp2, tmp3);
1214 XORI_B4_128_UB(src0, src1, src2, src3);
1215
1216 out0 = __msa_adds_s_b((v16i8) src0, tmp0);
1217 out1 = __msa_adds_s_b((v16i8) src1, tmp1);
1218 out2 = __msa_adds_s_b((v16i8) src2, tmp2);
1219 out3 = __msa_adds_s_b((v16i8) src3, tmp3);
1220
1221 /* load for next iteration */
1222 LD_UB4(src + w_cnt, src_stride, src0, src1, src2, src3);
1223
1224 XORI_B4_128_SB(out0, out1, out2, out3);
1225
1226 ST_SB4(out0, out1, out2, out3, dst + w_cnt - 16, dst_stride);
1227 }
1228
1229 mask0 = __msa_srli_b((v16i8) src0, 3);
1230 mask1 = __msa_srli_b((v16i8) src1, 3);
1231 mask2 = __msa_srli_b((v16i8) src2, 3);
1232 mask3 = __msa_srli_b((v16i8) src3, 3);
1233
1234 VSHF_B2_SB(offset0, offset1, offset0, offset1, mask0, mask1, tmp0,
1235 tmp1);
1236 VSHF_B2_SB(offset0, offset1, offset0, offset1, mask2, mask3, tmp2,
1237 tmp3);
1238 XORI_B4_128_UB(src0, src1, src2, src3);
1239
1240 out0 = __msa_adds_s_b((v16i8) src0, tmp0);
1241 out1 = __msa_adds_s_b((v16i8) src1, tmp1);
1242 out2 = __msa_adds_s_b((v16i8) src2, tmp2);
1243 out3 = __msa_adds_s_b((v16i8) src3, tmp3);
1244
1245 XORI_B4_128_SB(out0, out1, out2, out3);
1246
1247 ST_SB4(out0, out1, out2, out3, dst + w_cnt - 16, dst_stride);
1248
1249 src += src_stride << 2;
1250 dst += dst_stride << 2;
1251 height -= 4;
1252 }
1253}
1254
1255static void hevc_sao_edge_filter_0degree_4width_msa(uint8_t *dst,
1256 int32_t dst_stride,
1257 const uint8_t *src,
1258 int32_t src_stride,
1259 const int16_t *sao_offset_val,
1260 int32_t height)
1261{
1262 uint32_t dst_val0, dst_val1;
1263 v16u8 cmp_minus10, diff_minus10, diff_minus11, src_minus10, src_minus11;
1264 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1265 v16i8 sao_offset = LD_SB(sao_offset_val);
1266 v16i8 src_plus10, offset, src0, dst0;
1267 v16u8 const1 = (v16u8) __msa_ldi_b(1);
1268 v16i8 zero = { 0 };
1269
1270 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
1271 src -= 1;
1272
1273 /* load in advance */
1274 LD_UB2(src, src_stride, src_minus10, src_minus11);
1275
1276 for (height -= 2; height; height -= 2) {
1277 src += (2 * src_stride);
1278
1279 src_minus10 = (v16u8) __msa_pckev_d((v2i64) src_minus11,
1280 (v2i64) src_minus10);
1281
1282 src0 = (v16i8) __msa_sldi_b(zero, (v16i8) src_minus10, 1);
1283 src_plus10 = (v16i8) __msa_sldi_b(zero, (v16i8) src_minus10, 2);
1284
1285 cmp_minus10 = ((v16u8) src0 == src_minus10);
1286 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1287 cmp_minus10 = (src_minus10 < (v16u8) src0);
1288 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1289
1290 cmp_minus10 = ((v16u8) src0 == (v16u8) src_plus10);
1291 diff_minus11 = __msa_nor_v(cmp_minus10, cmp_minus10);
1292 cmp_minus10 = ((v16u8) src_plus10 < (v16u8) src0);
1293 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus10);
1294
1295 offset = (v16i8) diff_minus10 + (v16i8) diff_minus11 + 2;
1296
1297 /* load in advance */
1298 LD_UB2(src, src_stride, src_minus10, src_minus11);
1299
1300 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
1301 offset, offset);
1302
1303 src0 = (v16i8) __msa_xori_b((v16u8) src0, 128);
1304 dst0 = __msa_adds_s_b(src0, offset);
1305 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1306
1307 dst_val0 = __msa_copy_u_w((v4i32) dst0, 0);
1308 dst_val1 = __msa_copy_u_w((v4i32) dst0, 2);
1309 SW(dst_val0, dst);
1310 dst += dst_stride;
1311 SW(dst_val1, dst);
1312 dst += dst_stride;
1313 }
1314
1315 src_minus10 = (v16u8) __msa_pckev_d((v2i64) src_minus11,
1316 (v2i64) src_minus10);
1317
1318 src0 = (v16i8) __msa_sldi_b(zero, (v16i8) src_minus10, 1);
1319 src_plus10 = (v16i8) __msa_sldi_b(zero, (v16i8) src_minus10, 2);
1320
1321 cmp_minus10 = ((v16u8) src0 == src_minus10);
1322 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1323 cmp_minus10 = (src_minus10 < (v16u8) src0);
1324 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1325
1326 cmp_minus10 = ((v16u8) src0 == (v16u8) src_plus10);
1327 diff_minus11 = __msa_nor_v(cmp_minus10, cmp_minus10);
1328 cmp_minus10 = ((v16u8) src_plus10 < (v16u8) src0);
1329 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus10);
1330
1331 offset = (v16i8) diff_minus10 + (v16i8) diff_minus11 + 2;
1332 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
1333 offset, offset);
1334
1335 src0 = (v16i8) __msa_xori_b((v16u8) src0, 128);
1336 dst0 = __msa_adds_s_b(src0, offset);
1337 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1338
1339 dst_val0 = __msa_copy_u_w((v4i32) dst0, 0);
1340 dst_val1 = __msa_copy_u_w((v4i32) dst0, 2);
1341
1342 SW(dst_val0, dst);
1343 dst += dst_stride;
1344 SW(dst_val1, dst);
1345}
1346
1347static void hevc_sao_edge_filter_0degree_8width_msa(uint8_t *dst,
1348 int32_t dst_stride,
1349 const uint8_t *src,
1350 int32_t src_stride,
1351 const int16_t *sao_offset_val,
1352 int32_t height)
1353{
1354 uint64_t dst_val0, dst_val1;
1355 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1356 v16u8 const1 = (v16u8) __msa_ldi_b(1);
1357 v16u8 cmp_minus10, diff_minus10, diff_minus11;
1358 v16u8 src0, src1, dst0, src_minus10, src_minus11, src_plus10, src_plus11;
1359 v16i8 offset, sao_offset = LD_SB(sao_offset_val);
1360 v16i8 zeros = { 0 };
1361
1362 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
1363 src -= 1;
1364
1365 /* load in advance */
1366 LD_UB2(src, src_stride, src_minus10, src_minus11);
1367
1368 for (height -= 2; height; height -= 2) {
1369 src += (src_stride << 1);
1370
1371 SLDI_B2_UB(zeros, src_minus10, zeros, src_minus11, 1, src0, src1);
1372 SLDI_B2_UB(zeros, src_minus10, zeros, src_minus11, 2, src_plus10, src_plus11);
1373
1374 PCKEV_D2_UB(src_minus11, src_minus10, src_plus11, src_plus10,
1375 src_minus10, src_plus10);
1376 src0 = (v16u8) __msa_pckev_d((v2i64) src1, (v2i64) src0);
1377
1378 cmp_minus10 = (src0 == src_minus10);
1379 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1380 cmp_minus10 = (src_minus10 < src0);
1381 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1382
1383 cmp_minus10 = (src0 == src_plus10);
1384 diff_minus11 = __msa_nor_v(cmp_minus10, cmp_minus10);
1385 cmp_minus10 = (src_plus10 < src0);
1386 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus10);
1387
1388 offset = (v16i8) diff_minus10 + (v16i8) diff_minus11 + 2;
1389
1390 /* load in advance */
1391 LD_UB2(src, src_stride, src_minus10, src_minus11);
1392
1393 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
1394 offset, offset);
1395
1396 src0 = __msa_xori_b(src0, 128);
1397 dst0 = (v16u8) __msa_adds_s_b((v16i8) src0, offset);
1398 dst0 = __msa_xori_b(dst0, 128);
1399
1400 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
1401 dst_val1 = __msa_copy_u_d((v2i64) dst0, 1);
1402 SD(dst_val0, dst);
1403 dst += dst_stride;
1404 SD(dst_val1, dst);
1405 dst += dst_stride;
1406 }
1407
1408 SLDI_B2_UB(zeros, src_minus10, zeros, src_minus11, 1, src0, src1);
1409 SLDI_B2_UB(zeros, src_minus10, zeros, src_minus11, 2, src_plus10, src_plus11);
1410
1411 PCKEV_D2_UB(src_minus11, src_minus10, src_plus11, src_plus10, src_minus10,
1412 src_plus10);
1413 src0 = (v16u8) __msa_pckev_d((v2i64) src1, (v2i64) src0);
1414
1415 cmp_minus10 = ((v16u8) src0 == src_minus10);
1416 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1417 cmp_minus10 = (src_minus10 < (v16u8) src0);
1418 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1419
1420 cmp_minus10 = (src0 == src_plus10);
1421 diff_minus11 = __msa_nor_v(cmp_minus10, cmp_minus10);
1422 cmp_minus10 = (src_plus10 < src0);
1423 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus10);
1424
1425 offset = (v16i8) diff_minus10 + (v16i8) diff_minus11 + 2;
1426
1427 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
1428 offset, offset);
1429
1430 src0 = __msa_xori_b(src0, 128);
1431 dst0 = (v16u8) __msa_adds_s_b((v16i8) src0, offset);
1432 dst0 = __msa_xori_b(dst0, 128);
1433
1434 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
1435 dst_val1 = __msa_copy_u_d((v2i64) dst0, 1);
1436 SD(dst_val0, dst);
1437 dst += dst_stride;
1438 SD(dst_val1, dst);
1439}
1440
1441static void hevc_sao_edge_filter_0degree_16multiple_msa(uint8_t *dst,
1442 int32_t dst_stride,
1443 const uint8_t *src,
1444 int32_t src_stride,
1445 const int16_t *sao_offset_val,
1446 int32_t width,
1447 int32_t height)
1448{
1449 const uint8_t *src_minus1;
1450 uint8_t *dst_ptr;
1451 int32_t v_cnt;
1452 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1453 v16u8 const1 = (v16u8) __msa_ldi_b(1);
1454 v16i8 sao_offset;
1455 v16u8 cmp_minus10, cmp_plus10, diff_minus10, diff_plus10, cmp_minus11;
1456 v16u8 cmp_plus11, diff_minus11, diff_plus11, cmp_minus12, cmp_plus12;
1457 v16u8 diff_minus12, diff_plus12, cmp_minus13, cmp_plus13, diff_minus13;
1458 v16u8 diff_plus13;
1459 v16u8 src10, src11, src12, src13, dst0, dst1, dst2, dst3;
1460 v16u8 src_minus10, src_minus11, src_minus12, src_minus13;
1461 v16i8 offset_mask0, offset_mask1, offset_mask2, offset_mask3;
1462 v16i8 src_zero0, src_zero1, src_zero2, src_zero3;
1463 v16i8 src_plus10, src_plus11, src_plus12, src_plus13;
1464
1465 sao_offset = LD_SB(sao_offset_val);
1466 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
1467
1468 for (; height; height -= 4) {
1469 src_minus1 = src - 1;
1470 LD_UB4(src_minus1, src_stride,
1471 src_minus10, src_minus11, src_minus12, src_minus13);
1472
1473 for (v_cnt = 0; v_cnt < width; v_cnt += 16) {
1474 src_minus1 += 16;
1475 dst_ptr = dst + v_cnt;
1476 LD_UB4(src_minus1, src_stride, src10, src11, src12, src13);
1477
1478 SLDI_B4_SB(src10, src_minus10, src11, src_minus11,
1479 src12, src_minus12, src13, src_minus13, 1,
1480 src_zero0, src_zero1, src_zero2, src_zero3);
1481 SLDI_B4_SB(src10, src_minus10, src11, src_minus11,
1482 src12, src_minus12, src13, src_minus13, 2,
1483 src_plus10, src_plus11, src_plus12, src_plus13);
1484
1485 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
1486 cmp_plus10 = ((v16u8) src_zero0 == (v16u8) src_plus10);
1487 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
1488 cmp_plus11 = ((v16u8) src_zero1 == (v16u8) src_plus11);
1489 cmp_minus12 = ((v16u8) src_zero2 == src_minus12);
1490 cmp_plus12 = ((v16u8) src_zero2 == (v16u8) src_plus12);
1491 cmp_minus13 = ((v16u8) src_zero3 == src_minus13);
1492 cmp_plus13 = ((v16u8) src_zero3 == (v16u8) src_plus13);
1493
1494 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1495 diff_plus10 = __msa_nor_v(cmp_plus10, cmp_plus10);
1496 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
1497 diff_plus11 = __msa_nor_v(cmp_plus11, cmp_plus11);
1498 diff_minus12 = __msa_nor_v(cmp_minus12, cmp_minus12);
1499 diff_plus12 = __msa_nor_v(cmp_plus12, cmp_plus12);
1500 diff_minus13 = __msa_nor_v(cmp_minus13, cmp_minus13);
1501 diff_plus13 = __msa_nor_v(cmp_plus13, cmp_plus13);
1502
1503 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
1504 cmp_plus10 = ((v16u8) src_plus10 < (v16u8) src_zero0);
1505 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
1506 cmp_plus11 = ((v16u8) src_plus11 < (v16u8) src_zero1);
1507 cmp_minus12 = (src_minus12 < (v16u8) src_zero2);
1508 cmp_plus12 = ((v16u8) src_plus12 < (v16u8) src_zero2);
1509 cmp_minus13 = (src_minus13 < (v16u8) src_zero3);
1510 cmp_plus13 = ((v16u8) src_plus13 < (v16u8) src_zero3);
1511
1512 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1513 diff_plus10 = __msa_bmnz_v(diff_plus10, const1, cmp_plus10);
1514 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
1515 diff_plus11 = __msa_bmnz_v(diff_plus11, const1, cmp_plus11);
1516 diff_minus12 = __msa_bmnz_v(diff_minus12, const1, cmp_minus12);
1517 diff_plus12 = __msa_bmnz_v(diff_plus12, const1, cmp_plus12);
1518 diff_minus13 = __msa_bmnz_v(diff_minus13, const1, cmp_minus13);
1519 diff_plus13 = __msa_bmnz_v(diff_plus13, const1, cmp_plus13);
1520
1521 offset_mask0 = 2 + (v16i8) diff_minus10 + (v16i8) diff_plus10;
1522 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset_mask0,
1523 offset_mask0, offset_mask0, offset_mask0);
1524 offset_mask1 = 2 + (v16i8) diff_minus11 + (v16i8) diff_plus11;
1525 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset_mask1,
1526 offset_mask1, offset_mask1, offset_mask1);
1527 offset_mask2 = 2 + (v16i8) diff_minus12 + (v16i8) diff_plus12;
1528 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset_mask2,
1529 offset_mask2, offset_mask2, offset_mask2);
1530 offset_mask3 = 2 + (v16i8) diff_minus13 + (v16i8) diff_plus13;
1531 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset_mask3,
1532 offset_mask3, offset_mask3, offset_mask3);
1533
1534 XORI_B4_128_SB(src_zero0, src_zero1, src_zero2, src_zero3);
1535
1536 dst0 = (v16u8) __msa_adds_s_b((v16i8) src_zero0, offset_mask0);
1537 dst1 = (v16u8) __msa_adds_s_b((v16i8) src_zero1, offset_mask1);
1538 dst2 = (v16u8) __msa_adds_s_b((v16i8) src_zero2, offset_mask2);
1539 dst3 = (v16u8) __msa_adds_s_b((v16i8) src_zero3, offset_mask3);
1540
1541 XORI_B4_128_UB(dst0, dst1, dst2, dst3);
1542
1543 src_minus10 = src10;
1544 ST_UB(dst0, dst_ptr);
1545 src_minus11 = src11;
1546 ST_UB(dst1, dst_ptr + dst_stride);
1547 src_minus12 = src12;
1548 ST_UB(dst2, dst_ptr + (dst_stride << 1));
1549 src_minus13 = src13;
1550 ST_UB(dst3, dst_ptr + (dst_stride * 3));
1551 }
1552
1553 src += (src_stride << 2);
1554 dst += (dst_stride << 2);
1555 }
1556}
1557
1558static void hevc_sao_edge_filter_90degree_4width_msa(uint8_t *dst,
1559 int32_t dst_stride,
1560 const uint8_t *src,
1561 int32_t src_stride,
1562 const int16_t *sao_offset_val,
1563 int32_t height)
1564{
1565 uint32_t dst_val0, dst_val1;
1566 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1567 v16u8 const1 = (v16u8) __msa_ldi_b(1);
1568 v16i8 dst0;
1569 v16i8 sao_offset = LD_SB(sao_offset_val);
1570 v16u8 cmp_minus10, diff_minus10, cmp_minus11, diff_minus11;
1571 v16u8 src_minus10, src_minus11, src10, src11;
1572 v16i8 src_zero0, src_zero1;
1573 v16i8 offset;
1574 v8i16 offset_mask0, offset_mask1;
1575
1576 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
1577
1578 /* load in advance */
1579 LD_UB2(src - src_stride, src_stride, src_minus10, src_minus11);
1580 LD_UB2(src + src_stride, src_stride, src10, src11);
1581
1582 for (height -= 2; height; height -= 2) {
1583 src += (src_stride << 1);
1584
1585 src_minus10 = (v16u8) __msa_ilvr_b((v16i8) src10, (v16i8) src_minus10);
1586 src_zero0 = __msa_ilvr_b((v16i8) src_minus11, (v16i8) src_minus11);
1587 src_minus11 = (v16u8) __msa_ilvr_b((v16i8) src11, (v16i8) src_minus11);
1588 src_zero1 = __msa_ilvr_b((v16i8) src10, (v16i8) src10);
1589
1590 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
1591 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1592 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
1593 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1594
1595 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
1596 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
1597 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
1598 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
1599
1600 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
1601 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
1602
1603 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
1604 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
1605
1606 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
1607 offset, offset);
1608
1609 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1610 dst0 = __msa_adds_s_b(dst0, offset);
1611 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1612
1613 src_minus10 = src10;
1614 src_minus11 = src11;
1615
1616 /* load in advance */
1617 LD_UB2(src + src_stride, src_stride, src10, src11);
1618
1619 dst_val0 = __msa_copy_u_w((v4i32) dst0, 0);
1620 dst_val1 = __msa_copy_u_w((v4i32) dst0, 2);
1621 SW(dst_val0, dst);
1622 dst += dst_stride;
1623 SW(dst_val1, dst);
1624
1625 dst += dst_stride;
1626 }
1627
1628 src_minus10 = (v16u8) __msa_ilvr_b((v16i8) src10, (v16i8) src_minus10);
1629 src_zero0 = __msa_ilvr_b((v16i8) src_minus11, (v16i8) src_minus11);
1630 src_minus11 = (v16u8) __msa_ilvr_b((v16i8) src11, (v16i8) src_minus11);
1631 src_zero1 = __msa_ilvr_b((v16i8) src10, (v16i8) src10);
1632
1633 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
1634 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1635 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
1636 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1637
1638 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
1639 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
1640 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
1641 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
1642
1643 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
1644 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
1645
1646 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
1647 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
1648
1649 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset,
1650 offset, offset, offset);
1651
1652 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1653 dst0 = __msa_adds_s_b(dst0, offset);
1654 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1655
1656 dst_val0 = __msa_copy_u_w((v4i32) dst0, 0);
1657 dst_val1 = __msa_copy_u_w((v4i32) dst0, 2);
1658 SW(dst_val0, dst);
1659 dst += dst_stride;
1660 SW(dst_val1, dst);
1661}
1662
1663static void hevc_sao_edge_filter_90degree_8width_msa(uint8_t *dst,
1664 int32_t dst_stride,
1665 const uint8_t *src,
1666 int32_t src_stride,
1667 const int16_t *sao_offset_val,
1668 int32_t height)
1669{
1670 uint64_t dst_val0, dst_val1;
1671 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1672 v16u8 const1 = (v16u8) __msa_ldi_b(1);
1673 v16i8 offset, sao_offset = LD_SB(sao_offset_val);
1674 v16i8 src_zero0, src_zero1, dst0;
1675 v16u8 cmp_minus10, diff_minus10, cmp_minus11, diff_minus11;
1676 v16u8 src_minus10, src_minus11, src10, src11;
1677 v8i16 offset_mask0, offset_mask1;
1678
1679 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
1680
1681 /* load in advance */
1682 LD_UB2(src - src_stride, src_stride, src_minus10, src_minus11);
1683 LD_UB2(src + src_stride, src_stride, src10, src11);
1684
1685 for (height -= 2; height; height -= 2) {
1686 src += (src_stride << 1);
1687
1688 src_minus10 = (v16u8) __msa_ilvr_b((v16i8) src10, (v16i8) src_minus10);
1689 src_zero0 = __msa_ilvr_b((v16i8) src_minus11, (v16i8) src_minus11);
1690 src_minus11 = (v16u8) __msa_ilvr_b((v16i8) src11, (v16i8) src_minus11);
1691 src_zero1 = __msa_ilvr_b((v16i8) src10, (v16i8) src10);
1692
1693 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
1694 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1695 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
1696 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1697
1698 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
1699 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
1700 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
1701 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
1702
1703 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
1704 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
1705
1706 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
1707 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
1708
1709 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset,
1710 offset, offset, offset);
1711
1712 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1713 dst0 = __msa_adds_s_b(dst0, offset);
1714 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1715
1716 src_minus10 = src10;
1717 src_minus11 = src11;
1718
1719 /* load in advance */
1720 LD_UB2(src + src_stride, src_stride, src10, src11);
1721
1722 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
1723 dst_val1 = __msa_copy_u_d((v2i64) dst0, 1);
1724 SD(dst_val0, dst);
1725 dst += dst_stride;
1726 SD(dst_val1, dst);
1727 dst += dst_stride;
1728 }
1729
1730 src_minus10 = (v16u8) __msa_ilvr_b((v16i8) src10, (v16i8) src_minus10);
1731 src_zero0 = __msa_ilvr_b((v16i8) src_minus11, (v16i8) src_minus11);
1732 src_minus11 = (v16u8) __msa_ilvr_b((v16i8) src11, (v16i8) src_minus11);
1733 src_zero1 = __msa_ilvr_b((v16i8) src10, (v16i8) src10);
1734
1735 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
1736 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1737 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
1738 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1739
1740 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
1741 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
1742 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
1743 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
1744
1745 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
1746 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
1747
1748 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
1749 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
1750
1751 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
1752 offset, offset);
1753
1754 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1755 dst0 = __msa_adds_s_b(dst0, offset);
1756 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1757
1758 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
1759 dst_val1 = __msa_copy_u_d((v2i64) dst0, 1);
1760 SD(dst_val0, dst);
1761 dst += dst_stride;
1762 SD(dst_val1, dst);
1763}
1764
1765static void hevc_sao_edge_filter_90degree_16multiple_msa(uint8_t *dst,
1766 int32_t dst_stride,
1767 const uint8_t *src,
1768 int32_t src_stride,
1769 const int16_t *
1770 sao_offset_val,
1771 int32_t width,
1772 int32_t height)
1773{
1774 const uint8_t *src_orig = src;
1775 uint8_t *dst_orig = dst;
1776 int32_t h_cnt, v_cnt;
1777 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1778 v16u8 const1 = (v16u8) __msa_ldi_b(1);
1779 v16u8 cmp_minus10, cmp_plus10, diff_minus10, diff_plus10, cmp_minus11;
1780 v16u8 cmp_plus11, diff_minus11, diff_plus11, cmp_minus12, cmp_plus12;
1781 v16u8 diff_minus12, diff_plus12, cmp_minus13, cmp_plus13, diff_minus13;
1782 v16u8 diff_plus13;
1783 v16u8 src10, src_minus10, dst0, src11, src_minus11, dst1;
1784 v16u8 src12, dst2, src13, dst3;
1785 v16i8 offset_mask0, offset_mask1, offset_mask2, offset_mask3, sao_offset;
1786
1787 sao_offset = LD_SB(sao_offset_val);
1788 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
1789
1790 for (v_cnt = 0; v_cnt < width; v_cnt += 16) {
1791 src = src_orig + v_cnt;
1792 dst = dst_orig + v_cnt;
1793
1794 LD_UB2(src - src_stride, src_stride, src_minus10, src_minus11);
1795
1796 for (h_cnt = (height >> 2); h_cnt--;) {
1797 LD_UB4(src + src_stride, src_stride, src10, src11, src12, src13);
1798
1799 cmp_minus10 = (src_minus11 == src_minus10);
1800 cmp_plus10 = (src_minus11 == src10);
1801 cmp_minus11 = (src10 == src_minus11);
1802 cmp_plus11 = (src10 == src11);
1803 cmp_minus12 = (src11 == src10);
1804 cmp_plus12 = (src11 == src12);
1805 cmp_minus13 = (src12 == src11);
1806 cmp_plus13 = (src12 == src13);
1807
1808 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1809 diff_plus10 = __msa_nor_v(cmp_plus10, cmp_plus10);
1810 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
1811 diff_plus11 = __msa_nor_v(cmp_plus11, cmp_plus11);
1812 diff_minus12 = __msa_nor_v(cmp_minus12, cmp_minus12);
1813 diff_plus12 = __msa_nor_v(cmp_plus12, cmp_plus12);
1814 diff_minus13 = __msa_nor_v(cmp_minus13, cmp_minus13);
1815 diff_plus13 = __msa_nor_v(cmp_plus13, cmp_plus13);
1816
1817 cmp_minus10 = (src_minus10 < src_minus11);
1818 cmp_plus10 = (src10 < src_minus11);
1819 cmp_minus11 = (src_minus11 < src10);
1820 cmp_plus11 = (src11 < src10);
1821 cmp_minus12 = (src10 < src11);
1822 cmp_plus12 = (src12 < src11);
1823 cmp_minus13 = (src11 < src12);
1824 cmp_plus13 = (src13 < src12);
1825
1826 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1827 diff_plus10 = __msa_bmnz_v(diff_plus10, const1, cmp_plus10);
1828 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
1829 diff_plus11 = __msa_bmnz_v(diff_plus11, const1, cmp_plus11);
1830 diff_minus12 = __msa_bmnz_v(diff_minus12, const1, cmp_minus12);
1831 diff_plus12 = __msa_bmnz_v(diff_plus12, const1, cmp_plus12);
1832 diff_minus13 = __msa_bmnz_v(diff_minus13, const1, cmp_minus13);
1833 diff_plus13 = __msa_bmnz_v(diff_plus13, const1, cmp_plus13);
1834
1835 offset_mask0 = 2 + (v16i8) diff_minus10 + (v16i8) diff_plus10;
1836 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
1837 offset_mask0, offset_mask0, offset_mask0, offset_mask0);
1838 offset_mask1 = 2 + (v16i8) diff_minus11 + (v16i8) diff_plus11;
1839 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
1840 offset_mask1, offset_mask1, offset_mask1, offset_mask1);
1841 offset_mask2 = 2 + (v16i8) diff_minus12 + (v16i8) diff_plus12;
1842 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
1843 offset_mask2, offset_mask2, offset_mask2, offset_mask2);
1844 offset_mask3 = 2 + (v16i8) diff_minus13 + (v16i8) diff_plus13;
1845 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
1846 offset_mask3, offset_mask3, offset_mask3, offset_mask3);
1847
1848 src_minus10 = src12;
1849 XORI_B4_128_UB(src_minus11, src10, src11, src12);
1850
1851 dst0 = (v16u8) __msa_adds_s_b((v16i8) src_minus11, offset_mask0);
1852 dst1 = (v16u8) __msa_adds_s_b((v16i8) src10, offset_mask1);
1853 dst2 = (v16u8) __msa_adds_s_b((v16i8) src11, offset_mask2);
1854 dst3 = (v16u8) __msa_adds_s_b((v16i8) src12, offset_mask3);
1855
1856 XORI_B4_128_UB(dst0, dst1, dst2, dst3);
1857 src_minus11 = src13;
1858
1859 ST_UB4(dst0, dst1, dst2, dst3, dst, dst_stride);
1860
1861 src += (src_stride << 2);
1862 dst += (dst_stride << 2);
1863 }
1864 }
1865}
1866
1867static void hevc_sao_edge_filter_45degree_4width_msa(uint8_t *dst,
1868 int32_t dst_stride,
1869 const uint8_t *src,
1870 int32_t src_stride,
1871 const int16_t *sao_offset_val,
1872 int32_t height)
1873{
1874 const uint8_t *src_orig;
1875 uint32_t dst_val0, dst_val1;
1876 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1877 v16u8 const1 = (v16u8) __msa_ldi_b(1);
1878 v16i8 offset, sao_offset = LD_SB(sao_offset_val);
1879 v16u8 cmp_minus10, diff_minus10, src_minus10, cmp_minus11, diff_minus11;
1880 v16u8 src_minus11, src10, src11;
1881 v16i8 src_plus0, src_zero0, src_plus1, src_zero1, dst0;
1882 v8i16 offset_mask0, offset_mask1;
1883 v16i8 zeros = { 0 };
1884
1885 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
1886
1887 src_orig = src - 1;
1888
1889 /* load in advance */
1890 LD_UB2(src_orig - src_stride, src_stride, src_minus10, src_minus11);
1891 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
1892
1893 for (height -= 2; height; height -= 2) {
1894 src_orig += (src_stride << 1);
1895
1896 SLDI_B2_SB(zeros, src_minus11, zeros, src10, 1, src_zero0, src_zero1);
1897 SLDI_B2_SB(zeros, src10, zeros, src11, 2, src_plus0, src_plus1);
1898
1899 ILVR_B2_UB(src_plus0, src_minus10, src_plus1, src_minus11, src_minus10,
1900 src_minus11);
1901 ILVR_B2_SB(src_zero0, src_zero0, src_zero1, src_zero1, src_zero0,
1902 src_zero1);
1903
1904 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
1905 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1906 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
1907 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1908
1909 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
1910 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
1911 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
1912 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
1913
1914 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
1915 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
1916
1917 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
1918 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
1919
1920 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset,
1921 offset, offset, offset);
1922
1923 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1924 dst0 = __msa_adds_s_b(dst0, offset);
1925 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1926
1927 src_minus10 = src10;
1928 src_minus11 = src11;
1929
1930 /* load in advance */
1931 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
1932
1933 dst_val0 = __msa_copy_u_w((v4i32) dst0, 0);
1934 dst_val1 = __msa_copy_u_w((v4i32) dst0, 2);
1935 SW(dst_val0, dst);
1936 dst += dst_stride;
1937 SW(dst_val1, dst);
1938
1939 dst += dst_stride;
1940 }
1941
1942 SLDI_B2_SB(zeros, src_minus11, zeros, src10, 1, src_zero0, src_zero1);
1943 SLDI_B2_SB(zeros, src10, zeros, src11, 2, src_plus0, src_plus1);
1944
1945 ILVR_B2_UB(src_plus0, src_minus10, src_plus1, src_minus11, src_minus10,
1946 src_minus11);
1947 ILVR_B2_SB(src_zero0, src_zero0, src_zero1, src_zero1, src_zero0,
1948 src_zero1);
1949
1950 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
1951 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
1952 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
1953 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
1954
1955 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
1956 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
1957 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
1958 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
1959
1960 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
1961 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
1962
1963 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
1964 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
1965
1966 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
1967 offset, offset);
1968
1969 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1970 dst0 = __msa_adds_s_b(dst0, offset);
1971 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
1972
1973 dst_val0 = __msa_copy_u_w((v4i32) dst0, 0);
1974 dst_val1 = __msa_copy_u_w((v4i32) dst0, 2);
1975 SW(dst_val0, dst);
1976 dst += dst_stride;
1977 SW(dst_val1, dst);
1978}
1979
1980static void hevc_sao_edge_filter_45degree_8width_msa(uint8_t *dst,
1981 int32_t dst_stride,
1982 const uint8_t *src,
1983 int32_t src_stride,
1984 const int16_t *sao_offset_val,
1985 int32_t height)
1986{
1987 const uint8_t *src_orig;
1988 uint64_t dst_val0, dst_val1;
1989 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
1990 v16u8 const1 = (v16u8) __msa_ldi_b(1);
1991 v16i8 offset, sao_offset = LD_SB(sao_offset_val);
1992 v16u8 cmp_minus10, diff_minus10, cmp_minus11, diff_minus11;
1993 v16u8 src_minus10, src10, src_minus11, src11;
1994 v16i8 src_zero0, src_plus10, src_zero1, src_plus11, dst0;
1995 v8i16 offset_mask0, offset_mask1;
1996 v16i8 zeros = { 0 };
1997
1998 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
1999 src_orig = src - 1;
2000
2001 /* load in advance */
2002 LD_UB2(src_orig - src_stride, src_stride, src_minus10, src_minus11);
2003 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
2004
2005 for (height -= 2; height; height -= 2) {
2006 src_orig += (src_stride << 1);
2007
2008 SLDI_B2_SB(zeros, src_minus11, zeros, src10, 1, src_zero0, src_zero1);
2009 SLDI_B2_SB(zeros, src10, zeros, src11, 2, src_plus10, src_plus11);
2010
2011 ILVR_B2_UB(src_plus10, src_minus10, src_plus11, src_minus11,
2012 src_minus10, src_minus11);
2013 ILVR_B2_SB(src_zero0, src_zero0, src_zero1, src_zero1,
2014 src_zero0, src_zero1);
2015
2016 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
2017 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
2018 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
2019 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
2020
2021 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
2022 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
2023 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
2024 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
2025
2026 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
2027 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
2028
2029 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
2030 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
2031
2032 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
2033 offset, offset);
2034
2035 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2036 dst0 = __msa_adds_s_b(dst0, offset);
2037 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2038
2039 src_minus10 = src10;
2040 src_minus11 = src11;
2041
2042 /* load in advance */
2043 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
2044
2045 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
2046 dst_val1 = __msa_copy_u_d((v2i64) dst0, 1);
2047 SD(dst_val0, dst);
2048 dst += dst_stride;
2049 SD(dst_val1, dst);
2050 dst += dst_stride;
2051 }
2052
2053 SLDI_B2_SB(zeros, src_minus11, zeros, src10, 1, src_zero0, src_zero1);
2054 SLDI_B2_SB(zeros, src10, zeros, src11, 2, src_plus10, src_plus11);
2055 ILVR_B2_UB(src_plus10, src_minus10, src_plus11, src_minus11, src_minus10,
2056 src_minus11);
2057 ILVR_B2_SB(src_zero0, src_zero0, src_zero1, src_zero1, src_zero0,
2058 src_zero1);
2059
2060 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
2061 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
2062 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
2063 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
2064
2065 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
2066 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
2067 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
2068 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
2069
2070 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
2071 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
2072
2073 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
2074 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
2075
2076 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
2077 offset, offset);
2078
2079 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2080 dst0 = __msa_adds_s_b(dst0, offset);
2081 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2082
2083 src_minus10 = src10;
2084 src_minus11 = src11;
2085
2086 /* load in advance */
2087 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
2088
2089 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
2090 dst_val1 = __msa_copy_u_d((v2i64) dst0, 1);
2091 SD(dst_val0, dst);
2092 dst += dst_stride;
2093 SD(dst_val1, dst);
2094}
2095
2096static void hevc_sao_edge_filter_45degree_16multiple_msa(uint8_t *dst,
2097 int32_t dst_stride,
2098 const uint8_t *src,
2099 int32_t src_stride,
2100 const int16_t *
2101 sao_offset_val,
2102 int32_t width,
2103 int32_t height)
2104{
2105 const uint8_t *src_orig = src;
2106 uint8_t *dst_orig = dst;
2107 int32_t v_cnt;
2108 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
2109 v16u8 const1 = (v16u8) __msa_ldi_b(1);
2110 v16u8 cmp_minus10, cmp_plus10, diff_minus10, diff_plus10, cmp_minus11;
2111 v16u8 cmp_plus11, diff_minus11, diff_plus11, cmp_minus12, cmp_plus12;
2112 v16u8 diff_minus12, diff_plus12, cmp_minus13, cmp_plus13, diff_minus13;
2113 v16u8 diff_plus13, src_minus14, src_plus13;
2114 v16i8 offset_mask0, offset_mask1, offset_mask2, offset_mask3;
2115 v16u8 src10, src_minus10, dst0, src11, src_minus11, dst1;
2116 v16u8 src12, src_minus12, dst2, src13, src_minus13, dst3;
2117 v16i8 src_zero0, src_plus10, src_zero1, src_plus11, src_zero2, src_plus12;
2118 v16i8 src_zero3, sao_offset;
2119
2120 sao_offset = LD_SB(sao_offset_val);
2121 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
2122
2123 for (; height; height -= 4) {
2124 src_orig = src - 1;
2125 dst_orig = dst;
2126 LD_UB4(src_orig, src_stride, src_minus11, src_minus12, src_minus13,
2127 src_minus14);
2128
2129 for (v_cnt = 0; v_cnt < width; v_cnt += 16) {
2130 src_minus10 = LD_UB(src_orig - src_stride);
2131 LD_UB4(src_orig + 16, src_stride, src10, src11, src12, src13);
2132 src_plus13 = LD_UB(src + 1 + v_cnt + (src_stride << 2));
2133 src_orig += 16;
2134
2135 SLDI_B4_SB(src10, src_minus11, src11, src_minus12,
2136 src12, src_minus13, src13, src_minus14, 1,
2137 src_zero0, src_zero1, src_zero2, src_zero3);
2138 SLDI_B2_SB(src11, src_minus12, src12, src_minus13, 2, src_plus10,
2139 src_plus11);
2140
2141 src_plus12 = __msa_sldi_b((v16i8) src13, (v16i8) src_minus14, 2);
2142
2143 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
2144 cmp_plus10 = ((v16u8) src_zero0 == (v16u8) src_plus10);
2145 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
2146 cmp_plus11 = ((v16u8) src_zero1 == (v16u8) src_plus11);
2147 cmp_minus12 = ((v16u8) src_zero2 == src_minus12);
2148 cmp_plus12 = ((v16u8) src_zero2 == (v16u8) src_plus12);
2149 cmp_minus13 = ((v16u8) src_zero3 == src_minus13);
2150 cmp_plus13 = ((v16u8) src_zero3 == src_plus13);
2151
2152 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
2153 diff_plus10 = __msa_nor_v(cmp_plus10, cmp_plus10);
2154 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
2155 diff_plus11 = __msa_nor_v(cmp_plus11, cmp_plus11);
2156 diff_minus12 = __msa_nor_v(cmp_minus12, cmp_minus12);
2157 diff_plus12 = __msa_nor_v(cmp_plus12, cmp_plus12);
2158 diff_minus13 = __msa_nor_v(cmp_minus13, cmp_minus13);
2159 diff_plus13 = __msa_nor_v(cmp_plus13, cmp_plus13);
2160
2161 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
2162 cmp_plus10 = ((v16u8) src_plus10 < (v16u8) src_zero0);
2163 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
2164 cmp_plus11 = ((v16u8) src_plus11 < (v16u8) src_zero1);
2165 cmp_minus12 = (src_minus12 < (v16u8) src_zero2);
2166 cmp_plus12 = ((v16u8) src_plus12 < (v16u8) src_zero2);
2167 cmp_minus13 = (src_minus13 < (v16u8) src_zero3);
2168 cmp_plus13 = (src_plus13 < (v16u8) src_zero3);
2169
2170 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
2171 diff_plus10 = __msa_bmnz_v(diff_plus10, const1, cmp_plus10);
2172 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
2173 diff_plus11 = __msa_bmnz_v(diff_plus11, const1, cmp_plus11);
2174 diff_minus12 = __msa_bmnz_v(diff_minus12, const1, cmp_minus12);
2175 diff_plus12 = __msa_bmnz_v(diff_plus12, const1, cmp_plus12);
2176 diff_minus13 = __msa_bmnz_v(diff_minus13, const1, cmp_minus13);
2177 diff_plus13 = __msa_bmnz_v(diff_plus13, const1, cmp_plus13);
2178
2179 offset_mask0 = 2 + (v16i8) diff_minus10 + (v16i8) diff_plus10;
2180 offset_mask1 = 2 + (v16i8) diff_minus11 + (v16i8) diff_plus11;
2181 offset_mask2 = 2 + (v16i8) diff_minus12 + (v16i8) diff_plus12;
2182 offset_mask3 = 2 + (v16i8) diff_minus13 + (v16i8) diff_plus13;
2183
2184 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
2185 offset_mask0, offset_mask0, offset_mask0, offset_mask0);
2186 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
2187 offset_mask1, offset_mask1, offset_mask1, offset_mask1);
2188 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
2189 offset_mask2, offset_mask2, offset_mask2, offset_mask2);
2190 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
2191 offset_mask3, offset_mask3, offset_mask3, offset_mask3);
2192
2193 XORI_B4_128_SB(src_zero0, src_zero1, src_zero2, src_zero3);
2194
2195 dst0 = (v16u8) __msa_adds_s_b((v16i8) src_zero0, offset_mask0);
2196 dst1 = (v16u8) __msa_adds_s_b((v16i8) src_zero1, offset_mask1);
2197 dst2 = (v16u8) __msa_adds_s_b((v16i8) src_zero2, offset_mask2);
2198 dst3 = (v16u8) __msa_adds_s_b((v16i8) src_zero3, offset_mask3);
2199
2200 XORI_B4_128_UB(dst0, dst1, dst2, dst3);
2201
2202 src_minus11 = src10;
2203 src_minus12 = src11;
2204 src_minus13 = src12;
2205 src_minus14 = src13;
2206
2207 ST_UB4(dst0, dst1, dst2, dst3, dst_orig, dst_stride);
2208 dst_orig += 16;
2209 }
2210
2211 src += (src_stride << 2);
2212 dst += (dst_stride << 2);
2213 }
2214}
2215
2216static void hevc_sao_edge_filter_135degree_4width_msa(uint8_t *dst,
2217 int32_t dst_stride,
2218 const uint8_t *src,
2219 int32_t src_stride,
2220 const int16_t *sao_offset_val,
2221 int32_t height)
2222{
2223 const uint8_t *src_orig;
2224 uint32_t dst_val0, dst_val1;
2225 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
2226 v16u8 const1 = (v16u8) __msa_ldi_b(1);
2227 v16i8 offset, sao_offset = LD_SB(sao_offset_val);
2228 v16i8 src_zero0, src_zero1, dst0;
2229 v16u8 cmp_minus10, diff_minus10, cmp_minus11, diff_minus11;
2230 v16u8 src_minus10, src10, src_minus11, src11;
2231 v8i16 offset_mask0, offset_mask1;
2232 v16i8 zeros = { 0 };
2233
2234 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
2235 src_orig = src - 1;
2236
2237 /* load in advance */
2238 LD_UB2(src_orig - src_stride, src_stride, src_minus10, src_minus11);
2239 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
2240
2241 for (height -= 2; height; height -= 2) {
2242 src_orig += (src_stride << 1);
2243
2244 SLDI_B2_SB(zeros, src_minus11, zeros, src10, 1, src_zero0, src_zero1);
2245 SLDI_B2_UB(zeros, src_minus10, zeros, src_minus11, 2, src_minus10, src_minus11);
2246
2247 ILVR_B2_UB(src10, src_minus10, src11, src_minus11, src_minus10,
2248 src_minus11);
2249 ILVR_B2_SB(src_zero0, src_zero0, src_zero1, src_zero1, src_zero0,
2250 src_zero1);
2251
2252 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
2253 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
2254 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
2255 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
2256
2257 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
2258 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
2259 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
2260 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
2261
2262 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
2263 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
2264
2265 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
2266 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
2267
2268 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
2269 offset, offset);
2270
2271 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2272 dst0 = __msa_adds_s_b(dst0, offset);
2273 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2274
2275 src_minus10 = src10;
2276 src_minus11 = src11;
2277
2278 /* load in advance */
2279 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
2280
2281 dst_val0 = __msa_copy_u_w((v4i32) dst0, 0);
2282 dst_val1 = __msa_copy_u_w((v4i32) dst0, 2);
2283
2284 SW(dst_val0, dst);
2285 dst += dst_stride;
2286 SW(dst_val1, dst);
2287
2288 dst += dst_stride;
2289 }
2290
2291 SLDI_B2_SB(zeros, src_minus11, zeros, src10, 1, src_zero0, src_zero1);
2292 SLDI_B2_UB(zeros, src_minus10, zeros, src_minus11, 2, src_minus10, src_minus11);
2293
2294 ILVR_B2_UB(src10, src_minus10, src11, src_minus11, src_minus10,
2295 src_minus11);
2296 ILVR_B2_SB(src_zero0, src_zero0, src_zero1, src_zero1, src_zero0,
2297 src_zero1);
2298
2299 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
2300 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
2301 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
2302 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
2303
2304 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
2305 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
2306 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
2307 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
2308
2309 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
2310 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
2311
2312 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
2313 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
2314
2315 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
2316 offset, offset);
2317
2318 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2319 dst0 = __msa_adds_s_b(dst0, offset);
2320 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2321
2322 dst_val0 = __msa_copy_u_w((v4i32) dst0, 0);
2323 dst_val1 = __msa_copy_u_w((v4i32) dst0, 2);
2324
2325 SW(dst_val0, dst);
2326 dst += dst_stride;
2327 SW(dst_val1, dst);
2328 dst += dst_stride;
2329}
2330
2331static void hevc_sao_edge_filter_135degree_8width_msa(uint8_t *dst,
2332 int32_t dst_stride,
2333 const uint8_t *src,
2334 int32_t src_stride,
2335 const int16_t *sao_offset_val,
2336 int32_t height)
2337{
2338 const uint8_t *src_orig;
2339 uint64_t dst_val0, dst_val1;
2340 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
2341 v16u8 const1 = (v16u8) __msa_ldi_b(1);
2342 v16i8 offset, sao_offset = LD_SB(sao_offset_val);
2343 v16u8 cmp_minus10, diff_minus10, cmp_minus11, diff_minus11;
2344 v16u8 src_minus10, src10, src_minus11, src11;
2345 v16i8 src_zero0, src_zero1, dst0;
2346 v8i16 offset_mask0, offset_mask1;
2347 v16i8 zeros = { 0 };
2348
2349 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
2350 src_orig = src - 1;
2351
2352 /* load in advance */
2353 LD_UB2(src_orig - src_stride, src_stride, src_minus10, src_minus11);
2354 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
2355
2356 for (height -= 2; height; height -= 2) {
2357 src_orig += (src_stride << 1);
2358
2359 SLDI_B2_SB(zeros, src_minus11, zeros, src10, 1, src_zero0, src_zero1);
2360 SLDI_B2_UB(zeros, src_minus10, zeros, src_minus11, 2, src_minus10, src_minus11);
2361 ILVR_B2_UB(src10, src_minus10, src11, src_minus11, src_minus10,
2362 src_minus11);
2363 ILVR_B2_SB(src_zero0, src_zero0, src_zero1, src_zero1, src_zero0,
2364 src_zero1);
2365
2366 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
2367 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
2368 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
2369 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
2370
2371 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
2372 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
2373 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
2374 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
2375
2376 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
2377 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
2378
2379 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
2380 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
2381
2382 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
2383 offset, offset);
2384
2385 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2386 dst0 = __msa_adds_s_b(dst0, offset);
2387 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2388
2389 src_minus10 = src10;
2390 src_minus11 = src11;
2391
2392 /* load in advance */
2393 LD_UB2(src_orig + src_stride, src_stride, src10, src11);
2394
2395 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
2396 dst_val1 = __msa_copy_u_d((v2i64) dst0, 1);
2397
2398 SD(dst_val0, dst);
2399 dst += dst_stride;
2400 SD(dst_val1, dst);
2401 dst += dst_stride;
2402 }
2403
2404 SLDI_B2_SB(zeros, src_minus11, zeros, src10, 1, src_zero0, src_zero1);
2405 SLDI_B2_UB(zeros, src_minus10, zeros, src_minus11, 2, src_minus10, src_minus11);
2406 ILVR_B2_UB(src10, src_minus10, src11, src_minus11, src_minus10,
2407 src_minus11);
2408 ILVR_B2_SB(src_zero0, src_zero0, src_zero1, src_zero1, src_zero0,
2409 src_zero1);
2410
2411 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
2412 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
2413 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
2414 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
2415
2416 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
2417 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
2418 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
2419 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
2420
2421 offset_mask0 = (v8i16) (__msa_hadd_u_h(diff_minus10, diff_minus10) + 2);
2422 offset_mask1 = (v8i16) (__msa_hadd_u_h(diff_minus11, diff_minus11) + 2);
2423
2424 offset = __msa_pckev_b((v16i8) offset_mask1, (v16i8) offset_mask0);
2425 dst0 = __msa_pckev_b((v16i8) src_zero1, (v16i8) src_zero0);
2426
2427 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset, offset, offset,
2428 offset, offset);
2429
2430 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2431 dst0 = __msa_adds_s_b(dst0, offset);
2432 dst0 = (v16i8) __msa_xori_b((v16u8) dst0, 128);
2433
2434 dst_val0 = __msa_copy_u_d((v2i64) dst0, 0);
2435 dst_val1 = __msa_copy_u_d((v2i64) dst0, 1);
2436
2437 SD(dst_val0, dst);
2438 dst += dst_stride;
2439 SD(dst_val1, dst);
2440 dst += dst_stride;
2441}
2442
2443static void hevc_sao_edge_filter_135degree_16multiple_msa(uint8_t *dst,
2444 int32_t dst_stride,
2445 const uint8_t *src,
2446 int32_t src_stride,
2447 const int16_t *
2448 sao_offset_val,
2449 int32_t width,
2450 int32_t height)
2451{
2452 const uint8_t *src_orig;
2453 uint8_t *dst_orig;
2454 int32_t v_cnt;
2455 v16i8 edge_idx = { 1, 2, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
2456 v16u8 const1 = (v16u8) __msa_ldi_b(1);
2457 v16u8 dst0, dst1, dst2, dst3;
2458 v16u8 cmp_minus10, cmp_minus11, cmp_minus12, cmp_minus13, cmp_plus10;
2459 v16u8 cmp_plus11, cmp_plus12, cmp_plus13, diff_minus10, diff_minus11;
2460 v16u8 diff_minus12, diff_minus13, diff_plus10, diff_plus11, diff_plus12;
2461 v16u8 diff_plus13, src10, src11, src12, src13, src_minus10, src_minus11;
2462 v16u8 src_plus10, src_plus11, src_plus12, src_plus13;
2463 v16i8 src_minus12, src_minus13, src_zero0, src_zero1, src_zero2, src_zero3;
2464 v16i8 offset_mask0, offset_mask1, offset_mask2, offset_mask3, sao_offset;
2465
2466 sao_offset = LD_SB(sao_offset_val);
2467 sao_offset = __msa_pckev_b(sao_offset, sao_offset);
2468
2469 for (; height; height -= 4) {
2470 src_orig = src - 1;
2471 dst_orig = dst;
2472
2473 LD_UB4(src_orig, src_stride, src_minus11, src_plus10, src_plus11,
2474 src_plus12);
2475
2476 for (v_cnt = 0; v_cnt < width; v_cnt += 16) {
2477 src_minus10 = LD_UB(src_orig + 2 - src_stride);
2478 LD_UB4(src_orig + 16, src_stride, src10, src11, src12, src13);
2479 src_plus13 = LD_UB(src_orig + (src_stride << 2));
2480 src_orig += 16;
2481
2482 src_zero0 = __msa_sldi_b((v16i8) src10, (v16i8) src_minus11, 1);
2483 cmp_minus10 = ((v16u8) src_zero0 == src_minus10);
2484 cmp_plus10 = ((v16u8) src_zero0 == src_plus10);
2485
2486 src_zero1 = __msa_sldi_b((v16i8) src11, (v16i8) src_plus10, 1);
2487 src_minus11 = (v16u8) __msa_sldi_b((v16i8) src10,
2488 (v16i8) src_minus11, 2);
2489 cmp_minus11 = ((v16u8) src_zero1 == src_minus11);
2490 cmp_plus11 = ((v16u8) src_zero1 == src_plus11);
2491
2492 src_zero2 = __msa_sldi_b((v16i8) src12, (v16i8) src_plus11, 1);
2493 src_minus12 = __msa_sldi_b((v16i8) src11, (v16i8) src_plus10, 2);
2494 cmp_minus12 = ((v16u8) src_zero2 == (v16u8) src_minus12);
2495 cmp_plus12 = ((v16u8) src_zero2 == src_plus12);
2496
2497 src_zero3 = __msa_sldi_b((v16i8) src13, (v16i8) src_plus12, 1);
2498 src_minus13 = __msa_sldi_b((v16i8) src12, (v16i8) src_plus11, 2);
2499 cmp_minus13 = ((v16u8) src_zero3 == (v16u8) src_minus13);
2500 cmp_plus13 = ((v16u8) src_zero3 == src_plus13);
2501
2502 diff_minus10 = __msa_nor_v(cmp_minus10, cmp_minus10);
2503 diff_plus10 = __msa_nor_v(cmp_plus10, cmp_plus10);
2504 diff_minus11 = __msa_nor_v(cmp_minus11, cmp_minus11);
2505 diff_plus11 = __msa_nor_v(cmp_plus11, cmp_plus11);
2506 diff_minus12 = __msa_nor_v(cmp_minus12, cmp_minus12);
2507 diff_plus12 = __msa_nor_v(cmp_plus12, cmp_plus12);
2508 diff_minus13 = __msa_nor_v(cmp_minus13, cmp_minus13);
2509 diff_plus13 = __msa_nor_v(cmp_plus13, cmp_plus13);
2510
2511 cmp_minus10 = (src_minus10 < (v16u8) src_zero0);
2512 cmp_plus10 = (src_plus10 < (v16u8) src_zero0);
2513 cmp_minus11 = (src_minus11 < (v16u8) src_zero1);
2514 cmp_plus11 = (src_plus11 < (v16u8) src_zero1);
2515 cmp_minus12 = ((v16u8) src_minus12 < (v16u8) src_zero2);
2516 cmp_plus12 = (src_plus12 < (v16u8) src_zero2);
2517 cmp_minus13 = ((v16u8) src_minus13 < (v16u8) src_zero3);
2518 cmp_plus13 = (src_plus13 < (v16u8) src_zero3);
2519
2520 diff_minus10 = __msa_bmnz_v(diff_minus10, const1, cmp_minus10);
2521 diff_plus10 = __msa_bmnz_v(diff_plus10, const1, cmp_plus10);
2522 diff_minus11 = __msa_bmnz_v(diff_minus11, const1, cmp_minus11);
2523 diff_plus11 = __msa_bmnz_v(diff_plus11, const1, cmp_plus11);
2524 diff_minus12 = __msa_bmnz_v(diff_minus12, const1, cmp_minus12);
2525 diff_plus12 = __msa_bmnz_v(diff_plus12, const1, cmp_plus12);
2526 diff_minus13 = __msa_bmnz_v(diff_minus13, const1, cmp_minus13);
2527 diff_plus13 = __msa_bmnz_v(diff_plus13, const1, cmp_plus13);
2528
2529 offset_mask0 = 2 + (v16i8) diff_minus10 + (v16i8) diff_plus10;
2530 offset_mask1 = 2 + (v16i8) diff_minus11 + (v16i8) diff_plus11;
2531 offset_mask2 = 2 + (v16i8) diff_minus12 + (v16i8) diff_plus12;
2532 offset_mask3 = 2 + (v16i8) diff_minus13 + (v16i8) diff_plus13;
2533
2534 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
2535 offset_mask0, offset_mask0, offset_mask0, offset_mask0);
2536 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
2537 offset_mask1, offset_mask1, offset_mask1, offset_mask1);
2538 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
2539 offset_mask2, offset_mask2, offset_mask2, offset_mask2);
2540 VSHF_B2_SB(edge_idx, edge_idx, sao_offset, sao_offset,
2541 offset_mask3, offset_mask3, offset_mask3, offset_mask3);
2542
2543 XORI_B4_128_SB(src_zero0, src_zero1, src_zero2, src_zero3);
2544
2545 dst0 = (v16u8) __msa_adds_s_b((v16i8) src_zero0, offset_mask0);
2546 dst1 = (v16u8) __msa_adds_s_b((v16i8) src_zero1, offset_mask1);
2547 dst2 = (v16u8) __msa_adds_s_b((v16i8) src_zero2, offset_mask2);
2548 dst3 = (v16u8) __msa_adds_s_b((v16i8) src_zero3, offset_mask3);
2549
2550 XORI_B4_128_UB(dst0, dst1, dst2, dst3);
2551
2552 src_minus11 = src10;
2553 src_plus10 = src11;
2554 src_plus11 = src12;
2555 src_plus12 = src13;
2556
2557 ST_UB4(dst0, dst1, dst2, dst3, dst_orig, dst_stride);
2558 dst_orig += 16;
2559 }
2560
2561 src += (src_stride << 2);
2562 dst += (dst_stride << 2);
2563 }
2564}
2565
2566void ff_hevc_loop_filter_luma_h_8_msa(uint8_t *src,
2567 ptrdiff_t src_stride,
2568 int32_t beta, const int32_t *tc,
2569 const uint8_t *no_p, const uint8_t *no_q)
2570{
2571 hevc_loopfilter_luma_hor_msa(src, src_stride, beta, tc, no_p, no_q);
2572}
2573
2574void ff_hevc_loop_filter_luma_v_8_msa(uint8_t *src,
2575 ptrdiff_t src_stride,
2576 int32_t beta, const int32_t *tc,
2577 const uint8_t *no_p, const uint8_t *no_q)
2578{
2579 hevc_loopfilter_luma_ver_msa(src, src_stride, beta, tc, no_p, no_q);
2580}
2581
2582void ff_hevc_loop_filter_chroma_h_8_msa(uint8_t *src,
2583 ptrdiff_t src_stride,
2584 const int32_t *tc, const uint8_t *no_p,
2585 const uint8_t *no_q)
2586{
2587 hevc_loopfilter_chroma_hor_msa(src, src_stride, tc, no_p, no_q);
2588}
2589
2590void ff_hevc_loop_filter_chroma_v_8_msa(uint8_t *src,
2591 ptrdiff_t src_stride,
2592 const int32_t *tc, const uint8_t *no_p,
2593 const uint8_t *no_q)
2594{
2595 hevc_loopfilter_chroma_ver_msa(src, src_stride, tc, no_p, no_q);
2596}
2597
2598void ff_hevc_sao_band_filter_0_8_msa(uint8_t *dst, const uint8_t *src,
2599 ptrdiff_t stride_dst, ptrdiff_t stride_src,
2600 const int16_t *sao_offset_val, int sao_left_class,
2601 int width, int height)
2602{
2603 if (width >> 4) {
2604 hevc_sao_band_filter_16multiple_msa(dst, stride_dst, src, stride_src,
2605 sao_left_class, sao_offset_val,
2606 width - (width % 16), height);
2607 dst += width - (width % 16);
2608 src += width - (width % 16);
2609 width %= 16;
2610 }
2611
2612 if (width >> 3) {
2613 hevc_sao_band_filter_8width_msa(dst, stride_dst, src, stride_src,
2614 sao_left_class, sao_offset_val, height);
2615 dst += 8;
2616 src += 8;
2617 width %= 8;
2618 }
2619
2620 if (width) {
2621 hevc_sao_band_filter_4width_msa(dst, stride_dst, src, stride_src,
2622 sao_left_class, sao_offset_val, height);
2623 }
2624}
2625
2626void ff_hevc_sao_edge_filter_8_msa(uint8_t *dst, const uint8_t *src,
2627 ptrdiff_t stride_dst,
2628 const int16_t *sao_offset_val,
2629 int eo, int width, int height)
2630{
2631 ptrdiff_t stride_src = (2 * MAX_PB_SIZE + AV_INPUT_BUFFER_PADDING_SIZE) / sizeof(uint8_t);
2632
2633 switch (eo) {
2634 case 0:
2635 if (width >> 4) {
2636 hevc_sao_edge_filter_0degree_16multiple_msa(dst, stride_dst,
2637 src, stride_src,
2638 sao_offset_val,
2639 width - (width % 16),
2640 height);
2641 dst += width - (width % 16);
2642 src += width - (width % 16);
2643 width %= 16;
2644 }
2645
2646 if (width >> 3) {
2647 hevc_sao_edge_filter_0degree_8width_msa(dst, stride_dst,
2648 src, stride_src,
2649 sao_offset_val, height);
2650 dst += 8;
2651 src += 8;
2652 width %= 8;
2653 }
2654
2655 if (width) {
2656 hevc_sao_edge_filter_0degree_4width_msa(dst, stride_dst,
2657 src, stride_src,
2658 sao_offset_val, height);
2659 }
2660 break;
2661
2662 case 1:
2663 if (width >> 4) {
2664 hevc_sao_edge_filter_90degree_16multiple_msa(dst, stride_dst,
2665 src, stride_src,
2666 sao_offset_val,
2667 width - (width % 16),
2668 height);
2669 dst += width - (width % 16);
2670 src += width - (width % 16);
2671 width %= 16;
2672 }
2673
2674 if (width >> 3) {
2675 hevc_sao_edge_filter_90degree_8width_msa(dst, stride_dst,
2676 src, stride_src,
2677 sao_offset_val, height);
2678 dst += 8;
2679 src += 8;
2680 width %= 8;
2681 }
2682
2683 if (width) {
2684 hevc_sao_edge_filter_90degree_4width_msa(dst, stride_dst,
2685 src, stride_src,
2686 sao_offset_val, height);
2687 }
2688 break;
2689
2690 case 2:
2691 if (width >> 4) {
2692 hevc_sao_edge_filter_45degree_16multiple_msa(dst, stride_dst,
2693 src, stride_src,
2694 sao_offset_val,
2695 width - (width % 16),
2696 height);
2697 dst += width - (width % 16);
2698 src += width - (width % 16);
2699 width %= 16;
2700 }
2701
2702 if (width >> 3) {
2703 hevc_sao_edge_filter_45degree_8width_msa(dst, stride_dst,
2704 src, stride_src,
2705 sao_offset_val, height);
2706 dst += 8;
2707 src += 8;
2708 width %= 8;
2709 }
2710
2711 if (width) {
2712 hevc_sao_edge_filter_45degree_4width_msa(dst, stride_dst,
2713 src, stride_src,
2714 sao_offset_val, height);
2715 }
2716 break;
2717
2718 case 3:
2719 if (width >> 4) {
2720 hevc_sao_edge_filter_135degree_16multiple_msa(dst, stride_dst,
2721 src, stride_src,
2722 sao_offset_val,
2723 width - (width % 16),
2724 height);
2725 dst += width - (width % 16);
2726 src += width - (width % 16);
2727 width %= 16;
2728 }
2729
2730 if (width >> 3) {
2731 hevc_sao_edge_filter_135degree_8width_msa(dst, stride_dst,
2732 src, stride_src,
2733 sao_offset_val, height);
2734 dst += 8;
2735 src += 8;
2736 width %= 8;
2737 }
2738
2739 if (width) {
2740 hevc_sao_edge_filter_135degree_4width_msa(dst, stride_dst,
2741 src, stride_src,
2742 sao_offset_val, height);
2743 }
2744 break;
2745 }
2746}
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition dsp.h:87
int32_t
int32_t
Definition audioconvert.c:56
SD
#define SD
Definition ccaption_dec.c:944
abs
#define abs(x)
Definition cuda_runtime.h:35
generic_macros_msa.h
LD_UB4
#define LD_UB4(...)
Definition generic_macros_msa.h:296
ST_UB4
#define ST_UB4(...)
Definition generic_macros_msa.h:374
ILVR_B4_UH
#define ILVR_B4_UH(...)
Definition generic_macros_msa.h:1361
ILVR_B2_SB
#define ILVR_B2_SB(...)
Definition generic_macros_msa.h:1338
ILVRL_B2_UB
#define ILVRL_B2_UB(...)
Definition generic_macros_msa.h:1495
SH
#define SH(val, pdst)
Definition generic_macros_msa.h:154
LD_UB2
#define LD_UB2(...)
Definition generic_macros_msa.h:277
PCKEV_B4_UB
#define PCKEV_B4_UB(...)
Definition generic_macros_msa.h:1739
ST_D4
#define ST_D4(in0, in1, idx0, idx1, idx2, idx3, pdst, stride)
Definition generic_macros_msa.h:499
XORI_B2_128_SB
#define XORI_B2_128_SB(...)
Definition generic_macros_msa.h:1835
SWAP
#define SWAP(in0, in1)
Definition generic_macros_msa.h:2237
ILVEV_D2_SB
#define ILVEV_D2_SB(...)
Definition generic_macros_msa.h:1245
PCKEV_B2_UB
#define PCKEV_B2_UB(...)
Definition generic_macros_msa.h:1720
SLDI_B4_SB
#define SLDI_B4_SB(...)
Definition generic_macros_msa.h:644
XORI_B4_128_UB
#define XORI_B4_128_UB(...)
Definition generic_macros_msa.h:1850
TRANSPOSE8x8_UB_UH
#define TRANSPOSE8x8_UB_UH(...)
Definition generic_macros_msa.h:2376
XORI_B4_128_SB
#define XORI_B4_128_SB(...)
Definition generic_macros_msa.h:1851
TRANSPOSE8x4_UB_UH
#define TRANSPOSE8x4_UB_UH(...)
Definition generic_macros_msa.h:2349
PCKEV_D2_UB
#define PCKEV_D2_UB(...)
Definition generic_macros_msa.h:1787
ILVR_B2_UB
#define ILVR_B2_UB(...)
Definition generic_macros_msa.h:1337
ILVR_D2_SB
#define ILVR_D2_SB(...)
Definition generic_macros_msa.h:1444
ST_SB4
#define ST_SB4(...)
Definition generic_macros_msa.h:375
ILVRL_H2_UB
#define ILVRL_H2_UB(...)
Definition generic_macros_msa.h:1506
ST_H8
#define ST_H8(in, idx0, idx1, idx2, idx3, idx4, idx5, idx6, idx7, pdst, stride)
Definition generic_macros_msa.h:429
CLIP_SH
#define CLIP_SH(in, min, max)
Definition generic_macros_msa.h:923
SW
#define SW(val, pdst)
Definition generic_macros_msa.h:167
LD_UB8
#define LD_UB8(...)
Definition generic_macros_msa.h:335
VSHF_B2_SB
#define VSHF_B2_SB(...)
Definition generic_macros_msa.h:662
CLIP_SH_0_255
#define CLIP_SH_0_255(in)
Definition generic_macros_msa.h:935
LD_UH
#define LD_UH(...)
Definition generic_macros_msa.h:34
LD_SB
#define LD_SB(...)
Definition generic_macros_msa.h:33
LD_UH8
#define LD_UH8(...)
Definition generic_macros_msa.h:337
SLDI_B2_UB
#define SLDI_B2_UB(...)
Definition generic_macros_msa.h:622
ST_D2
#define ST_D2(in, idx0, idx1, pdst, stride)
Definition generic_macros_msa.h:491
ST_UB
#define ST_UB(...)
Definition generic_macros_msa.h:40
LD_UB
#define LD_UB(...)
Definition generic_macros_msa.h:32
SLDI_B2_SB
#define SLDI_B2_SB(...)
Definition generic_macros_msa.h:623
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Required number of additionally allocated bytes at the end of the input bitstream for decoding.
Definition defs.h:40
src2
const pixel * src2
Definition h264pred_template.c:421
MAX_PB_SIZE
#define MAX_PB_SIZE
Definition dsp.h:32
hevc_sao_edge_filter_90degree_16multiple_msa
static void hevc_sao_edge_filter_90degree_16multiple_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t width, int32_t height)
Definition hevc_lpf_sao_msa.c:1765
ff_hevc_loop_filter_chroma_h_8_msa
void ff_hevc_loop_filter_chroma_h_8_msa(uint8_t *src, ptrdiff_t src_stride, const int32_t *tc, const uint8_t *no_p, const uint8_t *no_q)
Definition hevc_lpf_sao_msa.c:2582
ff_hevc_loop_filter_luma_v_8_msa
void ff_hevc_loop_filter_luma_v_8_msa(uint8_t *src, ptrdiff_t src_stride, int32_t beta, const int32_t *tc, const uint8_t *no_p, const uint8_t *no_q)
Definition hevc_lpf_sao_msa.c:2574
hevc_loopfilter_luma_ver_msa
static void hevc_loopfilter_luma_ver_msa(uint8_t *src, int32_t stride, int32_t beta, const int32_t *tc, const uint8_t *p_is_pcm, const uint8_t *q_is_pcm)
Definition hevc_lpf_sao_msa.c:450
hevc_sao_edge_filter_135degree_8width_msa
static void hevc_sao_edge_filter_135degree_8width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:2331
hevc_sao_edge_filter_0degree_8width_msa
static void hevc_sao_edge_filter_0degree_8width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:1347
ff_hevc_loop_filter_luma_h_8_msa
void ff_hevc_loop_filter_luma_h_8_msa(uint8_t *src, ptrdiff_t src_stride, int32_t beta, const int32_t *tc, const uint8_t *no_p, const uint8_t *no_q)
Definition hevc_lpf_sao_msa.c:2566
hevc_sao_edge_filter_90degree_4width_msa
static void hevc_sao_edge_filter_90degree_4width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:1558
hevc_sao_edge_filter_45degree_4width_msa
static void hevc_sao_edge_filter_45degree_4width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:1867
hevc_sao_edge_filter_0degree_16multiple_msa
static void hevc_sao_edge_filter_0degree_16multiple_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t width, int32_t height)
Definition hevc_lpf_sao_msa.c:1441
hevc_loopfilter_luma_hor_msa
static void hevc_loopfilter_luma_hor_msa(uint8_t *src, int32_t stride, int32_t beta, const int32_t *tc, const uint8_t *p_is_pcm, const uint8_t *q_is_pcm)
Definition hevc_lpf_sao_msa.c:24
hevc_sao_edge_filter_45degree_8width_msa
static void hevc_sao_edge_filter_45degree_8width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:1980
ff_hevc_loop_filter_chroma_v_8_msa
void ff_hevc_loop_filter_chroma_v_8_msa(uint8_t *src, ptrdiff_t src_stride, const int32_t *tc, const uint8_t *no_p, const uint8_t *no_q)
Definition hevc_lpf_sao_msa.c:2590
hevc_sao_band_filter_8width_msa
static void hevc_sao_band_filter_8width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, int32_t sao_left_class, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:1102
ff_hevc_sao_band_filter_0_8_msa
void ff_hevc_sao_band_filter_0_8_msa(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst, ptrdiff_t stride_src, const int16_t *sao_offset_val, int sao_left_class, int width, int height)
Definition hevc_lpf_sao_msa.c:2598
hevc_loopfilter_chroma_hor_msa
static void hevc_loopfilter_chroma_hor_msa(uint8_t *src, int32_t stride, const int32_t *tc, const uint8_t *p_is_pcm, const uint8_t *q_is_pcm)
Definition hevc_lpf_sao_msa.c:916
hevc_sao_edge_filter_135degree_16multiple_msa
static void hevc_sao_edge_filter_135degree_16multiple_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t width, int32_t height)
Definition hevc_lpf_sao_msa.c:2443
ff_hevc_sao_edge_filter_8_msa
void ff_hevc_sao_edge_filter_8_msa(uint8_t *dst, const uint8_t *src, ptrdiff_t stride_dst, const int16_t *sao_offset_val, int eo, int width, int height)
Definition hevc_lpf_sao_msa.c:2626
hevc_sao_edge_filter_135degree_4width_msa
static void hevc_sao_edge_filter_135degree_4width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:2216
hevc_sao_band_filter_16multiple_msa
static void hevc_sao_band_filter_16multiple_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, int32_t sao_left_class, const int16_t *sao_offset_val, int32_t width, int32_t height)
Definition hevc_lpf_sao_msa.c:1173
hevc_sao_edge_filter_45degree_16multiple_msa
static void hevc_sao_edge_filter_45degree_16multiple_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t width, int32_t height)
Definition hevc_lpf_sao_msa.c:2096
hevc_sao_edge_filter_0degree_4width_msa
static void hevc_sao_edge_filter_0degree_4width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:1255
hevc_sao_edge_filter_90degree_8width_msa
static void hevc_sao_edge_filter_90degree_8width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:1663
hevc_loopfilter_chroma_ver_msa
static void hevc_loopfilter_chroma_ver_msa(uint8_t *src, int32_t stride, const int32_t *tc, const uint8_t *p_is_pcm, const uint8_t *q_is_pcm)
Definition hevc_lpf_sao_msa.c:979
hevc_sao_band_filter_4width_msa
static void hevc_sao_band_filter_4width_msa(uint8_t *dst, int32_t dst_stride, const uint8_t *src, int32_t src_stride, int32_t sao_left_class, const int16_t *sao_offset_val, int32_t height)
Definition hevc_lpf_sao_msa.c:1040
hevcdsp_mips.h
zero
static int zero(InterplayACMContext *s, unsigned ind, unsigned col)
Definition interplayacm.c:121
offset
unsigned offset
Definition libaomenc.c:763
mask
static const uint16_t mask[17]
Definition lzw.c:38
stride
#define stride
Definition h264pred_template.c:536
tmp
static uint8_t tmp[40]
Definition aes_ctr.c:52
src1
#define src1
Definition h264pred.c:141
src0
#define src0
Definition h264pred.c:140
src
#define src
Definition vp8dsp.c:248
height
#define height
Definition dsp.h:89
width
#define width
Definition dsp.h:89
q1
static const uint8_t q1[256]
Definition twofish.c:100
q0
static const uint8_t q0[256]
Definition twofish.c:81
delta
float delta
Definition vorbis_enc_data.h:430
ST_W4
#define ST_W4(in, idx0, idx1, idx2, idx3, pdst, stride)
Definition vp8_lpf_lsx.c:234
Generated on Sun Aug 23 2026 19:20:55 for FFmpeg by doxygen 1.13.2