FFmpeg
Loading...
Searching...
No Matches
ops_static.c
Go to the documentation of this file.
1/*
2 * Copyright (C) 2026 Ramiro Polla
3 *
4 * This file is part of FFmpeg.
5 *
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
10 *
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 */
20
21#include <assert.h>
22#include <limits.h>
23#include <stdint.h>
24#include <stdio.h>
25#include <stdlib.h>
26#include <string.h>
27
28#ifdef _WIN32
29#include <io.h>
30#include <fcntl.h>
31#endif
32
33/**
34 * This file is compiled as a standalone build-time tool and must not depend
35 * on internal FFmpeg libraries. The necessary utils are redefined below using
36 * standard C equivalents.
37 */
38
39#define AVUTIL_AVASSERT_H
40#define AVUTIL_LOG_H
41#define AVUTIL_MACROS_H
42#define AVUTIL_MEM_H
43#define AV_STRINGIFY(s) AV_TOSTRING(s)
44#define AV_TOSTRING(s) #s
45#define av_assert0(cond) do { \
46 if (!(cond)) { \
47 fprintf(stderr, "Assertion %s failed at %s:%d\n", \
48 AV_STRINGIFY(cond), __FILE__, __LINE__); \
49 abort(); \
50 } \
51} while (0)
52#define av_malloc(s) malloc(s)
53#define av_mallocz(s) calloc(1, s)
54#define av_realloc(p, s) realloc(p, s)
55#define av_strdup(s) strdup(s)
56#define av_free(p) free(p)
57#define FFMAX(a,b) ((a) > (b) ? (a) : (b))
58#define FFMIN(a,b) ((a) > (b) ? (b) : (a))
59#define FF_ARRAY_ELEMS(a) (sizeof(a) / sizeof((a)[0]))
60#define MKTAG(a,b,c,d) ((a) | ((b) << 8) | ((c) << 16) | ((unsigned)(d) << 24))
61#define FFALIGN(x, a) (((x)+(a)-1)&~((a)-1))
62
63static void av_freep(void *ptr)
64{
65 void **pptr = (void **) ptr;
66 if (pptr) {
67 ptr = *pptr;
68 if (ptr)
69 free(ptr);
70 *pptr = NULL;
71 }
72}
73
74static void *av_memdup(const void *p, size_t size)
75{
76 void *ptr = NULL;
77 if (p) {
78 ptr = av_malloc(size);
79 if (ptr)
80 memcpy(ptr, p, size);
81 }
82 return ptr;
83}
84
85#include "libavutil/dynarray.h"
86
87static void *av_dynarray2_add(void **tab_ptr, int *nb_ptr, size_t elem_size,
88 const uint8_t *elem_data)
89{
90 uint8_t *tab_elem_data = NULL;
91
92 FF_DYNARRAY_ADD(INT_MAX, elem_size, *tab_ptr, *nb_ptr, {
93 tab_elem_data = (uint8_t *)*tab_ptr + (*nb_ptr) * elem_size;
94 if (elem_data)
95 memcpy(tab_elem_data, elem_data, elem_size);
96 }, {
97 av_freep(tab_ptr);
98 *nb_ptr = 0;
99 });
100 return tab_elem_data;
101}
102
103#include "libavutil/bprint.c"
104
105/*********************************************************************/
106#include "rasm.c"
107#include "rasm_print.c"
108#include "ops_impl.h"
109
110/**
111 * Implementation parameters for all exported functions. This list is
112 * compiled by performing a dummy run of all conversions in sws_ops and
113 * collecting all functions that need to be generated. This is achieved
114 * by running:
115 * make fate-sws-ops-entries-aarch64 GEN=1
116 */
121
123#define ENTRY(fname, ...) { .name = #fname, .params = __VA_ARGS__ },
124#include "ops_entries.c"
125#undef ENTRY
126 { NULL }
127};
128
129#include "ops_asmgen.c"
130
131/*********************************************************************/
132#define IMPL_PRIV(s) a64op_off(s->impl, offsetof_impl_priv)
133
134/**
135 * Set node where the continuation address will be loaded and impl will
136 * be incremented. This should be done right after impl->priv has been
137 * used.
138 */
140{
141 RasmContext *r = s->rctx;
142 s->load_cont_node = rasm_get_current_node(r);
143}
144
145/*********************************************************************/
146/* SWS_UOP_READ_BIT: fractional read (1 bit) from single plane */
147/* SWS_UOP_READ_NIBBLE: fractional read (4 bits) from single plane */
148/* SWS_UOP_READ_PACKED: simple packed byte-aligned read */
149/* SWS_UOP_READ_PLANAR: simple planar byte-aligned read */
150
152 SwsAArch64OpRegs *regs)
153{
154 RasmContext *r = s->rctx;
155 AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]);
156 AArch64VecViews bitmask_vec = a64op_vec_views(regs->vk[1]);
157
158 rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;");
159 i_ldr(r, shift_vec.q, IMPL_PRIV(s));
161 if (p->block_size == 16) {
162 i_movi(r, bitmask_vec.b16, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 16 times>};");
163 } else {
164 i_movi(r, bitmask_vec.b8, IMM(1)); CMT("v128 bitmask_vec = {1 <repeats 8 times>, 0 <repeats 8 times>};");
165 }
166}
167
169 SwsAArch64OpRegs *regs)
170{
171 RasmContext *r = s->rctx;
173
174 rasm_annotate_next(r, "v128 nibble_mask = {0xf <repeats 8 times>, 0x0 <repeats 8 times>};");
175 i_movi(r, nibble_mask.b8, IMM(0x0f));
176}
177
179 SwsAArch64OpRegs *regs)
180{
181 RasmContext *r = s->rctx;
182 AArch64VecViews shift_vec = a64op_vec_views(regs->vk[0]);
183
184 rasm_annotate_next(r, "v128 shift_vec = impl->priv.v128;");
185 i_ldr(r, shift_vec.q, IMPL_PRIV(s));
187}
188
190 SwsAArch64OpRegs *regs)
191{
192 RasmContext *r = s->rctx;
193 RasmOp *vmask = regs->vk;
194 RasmOp mask_gpr = a64op_w(s->tmp0);
195 uint32_t mask_val[4] = { 0 };
196
197 /* Generate masks. */
198 rasm_add_comment(r, "generate masks");
199 LOOP_MASK(p, i) {
200 uint32_t val = (1u << p->par.pack.pattern[i]) - 1;
201 for (int j = 0; j < 4; j++) {
202 if (mask_val[j] == val) {
203 mask_val[i] = mask_val[j];
204 vmask[i] = vmask[j];
205 break;
206 }
207 }
208 if (!mask_val[i]) {
209 /**
210 * All-one values in movi only work up to 8-bit, and then
211 * at full 16- or 32-bit, but not for intermediate values
212 * like 10-bit. In those cases, we use mov + dup instead.
213 */
214 if (val <= 0xff || val == 0xffff) {
215 i_movi(r, vmask[i], IMM(val));
216 } else {
217 i_mov (r, mask_gpr, IMM(val));
218 i_dup (r, vmask[i], mask_gpr);
219 }
220 mask_val[i] = val;
221 vmask[i] = v_16b(vmask[i]);
222 }
223 }
224}
225
227 SwsAArch64OpRegs *regs)
228{
229 RasmContext *r = s->rctx;
230 RasmOp *vk = regs->vk;
231
232 /**
233 * TODO
234 * - pack elements in impl->priv and perform smaller loads
235 * - if only 1 element and not vh, load directly with ld1r
236 */
237
238 bool load_priv = false;
239 LOOP_MASK(p, i) {
240 if (!((p->par.clear.zero | p->par.clear.one) & SWS_COMP(i)))
241 load_priv = true;
242 }
243 if (load_priv) {
244 i_ldr(r, v_q(vk[0]), IMPL_PRIV(s)); CMT("v128 clear_vec = impl->priv.v128;");
246 }
247}
248
250 SwsAArch64OpRegs *regs)
251{
252 RasmContext *r = s->rctx;
253 RasmOp *vk = regs->vk;
254
255 RasmOp add_vec = regs->vt[0];
256 i_ldr(r, v_q(add_vec), IMPL_PRIV(s)); CMT("v128 add_vec = impl->priv.v128;");
258 LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(add_vec, i)); CMTF("v128 vadd%u = add_vec[%u];", i, i); }
259}
260
262 SwsAArch64OpRegs *regs)
263{
264 RasmContext *r = s->rctx;
265 RasmOp *vk = regs->vk;
266
267 RasmOp min_vec = regs->vt[0];
268 i_ldr(r, v_q(min_vec), IMPL_PRIV(s)); CMT("v128 min_vec = impl->priv.v128;");
270 LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(min_vec, i)); CMTF("v128 vmin%u = min_vec[%u];", i, i); }
271}
272
274 SwsAArch64OpRegs *regs)
275{
276 RasmContext *r = s->rctx;
277 RasmOp *vk = regs->vk;
278
279 RasmOp max_vec = regs->vt[0];
280 i_ldr(r, v_q(max_vec), IMPL_PRIV(s)); CMT("v128 max_vec = impl->priv.v128;");
282 LOOP_MASK(p, i) { i_dup(r, vk[i], a64op_elem(max_vec, i)); CMTF("v128 vmax%u = max_vec[%u];", i, i); }
283}
284
286 SwsAArch64OpRegs *regs)
287{
288 RasmContext *r = s->rctx;
289 RasmOp scale_vec = regs->vk[0];
290
291 RasmOp priv_ptr = s->tmp0;
292 i_add (r, priv_ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *scale_vec_ptr = &impl->priv;");
294 i_ld1r(r, vv_1(scale_vec), a64op_base(priv_ptr)); CMT("v128 scale_vec = broadcast(*scale_vec_ptr);");
295}
296
298 SwsAArch64OpRegs *regs)
299{
300 RasmContext *r = s->rctx;
301 RasmOp *sl = regs->sl;
302 RasmOp *sh = regs->sh;
303 RasmOp *vc = regs->vk;
304 RasmOp *vt = regs->vt;
305
306 RasmOp ptr = s->tmp0;
307 RasmOp coeff_veclist;
308
309 /* Preload coefficients from impl->priv. */
310 const int num_vregs = linear_num_vregs(p);
311 av_assert0(num_vregs <= 4);
312 switch (num_vregs) {
313 case 1: coeff_veclist = vv_1(vc[0]); break;
314 case 2: coeff_veclist = vv_2(vc[0], vc[1]); break;
315 case 3: coeff_veclist = vv_3(vc[0], vc[1], vc[2]); break;
316 case 4: coeff_veclist = vv_4(vc[0], vc[1], vc[2], vc[3]); break;
317 }
318 i_add(r, ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *vcoeff_ptr = &impl->priv;");
320 i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;");
321
322 /**
323 * Integer mul/mla by element does not exist for 8-bit elements, and
324 * requires the element register to be in v0-v15 for 16-bit elements.
325 * For these types, the coefficients used as multiplication operands
326 * are broadcast into full temp vectors instead. Temp vectors 8-11
327 * are otherwise only used by the floating-point fmul+fadd path, so
328 * they are free for integer types.
329 */
330 const bool dup_coeffs = (p->type == SWS_PIXEL_U8 || p->type == SWS_PIXEL_U16);
331 const int nelems = linear_vreg_nelems(p->type);
332 RasmOp *vint = &vt[8];
333 int i_vint = 0;
334
335 /**
336 * Populate operands matrix from packed data into linear_vcoeff matrix
337 * and compute mask for rows that must be saved before being overwritten.
338 */
339 SwsCompMask save_mask = 0;
340 bool overwritten[4] = { false, false, false, false };
341 int i_coeff = 0;
342 LOOP_MASK(p, i) {
343 bool first = true;
344 for (int j = 0; j < 5; j++) {
345 bool is_offset = (j == 0);
346 int src_j = is_offset ? 4 : (j - 1);
347 if (p->par.lin.zero & SWS_MASK(i, src_j))
348 continue;
349 uint8_t vc_i = i_coeff / nelems;
350 uint8_t vc_j = i_coeff & (nelems - 1);
351 RasmOp vcoeff = a64op_elem(vc[vc_i], vc_j);
352 /**
353 * The first coefficient of a row is not used as a multiplication
354 * operand if it is either an offset (broadcast) or one (move).
355 */
356 bool is_one = !is_offset && (p->par.lin.one & SWS_MASK(i, src_j));
357 if (dup_coeffs && !(first && (is_offset || is_one))) {
358 av_assert0(i_vint < 4);
359 i_dup(r, vint[i_vint], vcoeff); CMTF("vcoeff%u = broadcast(coeff[%u][%u]);", i_vint, i, src_j);
360 vcoeff = vint[i_vint++];
361 }
362 regs->linear_vcoeff[i][j] = vcoeff;
363 i_coeff++;
364 first = false;
365 if (!is_offset && overwritten[src_j])
366 save_mask |= SWS_COMP(src_j);
367 overwritten[i] = true;
368 }
369 }
370
371 /**
372 * Save rows that need to be used as input after they have been already
373 * written to.
374 */
375 RasmOp *tl = &vt[0];
376 RasmOp *th = &vt[4];
377 LOOP (save_mask, i) { i_mov16b(r, tl[i], sl[i]); CMTF("vsrcl[%u] = vl[%u];", i, i); }
378 LOOP_VH(s, save_mask, i) { i_mov16b(r, th[i], sh[i]); CMTF("vsrch[%u] = vh[%u];", i, i); }
379 LOOP (save_mask, i) { sl[i] = tl[i]; }
380 LOOP_VH(s, save_mask, i) { sh[i] = th[i]; }
381}
382
384 SwsAArch64OpRegs *regs)
385{
386 RasmContext *r = s->rctx;
387 RasmOp src_ptr = s->tmp0;
388
389 regs->dither_ptr = src_ptr;
390 i_ldr(r, src_ptr, IMPL_PRIV(s)); CMT("void *ptr = impl->priv.ptr;");
392}
393
394/*********************************************************************/
395/**
396 * Register assignment for CPS functions.
397 *
398 * The entry point of the SwsOpFunc is the `process` function. The
399 * first kernel function is called from `process`, and subsequent
400 * kernel functions are chained by directly branching to the next
401 * operation, using a continuation-passing style design. The last
402 * operation must be a write operation, which returns from the call
403 * to the `process` function.
404 *
405 * The GPRs used by the entire call-chain are listed below.
406 *
407 * Function arguments are passed in r0-r5. After the parameters from
408 * `exec` have been read, r0 is reused to branch to the continuation
409 * functions. After the original parameters from `impl` have been
410 * computed, r1 is reused as the `impl` pointer for each operation.
411 *
412 * Loop iterators are r6 for `bx` and r3 for `y`, reused from
413 * `y_start`, which doesn't need to be preserved.
414 *
415 * The intra-procedure-call temporary registers (r16 and r17) are used
416 * as scratch registers. They may be used by call veneers and PLT code
417 * inserted by the linker, so we cannot expect them to persist across
418 * branches between functions.
419 *
420 * The Platform Register (r18) is not used.
421 *
422 * The read/write data pointers and padding values first use up the
423 * remaining free caller-saved registers, and only then are the
424 * callee-saved registers (r19-r29) used.
425 *
426 * The Link Register (r30) is used when calling the first kernel, so it
427 * must be saved.
428 */
429
430static const int rw_gprs[] = {
431 9, 10, 11, 12,
432 13, 14, 15, 19,
433 20, 21, 22, 23,
434 24, 25, 26, 27,
435};
436
438{
439 /* Loop iterator variables. */
440 s->bx = a64op_gpw(6);
441 s->y = a64op_gpw(3); /* Reused from SwsOpFunc.y_start argument. */
442
443 /* Scratch registers. */
444 s->tmp0 = a64op_gpx(16); /* IP0 */
445 s->tmp1 = a64op_gpx(17); /* IP1 */
446
447 /* Read/Write data pointers. */
448 LOOP(imask, i) { s->in [i] = a64op_gpx(rw_gprs[(i * 4) + 0]); }
449 LOOP(omask, i) { s->out[i] = a64op_gpx(rw_gprs[(i * 4) + 1]); }
450}
451
453{
454 asmgen_common_frame(s, imask, omask);
455
456 /* SwsOpFunc arguments. */
457 s->exec = a64op_gpx(0); // const SwsOpExec *exec
458 s->impl = a64op_gpx(1); // const void *priv
459 s->bx_start = a64op_gpw(2); // int bx_start
460 s->y_start = a64op_gpw(3); // int y_start
461 s->bx_end = a64op_gpw(4); // int bx_end
462 s->y_end = a64op_gpw(5); // int y_end
463
464 /* CPS-related variables. */
465 s->op0_func = a64op_gpx(7);
466 s->op0_impl = a64op_gpx(8);
467
468 /* Read/Write data pointer padding. */
469 LOOP(imask, i) { s->in_bump [i] = a64op_gpx(rw_gprs[(i * 4) + 2]); }
470 LOOP(omask, i) { s->out_bump[i] = a64op_gpx(rw_gprs[(i * 4) + 3]); }
471}
472
474{
475 asmgen_common_frame(s, imask, omask);
476
477 /* CPS-related variables. */
478 s->cont = a64op_gpx(0); /* Reused from SwsOpFunc.exec argument. */
479 s->impl = a64op_gpx(1); /* Same as SwsOpFunc.impl argument. */
480}
481
482/*********************************************************************/
483/* Vector register assignment. */
485{
486 regs->sl[ 0] = a64op_vec( 0);
487 regs->sl[ 1] = a64op_vec( 1);
488 regs->sl[ 2] = a64op_vec( 2);
489 regs->sl[ 3] = a64op_vec( 3);
490 regs->sh[ 0] = a64op_vec( 4);
491 regs->sh[ 1] = a64op_vec( 5);
492 regs->sh[ 2] = a64op_vec( 6);
493 regs->sh[ 3] = a64op_vec( 7);
494 regs->dl[ 0] = a64op_vec( 0);
495 regs->dl[ 1] = a64op_vec( 1);
496 regs->dl[ 2] = a64op_vec( 2);
497 regs->dl[ 3] = a64op_vec( 3);
498 regs->dh[ 0] = a64op_vec( 4);
499 regs->dh[ 1] = a64op_vec( 5);
500 regs->dh[ 2] = a64op_vec( 6);
501 regs->dh[ 3] = a64op_vec( 7);
502 regs->vt[ 0] = a64op_vec(16);
503 regs->vt[ 1] = a64op_vec(17);
504 regs->vt[ 2] = a64op_vec(18);
505 regs->vt[ 3] = a64op_vec(19);
506 regs->vt[ 4] = a64op_vec(20);
507 regs->vt[ 5] = a64op_vec(21);
508 regs->vt[ 6] = a64op_vec(22);
509 regs->vt[ 7] = a64op_vec(23);
510 regs->vt[ 8] = a64op_vec(24);
511 regs->vt[ 9] = a64op_vec(25);
512 regs->vt[10] = a64op_vec(26);
513 regs->vt[11] = a64op_vec(27);
514 regs->vk[ 0] = a64op_vec(28);
515 regs->vk[ 1] = a64op_vec(29);
516 regs->vk[ 2] = a64op_vec(30);
517 regs->vk[ 3] = a64op_vec(31);
518}
519
520/*********************************************************************/
522{
523 RasmContext *r = s->rctx;
524 char func_name[128];
525
526 snprintf(func_name, sizeof(func_name), "ff_sws_process_%04x_neon", nibble_mask(mask));
527 rasm_func_begin(r, func_name, true, false);
529
531
532 /* Load values from impl. */
533 rasm_set_current_node(r, s->setup);
534 RasmOp impl_func = a64op_off(s->impl, offsetof_impl_func);
535 i_ldr(r, s->op0_func, impl_func); CMT("SwsFuncPtr op0_func = impl->func;");
536 i_mov(r, s->op0_impl, s->impl); CMT("SwsOpImpl *op0_impl = impl;");
537
538 /* Reset impl and call first kernel. */
539 rasm_set_current_node(r, s->loop);
540 i_mov(r, s->impl, s->op0_impl); CMT("impl = op0_impl;");
541 i_blr(r, s->op0_func); CMT("op0_func();");
542}
543
544/*********************************************************************/
546{
547 const SwsAArch64OpImplParams *p = &entry->params;
548 RasmContext *r = s->rctx;
549
550 bool is_read = false;
551 bool is_write = false;
552 switch (p->uop) {
553 case SWS_UOP_READ_BIT:
557 is_read = true;
558 break;
563 is_write = true;
564 break;
565 default:
566 break;
567 }
568
569 rasm_func_begin(r, entry->name, true, !is_read);
570 asmgen_op_frame(s, is_read ? p->mask : 0, is_write ? p->mask : 0);
571
572 /**
573 * Set up vector register dimensions and reshape all vectors
574 * accordingly.
575 */
576 size_t el_size = ff_sws_pixel_type_size(p->type);
577 size_t total_size = p->block_size * el_size;
578
579 s->vec_size = FFMIN(total_size, 16);
580 s->use_vh = (s->vec_size != total_size);
581
582 s->el_size = el_size;
583 s->el_count = s->vec_size / el_size;
584 init_vectors_cps(s, &s->regs);
585 reshape_io_vectors(&s->regs, s->el_count, el_size);
586 reshape_temp_vectors(&s->regs, s->el_count, el_size);
587 reshape_const_vectors(&s->regs, s->el_count, el_size);
588
589 /* Common start for continuation-passing style (CPS) functions. */
591
592 /* Set up constants. */
593 switch (p->uop) {
594 case SWS_UOP_READ_BIT: asmgen_setup_read_bit(s, p, &s->regs); break;
595 case SWS_UOP_READ_NIBBLE: asmgen_setup_read_nibble(s, p, &s->regs); break;
596 case SWS_UOP_WRITE_BIT: asmgen_setup_write_bit(s, p, &s->regs); break;
597 case SWS_UOP_UNPACK: asmgen_setup_unpack(s, p, &s->regs); break;
598 case SWS_UOP_CLEAR: asmgen_setup_clear(s, p, &s->regs); break;
599 case SWS_UOP_ADD: asmgen_setup_add(s, p, &s->regs); break;
600 case SWS_UOP_MIN: asmgen_setup_min(s, p, &s->regs); break;
601 case SWS_UOP_MAX: asmgen_setup_max(s, p, &s->regs); break;
602 case SWS_UOP_SCALE: asmgen_setup_scale(s, p, &s->regs); break;
603 case SWS_UOP_LINEAR: asmgen_setup_linear(s, p, &s->regs); break;
604 case SWS_UOP_LINEAR_FMA: asmgen_setup_linear(s, p, &s->regs); break;
605 case SWS_UOP_DITHER: asmgen_setup_dither(s, p, &s->regs); break;
606 default:
607 break;
608 }
609
610 /* Emit uop kernel. */
611 switch (p->uop) {
612 case SWS_UOP_READ_BIT: asmgen_op_read_bit(s, p, &s->regs); break;
613 case SWS_UOP_READ_NIBBLE: asmgen_op_read_nibble(s, p, &s->regs); break;
614 case SWS_UOP_READ_PACKED: asmgen_op_read_packed(s, p, &s->regs); break;
615 case SWS_UOP_READ_PLANAR: asmgen_op_read_planar(s, p, &s->regs); break;
616 case SWS_UOP_WRITE_BIT: asmgen_op_write_bit(s, p, &s->regs); break;
617 case SWS_UOP_WRITE_NIBBLE: asmgen_op_write_nibble(s, p, &s->regs); break;
618 case SWS_UOP_WRITE_PACKED: asmgen_op_write_packed(s, p, &s->regs); break;
619 case SWS_UOP_WRITE_PLANAR: asmgen_op_write_planar(s, p, &s->regs); break;
620 case SWS_UOP_SWAP_BYTES: asmgen_op_swap_bytes(s, p, &s->regs); break;
621 case SWS_UOP_PERMUTE: asmgen_op_move(s, p, &s->regs); break;
622 case SWS_UOP_COPY: asmgen_op_move(s, p, &s->regs); break;
623 case SWS_UOP_UNPACK: asmgen_op_unpack(s, p, &s->regs); break;
624 case SWS_UOP_PACK: asmgen_op_pack(s, p, &s->regs); break;
625 case SWS_UOP_LSHIFT: asmgen_op_lshift(s, p, &s->regs); break;
626 case SWS_UOP_RSHIFT: asmgen_op_rshift(s, p, &s->regs); break;
627 case SWS_UOP_CLEAR: asmgen_op_clear(s, p, &s->regs); break;
628 case SWS_UOP_TO_U8: asmgen_op_convert(s, p, &s->regs); break;
629 case SWS_UOP_TO_U16: asmgen_op_convert(s, p, &s->regs); break;
630 case SWS_UOP_TO_U32: asmgen_op_convert(s, p, &s->regs); break;
631 case SWS_UOP_TO_F32: asmgen_op_convert(s, p, &s->regs); break;
632 case SWS_UOP_EXPAND_PAIR: asmgen_op_expand(s, p, &s->regs); break;
633 case SWS_UOP_EXPAND_QUAD: asmgen_op_expand(s, p, &s->regs); break;
634 case SWS_UOP_ADD: asmgen_op_add(s, p, &s->regs); break;
635 case SWS_UOP_MIN: asmgen_op_min(s, p, &s->regs); break;
636 case SWS_UOP_MAX: asmgen_op_max(s, p, &s->regs); break;
637 case SWS_UOP_SCALE: asmgen_op_scale(s, p, &s->regs); break;
638 case SWS_UOP_LINEAR: asmgen_op_linear(s, p, &s->regs); break;
639 case SWS_UOP_LINEAR_FMA: asmgen_op_linear(s, p, &s->regs); break;
640 case SWS_UOP_DITHER: asmgen_op_dither(s, p, &s->regs); break;
641 /* TODO implement SWS_UOP_SHUFFLE */
642 default:
643 break;
644 }
645
646 if (is_write) {
647 /* Write functions return directly. */
648 i_ret(r);
649 } else {
650 /* Load continuation address and increment impl pointer. */
651 const size_t priv_size = FFALIGN(aarch64_priv_size(p), alignof_impl);
652 RasmNode *node = rasm_set_current_node(r, s->load_cont_node);
653 RasmOp impl_pre = a64op_pre(s->impl, offsetof_impl_priv + priv_size);
654 i_ldr(r, s->cont, impl_pre); CMT("SwsFuncPtr cont = (++impl)->func;");
656 /* Common end for remaining CPS functions. */
657 i_br (r, s->cont); CMT("jump to cont");
658 }
659}
660
661/*********************************************************************/
662
663/* Generate all functions described by ops_entries.c */
664static int asmgen(void)
665{
666 RasmContext *rctx = rasm_alloc();
667 if (!rctx)
668 return AVERROR(ENOMEM);
669
670 SwsAArch64Context s = { .rctx = rctx };
671 AVBPrint bp;
672 int ret;
673
675
676 /* Generate all process functions using rasm. */
681
682 /* Generate all functions from ops_entries.c using rasm. */
683 const SwsAArch64OpEntry *entries = ops_entries;
684 while (entries->name) {
685 asmgen_op_cps(&s, entries++);
686 if (rctx->error) {
687 ret = rctx->error;
688 goto error;
689 }
690 }
691
692 /* Print all rasm functions to stdout. */
693 printf("#include \"libavutil/aarch64/asm.S\"\n");
694 printf("\n");
695 ret = rasm_print(s.rctx, &bp);
696 if (ret < 0)
697 goto error;
698 fputs(bp.str, stdout);
699
700error:
702 rasm_free(&s.rctx);
703 return ret;
704}
705
706/*********************************************************************/
707int main(int argc, char *argv[])
708{
709#ifdef _WIN32
710 _setmode(_fileno(stdout), _O_BINARY);
711#endif
712
713 return asmgen();
714}
static const struct @070174075374214234277303002223103136214276265127 ops_entries[]
static double val(void *priv, double ch)
Definition aeval.c:77
#define entry
void av_bprint_init(AVBPrint *buf, unsigned size_init, unsigned size_max)
Definition bprint.c:68
#define AV_BPRINT_SIZE_UNLIMITED
#define i(width, name, range_min, range_max)
Definition cbs_h264.c:63
#define s(width, name)
Definition cbs_vp9.c:198
#define NULL
Definition coverity.c:32
__device__ int printf(const char *,...)
int main
Definition dovi_rpuenc.c:38
#define FF_DYNARRAY_ADD(av_size_max, av_elt_size, av_array, av_size, av_success, av_failure)
Add an element to a dynamic array.
Definition dynarray.h:45
int ret
Definition ffplay.c:391
int av_bprint_finalize(AVBPrint *buf, char **ret_str)
Finalize a print buffer.
Definition bprint.c:234
#define AVERROR(e)
Definition error.h:45
#define r
Definition input.c:42
static const uint16_t mask[17]
Definition lzw.c:38
static void asmgen_op_write_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:375
static void asmgen_op_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:603
static void asmgen_op_read_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:276
static void asmgen_op_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:243
#define CMT(comment)
Definition ops_asmgen.c:32
static void asmgen_op_add(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:759
static void reshape_io_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
Definition ops_asmgen.c:36
static void asmgen_op_write_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:332
static void asmgen_op_move(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:452
static void reshape_temp_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
Definition ops_asmgen.c:47
static void asmgen_op_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:462
static void reshape_const_vectors(SwsAArch64OpRegs *regs, int el_count, int el_size)
Definition ops_asmgen.c:54
static void asmgen_op_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:930
static void asmgen_process(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask)
Definition ops_asmgen.c:139
static void asmgen_op_lshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:550
static void asmgen_op_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:919
static void asmgen_op_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:778
static void asmgen_op_convert(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:620
static void asmgen_op_swap_bytes(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:395
static void asmgen_op_read_planar(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:285
static void asmgen_op_expand(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:717
static void asmgen_op_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:822
#define LOOP_VH(s, mask, idx)
Definition ops_asmgen.c:27
static void asmgen_op_rshift(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:567
#define CMTF(fmt,...)
Definition ops_asmgen.c:33
static void asmgen_op_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:214
static void asmgen_op_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:308
static void asmgen_op_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:800
static void asmgen_op_write_packed(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:366
static void asmgen_op_pack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_asmgen.c:503
#define LOOP(mask, idx)
Definition ops_impl.h:56
static uint16_t nibble_mask(SwsCompMask mask)
Definition ops_impl.h:34
static int linear_vreg_nelems(SwsPixelType type)
Definition ops_impl.h:67
static int linear_num_vregs(const SwsAArch64OpImplParams *params)
Definition ops_impl.h:73
#define offsetof_impl_func
Definition ops_impl.h:120
#define alignof_impl
Definition ops_impl.h:122
#define LOOP_MASK(p, idx)
Definition ops_impl.h:63
static int aarch64_priv_size(const SwsAArch64OpImplParams *params)
Number of bytes of private data consumed by each kernel; must match what aarch64_setup() packs.
Definition ops_impl.h:89
#define offsetof_impl_priv
Definition ops_impl.h:121
static void * av_memdup(const void *p, size_t size)
Definition ops_static.c:74
static void asmgen_setup_clear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:226
#define FFMIN(a, b)
Definition ops_static.c:58
static void asmgen_setup_scale(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:285
static void asmgen_setup_read_nibble(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:168
static void asmgen_setup_max(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:273
static void asmgen_setup_read_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:151
static void asmgen_common_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask)
Definition ops_static.c:437
static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:297
static void asmgen_setup_write_bit(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:178
static void asmgen_op_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask)
Definition ops_static.c:473
static void asmgen_setup_min(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:261
static void asmgen_process_frame(SwsAArch64Context *s, SwsCompMask imask, SwsCompMask omask)
Definition ops_static.c:452
static void init_vectors_cps(SwsAArch64Context *s, SwsAArch64OpRegs *regs)
Definition ops_static.c:484
static void asmgen_set_load_cont_node(SwsAArch64Context *s)
Set node where the continuation address will be loaded and impl will be incremented.
Definition ops_static.c:139
static void asmgen_setup_unpack(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:189
static void asmgen_setup_add(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:249
#define av_assert0(cond)
Definition ops_static.c:45
static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask)
Definition ops_static.c:521
static void asmgen_setup_dither(SwsAArch64Context *s, const SwsAArch64OpImplParams *p, SwsAArch64OpRegs *regs)
Definition ops_static.c:383
#define FFALIGN(x, a)
Definition ops_static.c:61
#define IMPL_PRIV(s)
Definition ops_static.c:132
static void * av_dynarray2_add(void **tab_ptr, int *nb_ptr, size_t elem_size, const uint8_t *elem_data)
Definition ops_static.c:87
#define av_malloc(s)
Definition ops_static.c:52
static int asmgen(void)
Definition ops_static.c:664
static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry)
Definition ops_static.c:545
static const int rw_gprs[]
Register assignment for CPS functions.
Definition ops_static.c:430
RasmNode * rasm_get_current_node(RasmContext *rctx)
Definition rasm.c:194
AArch64VecViews a64op_vec_views(RasmOp op)
Definition rasm.c:330
void rasm_free(RasmContext **prctx)
Definition rasm.c:37
RasmContext * rasm_alloc(void)
Definition rasm.c:32
int rasm_func_begin(RasmContext *rctx, const char *name, bool export, bool jumpable)
Definition rasm.c:209
RasmNode * rasm_add_comment(RasmContext *rctx, const char *comment)
Definition rasm.c:117
void rasm_annotate_next(RasmContext *rctx, const char *comment)
Definition rasm.c:263
RasmNode * rasm_set_current_node(RasmContext *rctx, RasmNode *node)
Definition rasm.c:199
#define i_mov16b(rctx, op0, op1)
Definition rasm.h:629
#define i_br(rctx, op0)
Definition rasm.h:555
#define i_dup(rctx, op0, op1)
Definition rasm.h:558
static RasmOp a64op_base(RasmOp op)
Definition rasm.h:507
static RasmOp vv_4(RasmOp op0, RasmOp op1, RasmOp op2, RasmOp op3)
Definition rasm.h:450
static RasmOp a64op_vec(uint8_t n)
Definition rasm.h:384
#define i_movi(rctx, op0, op1)
Definition rasm.h:578
static RasmOp a64op_pre(RasmOp op, int16_t imm)
Definition rasm.h:509
static RasmOp a64op_elem(RasmOp op, uint8_t idx)
Definition rasm.h:423
static RasmOp a64op_gpx(uint8_t n)
Definition rasm.h:356
#define i_ld1(rctx, op0, op1)
Definition rasm.h:566
static RasmOp v_q(RasmOp op)
Definition rasm.h:435
static RasmOp a64op_off(RasmOp op, int16_t imm)
Definition rasm.h:508
#define i_ldr(rctx, op0, op1)
Definition rasm.h:572
static RasmOp vv_2(RasmOp op0, RasmOp op1)
Definition rasm.h:448
static RasmOp v_16b(RasmOp op)
Definition rasm.h:439
#define i_add(rctx, op0, op1, op2)
Definition rasm.h:548
#define i_ld1r(rctx, op0, op1)
Definition rasm.h:567
void int rasm_print(RasmContext *rctx, AVBPrint *bp)
Definition rasm_print.c:423
#define i_blr(rctx, op0)
Definition rasm.h:554
static RasmOp vv_1(RasmOp op0)
Definition rasm.h:447
#define i_ret(rctx)
Definition rasm.h:581
static RasmOp vv_3(RasmOp op0, RasmOp op1, RasmOp op2)
Definition rasm.h:449
static RasmOp a64op_w(RasmOp op)
Definition rasm.h:361
#define i_mov(rctx, op0, op1)
Definition rasm.h:577
static RasmOp a64op_gpw(uint8_t n)
Definition rasm.h:355
#define IMM(val)
Definition rasm.h:92
This helper structure is used to mimic the assembler syntax for vector register modifiers.
Definition rasm.h:457
RasmOp b16
Definition rasm.h:466
RasmOp b8
Definition rasm.h:465
RasmOp q
Definition rasm.h:463
int error
Definition rasm.h:192
Implementation parameters for all exported functions.
Definition ops_static.c:117
const char * name
Definition ops_static.c:118
SwsAArch64OpImplParams params
Definition ops_static.c:119
SwsAArch64OpImplParams describes the parameters for an SwsUOpType operation.
Definition ops_impl.h:47
RasmOp vt[12]
Definition ops_asmgen.h:32
RasmOp linear_vcoeff[4][5]
Definition ops_asmgen.h:38
#define av_freep(p)
static void error(const char *err)
int size
Runtime assembler for AArch64.
Definition rasm.h:44
@ SWS_PIXEL_U16
Definition uops.h:43
@ SWS_PIXEL_U8
Definition uops.h:42
#define SWS_COMP(X)
Definition uops.h:133
#define SWS_COMP_ELEMS(N)
Definition uops.h:136
#define SWS_MASK(I, J)
Definition uops.h:275
@ SWS_UOP_TO_U8
Definition uops.h:199
@ SWS_UOP_PACK
Definition uops.h:212
@ SWS_UOP_PERMUTE
Definition uops.h:191
@ SWS_UOP_EXPAND_QUAD
Definition uops.h:198
@ SWS_UOP_READ_PLANAR
Definition uops.h:173
@ SWS_UOP_WRITE_PLANAR
Definition uops.h:182
@ SWS_UOP_READ_NIBBLE
Definition uops.h:178
@ SWS_UOP_MAX
Definition uops.h:208
@ SWS_UOP_READ_BIT
Definition uops.h:179
@ SWS_UOP_SWAP_BYTES
Definition uops.h:195
@ SWS_UOP_MIN
Definition uops.h:207
@ SWS_UOP_LINEAR
Definition uops.h:216
@ SWS_UOP_RSHIFT
Definition uops.h:214
@ SWS_UOP_COPY
Definition uops.h:192
@ SWS_UOP_WRITE_NIBBLE
Definition uops.h:184
@ SWS_UOP_LINEAR_FMA
Definition uops.h:217
@ SWS_UOP_SCALE
Definition uops.h:205
@ SWS_UOP_WRITE_PACKED
Definition uops.h:183
@ SWS_UOP_WRITE_BIT
Definition uops.h:185
@ SWS_UOP_READ_PACKED
Definition uops.h:177
@ SWS_UOP_EXPAND_PAIR
Definition uops.h:197
@ SWS_UOP_TO_U16
Definition uops.h:200
@ SWS_UOP_DITHER
Definition uops.h:218
@ SWS_UOP_ADD
Definition uops.h:206
@ SWS_UOP_CLEAR
Definition uops.h:215
@ SWS_UOP_TO_U32
Definition uops.h:201
@ SWS_UOP_UNPACK
Definition uops.h:211
@ SWS_UOP_TO_F32
Definition uops.h:202
@ SWS_UOP_LSHIFT
Definition uops.h:213
uint8_t SwsCompMask
Bit-mask of components.
Definition uops.h:129
static av_const int ff_sws_pixel_type_size(SwsPixelType type)
Definition uops.h:51