[FFmpeg-devel] [PATCH] lavfi/vf_transpose: fix regression with semiplanar formats

[Rodger Combs]

(e.g. nv12)

Regression since 7b19e76aeb0ace57b99aaef156bbfe592e43e65e
---
 libavfilter/vf_transpose.c              | 50 +++++++++++++++++++--------------
 tests/ref/fate/filter-pixfmts-transpose |  8 +++---
 2 files changed, 33 insertions(+), 25 deletions(-)

diff --git a/libavfilter/vf_transpose.c b/libavfilter/vf_transpose.c
index 3ff4cb4249..74a4bbcf58 100644
--- a/libavfilter/vf_transpose.c
+++ b/libavfilter/vf_transpose.c
@@ -52,6 +52,14 @@ enum TransposeDir {
     TRANSPOSE_CLOCK_FLIP,
 };
 
+typedef struct TransVtable {
+    void (*transpose_8x8)(uint8_t *src, ptrdiff_t src_linesize,
+                          uint8_t *dst, ptrdiff_t dst_linesize);
+    void (*transpose_block)(uint8_t *src, ptrdiff_t src_linesize,
+                            uint8_t *dst, ptrdiff_t dst_linesize,
+                            int w, int h);
+} TransVtable;
+
 typedef struct TransContext {
     const AVClass *class;
     int hsub, vsub;
@@ -61,11 +69,7 @@ typedef struct TransContext {
     int passthrough;    ///< PassthroughType, landscape passthrough mode enabled
     int dir;            ///< TransposeDir
 
-    void (*transpose_8x8)(uint8_t *src, ptrdiff_t src_linesize,
-                          uint8_t *dst, ptrdiff_t dst_linesize);
-    void (*transpose_block)(uint8_t *src, ptrdiff_t src_linesize,
-                            uint8_t *dst, ptrdiff_t dst_linesize,
-                            int w, int h);
+    TransVtable vtables[4];
 } TransContext;
 
 static int query_formats(AVFilterContext *ctx)
@@ -233,19 +237,22 @@ static int config_props_output(AVFilterLink *outlink)
     else
         outlink->sample_aspect_ratio = inlink->sample_aspect_ratio;
 
-    switch (s->pixsteps[0]) {
-    case 1: s->transpose_block = transpose_block_8_c;
-            s->transpose_8x8   = transpose_8x8_8_c;  break;
-    case 2: s->transpose_block = transpose_block_16_c;
-            s->transpose_8x8   = transpose_8x8_16_c; break;
-    case 3: s->transpose_block = transpose_block_24_c;
-            s->transpose_8x8   = transpose_8x8_24_c; break;
-    case 4: s->transpose_block = transpose_block_32_c;
-            s->transpose_8x8   = transpose_8x8_32_c; break;
-    case 6: s->transpose_block = transpose_block_48_c;
-            s->transpose_8x8   = transpose_8x8_48_c; break;
-    case 8: s->transpose_block = transpose_block_64_c;
-            s->transpose_8x8   = transpose_8x8_64_c; break;
+    for (int i = 0; i < 4; i++) {
+        TransVtable *v = &s->vtables[i];
+        switch (s->pixsteps[i]) {
+        case 1: v->transpose_block = transpose_block_8_c;
+                v->transpose_8x8   = transpose_8x8_8_c;  break;
+        case 2: v->transpose_block = transpose_block_16_c;
+                v->transpose_8x8   = transpose_8x8_16_c; break;
+        case 3: v->transpose_block = transpose_block_24_c;
+                v->transpose_8x8   = transpose_8x8_24_c; break;
+        case 4: v->transpose_block = transpose_block_32_c;
+                v->transpose_8x8   = transpose_8x8_32_c; break;
+        case 6: v->transpose_block = transpose_block_48_c;
+                v->transpose_8x8   = transpose_8x8_48_c; break;
+        case 8: v->transpose_block = transpose_block_64_c;
+                v->transpose_8x8   = transpose_8x8_64_c; break;
+        }
     }
 
     av_log(ctx, AV_LOG_VERBOSE,
@@ -290,6 +297,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr,
         uint8_t *dst, *src;
         int dstlinesize, srclinesize;
         int x, y;
+        TransVtable *v = &s->vtables[plane];
 
         dstlinesize = out->linesize[plane];
         dst         = out->data[plane] + start * dstlinesize;
@@ -308,20 +316,20 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr,
 
         for (y = start; y < end - 7; y += 8) {
             for (x = 0; x < outw - 7; x += 8) {
-                s->transpose_8x8(src + x * srclinesize + y * pixstep,
+                v->transpose_8x8(src + x * srclinesize + y * pixstep,
                                  srclinesize,
                                  dst + (y - start) * dstlinesize + x * pixstep,
                                  dstlinesize);
             }
             if (outw - x > 0 && end - y > 0)
-                s->transpose_block(src + x * srclinesize + y * pixstep,
+                v->transpose_block(src + x * srclinesize + y * pixstep,
                                    srclinesize,
                                    dst + (y - start) * dstlinesize + x * pixstep,
                                    dstlinesize, outw - x, end - y);
         }
 
         if (end - y > 0)
-            s->transpose_block(src + 0 * srclinesize + y * pixstep,
+            v->transpose_block(src + 0 * srclinesize + y * pixstep,
                                srclinesize,
                                dst + (y - start) * dstlinesize + 0 * pixstep,
                                dstlinesize, outw, end - y);
diff --git a/tests/ref/fate/filter-pixfmts-transpose b/tests/ref/fate/filter-pixfmts-transpose
index 6f25f3dd27..fcad28b9ff 100644
--- a/tests/ref/fate/filter-pixfmts-transpose
+++ b/tests/ref/fate/filter-pixfmts-transpose
@@ -45,10 +45,10 @@ gray16be            4aef307021a91b1de67f1d4381a39132
 gray16le            76f2afe156edca7ae05cfa4e5867126e
 gray9be             2c425fa532c940d226822da8b3592310
 gray9le             bcc575942910b3c72eaa72e8794f3acd
-nv12                aca847644e5dc0e942419183014981a4
-nv21                098884e968d27286c8cf0d2fb1557dcd
-p010be              5ff62dffa5dfdf823978c4f563f69c94
-p010le              20131abe34e084b04f1d169c66447825
+nv12                1965e3826144686748f2f6b516fca5ba
+nv21                292adaf5271c5c8516b71640458c01f4
+p010be              ad0de2cc9bff81688b182a870fcf7000
+p010le              e7ff5143595021246733ce6bd0a769e8
 rgb0                31ea5da7fe779c6ea0a33f1d28aad918
 rgb24               47654cabaaad79170b90afd5a02161dd
 rgb444be            3cac1f0c43a74d2a95eb02e187070845
-- 
2.16.1