[FFmpeg-devel] [PATCH 3/3] x86/hevc: add ff_hevc_sao_band_filter_{8, 10, 12}_{sse2, avx2}

James Almer jamrial at gmail.com
Fri Jan 30 19:50:14 CET 2015


Original x86 intrinsics code and initial 8bit yasm port by Pierre-Edouard Lepere.
10/12bit yasm ports, refactoring and optimizations by James Almer

Benchmarks of BQTerrace_1920x1080_60_qp22.bin with an Intel Core i5-4200U

width 32
40338 decicycles in sao_band_filter_0_8, 2048 runs, 0 skips
8585 decicycles in ff_hevc_sao_band_filter_8_sse2, 2048 runs, 0 skips
4543 decicycles in ff_hevc_sao_band_filter_8_avx2, 2048 runs, 0 skips

width 64
136046 decicycles in sao_band_filter_0_8, 16384 runs, 0 skips
29366 decicycles in ff_hevc_sao_band_filter_8_sse2, 16384 runs, 0 skips
15357 decicycles in ff_hevc_sao_band_filter_8_avx2, 16383 runs, 1 skips

Signed-off-by: James Almer <jamrial at gmail.com>
---
For reference:
Original intrinsics code: https://github.com/OpenHEVC/openHEVC/blob/shm6.1/libavcodec/x86/hevc_sao_sse.c
Initial 8bit yasm port: https://github.com/OpenHEVC/FFmpeg/blob/rext/libavcodec/x86/hevc_sao.asm

x86_32 port is pretty much done and will follow soon (I just need to make sure the macros are not ugly).
And having it separate will make reviewing easier.

 libavcodec/x86/Makefile       |   3 +-
 libavcodec/x86/hevc_sao.asm   | 262 ++++++++++++++++++++++++++++++++++++++++++
 libavcodec/x86/hevcdsp_init.c |  25 ++++
 3 files changed, 289 insertions(+), 1 deletion(-)
 create mode 100644 libavcodec/x86/hevc_sao.asm

diff --git a/libavcodec/x86/Makefile b/libavcodec/x86/Makefile
index 7c8e7aa..00dacda 100644
--- a/libavcodec/x86/Makefile
+++ b/libavcodec/x86/Makefile
@@ -134,7 +134,8 @@ YASM-OBJS-$(CONFIG_DCA_DECODER)        += x86/dcadsp.o
 YASM-OBJS-$(CONFIG_HEVC_DECODER)       += x86/hevc_mc.o                 \
                                           x86/hevc_deblock.o            \
                                           x86/hevc_idct.o               \
-                                          x86/hevc_res_add.o
+                                          x86/hevc_res_add.o            \
+                                          x86/hevc_sao.o
 YASM-OBJS-$(CONFIG_MLP_DECODER)        += x86/mlpdsp.o
 YASM-OBJS-$(CONFIG_PNG_DECODER)        += x86/pngdsp.o
 YASM-OBJS-$(CONFIG_PRORES_DECODER)     += x86/proresdsp.o
diff --git a/libavcodec/x86/hevc_sao.asm b/libavcodec/x86/hevc_sao.asm
new file mode 100644
index 0000000..f64e70e
--- /dev/null
+++ b/libavcodec/x86/hevc_sao.asm
@@ -0,0 +1,262 @@
+;******************************************************************************
+;* SIMD optimized SAO functions for HEVC decoding
+;*
+;* Copyright (c) 2013 Pierre-Edouard LEPERE
+;* Copyright (c) 2014 James Almer
+;*
+;* This file is part of FFmpeg.
+;*
+;* FFmpeg is free software; you can redistribute it and/or
+;* modify it under the terms of the GNU Lesser General Public
+;* License as published by the Free Software Foundation; either
+;* version 2.1 of the License, or (at your option) any later version.
+;*
+;* FFmpeg is distributed in the hope that it will be useful,
+;* but WITHOUT ANY WARRANTY; without even the implied warranty of
+;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+;* Lesser General Public License for more details.
+;*
+;* You should have received a copy of the GNU Lesser General Public
+;* License along with FFmpeg; if not, write to the Free Software
+;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+;******************************************************************************
+
+%include "libavutil/x86/x86util.asm"
+
+%if ARCH_X86_64
+SECTION_RODATA 32
+
+pw_mask10: times 16 dw 0x03FF
+pw_mask12: times 16 dw 0x0FFF
+
+SECTION_TEXT
+
+%macro HEVC_SAO_BAND_FILTER_INIT 1
+    movsxd        widthq, dword widthm
+    mov          heightd, heightm
+    pxor             m14, m14
+%if %1 > 8
+    mova             m13, [pw_mask %+ %1]
+%endif
+    and            leftq, 31
+    movd             xm0, leftd
+    add            leftq, 1
+    and            leftq, 31
+    movd             xm1, leftd
+    add            leftq, 1
+    and            leftq, 31
+    movd             xm2, leftd
+    add            leftq, 1
+    and            leftq, 31
+    movd             xm3, leftd
+
+%if mmsize > 16
+    cmp           widthq, 16
+    je hevc_sao_band_filter_16_%1 %+ SUFFIX
+%endif
+    cmp           widthq, 8
+    je hevc_sao_band_filter_8_%1 %+ SUFFIX
+%endmacro
+
+%macro HEVC_SAO_BAND_FILTER_INIT_OFFSETS 1
+%if %1 > 8
+    shl           widthd, 1
+%endif
+    SPLATW            m0, xm0
+    SPLATW            m1, xm1
+    SPLATW            m2, xm2
+    SPLATW            m3, xm3
+%if cpuflag(avx2)
+    SPLATW            m4, [offsetq + 2]
+    SPLATW            m5, [offsetq + 4]
+    SPLATW            m6, [offsetq + 6]
+    SPLATW            m7, [offsetq + 8]
+%else
+    movd              m4, [offsetq + 2]
+    movd              m5, [offsetq + 4]
+    movd              m6, [offsetq + 6]
+    movd              m7, [offsetq + 8]
+    SPLATW            m4, m4
+    SPLATW            m5, m5
+    SPLATW            m6, m6
+    SPLATW            m7, m7
+%endif
+
+    add             srcq, widthq
+    add             dstq, widthq
+    neg           widthq
+    mov             tmpq, widthq
+%endmacro
+
+%macro HEVC_SAO_BAND_FILTER_COMPUTE 3
+    psraw             %2, %3, %1-5
+    pcmpeqw          m10, %2, m0
+    pcmpeqw          m11, %2, m1
+    pcmpeqw          m12, %2, m2
+    pcmpeqw           %2, m3
+    pand             m10, m4
+    pand             m11, m5
+    pand             m12, m6
+    pand              %2, m7
+    por              m10, m11
+    por              m12, %2
+    por              m10, m12
+    paddw             %3, m10
+%endmacro
+
+;void ff_hevc_sao_band_filter_8_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src,
+;                                     int16_t *sao_offset_val, int sao_left_class, int width, int height);
+%macro HEVC_SAO_BAND_FILTER_8 0
+cglobal hevc_sao_band_filter_8, 6, 9, 15, dst, src, dststride, srcstride, offset, left, width, height, tmp
+    HEVC_SAO_BAND_FILTER_INIT 8
+    HEVC_SAO_BAND_FILTER_INIT_OFFSETS 8
+
+; width >= mmsize
+align 16
+.loop:
+    movu             m13, [srcq+widthq]
+    punpcklbw         m8, m13, m14
+    HEVC_SAO_BAND_FILTER_COMPUTE 8, m9,  m8
+    punpckhbw        m13, m14
+    HEVC_SAO_BAND_FILTER_COMPUTE 8, m9, m13
+    packuswb          m8, m13
+    movu      [dstq+widthq], m8
+
+    add  widthq,  mmsize
+    jl .loop
+
+    add             dstq, dststrideq
+    add             srcq, srcstrideq
+    mov           widthq, tmpq
+    dec          heightd
+    jg .loop
+    REP_RET
+
+%if mmsize > 16
+INIT_XMM cpuname
+; width 16, AVX2 only
+hevc_sao_band_filter_16_8 %+ SUFFIX
+    HEVC_SAO_BAND_FILTER_INIT_OFFSETS 8
+
+align 16
+.loop:
+    movu             m13, [srcq+widthq]
+    punpcklbw         m8, m13, m14
+    HEVC_SAO_BAND_FILTER_COMPUTE 8, m9,  m8
+    punpckhbw        m13, m14
+    HEVC_SAO_BAND_FILTER_COMPUTE 8, m9, m13
+    packuswb          m8, m13
+    movu   [dstq+widthq], m8
+
+    add             dstq, dststrideq
+    add             srcq, srcstrideq
+    dec          heightd
+    jg .loop
+    REP_RET
+%endif
+
+; width 8
+hevc_sao_band_filter_8_8 %+ SUFFIX
+    HEVC_SAO_BAND_FILTER_INIT_OFFSETS 8
+
+align 16
+.loop:
+    movq              m8, [srcq+widthq]
+    punpcklbw         m8, m14
+    HEVC_SAO_BAND_FILTER_COMPUTE 8, m9, m8
+    packuswb          m8, m14
+    movq      [dstq+widthq], m8
+
+    add             dstq, dststrideq
+    add             srcq, srcstrideq
+    dec          heightd
+    jg .loop
+    REP_RET
+%endmacro
+
+;void ff_hevc_sao_band_filter_<depth>_<opt>(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src,
+;                                           int16_t *sao_offset_val, int sao_left_class, int width, int height);
+%macro HEVC_SAO_BAND_FILTER_16 1
+cglobal hevc_sao_band_filter_%1, 6, 9, 15, dst, src, dststride, srcstride, offset, left, width, height, tmp
+    HEVC_SAO_BAND_FILTER_INIT %1
+    HEVC_SAO_BAND_FILTER_INIT_OFFSETS %1
+
+; width >= mmsize
+align 16
+.loop:
+    movu              m8, [srcq+widthq]
+    HEVC_SAO_BAND_FILTER_COMPUTE %1, m9, m8
+    CLIPW             m8, m14, m13
+    movu      [dstq+widthq], m8
+
+    movu              m9, [srcq+widthq+mmsize]
+    HEVC_SAO_BAND_FILTER_COMPUTE %1, m8, m9
+    CLIPW             m9, m14, m13
+    movu      [dstq+widthq+mmsize], m9
+
+    add  widthq,  mmsize*2
+    jl .loop
+
+    add             dstq, dststrideq
+    add             srcq, srcstrideq
+    mov           widthq, tmpq
+    dec          heightd
+    jg .loop
+    REP_RET
+
+%if mmsize > 16
+INIT_XMM cpuname
+; width 16, AVX2 only
+hevc_sao_band_filter_16_%1 %+ SUFFIX
+    HEVC_SAO_BAND_FILTER_INIT_OFFSETS %1
+
+align 16
+.loop:
+    movu              m8, [srcq+widthq]
+    HEVC_SAO_BAND_FILTER_COMPUTE %1, m9, m8
+    CLIPW             m8, m14, m13
+    movu   [dstq+widthq], m8
+
+    movu              m9, [srcq+widthq+mmsize]
+    HEVC_SAO_BAND_FILTER_COMPUTE %1, m8, m9
+    CLIPW             m9, m14, m13
+    movu   [dstq+widthq+mmsize], m9
+
+    add             dstq, dststrideq
+    add             srcq, srcstrideq
+    dec          heightd
+    jg .loop
+    REP_RET
+%endif
+
+; width 8
+hevc_sao_band_filter_8_%1 %+ SUFFIX
+    HEVC_SAO_BAND_FILTER_INIT_OFFSETS %1
+
+align 16
+.loop:
+    movu              m8, [srcq+widthq]
+    HEVC_SAO_BAND_FILTER_COMPUTE %1, m9, m8
+    CLIPW             m8, m14, m13
+    movu      [dstq+widthq], m8
+
+    add             dstq, dststrideq
+    add             srcq, srcstrideq
+    dec          heightd
+    jg .loop
+    REP_RET
+%endmacro
+
+INIT_XMM sse2
+HEVC_SAO_BAND_FILTER_8
+HEVC_SAO_BAND_FILTER_16 10
+HEVC_SAO_BAND_FILTER_16 12
+%if HAVE_AVX2_EXTERNAL
+INIT_YMM avx2
+HEVC_SAO_BAND_FILTER_8
+INIT_YMM avx2
+HEVC_SAO_BAND_FILTER_16 10
+INIT_YMM avx2
+HEVC_SAO_BAND_FILTER_16 12
+%endif
+%endif
diff --git a/libavcodec/x86/hevcdsp_init.c b/libavcodec/x86/hevcdsp_init.c
index eaa97e1..04b9439 100644
--- a/libavcodec/x86/hevcdsp_init.c
+++ b/libavcodec/x86/hevcdsp_init.c
@@ -478,6 +478,16 @@ mc_bi_w_funcs(qpel_v, 12, sse4);
 mc_bi_w_funcs(qpel_hv, 12, sse4);
 #endif //ARCH_X86_64 && HAVE_SSE4_EXTERNAL
 
+#define SAO_BAND_FILTER_FUNCS(bitd, opt) \
+void ff_hevc_sao_band_filter_##bitd##_##opt(uint8_t *_dst, uint8_t *_src, ptrdiff_t _stride_dst, ptrdiff_t _stride_src, \
+                                            int16_t *sao_offset_val, int sao_left_class, int width, int height)
+
+SAO_BAND_FILTER_FUNCS(8,  sse2);
+SAO_BAND_FILTER_FUNCS(10, sse2);
+SAO_BAND_FILTER_FUNCS(12, sse2);
+SAO_BAND_FILTER_FUNCS(8,  avx2);
+SAO_BAND_FILTER_FUNCS(10, avx2);
+SAO_BAND_FILTER_FUNCS(12, avx2);
 
 #define EPEL_LINKS(pointer, my, mx, fname, bitd, opt )           \
         PEL_LINK(pointer, 1, my , mx , fname##4 ,  bitd, opt ); \
@@ -516,6 +526,8 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
             if (ARCH_X86_64) {
                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_8_sse2;
                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_8_sse2;
+
+                c->sao_band_filter         = ff_hevc_sao_band_filter_8_sse2;
             }
             c->idct_dc[1] = ff_hevc_idct8x8_dc_8_sse2;
             c->idct_dc[2] = ff_hevc_idct16x16_dc_8_sse2;
@@ -555,6 +567,9 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
         if (EXTERNAL_AVX2(cpu_flags)) {
             c->idct_dc[2] = ff_hevc_idct16x16_dc_8_avx2;
             c->idct_dc[3] = ff_hevc_idct32x32_dc_8_avx2;
+            if (ARCH_X86_64) {
+                c->sao_band_filter = ff_hevc_sao_band_filter_8_avx2;
+            }
 
             c->transform_add[3]    = ff_hevc_transform_add32_8_avx2;
         }
@@ -570,6 +585,8 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
             if (ARCH_X86_64) {
                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_10_sse2;
                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_10_sse2;
+
+                c->sao_band_filter         = ff_hevc_sao_band_filter_10_sse2;
             }
 
             c->idct_dc[1] = ff_hevc_idct8x8_dc_10_sse2;
@@ -607,6 +624,9 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
 
             c->idct_dc[2] = ff_hevc_idct16x16_dc_10_avx2;
             c->idct_dc[3] = ff_hevc_idct32x32_dc_10_avx2;
+            if (ARCH_X86_64) {
+                c->sao_band_filter = ff_hevc_sao_band_filter_10_avx2;
+            }
 
             c->transform_add[2] = ff_hevc_transform_add16_10_avx2;
             c->transform_add[3] = ff_hevc_transform_add32_10_avx2;
@@ -623,6 +643,8 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
             if (ARCH_X86_64) {
                 c->hevc_v_loop_filter_luma = ff_hevc_v_loop_filter_luma_12_sse2;
                 c->hevc_h_loop_filter_luma = ff_hevc_h_loop_filter_luma_12_sse2;
+
+                c->sao_band_filter         = ff_hevc_sao_band_filter_12_sse2;
             }
 
             c->idct_dc[1] = ff_hevc_idct8x8_dc_12_sse2;
@@ -655,6 +677,9 @@ void ff_hevc_dsp_init_x86(HEVCDSPContext *c, const int bit_depth)
         if (EXTERNAL_AVX2(cpu_flags)) {
             c->idct_dc[2] = ff_hevc_idct16x16_dc_12_avx2;
             c->idct_dc[3] = ff_hevc_idct32x32_dc_12_avx2;
+            if (ARCH_X86_64) {
+                c->sao_band_filter = ff_hevc_sao_band_filter_12_avx2;
+            }
         }
     }
 }
-- 
2.2.2



More information about the ffmpeg-devel mailing list