[x265] [PATCH v2 3/3] AArch64: Add Armv8.4 Neon DotProd implementations of sse_pp
Hari Limaye
hari.limaye at arm.com
Fri Jul 19 17:15:56 UTC 2024
Add implementations of sse_pp primitives using Neon DotProd
instructions, which are mandatory from Armv8.4. These implementations
use a single UDOT instruction in place of multiple UMULL, UADALP
sequences.
---
source/common/CMakeLists.txt | 2 +-
source/common/aarch64/asm-primitives.cpp | 11 ++
source/common/aarch64/fun-decls.h | 1 +
source/common/aarch64/ssd-neon-dotprod.S | 169 +++++++++++++++++++++++
4 files changed, 182 insertions(+), 1 deletion(-)
create mode 100644 source/common/aarch64/ssd-neon-dotprod.S
diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt
index 568c1a50c..d94ea8795 100644
--- a/source/common/CMakeLists.txt
+++ b/source/common/CMakeLists.txt
@@ -112,7 +112,7 @@ if(ENABLE_ASSEMBLY AND (ARM64 OR CROSS_COMPILE_ARM64))
set(A_SRCS asm.S mc-a.S mc-a-common.S sad-a.S pixel-util.S pixel-util-common.S p2s.S p2s-common.S ipfilter.S ipfilter-common.S blockcopy8.S blockcopy8-common.S ssd-a.S ssd-a-common.S)
set(A_SRCS_SVE asm-sve.S blockcopy8-sve.S p2s-sve.S pixel-util-sve.S)
set(A_SRCS_SVE2 mc-a-sve2.S pixel-util-sve2.S ipfilter-sve2.S ssd-a-sve2.S)
- set(A_SRCS_NEON_DOTPROD sad-neon-dotprod.S)
+ set(A_SRCS_NEON_DOTPROD sad-neon-dotprod.S ssd-neon-dotprod.S)
set(VEC_PRIMITIVES)
set(ARM_ASMS "${A_SRCS}" CACHE INTERNAL "ARM Assembly Sources")
diff --git a/source/common/aarch64/asm-primitives.cpp b/source/common/aarch64/asm-primitives.cpp
index fc7d205db..32d75ee35 100644
--- a/source/common/aarch64/asm-primitives.cpp
+++ b/source/common/aarch64/asm-primitives.cpp
@@ -1172,6 +1172,17 @@ void setupNeonDotProdPrimitives(EncoderPrimitives &p)
LUMA_PU_MULTIPLE_16(sad, pixel_sad, neon_dotprod);
LUMA_PU_MULTIPLE_16(sad_x3, sad_x3, neon_dotprod);
LUMA_PU_MULTIPLE_16(sad_x4, sad_x4, neon_dotprod);
+
+ // sse_pp
+ ALL_LUMA_TU(sse_pp, pixel_sse_pp, neon_dotprod);
+ p.chroma[X265_CSP_I420].cu[BLOCK_420_4x4].sse_pp = PFX(pixel_sse_pp_4x4_neon_dotprod);
+ p.chroma[X265_CSP_I420].cu[BLOCK_420_8x8].sse_pp = PFX(pixel_sse_pp_8x8_neon_dotprod);
+ p.chroma[X265_CSP_I420].cu[BLOCK_420_16x16].sse_pp = PFX(pixel_sse_pp_16x16_neon_dotprod);
+ p.chroma[X265_CSP_I420].cu[BLOCK_420_32x32].sse_pp = PFX(pixel_sse_pp_32x32_neon_dotprod);
+ p.chroma[X265_CSP_I422].cu[BLOCK_422_4x8].sse_pp = PFX(pixel_sse_pp_4x8_neon_dotprod);
+ p.chroma[X265_CSP_I422].cu[BLOCK_422_8x16].sse_pp = PFX(pixel_sse_pp_8x16_neon_dotprod);
+ p.chroma[X265_CSP_I422].cu[BLOCK_422_16x32].sse_pp = PFX(pixel_sse_pp_16x32_neon_dotprod);
+ p.chroma[X265_CSP_I422].cu[BLOCK_422_32x64].sse_pp = PFX(pixel_sse_pp_32x64_neon_dotprod);
}
#else // !HIGH_BIT_DEPTH
void setupNeonDotProdPrimitives(EncoderPrimitives &)
diff --git a/source/common/aarch64/fun-decls.h b/source/common/aarch64/fun-decls.h
index ba5496032..363725fef 100644
--- a/source/common/aarch64/fun-decls.h
+++ b/source/common/aarch64/fun-decls.h
@@ -175,6 +175,7 @@ DECLS(sve2);
FUNCDEF_PU_MULT_16(int, pixel_sad, neon_dotprod, const pixel*, intptr_t, const pixel*, intptr_t);
FUNCDEF_PU_MULT_16(void, sad_x3, neon_dotprod, const pixel*, const pixel*, const pixel*, const pixel*, intptr_t, int32_t*);
FUNCDEF_PU_MULT_16(void, sad_x4, neon_dotprod, const pixel*, const pixel*, const pixel*, const pixel*, const pixel*, intptr_t, int32_t*);
+FUNCDEF_PU(sse_t, pixel_sse_pp, neon_dotprod, const pixel*, intptr_t, const pixel*, intptr_t);
void PFX(pixel_planecopy_cp_neon(const uint8_t* src, intptr_t srcStride, pixel* dst, intptr_t dstStride, int width, int height, int shift));
diff --git a/source/common/aarch64/ssd-neon-dotprod.S b/source/common/aarch64/ssd-neon-dotprod.S
new file mode 100644
index 000000000..044412fba
--- /dev/null
+++ b/source/common/aarch64/ssd-neon-dotprod.S
@@ -0,0 +1,169 @@
+/*****************************************************************************
+ * Copyright (C) 2024 MulticoreWare, Inc
+ *
+ * Authors: Hari Limaye <hari.limaye at arm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.
+ *
+ * This program is also available under a commercial proprietary license.
+ * For more information, contact us at license @ x265.com.
+ *****************************************************************************/
+
+#include "asm.S"
+
+.arch armv8.2-a+dotprod
+
+#ifdef __APPLE__
+.section __RODATA,__rodata
+#else
+.section .rodata
+#endif
+
+.align 4
+
+.text
+
+// Fully unrolled.
+.macro SSE_PP_4xN h
+function PFX(pixel_sse_pp_4x\h\()_neon_dotprod)
+ movi v0.4s, #0
+.rept \h / 4
+ ldr s16, [x0]
+ ldr s17, [x2]
+ add x0, x0, x1
+ add x2, x2, x3
+ ld1 {v16.s}[1], [x0], x1
+ ld1 {v16.s}[2], [x0], x1
+ ld1 {v16.s}[3], [x0], x1
+ ld1 {v17.s}[1], [x2], x3
+ ld1 {v17.s}[2], [x2], x3
+ ld1 {v17.s}[3], [x2], x3
+
+ uabd v1.16b, v16.16b, v17.16b
+ udot v0.4s, v1.16b, v1.16b
+.endr
+ addv s0, v0.4s
+ fmov w0, s0
+ ret
+endfunc
+.endm
+
+SSE_PP_4xN 4
+SSE_PP_4xN 8
+
+// Fully unrolled.
+.macro SSE_PP_8xN h
+function PFX(pixel_sse_pp_8x\h\()_neon_dotprod)
+ movi v0.4s, #0
+.rept \h
+ ld1 {v16.8b}, [x0], x1
+ ld1 {v17.8b}, [x2], x3
+
+ uabd v1.8b, v16.8b, v17.8b
+ udot v0.2s, v1.8b, v1.8b
+.endr
+ addv s0, v0.4s
+ fmov w0, s0
+ ret
+endfunc
+.endm
+
+SSE_PP_8xN 8
+SSE_PP_8xN 16
+
+// Fully unrolled.
+.macro SSE_PP_16xN h
+function PFX(pixel_sse_pp_16x\h\()_neon_dotprod)
+ movi v0.4s, #0
+ movi v1.4s, #0
+.rept \h / 2
+ ld1 {v16.16b}, [x0], x1
+ ld1 {v17.16b}, [x2], x3
+ ld1 {v18.16b}, [x0], x1
+ ld1 {v19.16b}, [x2], x3
+
+ uabd v2.16b, v16.16b, v17.16b
+ udot v0.4s, v2.16b, v2.16b
+ uabd v3.16b, v18.16b, v19.16b
+ udot v1.4s, v3.16b, v3.16b
+.endr
+ add v0.4s, v0.4s, v1.4s
+ addv s0, v0.4s
+ fmov w0, s0
+ ret
+endfunc
+.endm
+
+SSE_PP_16xN 16
+SSE_PP_16xN 32
+
+// Loop unrolled to process 4 rows per iteration.
+function PFX(pixel_sse_pp_32xh_neon_dotprod), export=0
+ movi v0.4s, #0
+ movi v1.4s, #0
+.Loop_sse_pp_32xh:
+ sub w4, w4, #1
+.rept 4
+ ld1 {v16.16b,v17.16b}, [x0], x1
+ ld1 {v18.16b,v19.16b}, [x2], x3
+
+ uabd v2.16b, v16.16b, v18.16b
+ udot v0.4s, v2.16b, v2.16b
+ uabd v3.16b, v17.16b, v19.16b
+ udot v1.4s, v3.16b, v3.16b
+.endr
+ cbnz w4, .Loop_sse_pp_32xh
+ add v0.4s, v0.4s, v1.4s
+ addv s0, v0.4s
+ fmov w0, s0
+ ret
+endfunc
+
+.macro SSE_PP_32xN h
+function PFX(pixel_sse_pp_32x\h\()_neon_dotprod)
+ mov w4, \h / 4
+ b PFX(pixel_sse_pp_32xh_neon_dotprod)
+endfunc
+.endm
+
+SSE_PP_32xN 32
+SSE_PP_32xN 64
+
+// Loop unrolled to process 4 rows per iteration.
+function PFX(pixel_sse_pp_64x64_neon_dotprod)
+ mov w12, #16
+ movi v0.4s, #0
+ movi v1.4s, #0
+.Loop_sse_pp_64:
+ sub w12, w12, #1
+.rept 4
+ ld1 {v16.16b-v19.16b}, [x0], x1
+ ld1 {v20.16b-v23.16b}, [x2], x3
+
+ uabd v2.16b, v16.16b, v20.16b
+ udot v0.4s, v2.16b, v2.16b
+ uabd v3.16b, v17.16b, v21.16b
+ udot v1.4s, v3.16b, v3.16b
+ uabd v4.16b, v18.16b, v22.16b
+ udot v0.4s, v4.16b, v4.16b
+ uabd v5.16b, v19.16b, v23.16b
+ udot v1.4s, v5.16b, v5.16b
+.endr
+ cbnz w12, .Loop_sse_pp_64
+ add v0.4s, v0.4s, v1.4s
+ addv s0, v0.4s
+ fmov w0, s0
+ ret
+endfunc
--
2.42.1
-------------- next part --------------
A non-text attachment was scrubbed...
Name: v2-0003-AArch64-Add-Armv8.4-Neon-DotProd-implementations-.patch
Type: text/x-patch
Size: 9026 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20240719/2c857475/attachment-0001.bin>
More information about the x265-devel
mailing list