[x265] [PATCH 04/14] AArch64: Add Armv8.4 Neon DotProd implementations of filter_hpp

Hari Limaye hari.limaye at arm.com
Fri Sep 6 13:33:24 UTC 2024


Add implementations of filter_hpp primitives using Neon DotProd
instructions, which are mandatory from Armv8.4.

Geomean uplift across all block sizes for chroma filters, relative to
Armv8.0 Neon implementation:

    Neoverse N1: 1.123x
    Neoverse N2: 0.921x
    Neoverse V1: 1.021x
    Neoverse V2: 0.995x

For N2, V1 and V2 this implementation will be superseded by a Neon I8MM
implementation in a subsequent patch.
---
 source/common/aarch64/filter-neon-dotprod.cpp | 212 ++++++++++++++++++
 1 file changed, 212 insertions(+)

diff --git a/source/common/aarch64/filter-neon-dotprod.cpp b/source/common/aarch64/filter-neon-dotprod.cpp
index 4cd052fb0..8247dc29a 100644
--- a/source/common/aarch64/filter-neon-dotprod.cpp
+++ b/source/common/aarch64/filter-neon-dotprod.cpp
@@ -186,6 +186,28 @@ int16x8_t inline filter8_8_ps_reuse(uint8x16_t samples, const int8x8_t filter,
     // Narrow and combine.
     return vcombine_s16(vmovn_s32(dotprod_lo), vmovn_s32(dotprod_hi));
 }
+
+uint8x8_t inline filter4_8_pp(uint8x16_t samples, const int8x8_t filter,
+                              const int32x4_t constant, const uint8x16x2_t tbl)
+{
+    // Transform sample range from uint8_t to int8_t for signed dot product.
+    int8x16_t samples_s8 =
+        vreinterpretq_s8_u8(vsubq_u8(samples, vdupq_n_u8(128)));
+
+    // Permute input samples for dot product.
+    // {0,  1,  2,  3 ,  1,  2,  3,  4 ,  2,  3,  4,  5 ,  3,  4,  5,  6}
+    int8x16_t perm_samples_0 = vqtbl1q_s8(samples_s8, tbl.val[0]);
+    // {4,  5,  6,  7 ,  5,  6,  7,  8 ,  6,  7,  8,  9 ,  7,  8,  9, 10}
+    int8x16_t perm_samples_1 = vqtbl1q_s8(samples_s8, tbl.val[1]);
+
+    int32x4_t dotprod_lo = vdotq_lane_s32(constant, perm_samples_0, filter, 0);
+    int32x4_t dotprod_hi = vdotq_lane_s32(constant, perm_samples_1, filter, 0);
+
+    // Narrow and combine.
+    int16x8_t dotprod = vcombine_s16(vmovn_s32(dotprod_lo),
+                                     vmovn_s32(dotprod_hi));
+    return vqrshrun_n_s16(dotprod, IF_FILTER_PREC);
+}
 } // Unnamed namespace.
 
 namespace X265_NS {
@@ -388,10 +410,124 @@ void interp8_horiz_ps_dotprod(const uint8_t *src, intptr_t srcStride,
     }
 }
 
+template<int N, int width, int height>
+void interp_horiz_pp_neon(const pixel *src, intptr_t srcStride, pixel *dst,
+                          intptr_t dstStride, int coeffIdx);
+
+template<int width, int height>
+void interp4_horiz_pp_dotprod(const uint8_t *src, intptr_t srcStride,
+                              uint8_t *dst, intptr_t dstStride, int coeffIdx)
+{
+    const int N_TAPS = 4;
+
+    if (coeffIdx == 4)
+        return interp_horiz_pp_neon<N_TAPS, width, height>(src, srcStride, dst,
+                                                           dstStride, coeffIdx);
+
+    src -= N_TAPS / 2 - 1;
+
+    const uint8x16x2_t tbl = vld1q_u8_x2(dotprod_permute_tbl);
+    const int16x4_t filter_16 = vld1_s16(g_chromaFilter[coeffIdx]);
+    const int8x8_t filter = vmovn_s16(vcombine_s16(filter_16, vdup_n_s16(0)));
+
+    // Correction accounting for sample range transform from uint8_t to int8_t.
+    const int32x4_t c = vdupq_n_s32(64 * 128);
+
+    for (int row = 0; row + 4 <= height; row += 4)
+    {
+        int col = 0;
+        for (; col + 16 <= width; col += 16)
+        {
+            uint8x16_t s0[4], s1[4];
+            load_u8x16xn<4>(src + col, srcStride, s0);
+            load_u8x16xn<4>(src + col + 8, srcStride, s1);
+
+            uint8x8_t d_lo[4];
+            d_lo[0] = filter4_8_pp(s0[0], filter, c, tbl);
+            d_lo[1] = filter4_8_pp(s0[1], filter, c, tbl);
+            d_lo[2] = filter4_8_pp(s0[2], filter, c, tbl);
+            d_lo[3] = filter4_8_pp(s0[3], filter, c, tbl);
+
+            uint8x8_t d_hi[4];
+            d_hi[0] = filter4_8_pp(s1[0], filter, c, tbl);
+            d_hi[1] = filter4_8_pp(s1[1], filter, c, tbl);
+            d_hi[2] = filter4_8_pp(s1[2], filter, c, tbl);
+            d_hi[3] = filter4_8_pp(s1[3], filter, c, tbl);
+
+            uint8x16_t d[4];
+            d[0] = vcombine_u8(d_lo[0], d_hi[0]);
+            d[1] = vcombine_u8(d_lo[1], d_hi[1]);
+            d[2] = vcombine_u8(d_lo[2], d_hi[2]);
+            d[3] = vcombine_u8(d_lo[3], d_hi[3]);
+
+            store_u8x16xn<4>(dst + col, dstStride, d);
+        }
+
+        for (; col + 8 <= width; col += 8)
+        {
+            uint8x16_t s[4];
+            load_u8x16xn<4>(src + col, srcStride, s);
+
+            uint8x8_t d[4];
+            d[0] = filter4_8_pp(s[0], filter, c, tbl);
+            d[1] = filter4_8_pp(s[1], filter, c, tbl);
+            d[2] = filter4_8_pp(s[2], filter, c, tbl);
+            d[3] = filter4_8_pp(s[3], filter, c, tbl);
+
+            store_u8x8xn<4>(dst + col, dstStride, d);
+        }
+
+        // Block sizes 12xH, 6xH, 4xH, 2xH.
+        if (width % 8 != 0)
+        {
+            uint8x16_t s[4];
+            load_u8x16xn<4>(src + col, srcStride, s);
+
+            uint8x8_t d[4];
+            d[0] = filter4_8_pp(s[0], filter, c, tbl);
+            d[1] = filter4_8_pp(s[1], filter, c, tbl);
+            d[2] = filter4_8_pp(s[2], filter, c, tbl);
+            d[3] = filter4_8_pp(s[3], filter, c, tbl);
+
+            const int n_store = width < 8 ? width : 4;
+            store_u8xnxm<n_store, 4>(dst + col, dstStride, d);
+        }
+
+        src += 4 * srcStride;
+        dst += 4 * dstStride;
+    }
+
+    // Block sizes 8x6, 8x2, 4x2.
+    if (height & 2)
+    {
+        uint8x16_t s[2];
+        load_u8x16xn<2>(src, srcStride, s);
+
+        uint8x8_t d[2];
+        d[0] = filter4_8_pp(s[0], filter, c, tbl);
+        d[1] = filter4_8_pp(s[1], filter, c, tbl);
+
+        const int n_store = width < 8 ? width : 8;
+        store_u8xnxm<n_store, 2>(dst, dstStride, d);
+    }
+}
+
 #define LUMA_DOTPROD(W, H) \
         p.pu[LUMA_ ## W ## x ## H].luma_hpp = interp8_horiz_pp_dotprod<W, H>; \
         p.pu[LUMA_ ## W ## x ## H].luma_hps = interp8_horiz_ps_dotprod<W, H>;
 
+#define CHROMA_420_DOTPROD(W, H) \
+        p.chroma[X265_CSP_I420].pu[CHROMA_420_ ## W ## x ## H].filter_hpp = \
+            interp4_horiz_pp_dotprod<W, H>;
+
+#define CHROMA_422_DOTPROD(W, H) \
+        p.chroma[X265_CSP_I422].pu[CHROMA_422_ ## W ## x ## H].filter_hpp = \
+            interp4_horiz_pp_dotprod<W, H>;
+
+#define CHROMA_444_DOTPROD(W, H) \
+        p.chroma[X265_CSP_I444].pu[LUMA_ ## W ## x ## H].filter_hpp = \
+            interp4_horiz_pp_dotprod<W, H>;
+
 void setupFilterPrimitives_neon_dotprod(EncoderPrimitives &p)
 {
     LUMA_DOTPROD(4, 4);
@@ -419,6 +555,82 @@ void setupFilterPrimitives_neon_dotprod(EncoderPrimitives &p)
     LUMA_DOTPROD(64, 32);
     LUMA_DOTPROD(64, 48);
     LUMA_DOTPROD(64, 64);
+
+    CHROMA_420_DOTPROD(2, 4);
+    CHROMA_420_DOTPROD(2, 8);
+    CHROMA_420_DOTPROD(4, 2);
+    CHROMA_420_DOTPROD(4, 4);
+    CHROMA_420_DOTPROD(4, 8);
+    CHROMA_420_DOTPROD(4, 16);
+    CHROMA_420_DOTPROD(6, 8);
+    CHROMA_420_DOTPROD(12, 16);
+    CHROMA_420_DOTPROD(8, 2);
+    CHROMA_420_DOTPROD(8, 4);
+    CHROMA_420_DOTPROD(8, 6);
+    CHROMA_420_DOTPROD(8, 8);
+    CHROMA_420_DOTPROD(8, 16);
+    CHROMA_420_DOTPROD(8, 32);
+    CHROMA_420_DOTPROD(16, 4);
+    CHROMA_420_DOTPROD(16, 8);
+    CHROMA_420_DOTPROD(16, 12);
+    CHROMA_420_DOTPROD(16, 16);
+    CHROMA_420_DOTPROD(16, 32);
+    CHROMA_420_DOTPROD(24, 32);
+    CHROMA_420_DOTPROD(32, 8);
+    CHROMA_420_DOTPROD(32, 16);
+    CHROMA_420_DOTPROD(32, 24);
+    CHROMA_420_DOTPROD(32, 32);
+
+    CHROMA_422_DOTPROD(2, 8);
+    CHROMA_422_DOTPROD(2, 16);
+    CHROMA_422_DOTPROD(4, 4);
+    CHROMA_422_DOTPROD(4, 8);
+    CHROMA_422_DOTPROD(4, 16);
+    CHROMA_422_DOTPROD(4, 32);
+    CHROMA_422_DOTPROD(6, 16);
+    CHROMA_422_DOTPROD(12, 32);
+    CHROMA_422_DOTPROD(8, 4);
+    CHROMA_422_DOTPROD(8, 8);
+    CHROMA_422_DOTPROD(8, 12);
+    CHROMA_422_DOTPROD(8, 16);
+    CHROMA_422_DOTPROD(8, 32);
+    CHROMA_422_DOTPROD(8, 64);
+    CHROMA_422_DOTPROD(16, 8);
+    CHROMA_422_DOTPROD(16, 16);
+    CHROMA_422_DOTPROD(16, 24);
+    CHROMA_422_DOTPROD(16, 32);
+    CHROMA_422_DOTPROD(16, 64);
+    CHROMA_422_DOTPROD(24, 64);
+    CHROMA_422_DOTPROD(32, 16);
+    CHROMA_422_DOTPROD(32, 32);
+    CHROMA_422_DOTPROD(32, 48);
+    CHROMA_422_DOTPROD(32, 64);
+
+    CHROMA_444_DOTPROD(4, 4);
+    CHROMA_444_DOTPROD(4, 8);
+    CHROMA_444_DOTPROD(4, 16);
+    CHROMA_444_DOTPROD(12, 16);
+    CHROMA_444_DOTPROD(8, 4);
+    CHROMA_444_DOTPROD(8, 8);
+    CHROMA_444_DOTPROD(8, 16);
+    CHROMA_444_DOTPROD(8, 32);
+    CHROMA_444_DOTPROD(16, 4);
+    CHROMA_444_DOTPROD(16, 8);
+    CHROMA_444_DOTPROD(16, 12);
+    CHROMA_444_DOTPROD(16, 16);
+    CHROMA_444_DOTPROD(16, 32);
+    CHROMA_444_DOTPROD(16, 64);
+    CHROMA_444_DOTPROD(24, 32);
+    CHROMA_444_DOTPROD(32, 8);
+    CHROMA_444_DOTPROD(32, 16);
+    CHROMA_444_DOTPROD(32, 24);
+    CHROMA_444_DOTPROD(32, 32);
+    CHROMA_444_DOTPROD(32, 64);
+    CHROMA_444_DOTPROD(48, 64);
+    CHROMA_444_DOTPROD(64, 16);
+    CHROMA_444_DOTPROD(64, 32);
+    CHROMA_444_DOTPROD(64, 48);
+    CHROMA_444_DOTPROD(64, 64);
 }
 }
 
-- 
2.42.1

-------------- next part --------------
A non-text attachment was scrubbed...
Name: 0004-AArch64-Add-Armv8.4-Neon-DotProd-implementations-of-.patch
Type: text/x-patch
Size: 9505 bytes
Desc: not available
URL: <http://mailman.videolan.org/pipermail/x265-devel/attachments/20240906/ce97c79f/attachment-0001.bin>


More information about the x265-devel mailing list