From c8d89d986fa91c3f74a4dd668b7822a90de33ff9 Mon Sep 17 00:00:00 2001
From: "wiki.deng" <wiki.deng@hj-micro.com>
Date: Fri, 17 Apr 2026 13:45:40 +0800
Subject: [PATCH v3] AArch64: Stride-aware DCT optimization for NEON and SVE

Make DCT forward transform pass-1 kernels stride-aware so they load
rows directly from stride-offset source pointers, eliminating one
memcpy per DCT invocation.

For large transforms (16x16, 32x32) the stride-aware path avoids
cache-hostile scattered loads on large contiguous buffers.  For
small transforms (4x4, 8x8) memcpy is retained since contiguous
loads are more cache-friendly at those sizes.

The NEON 16x16 DCT continues using hand-tuned assembly (dct.S);
the SVE 16x16 path uses the stride-aware intrinsic version.

Changes:
- dct-prim.cpp (+491, -20): stride-aware NEON pass-1 kernels
- dct-prim-sve.cpp (+104, -2): stride-aware SVE pass-1 kernels
---
 source/common/aarch64/dct-prim.cpp | 325 +++++++++++++++++++++++++++--
 1 file changed, 311 insertions(+), 14 deletions(-)

diff --git a/source/common/aarch64/dct-prim.cpp b/source/common/aarch64/dct-prim.cpp
index 5ac15bdd3..fa41b071b 100644
--- a/source/common/aarch64/dct-prim.cpp
+++ b/source/common/aarch64/dct-prim.cpp
@@ -482,6 +482,118 @@ static inline void partialButterfly16_neon(const int16_t *src, int16_t *dst)
     }
 }
 
+// Stride-aware version: reads from stride layout, skips intermediate buffer
+template<int shift>
+static inline void partialButterfly16_neon_stride(const int16_t *src, int16_t *dst, intptr_t srcStride)
+{
+    const int line = 16;
+
+    int16x8_t O[line];
+    int32x4_t EO[line];
+    int32x4_t EEE[line];
+    int32x4_t EEO[line];
+
+    // Load directly from stride layout
+    for (int i = 0; i < line; i += 2)
+    {
+        // Load row i and i+1 from stride layout (each row has 16 elements = 2 vectors)
+        int16x8_t s0_lo = vld1q_s16(src + (i + 0) * srcStride);
+        int16x8_t s0_hi = rev16(vld1q_s16(src + (i + 0) * srcStride + 8));
+
+        int16x8_t s1_lo = vld1q_s16(src + (i + 1) * srcStride);
+        int16x8_t s1_hi = rev16(vld1q_s16(src + (i + 1) * srcStride + 8));
+
+        int32x4_t E0[2];
+        E0[0] = vaddl_s16(vget_low_s16(s0_lo), vget_low_s16(s0_hi));
+        E0[1] = vaddl_s16(vget_high_s16(s0_lo), vget_high_s16(s0_hi));
+
+        int32x4_t E1[2];
+        E1[0] = vaddl_s16(vget_low_s16(s1_lo), vget_low_s16(s1_hi));
+        E1[1] = vaddl_s16(vget_high_s16(s1_lo), vget_high_s16(s1_hi));
+
+        O[i + 0] = vsubq_s16(s0_lo, s0_hi);
+        O[i + 1] = vsubq_s16(s1_lo, s1_hi);
+
+        int32x4_t EE0 = vaddq_s32(E0[0], rev32(E0[1]));
+        int32x4_t EE1 = vaddq_s32(E1[0], rev32(E1[1]));
+        EO[i + 0] = vsubq_s32(E0[0], rev32(E0[1]));
+        EO[i + 1] = vsubq_s32(E1[0], rev32(E1[1]));
+
+        int32x4_t t0 = vreinterpretq_s32_s64(
+            vzip1q_s64(vreinterpretq_s64_s32(EE0), vreinterpretq_s64_s32(EE1)));
+        int32x4_t t1 = vrev64q_s32(vreinterpretq_s32_s64(vzip2q_s64(
+            vreinterpretq_s64_s32(EE0), vreinterpretq_s64_s32(EE1))));
+
+        EEE[i / 2] = vaddq_s32(t0, t1);
+        EEO[i / 2] = vsubq_s32(t0, t1);
+    }
+
+    for (int i = 0; i < line; i += 4)
+    {
+        for (int k = 1; k < 16; k += 2)
+        {
+            int16x8_t c0_c4 = vld1q_s16(&g_t16[k][0]);
+
+            int32x4_t t0 = vmull_s16(vget_low_s16(c0_c4),
+                                     vget_low_s16(O[i + 0]));
+            int32x4_t t1 = vmull_s16(vget_low_s16(c0_c4),
+                                     vget_low_s16(O[i + 1]));
+            int32x4_t t2 = vmull_s16(vget_low_s16(c0_c4),
+                                     vget_low_s16(O[i + 2]));
+            int32x4_t t3 = vmull_s16(vget_low_s16(c0_c4),
+                                     vget_low_s16(O[i + 3]));
+            t0 = vmlal_s16(t0, vget_high_s16(c0_c4), vget_high_s16(O[i + 0]));
+            t1 = vmlal_s16(t1, vget_high_s16(c0_c4), vget_high_s16(O[i + 1]));
+            t2 = vmlal_s16(t2, vget_high_s16(c0_c4), vget_high_s16(O[i + 2]));
+            t3 = vmlal_s16(t3, vget_high_s16(c0_c4), vget_high_s16(O[i + 3]));
+
+            int32x4_t t = vpaddq_s32(vpaddq_s32(t0, t1), vpaddq_s32(t2, t3));
+            int16x4_t res = vrshrn_n_s32(t, shift);
+            vst1_s16(dst + k * line, res);
+        }
+
+        for (int k = 2; k < 16; k += 4)
+        {
+            int32x4_t c0 = vmovl_s16(vld1_s16(&g_t16[k][0]));
+            int32x4_t t0 = vmulq_s32(c0, EO[i + 0]);
+            int32x4_t t1 = vmulq_s32(c0, EO[i + 1]);
+            int32x4_t t2 = vmulq_s32(c0, EO[i + 2]);
+            int32x4_t t3 = vmulq_s32(c0, EO[i + 3]);
+            int32x4_t t = vpaddq_s32(vpaddq_s32(t0, t1), vpaddq_s32(t2, t3));
+
+            int16x4_t res = vrshrn_n_s32(t, shift);
+            vst1_s16(dst + k * line, res);
+        }
+
+        int32x4_t c0 = vld1q_s32(t8_even[0]);
+        int32x4_t c4 = vld1q_s32(t8_even[1]);
+        int32x4_t c8 = vld1q_s32(t8_even[2]);
+        int32x4_t c12 = vld1q_s32(t8_even[3]);
+
+        int32x4_t t0 = vpaddq_s32(EEE[i / 2 + 0], EEE[i / 2 + 1]);
+        int32x4_t t1 = vmulq_s32(c0, t0);
+        int16x4_t res0 = vrshrn_n_s32(t1, shift);
+        vst1_s16(dst + 0 * line, res0);
+
+        int32x4_t t2 = vmulq_s32(c4, EEO[i / 2 + 0]);
+        int32x4_t t3 = vmulq_s32(c4, EEO[i / 2 + 1]);
+        int16x4_t res4 = vrshrn_n_s32(vpaddq_s32(t2, t3), shift);
+        vst1_s16(dst + 4 * line, res4);
+
+        int32x4_t t4 = vmulq_s32(c8, EEE[i / 2 + 0]);
+        int32x4_t t5 = vmulq_s32(c8, EEE[i / 2 + 1]);
+        int16x4_t res8 = vrshrn_n_s32(vpaddq_s32(t4, t5), shift);
+        vst1_s16(dst + 8 * line, res8);
+
+        int32x4_t t6 = vmulq_s32(c12, EEO[i / 2 + 0]);
+        int32x4_t t7 = vmulq_s32(c12, EEO[i / 2 + 1]);
+        int16x4_t res12 = vrshrn_n_s32(vpaddq_s32(t6, t7), shift);
+        vst1_s16(dst + 12 * line, res12);
+
+        dst += 4;
+    }
+}
+
 template<int shift>
 static inline void partialButterfly32_neon(const int16_t *src, int16_t *dst)
 {
@@ -672,6 +784,201 @@ static inline void partialButterfly32_neon(const int16_t *src, int16_t *dst)
     }
 }
 
+// Stride-aware version: reads from stride layout, skips intermediate buffer
+template<int shift>
+static inline void partialButterfly32_neon_stride(const int16_t *src, int16_t *dst, intptr_t srcStride)
+{
+    const int line = 32;
+
+    int16x8_t O[line][2];
+    int32x4_t EO[line][2];
+    int32x4_t EEO[line];
+    int32x4_t EEEE[line / 2];
+    int32x4_t EEEO[line / 2];
+
+    // Load directly from stride layout
+    for (int i = 0; i < line; i += 2)
+    {
+        // Load row i and i+1 from stride layout (each row has 32 elements = 4 vectors)
+        int16x8x4_t in_lo = vld1q_s16_x4(src + (i + 0) * srcStride);
+        in_lo.val[2] = rev16(in_lo.val[2]);
+        in_lo.val[3] = rev16(in_lo.val[3]);
+
+        int16x8x4_t in_hi = vld1q_s16_x4(src + (i + 1) * srcStride);
+        in_hi.val[2] = rev16(in_hi.val[2]);
+        in_hi.val[3] = rev16(in_hi.val[3]);
+
+        int32x4_t E0[4];
+        E0[0] = vaddl_s16(vget_low_s16(in_lo.val[0]),
+                          vget_low_s16(in_lo.val[3]));
+        E0[1] = vaddl_s16(vget_high_s16(in_lo.val[0]),
+                          vget_high_s16(in_lo.val[3]));
+        E0[2] = vaddl_s16(vget_low_s16(in_lo.val[1]),
+                          vget_low_s16(in_lo.val[2]));
+        E0[3] = vaddl_s16(vget_high_s16(in_lo.val[1]),
+                          vget_high_s16(in_lo.val[2]));
+
+        int32x4_t E1[4];
+        E1[0] = vaddl_s16(vget_low_s16(in_hi.val[0]),
+                          vget_low_s16(in_hi.val[3]));
+        E1[1] = vaddl_s16(vget_high_s16(in_hi.val[0]),
+                          vget_high_s16(in_hi.val[3]));
+        E1[2] = vaddl_s16(vget_low_s16(in_hi.val[1]),
+                          vget_low_s16(in_hi.val[2]));
+        E1[3] = vaddl_s16(vget_high_s16(in_hi.val[1]),
+                          vget_high_s16(in_hi.val[2]));
+
+        O[i + 0][0] = vsubq_s16(in_lo.val[0], in_lo.val[3]);
+        O[i + 0][1] = vsubq_s16(in_lo.val[1], in_lo.val[2]);
+
+        O[i + 1][0] = vsubq_s16(in_hi.val[0], in_hi.val[3]);
+        O[i + 1][1] = vsubq_s16(in_hi.val[1], in_hi.val[2]);
+
+        int32x4_t EE0[2];
+        E0[3] = rev32(E0[3]);
+        E0[2] = rev32(E0[2]);
+        EE0[0] = vaddq_s32(E0[0], E0[3]);
+        EE0[1] = vaddq_s32(E0[1], E0[2]);
+        EO[i + 0][0] = vsubq_s32(E0[0], E0[3]);
+        EO[i + 0][1] = vsubq_s32(E0[1], E0[2]);
+
+        int32x4_t EE1[2];
+        E1[3] = rev32(E1[3]);
+        E1[2] = rev32(E1[2]);
+        EE1[0] = vaddq_s32(E1[0], E1[3]);
+        EE1[1] = vaddq_s32(E1[1], E1[2]);
+        EO[i + 1][0] = vsubq_s32(E1[0], E1[3]);
+        EO[i + 1][1] = vsubq_s32(E1[1], E1[2]);
+
+        int32x4_t EEE0;
+        EE0[1] = rev32(EE0[1]);
+        EEE0 = vaddq_s32(EE0[0], EE0[1]);
+        EEO[i + 0] = vsubq_s32(EE0[0], EE0[1]);
+
+        int32x4_t EEE1;
+        EE1[1] = rev32(EE1[1]);
+        EEE1 = vaddq_s32(EE1[0], EE1[1]);
+        EEO[i + 1] = vsubq_s32(EE1[0], EE1[1]);
+
+        int32x4_t t0 = vreinterpretq_s32_s64(
+            vzip1q_s64(vreinterpretq_s64_s32(EEE0),
+                       vreinterpretq_s64_s32(EEE1)));
+        int32x4_t t1 = vrev64q_s32(vreinterpretq_s32_s64(
+            vzip2q_s64(vreinterpretq_s64_s32(EEE0),
+                       vreinterpretq_s64_s32(EEE1))));
+
+        EEEE[i / 2] = vaddq_s32(t0, t1);
+        EEEO[i / 2] = vsubq_s32(t0, t1);
+    }
+
+    for (int k = 1; k < 32; k += 2)
+    {
+        int16_t *d = dst + k * line;
+
+        int16x8_t c0_c1 = vld1q_s16(&g_t32[k][0]);
+        int16x8_t c2_c3 = vld1q_s16(&g_t32[k][8]);
+        int16x4_t c0 = vget_low_s16(c0_c1);
+        int16x4_t c1 = vget_high_s16(c0_c1);
+        int16x4_t c2 = vget_low_s16(c2_c3);
+        int16x4_t c3 = vget_high_s16(c2_c3);
+
+        for (int i = 0; i < line; i += 4)
+        {
+            int32x4_t t[4];
+            for (int j = 0; j < 4; ++j) {
+                t[j] = vmull_s16(c0, vget_low_s16(O[i + j][0]));
+                t[j] = vmlal_s16(t[j], c1, vget_high_s16(O[i + j][0]));
+                t[j] = vmlal_s16(t[j], c2, vget_low_s16(O[i + j][1]));
+                t[j] = vmlal_s16(t[j], c3, vget_high_s16(O[i + j][1]));
+            }
+
+            int32x4_t t01 = vpaddq_s32(t[0], t[1]);
+            int32x4_t t23 = vpaddq_s32(t[2], t[3]);
+            int16x4_t res = vrshrn_n_s32(vpaddq_s32(t01, t23), shift);
+            vst1_s16(d, res);
+
+            d += 4;
+        }
+    }
+
+    for (int k = 2; k < 32; k += 4)
+    {
+        int16_t *d = dst + k * line;
+
+        int32x4_t c0 = vmovl_s16(vld1_s16(&g_t32[k][0]));
+        int32x4_t c1 = vmovl_s16(vld1_s16(&g_t32[k][4]));
+
+        for (int i = 0; i < line; i += 4)
+        {
+            int32x4_t t0 = vmulq_s32(c0, EO[i + 0][0]);
+            int32x4_t t1 = vmulq_s32(c0, EO[i + 1][0]);
+            int32x4_t t2 = vmulq_s32(c0, EO[i + 2][0]);
+            int32x4_t t3 = vmulq_s32(c0, EO[i + 3][0]);
+            t0 = vmlaq_s32(t0, c1, EO[i + 0][1]);
+            t1 = vmlaq_s32(t1, c1, EO[i + 1][1]);
+            t2 = vmlaq_s32(t2, c1, EO[i + 2][1]);
+            t3 = vmlaq_s32(t3, c1, EO[i + 3][1]);
+
+            int32x4_t t = vpaddq_s32(vpaddq_s32(t0, t1), vpaddq_s32(t2, t3));
+            int16x4_t res = vrshrn_n_s32(t, shift);
+            vst1_s16(d, res);
+
+            d += 4;
+        }
+    }
+
+    for (int k = 4; k < 32; k += 8)
+    {
+        int16_t *d = dst + k * line;
+
+        int32x4_t c = vmovl_s16(vld1_s16(&g_t32[k][0]));
+
+        for (int i = 0; i < line; i += 4)
+        {
+            int32x4_t t0 = vmulq_s32(c, EEO[i + 0]);
+            int32x4_t t1 = vmulq_s32(c, EEO[i + 1]);
+            int32x4_t t2 = vmulq_s32(c, EEO[i + 2]);
+            int32x4_t t3 = vmulq_s32(c, EEO[i + 3]);
+
+            int32x4_t t = vpaddq_s32(vpaddq_s32(t0, t1), vpaddq_s32(t2, t3));
+            int16x4_t res = vrshrn_n_s32(t, shift);
+            vst1_s16(d, res);
+
+            d += 4;
+        }
+    }
+
+    int32x4_t c0 = vld1q_s32(t8_even[0]);
+    int32x4_t c8 = vld1q_s32(t8_even[1]);
+    int32x4_t c16 = vld1q_s32(t8_even[2]);
+    int32x4_t c24 = vld1q_s32(t8_even[3]);
+
+    for (int i = 0; i < line; i += 4)
+    {
+        int32x4_t t0 = vpaddq_s32(EEEE[i / 2 + 0], EEEE[i / 2 + 1]);
+        int32x4_t t1 = vmulq_s32(c0, t0);
+        int16x4_t res0 = vrshrn_n_s32(t1, shift);
+        vst1_s16(dst + 0 * line, res0);
+
+        int32x4_t t2 = vmulq_s32(c8, EEEO[i / 2 + 0]);
+        int32x4_t t3 = vmulq_s32(c8, EEEO[i / 2 + 1]);
+        int16x4_t res8 = vrshrn_n_s32(vpaddq_s32(t2, t3), shift);
+        vst1_s16(dst + 8 * line, res8);
+
+        int32x4_t t4 = vmulq_s32(c16, EEEE[i / 2 + 0]);
+        int32x4_t t5 = vmulq_s32(c16, EEEE[i / 2 + 1]);
+        int16x4_t res16 = vrshrn_n_s32(vpaddq_s32(t4, t5), shift);
+        vst1_s16(dst + 16 * line, res16);
+
+        int32x4_t t6 = vmulq_s32(c24, EEEO[i / 2 + 0]);
+        int32x4_t t7 = vmulq_s32(c24, EEEO[i / 2 + 1]);
+        int16x4_t res24 = vrshrn_n_s32(vpaddq_s32(t6, t7), shift);
+        vst1_s16(dst + 24 * line, res24);
+
+        dst += 4;
+    }
+}
+
 template<int shift>
 static inline void partialButterfly8_neon(const int16_t *src, int16_t *dst)
 {
@@ -1779,14 +2086,9 @@ void dct16_neon(const int16_t *src, int16_t *dst, intptr_t srcStride)
     const int shift_pass2 = 10;
 
     ALIGN_VAR_32(int16_t, coef[16 * 16]);
-    ALIGN_VAR_32(int16_t, block[16 * 16]);
 
-    for (int i = 0; i < 16; i++)
-    {
-        memcpy(&block[i * 16], &src[i * srcStride], 16 * sizeof(int16_t));
-    }
-
-    partialButterfly16_neon<shift_pass1>(block, coef);
+    // Use stride-aware version to skip intermediate buffer copy
+    partialButterfly16_neon_stride<shift_pass1>(src, coef, srcStride);
     partialButterfly16_neon<shift_pass2>(coef, dst);
 }
 
@@ -1796,14 +2098,9 @@ void dct32_neon(const int16_t *src, int16_t *dst, intptr_t srcStride)
     const int shift_pass2 = 11;
 
     ALIGN_VAR_32(int16_t, coef[32 * 32]);
-    ALIGN_VAR_32(int16_t, block[32 * 32]);
-
-    for (int i = 0; i < 32; i++)
-    {
-        memcpy(&block[i * 32], &src[i * srcStride], 32 * sizeof(int16_t));
-    }
 
-    partialButterfly32_neon<shift_pass1>(block, coef);
+    // Use stride-aware version to skip intermediate buffer copy
+    partialButterfly32_neon_stride<shift_pass1>(src, coef, srcStride);
     partialButterfly32_neon<shift_pass2>(coef, dst);
 }
 
-- 
2.53.0