<div dir="ltr"><span style="color:rgb(0,0,0);font-family:arial;font-size:14px">Pushed to master branch.</span><br style="color:rgb(0,0,0);font-family:arial;font-size:14px"><div><div dir="ltr" class="gmail_signature"><div dir="ltr"></div></div></div><div><div dir="ltr" class="gmail_signature" data-smartmail="gmail_signature"><div dir="ltr"><div><b>__________________________</b></div><div><b>Karam Singh</b></div><div><b>Ph.D. IIT Guwahati</b></div><div><font size="1">Senior Software (Video Coding) Engineer </font></div><div><font size="1">Mobile: +91 8011279030</font></div><div><font size="1">Block 9A, 6th floor, DLF Cyber City</font></div><div><font size="1">Manapakkam, Chennai 600 089</font></div></div></div></div><br></div><br><div class="gmail_quote"><div dir="ltr" class="gmail_attr">On Mon, May 20, 2024 at 9:49 PM Hari Limaye <<a href="mailto:hari.limaye@arm.com">hari.limaye@arm.com</a>> wrote:<br></div><blockquote class="gmail_quote" style="margin:0px 0px 0px 0.8ex;border-left:1px solid rgb(204,204,204);padding-left:1ex">Add optimised implementations of saoCuStats primitives using SVE2<br>
instructions for both low and high bitdepth.<br>
<br>
Performance uplift, compared to the SVE implementations, when compiled<br>
with LLVM 17 on a Neoverse V2 machine (higher is better):<br>
<br>
Low bitdepth:<br>
<br>
| SVE -> SVE2 |<br>
saoCuStatsE0 | 1.08x |<br>
saoCuStatsE1 | 1.06x |<br>
saoCuStatsE2 | 1.06x |<br>
saoCuStatsE3 | 1.09x |<br>
<br>
High bitdepth:<br>
<br>
| SVE -> SVE2 |<br>
saoCuStatsE0 | 1.03x |<br>
saoCuStatsE1 | 1.10x |<br>
saoCuStatsE2 | 1.08x |<br>
saoCuStatsE3 | 1.09x |<br>
---<br>
source/common/CMakeLists.txt | 7 +<br>
source/common/aarch64/asm-primitives.cpp | 6 +<br>
source/common/aarch64/sao-prim-sve.cpp | 27 --<br>
source/common/aarch64/sao-prim-sve2.cpp | 317 +++++++++++++++++++++++<br>
source/common/aarch64/sao-prim.h | 35 +++<br>
5 files changed, 365 insertions(+), 27 deletions(-)<br>
create mode 100644 source/common/aarch64/sao-prim-sve2.cpp<br>
<br>
diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt<br>
index 40c932966..7d0506909 100644<br>
--- a/source/common/CMakeLists.txt<br>
+++ b/source/common/CMakeLists.txt<br>
@@ -105,6 +105,7 @@ if(ENABLE_ASSEMBLY AND (ARM64 OR CROSS_COMPILE_ARM64))<br>
<br>
set(C_SRCS_NEON asm-primitives.cpp pixel-prim.h pixel-prim.cpp filter-prim.h filter-prim.cpp dct-prim.h dct-prim.cpp loopfilter-prim.cpp loopfilter-prim.h intrapred-prim.cpp arm64-utils.cpp arm64-utils.h fun-decls.h sao-prim.cpp)<br>
set(C_SRCS_SVE sao-prim-sve.cpp)<br>
+ set(C_SRCS_SVE2 sao-prim-sve2.cpp)<br>
enable_language(ASM)<br>
<br>
# add ARM assembly/intrinsic files here<br>
@@ -126,6 +127,12 @@ if(ENABLE_ASSEMBLY AND (ARM64 OR CROSS_COMPILE_ARM64))<br>
endforeach()<br>
endif()<br>
<br>
+ if(CPU_HAS_SVE2 AND HAVE_SVE_BRIDGE)<br>
+ foreach(SRC ${C_SRCS_SVE2})<br>
+ set(ASM_PRIMITIVES ${ASM_PRIMITIVES} aarch64/${SRC})<br>
+ endforeach()<br>
+ endif()<br>
+<br>
source_group(Assembly FILES ${ASM_PRIMITIVES})<br>
endif(ENABLE_ASSEMBLY AND (ARM64 OR CROSS_COMPILE_ARM64))<br>
<br>
diff --git a/source/common/aarch64/asm-primitives.cpp b/source/common/aarch64/asm-primitives.cpp<br>
index bab34a493..356901dd9 100644<br>
--- a/source/common/aarch64/asm-primitives.cpp<br>
+++ b/source/common/aarch64/asm-primitives.cpp<br>
@@ -1958,6 +1958,12 @@ void setupIntrinsicPrimitives(EncoderPrimitives &p, int cpuMask)<br>
setupSaoPrimitives_sve(p);<br>
}<br>
#endif<br>
+#if defined(HAVE_SVE2) && HAVE_SVE_BRIDGE<br>
+ if (cpuMask & X265_CPU_SVE2)<br>
+ {<br>
+ setupSaoPrimitives_sve2(p);<br>
+ }<br>
+#endif<br>
}<br>
<br>
} // namespace X265_NS<br>
diff --git a/source/common/aarch64/sao-prim-sve.cpp b/source/common/aarch64/sao-prim-sve.cpp<br>
index 4b9e3c5d2..889b42a79 100644<br>
--- a/source/common/aarch64/sao-prim-sve.cpp<br>
+++ b/source/common/aarch64/sao-prim-sve.cpp<br>
@@ -22,33 +22,6 @@<br>
*****************************************************************************/<br>
<br>
#include "sao-prim.h"<br>
-#include <arm_neon_sve_bridge.h><br>
-<br>
-/* We can access instructions that are exclusive to the SVE instruction set from<br>
- * a predominantly Neon context by making use of the Neon-SVE bridge intrinsics<br>
- * to reinterpret Neon vectors as SVE vectors - with the high part of the SVE<br>
- * vector (if it's longer than 128 bits) being "don't care".<br>
- *<br>
- * While sub-optimal on machines that have SVE vector length > 128-bit - as the<br>
- * remainder of the vector is unused - this approach is still beneficial when<br>
- * compared to a Neon-only implementation. */<br>
-<br>
-static inline int8x16_t x265_sve_mask(const int x, const int endX,<br>
- const int8x16_t in)<br>
-{<br>
- // Use predicate to shift "unused lanes" outside of range [-2, 2]<br>
- svbool_t svpred = svwhilelt_b8(x, endX);<br>
- svint8_t edge_type = svsel_s8(svpred, svset_neonq_s8(svundef_s8(), in),<br>
- svdup_n_s8(-3));<br>
- return svget_neonq_s8(edge_type);<br>
-}<br>
-<br>
-static inline int64x2_t x265_sdotq_s16(int64x2_t acc, int16x8_t x, int16x8_t y)<br>
-{<br>
- return svget_neonq_s64(svdot_s64(svset_neonq_s64(svundef_s64(), acc),<br>
- svset_neonq_s16(svundef_s16(), x),<br>
- svset_neonq_s16(svundef_s16(), y)));<br>
-}<br>
<br>
/*<br>
* Compute Edge Offset statistics (count and stats).<br>
diff --git a/source/common/aarch64/sao-prim-sve2.cpp b/source/common/aarch64/sao-prim-sve2.cpp<br>
new file mode 100644<br>
index 000000000..0653537e5<br>
--- /dev/null<br>
+++ b/source/common/aarch64/sao-prim-sve2.cpp<br>
@@ -0,0 +1,317 @@<br>
+/*****************************************************************************<br>
+ * Copyright (C) 2024 MulticoreWare, Inc<br>
+ *<br>
+ * Authors: Hari Limaye <<a href="mailto:hari.limaye@arm.com" target="_blank">hari.limaye@arm.com</a>><br>
+ *<br>
+ * This program is free software; you can redistribute it and/or modify<br>
+ * it under the terms of the GNU General Public License as published by<br>
+ * the Free Software Foundation; either version 2 of the License, or<br>
+ * (at your option) any later version.<br>
+ *<br>
+ * This program is distributed in the hope that it will be useful,<br>
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of<br>
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the<br>
+ * GNU General Public License for more details.<br>
+ *<br>
+ * You should have received a copy of the GNU General Public License<br>
+ * along with this program; if not, write to the Free Software<br>
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA.<br>
+ *<br>
+ * This program is also available under a commercial proprietary license.<br>
+ * For more information, contact us at license @ <a href="http://x265.com" rel="noreferrer" target="_blank">x265.com</a>.<br>
+ *****************************************************************************/<br>
+<br>
+#include "sao-prim.h"<br>
+<br>
+static inline uint8x16_t sve_count(int8x16_t in)<br>
+{<br>
+ // We do not care about initialising the values in the rest of the vector,<br>
+ // for VL > 128, as HISTSEG counts matching elements in 128-bit segments.<br>
+ svint8_t edge_type = svset_neonq_s8(svundef_s8(), in);<br>
+<br>
+ // Use an arbitrary value outside of range [-2, 2] for lanes we don't<br>
+ // need to use the result from.<br>
+ const int DC = -3;<br>
+ // s_eoTable maps edge types to memory in order: {2, 0, 1, 3, 4}.<br>
+ // We use (edge_class - 2) resulting in {0, -2, -1, 1, 2}<br>
+ int8x16_t idx = { 0, -2, -1, 1, 2, DC, DC, DC, DC, DC, DC, DC, DC, DC, DC,<br>
+ DC };<br>
+ svint8_t svidx = svset_neonq_s8(svundef_s8(), idx);<br>
+<br>
+ svuint8_t count = svhistseg_s8(svidx, edge_type);<br>
+ return svget_neonq_u8(count);<br>
+}<br>
+<br>
+/*<br>
+ * Compute Edge Offset statistics (stats array).<br>
+ * To save some instructions compute stats as negative values - since output of<br>
+ * Neon comparison instructions for a matched condition is all 1s (-1).<br>
+ */<br>
+static inline void compute_eo_stats(const int8x16_t edge_type,<br>
+ const int16_t *diff, int64x2_t *stats)<br>
+{<br>
+ // Create a mask for each edge type.<br>
+ int8x16_t mask0 = vreinterpretq_s8_u8(vceqq_s8(edge_type, vdupq_n_s8(-2)));<br>
+ int8x16_t mask1 = vreinterpretq_s8_u8(vceqq_s8(edge_type, vdupq_n_s8(-1)));<br>
+ int8x16_t mask2 = vreinterpretq_s8_u8(vceqq_s8(edge_type, vdupq_n_s8(0)));<br>
+ int8x16_t mask3 = vreinterpretq_s8_u8(vceqq_s8(edge_type, vdupq_n_s8(1)));<br>
+ int8x16_t mask4 = vreinterpretq_s8_u8(vceqq_s8(edge_type, vdupq_n_s8(2)));<br>
+<br>
+ // Widen the masks to 16-bit.<br>
+ int16x8_t mask0_lo = vreinterpretq_s16_s8(vzip1q_s8(mask0, mask0));<br>
+ int16x8_t mask0_hi = vreinterpretq_s16_s8(vzip2q_s8(mask0, mask0));<br>
+ int16x8_t mask1_lo = vreinterpretq_s16_s8(vzip1q_s8(mask1, mask1));<br>
+ int16x8_t mask1_hi = vreinterpretq_s16_s8(vzip2q_s8(mask1, mask1));<br>
+ int16x8_t mask2_lo = vreinterpretq_s16_s8(vzip1q_s8(mask2, mask2));<br>
+ int16x8_t mask2_hi = vreinterpretq_s16_s8(vzip2q_s8(mask2, mask2));<br>
+ int16x8_t mask3_lo = vreinterpretq_s16_s8(vzip1q_s8(mask3, mask3));<br>
+ int16x8_t mask3_hi = vreinterpretq_s16_s8(vzip2q_s8(mask3, mask3));<br>
+ int16x8_t mask4_lo = vreinterpretq_s16_s8(vzip1q_s8(mask4, mask4));<br>
+ int16x8_t mask4_hi = vreinterpretq_s16_s8(vzip2q_s8(mask4, mask4));<br>
+<br>
+ int16x8_t diff_lo = vld1q_s16(diff);<br>
+ int16x8_t diff_hi = vld1q_s16(diff + 8);<br>
+<br>
+ // Compute negative stats for each edge type.<br>
+ stats[0] = x265_sdotq_s16(stats[0], diff_lo, mask0_lo);<br>
+ stats[0] = x265_sdotq_s16(stats[0], diff_hi, mask0_hi);<br>
+ stats[1] = x265_sdotq_s16(stats[1], diff_lo, mask1_lo);<br>
+ stats[1] = x265_sdotq_s16(stats[1], diff_hi, mask1_hi);<br>
+ stats[2] = x265_sdotq_s16(stats[2], diff_lo, mask2_lo);<br>
+ stats[2] = x265_sdotq_s16(stats[2], diff_hi, mask2_hi);<br>
+ stats[3] = x265_sdotq_s16(stats[3], diff_lo, mask3_lo);<br>
+ stats[3] = x265_sdotq_s16(stats[3], diff_hi, mask3_hi);<br>
+ stats[4] = x265_sdotq_s16(stats[4], diff_lo, mask4_lo);<br>
+ stats[4] = x265_sdotq_s16(stats[4], diff_hi, mask4_hi);<br>
+}<br>
+<br>
+/*<br>
+ * Reduce and store Edge Offset statistics (count and stats).<br>
+ */<br>
+static inline void reduce_eo_stats(int64x2_t *vstats, uint16x8_t vcount,<br>
+ int32_t *stats, int32_t *count)<br>
+{<br>
+ // s_eoTable maps edge types to memory in order: {2, 0, 1, 3, 4}.<br>
+ // We already have the count values in the correct order for the store,<br>
+ // so widen to 32-bit and accumulate to the destination.<br>
+ int32x4_t c0123 = vmovl_s16(vget_low_s16(vreinterpretq_s16_u16(vcount)));<br>
+ vst1q_s32(count, vaddq_s32(vld1q_s32(count), c0123));<br>
+ count[4] += vcount[4];<br>
+<br>
+ int32x4_t s01 = vcombine_s32(vmovn_s64(vstats[2]), vmovn_s64(vstats[0]));<br>
+ int32x4_t s23 = vcombine_s32(vmovn_s64(vstats[1]), vmovn_s64(vstats[3]));<br>
+ int32x4_t s0123 = vpaddq_s32(s01, s23);<br>
+ // Subtract from current stats, as we calculate the negation.<br>
+ vst1q_s32(stats, vsubq_s32(vld1q_s32(stats), s0123));<br>
+ stats[4] -= vaddvq_s64(vstats[4]);<br>
+}<br>
+<br>
+namespace X265_NS {<br>
+void saoCuStatsE0_sve2(const int16_t *diff, const pixel *rec, intptr_t stride,<br>
+ int endX, int endY, int32_t *stats, int32_t *count)<br>
+{<br>
+ // Separate buffers for each edge type, so that we can vectorise.<br>
+ int64x2_t tmp_stats[5] = { vdupq_n_s64(0), vdupq_n_s64(0), vdupq_n_s64(0),<br>
+ vdupq_n_s64(0), vdupq_n_s64(0) };<br>
+ uint16x8_t count_acc_u16 = vdupq_n_u16(0);<br>
+<br>
+ for (int y = 0; y < endY; y++)<br>
+ {<br>
+ uint8x16_t count_acc_u8 = vdupq_n_u8(0);<br>
+<br>
+ // Calculate negated sign_left(x) directly, to save negation when<br>
+ // reusing sign_right(x) as sign_left(x + 1).<br>
+ int8x16_t neg_sign_left = vdupq_n_s8(x265_signOf(rec[-1] - rec[0]));<br>
+ for (int x = 0; x < endX; x += 16)<br>
+ {<br>
+ int8x16_t sign_right = signOf_neon(rec + x, rec + x + 1);<br>
+<br>
+ // neg_sign_left(x) = sign_right(x + 1), reusing one from previous<br>
+ // iteration.<br>
+ neg_sign_left = vextq_s8(neg_sign_left, sign_right, 15);<br>
+<br>
+ // Subtract instead of add, as sign_left is negated.<br>
+ int8x16_t edge_type = vsubq_s8(sign_right, neg_sign_left);<br>
+<br>
+ // For reuse in the next iteration.<br>
+ neg_sign_left = sign_right;<br>
+<br>
+ edge_type = x265_sve_mask(x, endX, edge_type);<br>
+ count_acc_u8 = vaddq_u8(count_acc_u8, sve_count(edge_type));<br>
+ compute_eo_stats(edge_type, diff + x, tmp_stats);<br>
+ }<br>
+<br>
+ // The width (endX) can be a maximum of 64, so we can safely<br>
+ // widen from 8-bit count accumulators after one inner loop iteration.<br>
+ // Technically the largest an accumulator could reach after one inner<br>
+ // loop iteration is 64, if every input value had the same edge type, so<br>
+ // we could complete two iterations (2 * 64 = 128) before widening.<br>
+ count_acc_u16 = vaddw_u8(count_acc_u16, vget_low_u8(count_acc_u8));<br>
+<br>
+ diff += MAX_CU_SIZE;<br>
+ rec += stride;<br>
+ }<br>
+<br>
+ reduce_eo_stats(tmp_stats, count_acc_u16, stats, count);<br>
+}<br>
+<br>
+void saoCuStatsE1_sve2(const int16_t *diff, const pixel *rec, intptr_t stride,<br>
+ int8_t *upBuff1, int endX, int endY, int32_t *stats,<br>
+ int32_t *count)<br>
+{<br>
+ // Separate buffers for each edge type, so that we can vectorise.<br>
+ int64x2_t tmp_stats[5] = { vdupq_n_s64(0), vdupq_n_s64(0), vdupq_n_s64(0),<br>
+ vdupq_n_s64(0), vdupq_n_s64(0) };<br>
+ uint16x8_t count_acc_u16 = vdupq_n_u16(0);<br>
+<br>
+ // Negate upBuff1 (sign_up), so we can subtract and save repeated negations.<br>
+ for (int x = 0; x < endX; x += 16)<br>
+ {<br>
+ vst1q_s8(upBuff1 + x, vnegq_s8(vld1q_s8(upBuff1 + x)));<br>
+ }<br>
+<br>
+ for (int y = 0; y < endY; y++)<br>
+ {<br>
+ uint8x16_t count_acc_u8 = vdupq_n_u8(0);<br>
+<br>
+ for (int x = 0; x < endX; x += 16)<br>
+ {<br>
+ int8x16_t sign_up = vld1q_s8(upBuff1 + x);<br>
+ int8x16_t sign_down = signOf_neon(rec + x, rec + x + stride);<br>
+<br>
+ // Subtract instead of add, as sign_up is negated.<br>
+ int8x16_t edge_type = vsubq_s8(sign_down, sign_up);<br>
+<br>
+ // For reuse in the next iteration.<br>
+ vst1q_s8(upBuff1 + x, sign_down);<br>
+<br>
+ edge_type = x265_sve_mask(x, endX, edge_type);<br>
+ count_acc_u8 = vaddq_u8(count_acc_u8, sve_count(edge_type));<br>
+ compute_eo_stats(edge_type, diff + x, tmp_stats);<br>
+ }<br>
+<br>
+ // The width (endX) can be a maximum of 64, so we can safely<br>
+ // widen from 8-bit count accumulators after one inner loop iteration.<br>
+ // Technically the largest an accumulator could reach after one inner<br>
+ // loop iteration is 64, if every input value had the same edge type, so<br>
+ // we could complete two iterations (2 * 64 = 128) before widening.<br>
+ count_acc_u16 = vaddw_u8(count_acc_u16, vget_low_u8(count_acc_u8));<br>
+<br>
+ diff += MAX_CU_SIZE;<br>
+ rec += stride;<br>
+ }<br>
+<br>
+ reduce_eo_stats(tmp_stats, count_acc_u16, stats, count);<br>
+}<br>
+<br>
+void saoCuStatsE2_sve2(const int16_t *diff, const pixel *rec, intptr_t stride,<br>
+ int8_t *upBuff1, int8_t *upBufft, int endX, int endY,<br>
+ int32_t *stats, int32_t *count)<br>
+{<br>
+ // Separate buffers for each edge type, so that we can vectorise.<br>
+ int64x2_t tmp_stats[5] = { vdupq_n_s64(0), vdupq_n_s64(0), vdupq_n_s64(0),<br>
+ vdupq_n_s64(0), vdupq_n_s64(0) };<br>
+ uint16x8_t count_acc_u16 = vdupq_n_u16(0);<br>
+<br>
+ // Negate upBuff1 (sign_up) so we can subtract and save repeated negations.<br>
+ for (int x = 0; x < endX; x += 16)<br>
+ {<br>
+ vst1q_s8(upBuff1 + x, vnegq_s8(vld1q_s8(upBuff1 + x)));<br>
+ }<br>
+<br>
+ for (int y = 0; y < endY; y++)<br>
+ {<br>
+ uint8x16_t count_acc_u8 = vdupq_n_u8(0);<br>
+<br>
+ upBufft[0] = x265_signOf(rec[-1] - rec[stride]);<br>
+ for (int x = 0; x < endX; x += 16)<br>
+ {<br>
+ int8x16_t sign_up = vld1q_s8(upBuff1 + x);<br>
+ int8x16_t sign_down = signOf_neon(rec + x, rec + x + stride + 1);<br>
+<br>
+ // Subtract instead of add, as sign_up is negated.<br>
+ int8x16_t edge_type = vsubq_s8(sign_down, sign_up);<br>
+<br>
+ // For reuse in the next iteration.<br>
+ vst1q_s8(upBufft + x + 1, sign_down);<br>
+<br>
+ edge_type = x265_sve_mask(x, endX, edge_type);<br>
+ count_acc_u8 = vaddq_u8(count_acc_u8, sve_count(edge_type));<br>
+ compute_eo_stats(edge_type, diff + x, tmp_stats);<br>
+ }<br>
+<br>
+ std::swap(upBuff1, upBufft);<br>
+<br>
+ // The width (endX) can be a maximum of 64, so we can safely<br>
+ // widen from 8-bit count accumulators after one inner loop iteration.<br>
+ // Technically the largest an accumulator could reach after one inner<br>
+ // loop iteration is 64, if every input value had the same edge type, so<br>
+ // we could complete two iterations (2 * 64 = 128) before widening.<br>
+ count_acc_u16 = vaddw_u8(count_acc_u16, vget_low_u8(count_acc_u8));<br>
+<br>
+ rec += stride;<br>
+ diff += MAX_CU_SIZE;<br>
+ }<br>
+<br>
+ reduce_eo_stats(tmp_stats, count_acc_u16, stats, count);<br>
+}<br>
+<br>
+void saoCuStatsE3_sve2(const int16_t *diff, const pixel *rec, intptr_t stride,<br>
+ int8_t *upBuff1, int endX, int endY, int32_t *stats,<br>
+ int32_t *count)<br>
+{<br>
+ // Separate buffers for each edge type, so that we can vectorise.<br>
+ int64x2_t tmp_stats[5] = { vdupq_n_s64(0), vdupq_n_s64(0), vdupq_n_s64(0),<br>
+ vdupq_n_s64(0), vdupq_n_s64(0) };<br>
+ uint16x8_t count_acc_u16 = vdupq_n_u16(0);<br>
+<br>
+ // Negate upBuff1 (sign_up) so we can subtract and save repeated negations.<br>
+ for (int x = 0; x < endX; x += 16)<br>
+ {<br>
+ vst1q_s8(upBuff1 + x, vnegq_s8(vld1q_s8(upBuff1 + x)));<br>
+ }<br>
+<br>
+ for (int y = 0; y < endY; y++)<br>
+ {<br>
+ uint8x16_t count_acc_u8 = vdupq_n_u8(0);<br>
+<br>
+ for (int x = 0; x < endX; x += 16)<br>
+ {<br>
+ int8x16_t sign_up = vld1q_s8(upBuff1 + x);<br>
+ int8x16_t sign_down = signOf_neon(rec + x, rec + x + stride - 1);<br>
+<br>
+ // Subtract instead of add, as sign_up is negated.<br>
+ int8x16_t edge_type = vsubq_s8(sign_down, sign_up);<br>
+<br>
+ // For reuse in the next iteration.<br>
+ vst1q_s8(upBuff1 + x - 1, sign_down);<br>
+<br>
+ edge_type = x265_sve_mask(x, endX, edge_type);<br>
+ count_acc_u8 = vaddq_u8(count_acc_u8, sve_count(edge_type));<br>
+ compute_eo_stats(edge_type, diff + x, tmp_stats);<br>
+ }<br>
+<br>
+ upBuff1[endX - 1] = x265_signOf(rec[endX] - rec[endX - 1 + stride]);<br>
+<br>
+ // The width (endX) can be a maximum of 64, so we can safely<br>
+ // widen from 8-bit count accumulators after one inner loop iteration.<br>
+ // Technically the largest an accumulator could reach after one inner<br>
+ // loop iteration is 64, if every input value had the same edge type, so<br>
+ // we could complete two iterations (2 * 64 = 128) before widening.<br>
+ count_acc_u16 = vaddw_u8(count_acc_u16, vget_low_u8(count_acc_u8));<br>
+<br>
+ rec += stride;<br>
+ diff += MAX_CU_SIZE;<br>
+ }<br>
+<br>
+ reduce_eo_stats(tmp_stats, count_acc_u16, stats, count);<br>
+}<br>
+<br>
+void setupSaoPrimitives_sve2(EncoderPrimitives &p)<br>
+{<br>
+ p.saoCuStatsE0 = saoCuStatsE0_sve2;<br>
+ p.saoCuStatsE1 = saoCuStatsE1_sve2;<br>
+ p.saoCuStatsE2 = saoCuStatsE2_sve2;<br>
+ p.saoCuStatsE3 = saoCuStatsE3_sve2;<br>
+}<br>
+} // namespace X265_NS<br>
diff --git a/source/common/aarch64/sao-prim.h b/source/common/aarch64/sao-prim.h<br>
index e01dd28a5..4eba4bfda 100644<br>
--- a/source/common/aarch64/sao-prim.h<br>
+++ b/source/common/aarch64/sao-prim.h<br>
@@ -27,6 +27,37 @@<br>
#include "primitives.h"<br>
#include <arm_neon.h><br>
<br>
+#if defined(HAVE_SVE) && HAVE_SVE_BRIDGE<br>
+#include <arm_neon_sve_bridge.h><br>
+<br>
+/* We can access instructions that are exclusive to the SVE or SVE2 instruction<br>
+ * sets from a predominantly Neon context by making use of the Neon-SVE bridge<br>
+ * intrinsics to reinterpret Neon vectors as SVE vectors - with the high part of<br>
+ * the SVE vector (if it's longer than 128 bits) being "don't care".<br>
+ *<br>
+ * While sub-optimal on machines that have SVE vector length > 128-bit - as the<br>
+ * remainder of the vector is unused - this approach is still beneficial when<br>
+ * compared to a Neon-only implementation. */<br>
+<br>
+static inline int8x16_t x265_sve_mask(const int x, const int endX,<br>
+ const int8x16_t in)<br>
+{<br>
+ // Use predicate to shift "unused lanes" outside of range [-2, 2]<br>
+ svbool_t svpred = svwhilelt_b8(x, endX);<br>
+ svint8_t edge_type = svsel_s8(svpred, svset_neonq_s8(svundef_s8(), in),<br>
+ svdup_n_s8(-3));<br>
+ return svget_neonq_s8(edge_type);<br>
+}<br>
+<br>
+static inline int64x2_t x265_sdotq_s16(int64x2_t acc, int16x8_t x, int16x8_t y)<br>
+{<br>
+ return svget_neonq_s64(svdot_s64(svset_neonq_s64(svundef_s64(), acc),<br>
+ svset_neonq_s16(svundef_s16(), x),<br>
+ svset_neonq_s16(svundef_s16(), y)));<br>
+}<br>
+<br>
+#endif // defined(HAVE_SVE) && HAVE_SVE_BRIDGE<br>
+<br>
static inline int8x16_t signOf_neon(const pixel *a, const pixel *b)<br>
{<br>
#if HIGH_BIT_DEPTH<br>
@@ -60,6 +91,10 @@ void setupSaoPrimitives_neon(EncoderPrimitives &p);<br>
#if defined(HAVE_SVE) && HAVE_SVE_BRIDGE<br>
void setupSaoPrimitives_sve(EncoderPrimitives &p);<br>
#endif<br>
+<br>
+#if defined(HAVE_SVE2) && HAVE_SVE_BRIDGE<br>
+void setupSaoPrimitives_sve2(EncoderPrimitives &p);<br>
+#endif<br>
}<br>
<br>
#endif // X265_COMMON_AARCH64_SAO_PRIM_H<br>
-- <br>
2.42.1<br>
<br>
_______________________________________________<br>
x265-devel mailing list<br>
<a href="mailto:x265-devel@videolan.org" target="_blank">x265-devel@videolan.org</a><br>
<a href="https://mailman.videolan.org/listinfo/x265-devel" rel="noreferrer" target="_blank">https://mailman.videolan.org/listinfo/x265-devel</a><br>
</blockquote></div>