[x265] [PATCH] Replace sad_64 vector class function with intrinsic

yuvaraj at multicorewareinc.com yuvaraj at multicorewareinc.com
Fri Oct 4 12:34:28 CEST 2013


# HG changeset patch
# User yuvaraj
# Date 1380882745 -19800
#      Fri Oct 04 16:02:25 2013 +0530
# Node ID 5f7cf1250398451b97eaf72bcb418be4ca46d559
# Parent  0ca107fe5071b986404cf764a55ada128edb35d0
Replace sad_64 vector class function with intrinsic.

diff -r 0ca107fe5071 -r 5f7cf1250398 source/common/vec/pixel8.inc
--- a/source/common/vec/pixel8.inc	Fri Oct 04 15:32:59 2013 +0530
+++ b/source/common/vec/pixel8.inc	Fri Oct 04 16:02:25 2013 +0530
@@ -724,53 +724,857 @@
 
 #endif /* if INSTRSET >= X265_CPU_LEVEL_SSE41 */
 
-template<int size>
-ALWAYSINLINE void unrollFunc_64(pixel *fenc, intptr_t fencstride, pixel *fref, intptr_t frefstride, Vec8us& sad)
-{
-    unrollFunc_64<1>(fenc, fencstride, fref, frefstride, sad);
-    unrollFunc_64<size - 1>(fenc + fencstride, fencstride, fref + frefstride, frefstride, sad);
-}
-
-template<>
-ALWAYSINLINE void unrollFunc_64<1>(pixel *fenc, intptr_t, pixel *fref, intptr_t, Vec8us& sad)
-{
-    Vec16uc m1, n1;
-
-    m1.load_a(fenc);
-    n1.load(fref);
-    sad.addSumAbsDiff(m1, n1);
-
-    m1.load_a(fenc + 16);
-    n1.load(fref + 16);
-    sad.addSumAbsDiff(m1, n1);
-
-    m1.load_a(fenc + 32);
-    n1.load(fref + 32);
-    sad.addSumAbsDiff(m1, n1);
-
-    m1.load_a(fenc + 48);
-    n1.load(fref + 48);
-    sad.addSumAbsDiff(m1, n1);
-}
-
-template<int ly>
-int sad_64(pixel * fenc, intptr_t fencstride, pixel * fref, intptr_t frefstride)
-{
-    Vec4i sum(0);
-    Vec8us sad;
-    int max_iterators = (ly >> 2) << 2;
-    int row;
-    for (row = 0; row < max_iterators; row += 4)
-    {
-        sad = 0;
-        unrollFunc_64<4>(fenc, fencstride, fref, frefstride, sad);
-        sum += extend_low(sad) + extend_high(sad);
-        fenc += fencstride * 4;
-        fref += frefstride * 4;
-    }
-
-    return horizontal_add(sum);
-}
+#if INSTRSET >= X265_CPU_LEVEL_SSE41
+template<int ly>
+int sad_64(pixel * fenc, intptr_t fencstride, pixel * fref, intptr_t frefstride)
+{
+    assert((ly % 4) == 0);
+
+    __m128i sum0 = _mm_setzero_si128();
+    __m128i sum1 = _mm_setzero_si128();
+
+    if (ly == 4)
+    {
+        __m128i T00, T01, T02, T03;
+        __m128i T10, T11, T12, T13;
+        __m128i T20, T21, T22, T23;
+
+        T00 = _mm_load_si128((__m128i*)(fenc));              /*Loding 64 8-bit integer from fenc to local variables*/
+        T01 = _mm_load_si128((__m128i*)(fenc + 16));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref));              /*Loding 64 8-bit integer from fref to local variables*/
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (1) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (1) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (1) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (1) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (1) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (1) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (1) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (1) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (2) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (2) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (2) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (2) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (2) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (2) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (2) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (3) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (3) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (3) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (3) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (3) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (3) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (3) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+    }
+    else if (ly == 8)
+    {
+        __m128i T00, T01, T02, T03;
+        __m128i T10, T11, T12, T13;
+        __m128i T20, T21, T22, T23;
+
+        T00 = _mm_load_si128((__m128i*)(fenc));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (1) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (1) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (1) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (1) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (1) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (1) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (1) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (1) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (2) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (2) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (2) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (2) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (2) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (2) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (2) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (3) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (3) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (3) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (3) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (3) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (3) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (3) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (4) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (4) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (4) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (4) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (4) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (4) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (4) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (5) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (5) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (5) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (5) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (5) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (5) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (5) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (6) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (6) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (6) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (6) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (6) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (6) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (6) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (7) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (7) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (7) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (7) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (7) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (7) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (7) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+    }
+    else if (ly == 16)
+    {
+        __m128i T00, T01, T02, T03;
+        __m128i T10, T11, T12, T13;
+        __m128i T20, T21, T22, T23;
+
+        T00 = _mm_load_si128((__m128i*)(fenc));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (1) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (1) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (1) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (1) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (1) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (1) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (1) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (1) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (2) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (2) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (2) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (2) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (2) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (2) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (2) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (3) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (3) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (3) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (3) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (3) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (3) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (3) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (4) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (4) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (4) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (4) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (4) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (4) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (4) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (5) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (5) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (5) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (5) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (5) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (5) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (5) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (6) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (6) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (6) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (6) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (6) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (6) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (6) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (7) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (7) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (7) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (7) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (7) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (7) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (7) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (8) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (8) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (8) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (8) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (8) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (8) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (8) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (8) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (9) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (9) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (9) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (9) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (9) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (9) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (9) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (9) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (10) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (10) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (10) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (10) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (10) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (10) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (10) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (10) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (11) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (11) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (11) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (11) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (11) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (11) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (11) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (11) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (12) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (12) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (12) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (12) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (12) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (12) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (12) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (12) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (13) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (13) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (13) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (13) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (13) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (13) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (13) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (13) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (14) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (14) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (14) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (14) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (14) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (14) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (14) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (14) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (15) * fencstride));
+        T01 = _mm_load_si128((__m128i*)(fenc + 16 + (15) * fencstride));
+        T02 = _mm_load_si128((__m128i*)(fenc + 32 + (15) * fencstride));
+        T03 = _mm_load_si128((__m128i*)(fenc + 48 + (15) * fencstride));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref + (15) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (15) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (15) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (15) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        sum0 = _mm_add_epi16(sum0, T20);
+        sum0 = _mm_add_epi16(sum0, T21);
+        sum0 = _mm_add_epi16(sum0, T22);
+        sum0 = _mm_add_epi16(sum0, T23);
+    }
+    else if ((ly % 8) == 0)
+    {
+        for (int i = 0; i < ly; i += 8)
+        {
+            __m128i T00, T01, T02, T03;
+            __m128i T10, T11, T12, T13;
+            __m128i T20, T21, T22, T23;
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 0) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 0) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 0) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 0) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 0) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 0) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 0) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 1) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 1) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 1) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 1) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 1) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 1) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 1) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 2) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 2) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 2) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 2) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 2) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 2) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 2) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 3) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 3) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 3) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 3) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 3) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 3) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 3) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 4) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 4) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 4) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 4) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 4) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 4) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 4) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 4) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 5) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 5) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 5) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 5) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 5) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 5) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 5) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 5) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 6) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 6) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 6) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 6) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 6) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 6) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 6) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 6) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 7) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 7) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 7) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 7) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 7) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 7) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 7) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 7) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+        }
+    }
+    else
+    {
+        for (int i = 0; i < ly; i += 4)
+        {
+            __m128i T00, T01, T02, T03;
+            __m128i T10, T11, T12, T13;
+            __m128i T20, T21, T22, T23;
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 0) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 0) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 0) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 0) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 0) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 0) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 0) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 1) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 1) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 1) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 1) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 1) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 1) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 1) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 2) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 2) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 2) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 2) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 2) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 2) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 2) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 3) * fencstride));
+            T01 = _mm_load_si128((__m128i*)(fenc + 16 + (i + 3) * fencstride));
+            T02 = _mm_load_si128((__m128i*)(fenc + 32 + (i + 3) * fencstride));
+            T03 = _mm_load_si128((__m128i*)(fenc + 48 + (i + 3) * fencstride));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref + (i + 3) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref + 16 + (i + 3) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref + 32 + (i + 3) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref + 48 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            sum0 = _mm_add_epi32(sum0, T20);
+            sum0 = _mm_add_epi32(sum0, T21);
+            sum0 = _mm_add_epi32(sum0, T22);
+            sum0 = _mm_add_epi32(sum0, T23);
+        }
+    }
+    sum1 = _mm_shuffle_epi32(sum0, 2);
+    sum0 = _mm_add_epi32(sum0, sum1);
+    return _mm_cvtsi128_si32(sum0);
+}
+
+#endif /* if INSTRSET >= X265_CPU_LEVEL_SSE41 */
 
 template<int ly>
 void sad_x3_48(pixel *fenc, pixel *fref1, pixel *fref2, pixel *fref3, intptr_t frefstride, int *res)



More information about the x265-devel mailing list