[x265] [PATCH] pixel8.inc: SAD_x4_16XN optimized

praveen at multicorewareinc.com praveen at multicorewareinc.com
Tue Sep 3 08:24:29 CEST 2013


# HG changeset patch
# User praveen Tiwari
# Date 1378189457 -19800
# Node ID e52c71811663d5360e150af2c21c79835a39304a
# Parent  1b3cc26779af3b485b1193c4becdd42d294d1f83
pixel8.inc: SAD_x4_16XN optimized

diff -r 1b3cc26779af -r e52c71811663 source/common/vec/pixel8.inc
--- a/source/common/vec/pixel8.inc	Tue Sep 03 11:46:18 2013 +0530
+++ b/source/common/vec/pixel8.inc	Tue Sep 03 11:54:17 2013 +0530
@@ -7137,805 +7137,824 @@
     res[3] = horizontal_add(sum4);
 }
 
-template<int ly>
-void sad_x4_16(pixel *fenc, pixel *fref1, pixel *fref2, pixel *fref3, pixel *fref4, intptr_t frefstride, int *res)
-{
-    assert((ly % 4) == 0);
-
-    __m128i sum0, sum1;
-
-    __m128i T00, T01, T02, T03;
-    __m128i T10, T11, T12, T13;
-    __m128i T20, T21, T22, T23;
-
-    if (ly == 4)
-    {
-        T00 = _mm_load_si128((__m128i*)(fenc + (0) * FENC_STRIDE));
-        T01 = _mm_load_si128((__m128i*)(fenc + (1) * FENC_STRIDE));
-        T02 = _mm_load_si128((__m128i*)(fenc + (2) * FENC_STRIDE));
-        T03 = _mm_load_si128((__m128i*)(fenc + (3) * FENC_STRIDE));
-
-        T10 = _mm_loadu_si128((__m128i*)(fref1 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref1 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref1 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref1 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[0] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref2 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref2 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref2 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref2 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[1] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref3 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref3 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref3 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref3 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[2] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref4 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref4 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref4 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref4 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[3] = _mm_cvtsi128_si32(sum0);
-    }
-    else if (ly == 8)
-    {
-        T00 = _mm_load_si128((__m128i*)(fenc + (0) * FENC_STRIDE));
-        T01 = _mm_load_si128((__m128i*)(fenc + (1) * FENC_STRIDE));
-        T02 = _mm_load_si128((__m128i*)(fenc + (2) * FENC_STRIDE));
-        T03 = _mm_load_si128((__m128i*)(fenc + (3) * FENC_STRIDE));
-
-        T10 = _mm_loadu_si128((__m128i*)(fref1 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref1 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref1 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref1 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[0] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref2 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref2 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref2 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref2 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[1] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref3 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref3 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref3 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref3 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[2] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref4 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref4 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref4 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref4 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[3] = _mm_cvtsi128_si32(sum0);
-
-        T00 = _mm_load_si128((__m128i*)(fenc + (4) * FENC_STRIDE));
-        T01 = _mm_load_si128((__m128i*)(fenc + (5) * FENC_STRIDE));
-        T02 = _mm_load_si128((__m128i*)(fenc + (6) * FENC_STRIDE));
-        T03 = _mm_load_si128((__m128i*)(fenc + (7) * FENC_STRIDE));
-
-        T10 = _mm_loadu_si128((__m128i*)(fref1 + (4) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref1 + (5) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref1 + (6) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref1 + (7) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[0] = res[0] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref2 + (4) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref2 + (5) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref2 + (6) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref2 + (7) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[1] = res[1] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref3 + (4) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref3 + (5) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref3 + (6) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref3 + (7) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[2] = res[2] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref4 + (4) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref4 + (5) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref4 + (6) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref4 + (7) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[3] = res[3] + _mm_cvtsi128_si32(sum0);
-    }
-    else if (ly == 16)
-    {
-        T00 = _mm_load_si128((__m128i*)(fenc + (0) * FENC_STRIDE));
-        T01 = _mm_load_si128((__m128i*)(fenc + (1) * FENC_STRIDE));
-        T02 = _mm_load_si128((__m128i*)(fenc + (2) * FENC_STRIDE));
-        T03 = _mm_load_si128((__m128i*)(fenc + (3) * FENC_STRIDE));
-
-        T10 = _mm_loadu_si128((__m128i*)(fref1 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref1 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref1 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref1 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[0] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref2 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref2 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref2 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref2 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[1] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref3 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref3 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref3 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref3 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[2] = _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref4 + (0) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref4 + (1) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref4 + (2) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref4 + (3) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[3] = _mm_cvtsi128_si32(sum0);
-
-        T00 = _mm_load_si128((__m128i*)(fenc + (4) * FENC_STRIDE));
-        T01 = _mm_load_si128((__m128i*)(fenc + (5) * FENC_STRIDE));
-        T02 = _mm_load_si128((__m128i*)(fenc + (6) * FENC_STRIDE));
-        T03 = _mm_load_si128((__m128i*)(fenc + (7) * FENC_STRIDE));
-
-        T10 = _mm_loadu_si128((__m128i*)(fref1 + (4) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref1 + (5) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref1 + (6) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref1 + (7) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[0] = res[0] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref2 + (4) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref2 + (5) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref2 + (6) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref2 + (7) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[1] = res[1] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref3 + (4) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref3 + (5) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref3 + (6) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref3 + (7) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[2] = res[2] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref4 + (4) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref4 + (5) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref4 + (6) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref4 + (7) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[3] = res[3] + _mm_cvtsi128_si32(sum0);
-
-        T00 = _mm_load_si128((__m128i*)(fenc + (8) * FENC_STRIDE));
-        T01 = _mm_load_si128((__m128i*)(fenc + (9) * FENC_STRIDE));
-        T02 = _mm_load_si128((__m128i*)(fenc + (10) * FENC_STRIDE));
-        T03 = _mm_load_si128((__m128i*)(fenc + (11) * FENC_STRIDE));
-
-        T10 = _mm_loadu_si128((__m128i*)(fref1 + (8) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref1 + (9) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref1 + (10) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref1 + (11) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[0] = res[0] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref2 + (8) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref2 + (9) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref2 + (10) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref2 + (11) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[1] = res[1] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref3 + (8) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref3 + (9) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref3 + (10) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref3 + (11) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[2] = res[2] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref4 + (8) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref4 + (9) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref4 + (10) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref4 + (11) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[3] = res[3] + _mm_cvtsi128_si32(sum0);
-
-        T00 = _mm_load_si128((__m128i*)(fenc + (12) * FENC_STRIDE));
-        T01 = _mm_load_si128((__m128i*)(fenc + (13) * FENC_STRIDE));
-        T02 = _mm_load_si128((__m128i*)(fenc + (14) * FENC_STRIDE));
-        T03 = _mm_load_si128((__m128i*)(fenc + (15) * FENC_STRIDE));
-
-        T10 = _mm_loadu_si128((__m128i*)(fref1 + (12) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref1 + (13) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref1 + (14) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref1 + (15) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[0] = res[0] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref2 + (12) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref2 + (13) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref2 + (14) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref2 + (15) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[1] = res[1] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref3 + (12) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref3 + (13) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref3 + (14) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref3 + (15) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[2] = res[2] + _mm_cvtsi128_si32(sum0);
-
-        T10 = _mm_loadu_si128((__m128i*)(fref4 + (12) * frefstride));
-        T11 = _mm_loadu_si128((__m128i*)(fref4 + (13) * frefstride));
-        T12 = _mm_loadu_si128((__m128i*)(fref4 + (14) * frefstride));
-        T13 = _mm_loadu_si128((__m128i*)(fref4 + (15) * frefstride));
-
-        T20 = _mm_sad_epu8(T00, T10);
-        T21 = _mm_sad_epu8(T01, T11);
-        T22 = _mm_sad_epu8(T02, T12);
-        T23 = _mm_sad_epu8(T03, T13);
-
-        T20 = _mm_add_epi16(T20, T21);
-        T22 = _mm_add_epi16(T22, T23);
-        sum0 = _mm_add_epi16(T20, T22);
-
-        sum1 = _mm_shuffle_epi32(sum0, 2);
-        sum0 = _mm_add_epi32(sum0, sum1);
-        res[3] = res[3] + _mm_cvtsi128_si32(sum0);
-    }
-    else if ((ly % 8) == 0)
-    {
-        res[0] = res[1] = res[2] = res[3] = 0;
-        for (int i = 0; i < ly; i += 8)
-        {
-            T00 = _mm_load_si128((__m128i*)(fenc + (i + 0) * FENC_STRIDE));
-            T01 = _mm_load_si128((__m128i*)(fenc + (i + 1) * FENC_STRIDE));
-            T02 = _mm_load_si128((__m128i*)(fenc + (i + 2) * FENC_STRIDE));
-            T03 = _mm_load_si128((__m128i*)(fenc + (i + 3) * FENC_STRIDE));
-
-            T10 = _mm_loadu_si128((__m128i*)(fref1 + (i + 0) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref1 + (i + 1) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref1 + (i + 2) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref1 + (i + 3) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[0] = res[0] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref2 + (i + 0) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref2 + (i + 1) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref2 + (i + 2) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref2 + (i + 3) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[1] = res[1] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref3 + (i + 0) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref3 + (i + 1) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref3 + (i + 2) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref3 + (i + 3) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[2] = res[2] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref4 + (i + 0) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref4 + (i + 1) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref4 + (i + 2) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref4 + (i + 3) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[3] = res[3] + _mm_cvtsi128_si32(sum0);
-
-            T00 = _mm_load_si128((__m128i*)(fenc + (i + 4) * FENC_STRIDE));
-            T01 = _mm_load_si128((__m128i*)(fenc + (i + 5) * FENC_STRIDE));
-            T02 = _mm_load_si128((__m128i*)(fenc + (i + 6) * FENC_STRIDE));
-            T03 = _mm_load_si128((__m128i*)(fenc + (i + 7) * FENC_STRIDE));
-
-            T10 = _mm_loadu_si128((__m128i*)(fref1 + (i + 4) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref1 + (i + 5) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref1 + (i + 6) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref1 + (i + 7) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[0] = res[0] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref2 + (i + 4) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref2 + (i + 5) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref2 + (i + 6) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref2 + (i + 7) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[1] = res[1] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref3 + (i + 4) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref3 + (i + 5) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref3 + (i + 6) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref3 + (i + 7) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[2] = res[2] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref4 + (i + 4) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref4 + (i + 5) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref4 + (i + 6) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref4 + (i + 7) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[3] = res[3] + _mm_cvtsi128_si32(sum0);
-        }
-    }
-    else
-    {
-        res[0] = res[1] = res[2] = res[3] = 0;
-        for (int i = 0; i < ly; i += 4)
-        {
-            T00 = _mm_load_si128((__m128i*)(fenc + (i + 0) * FENC_STRIDE));
-            T01 = _mm_load_si128((__m128i*)(fenc + (i + 1) * FENC_STRIDE));
-            T02 = _mm_load_si128((__m128i*)(fenc + (i + 2) * FENC_STRIDE));
-            T03 = _mm_load_si128((__m128i*)(fenc + (i + 3) * FENC_STRIDE));
-
-            T10 = _mm_loadu_si128((__m128i*)(fref1 + (i + 0) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref1 + (i + 1) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref1 + (i + 2) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref1 + (i + 3) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[0] = res[0] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref2 + (i + 0) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref2 + (i + 1) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref2 + (i + 2) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref2 + (i + 3) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[1] = res[1] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref3 + (i + 0) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref3 + (i + 1) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref3 + (i + 2) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref3 + (i + 3) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[2] = res[2] + _mm_cvtsi128_si32(sum0);
-
-            T10 = _mm_loadu_si128((__m128i*)(fref4 + (i + 0) * frefstride));
-            T11 = _mm_loadu_si128((__m128i*)(fref4 + (i + 1) * frefstride));
-            T12 = _mm_loadu_si128((__m128i*)(fref4 + (i + 2) * frefstride));
-            T13 = _mm_loadu_si128((__m128i*)(fref4 + (i + 3) * frefstride));
-
-            T20 = _mm_sad_epu8(T00, T10);
-            T21 = _mm_sad_epu8(T01, T11);
-            T22 = _mm_sad_epu8(T02, T12);
-            T23 = _mm_sad_epu8(T03, T13);
-
-            T20 = _mm_add_epi16(T20, T21);
-            T22 = _mm_add_epi16(T22, T23);
-            sum0 = _mm_add_epi16(T20, T22);
-
-            sum1 = _mm_shuffle_epi32(sum0, 2);
-            sum0 = _mm_add_epi32(sum0, sum1);
-            res[3] = res[3] + _mm_cvtsi128_si32(sum0);
-        }
-    }
+template<int ly>
+void sad_x4_16(pixel *fenc, pixel *fref1, pixel *fref2, pixel *fref3, pixel *fref4, intptr_t frefstride, int *res)
+{
+    assert((ly % 4) == 0);
+
+    if (ly == 4)
+    {
+        __m128i T00, T01, T02, T03;
+        __m128i T10, T11, T12, T13;
+        __m128i T20, T21, T22, T23;
+        __m128i sum0, sum1;
+
+        T00 = _mm_load_si128((__m128i*)(fenc));
+        T01 = _mm_load_si128((__m128i*)(fenc + FENC_STRIDE));
+        T02 = _mm_load_si128((__m128i*)(fenc + (2) * FENC_STRIDE));
+        T03 = _mm_load_si128((__m128i*)(fenc + (3) * FENC_STRIDE));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref1));
+        T11 = _mm_loadu_si128((__m128i*)(fref1 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref1 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref1 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[0] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref2));
+        T11 = _mm_loadu_si128((__m128i*)(fref2 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref2 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref2 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[1] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref3));
+        T11 = _mm_loadu_si128((__m128i*)(fref3 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref3 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref3 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[2] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref4));
+        T11 = _mm_loadu_si128((__m128i*)(fref4 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref4 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref4 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[3] = _mm_cvtsi128_si32(sum0);
+    }
+    else if (ly == 8)
+    {
+        __m128i T00, T01, T02, T03;
+        __m128i T10, T11, T12, T13;
+        __m128i T20, T21, T22, T23;
+        __m128i sum0, sum1;
+
+        T00 = _mm_load_si128((__m128i*)(fenc));
+        T01 = _mm_load_si128((__m128i*)(fenc + FENC_STRIDE));
+        T02 = _mm_load_si128((__m128i*)(fenc + (2) * FENC_STRIDE));
+        T03 = _mm_load_si128((__m128i*)(fenc + (3) * FENC_STRIDE));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref1));
+        T11 = _mm_loadu_si128((__m128i*)(fref1 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref1 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref1 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[0] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref2));
+        T11 = _mm_loadu_si128((__m128i*)(fref2 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref2 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref2 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[1] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref3));
+        T11 = _mm_loadu_si128((__m128i*)(fref3 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref3 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref3 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[2] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref4));
+        T11 = _mm_loadu_si128((__m128i*)(fref4 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref4 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref4 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[3] = _mm_cvtsi128_si32(sum0);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (4) * FENC_STRIDE));
+        T01 = _mm_load_si128((__m128i*)(fenc + (5) * FENC_STRIDE));
+        T02 = _mm_load_si128((__m128i*)(fenc + (6) * FENC_STRIDE));
+        T03 = _mm_load_si128((__m128i*)(fenc + (7) * FENC_STRIDE));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref1 + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref1 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref1 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref1 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref2 + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref2 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref2 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref2 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref3 + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref3 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref3 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref3 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref4 + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref4 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref4 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref4 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[3] = res[3] + _mm_cvtsi128_si32(sum0);
+    }
+    else if (ly == 16)
+    {
+        __m128i T00, T01, T02, T03;
+        __m128i T10, T11, T12, T13;
+        __m128i T20, T21, T22, T23;
+        __m128i sum0, sum1;
+
+        T00 = _mm_load_si128((__m128i*)(fenc));
+        T01 = _mm_load_si128((__m128i*)(fenc + FENC_STRIDE));
+        T02 = _mm_load_si128((__m128i*)(fenc + (2) * FENC_STRIDE));
+        T03 = _mm_load_si128((__m128i*)(fenc + (3) * FENC_STRIDE));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref1));
+        T11 = _mm_loadu_si128((__m128i*)(fref1 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref1 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref1 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[0] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref2));
+        T11 = _mm_loadu_si128((__m128i*)(fref2 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref2 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref2 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[1] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref3));
+        T11 = _mm_loadu_si128((__m128i*)(fref3 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref3 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref3 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[2] = _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref4));
+        T11 = _mm_loadu_si128((__m128i*)(fref4 + frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref4 + (2) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref4 + (3) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[3] = _mm_cvtsi128_si32(sum0);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (4) * FENC_STRIDE));
+        T01 = _mm_load_si128((__m128i*)(fenc + (5) * FENC_STRIDE));
+        T02 = _mm_load_si128((__m128i*)(fenc + (6) * FENC_STRIDE));
+        T03 = _mm_load_si128((__m128i*)(fenc + (7) * FENC_STRIDE));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref1 + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref1 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref1 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref1 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref2 + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref2 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref2 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref2 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref3 + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref3 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref3 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref3 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref4 + (4) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref4 + (5) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref4 + (6) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref4 + (7) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[3] = res[3] + _mm_cvtsi128_si32(sum0);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (8) * FENC_STRIDE));
+        T01 = _mm_load_si128((__m128i*)(fenc + (9) * FENC_STRIDE));
+        T02 = _mm_load_si128((__m128i*)(fenc + (10) * FENC_STRIDE));
+        T03 = _mm_load_si128((__m128i*)(fenc + (11) * FENC_STRIDE));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref1 + (8) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref1 + (9) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref1 + (10) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref1 + (11) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref2 + (8) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref2 + (9) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref2 + (10) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref2 + (11) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref3 + (8) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref3 + (9) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref3 + (10) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref3 + (11) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref4 + (8) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref4 + (9) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref4 + (10) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref4 + (11) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[3] = res[3] + _mm_cvtsi128_si32(sum0);
+
+        T00 = _mm_load_si128((__m128i*)(fenc + (12) * FENC_STRIDE));
+        T01 = _mm_load_si128((__m128i*)(fenc + (13) * FENC_STRIDE));
+        T02 = _mm_load_si128((__m128i*)(fenc + (14) * FENC_STRIDE));
+        T03 = _mm_load_si128((__m128i*)(fenc + (15) * FENC_STRIDE));
+
+        T10 = _mm_loadu_si128((__m128i*)(fref1 + (12) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref1 + (13) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref1 + (14) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref1 + (15) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref2 + (12) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref2 + (13) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref2 + (14) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref2 + (15) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref3 + (12) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref3 + (13) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref3 + (14) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref3 + (15) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+        T10 = _mm_loadu_si128((__m128i*)(fref4 + (12) * frefstride));
+        T11 = _mm_loadu_si128((__m128i*)(fref4 + (13) * frefstride));
+        T12 = _mm_loadu_si128((__m128i*)(fref4 + (14) * frefstride));
+        T13 = _mm_loadu_si128((__m128i*)(fref4 + (15) * frefstride));
+
+        T20 = _mm_sad_epu8(T00, T10);
+        T21 = _mm_sad_epu8(T01, T11);
+        T22 = _mm_sad_epu8(T02, T12);
+        T23 = _mm_sad_epu8(T03, T13);
+
+        T20 = _mm_add_epi16(T20, T21);
+        T22 = _mm_add_epi16(T22, T23);
+        sum0 = _mm_add_epi16(T20, T22);
+
+        sum1 = _mm_shuffle_epi32(sum0, 2);
+        sum0 = _mm_add_epi32(sum0, sum1);
+        res[3] = res[3] + _mm_cvtsi128_si32(sum0);
+    }
+    else if ((ly % 8) == 0)
+    {
+        __m128i T00, T01, T02, T03;
+        __m128i T10, T11, T12, T13;
+        __m128i T20, T21, T22, T23;
+        __m128i sum0, sum1;
+
+        res[0] = res[1] = res[2] = res[3] = 0;
+        for (int i = 0; i < ly; i += 8)
+        {
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 0) * FENC_STRIDE));
+            T01 = _mm_load_si128((__m128i*)(fenc + (i + 1) * FENC_STRIDE));
+            T02 = _mm_load_si128((__m128i*)(fenc + (i + 2) * FENC_STRIDE));
+            T03 = _mm_load_si128((__m128i*)(fenc + (i + 3) * FENC_STRIDE));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref1 + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref1 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref1 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref1 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref2 + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref2 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref2 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref2 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref3 + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref3 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref3 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref3 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref4 + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref4 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref4 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref4 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[3] = res[3] + _mm_cvtsi128_si32(sum0);
+
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 4) * FENC_STRIDE));
+            T01 = _mm_load_si128((__m128i*)(fenc + (i + 5) * FENC_STRIDE));
+            T02 = _mm_load_si128((__m128i*)(fenc + (i + 6) * FENC_STRIDE));
+            T03 = _mm_load_si128((__m128i*)(fenc + (i + 7) * FENC_STRIDE));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref1 + (i + 4) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref1 + (i + 5) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref1 + (i + 6) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref1 + (i + 7) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref2 + (i + 4) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref2 + (i + 5) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref2 + (i + 6) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref2 + (i + 7) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref3 + (i + 4) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref3 + (i + 5) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref3 + (i + 6) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref3 + (i + 7) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref4 + (i + 4) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref4 + (i + 5) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref4 + (i + 6) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref4 + (i + 7) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[3] = res[3] + _mm_cvtsi128_si32(sum0);
+        }
+    }
+    else
+    {
+        __m128i T00, T01, T02, T03;
+        __m128i T10, T11, T12, T13;
+        __m128i T20, T21, T22, T23;
+        __m128i sum0, sum1;
+
+        res[0] = res[1] = res[2] = res[3] = 0;
+        for (int i = 0; i < ly; i += 4)
+        {
+            T00 = _mm_load_si128((__m128i*)(fenc + (i + 0) * FENC_STRIDE));
+            T01 = _mm_load_si128((__m128i*)(fenc + (i + 1) * FENC_STRIDE));
+            T02 = _mm_load_si128((__m128i*)(fenc + (i + 2) * FENC_STRIDE));
+            T03 = _mm_load_si128((__m128i*)(fenc + (i + 3) * FENC_STRIDE));
+
+            T10 = _mm_loadu_si128((__m128i*)(fref1 + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref1 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref1 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref1 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref2 + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref2 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref2 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref2 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref3 + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref3 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref3 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref3 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+            T10 = _mm_loadu_si128((__m128i*)(fref4 + (i + 0) * frefstride));
+            T11 = _mm_loadu_si128((__m128i*)(fref4 + (i + 1) * frefstride));
+            T12 = _mm_loadu_si128((__m128i*)(fref4 + (i + 2) * frefstride));
+            T13 = _mm_loadu_si128((__m128i*)(fref4 + (i + 3) * frefstride));
+
+            T20 = _mm_sad_epu8(T00, T10);
+            T21 = _mm_sad_epu8(T01, T11);
+            T22 = _mm_sad_epu8(T02, T12);
+            T23 = _mm_sad_epu8(T03, T13);
+
+            T20 = _mm_add_epi16(T20, T21);
+            T22 = _mm_add_epi16(T22, T23);
+            sum0 = _mm_add_epi16(T20, T22);
+
+            sum1 = _mm_shuffle_epi32(sum0, 2);
+            sum0 = _mm_add_epi32(sum0, sum1);
+            res[3] = res[3] + _mm_cvtsi128_si32(sum0);
+        }
+    }
 }
 
 #endif /* if INSTRSET >= X265_CPU_LEVEL_SSE41 */


More information about the x265-devel mailing list