[x265] [PATCH] pixel8inc: sad_x3 8xN, replace vector with intrinsic
praveen at multicorewareinc.com
praveen at multicorewareinc.com
Fri Aug 23 15:25:32 CEST 2013
# HG changeset patch
# User praveentiwari
# Date 1377264321 -19800
# Node ID 76b9c59e69aada33370b0d47a6d5c4ddbe3701d8
# Parent 7e5c568e480489d74c50673d538ddd323218cf97
pixel8inc: sad_x3 8xN, replace vector with intrinsic
diff -r 7e5c568e4804 -r 76b9c59e69aa source/common/vec/pixel8.inc
--- a/source/common/vec/pixel8.inc Fri Aug 23 17:30:54 2013 +0530
+++ b/source/common/vec/pixel8.inc Fri Aug 23 18:55:21 2013 +0530
@@ -1818,68 +1818,527 @@
}
}
-/* For performance - This function assumes that the *last load* can access 16 elements. */
template<int ly>
void sad_x3_8(pixel *fenc, pixel *fref1, pixel *fref2, pixel *fref3, intptr_t frefstride, int *res)
{
- Vec16uc m1, n1, n2, n3;
-
- Vec4i sum1(0), sum2(0), sum3(0);
- Vec8us sad1(0), sad2(0), sad3(0);
- int max_iterators = (ly >> 4) << 4;
- int row;
-
- for (row = 0; row < max_iterators; row += 16)
+ assert((ly % 4) == 0);
+ __m128i sum0 = _mm_setzero_si128();
+
+ __m128i T00, T01, T02, T03;
+ __m128i T10, T11, T12, T13;
+ __m128i T20, T21;
+
+ if (ly == 4)
{
- for (int i = 0; i < 16; i++)
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (0) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (1) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (2) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (3) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = _mm_cvtsi128_si32(sum0);
+ }
+
+ if (ly == 8)
+ {
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (0) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (1) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (2) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (3) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = _mm_cvtsi128_si32(sum0);
+
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (4) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (5) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (6) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (7) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+ }
+ if (ly == 16)
+ {
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (0) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (1) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (2) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (3) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = _mm_cvtsi128_si32(sum0);
+
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (4) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (5) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (6) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (7) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (8) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (9) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (10) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (11) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (8) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (9) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (10) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (11) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (8) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (9) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (10) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (11) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (8) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (9) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (10) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (11) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (12) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (13) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (14) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (15) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (12) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (13) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (14) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (15) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (12) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (13) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (14) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (15) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (12) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (13) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (14) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (15) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+ }
+
+ else if ((ly % 8) == 0)
+ {
+ res[0] = res[1] = res[2] = 0;
+ for (int i = 0; i < ly; i += 8)
{
- m1.load_a(fenc);
- n1.load(fref1);
- n2.load(fref2);
- n3.load(fref3);
-
- sad1.addSumAbsDiff(m1, n1);
- sad2.addSumAbsDiff(m1, n2);
- sad3.addSumAbsDiff(m1, n3);
-
- fenc += FENC_STRIDE;
- fref1 += frefstride;
- fref2 += frefstride;
- fref3 += frefstride;
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (i + 0) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (i + 1) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (i + 2) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (i + 3) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (i + 4) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (i + 5) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (i + 6) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (i + 7) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 4) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 5) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 6) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 7) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = res[2] + _mm_cvtsi128_si32(sum0);
}
-
- sum1 += extend_low(sad1);
- sum2 += extend_low(sad2);
- sum3 += extend_low(sad3);
- sad1 = 0;
- sad2 = 0;
- sad3 = 0;
}
- while (row++ < ly)
+ else
{
- m1.load_a(fenc);
- n1.load(fref1);
- n2.load(fref2);
- n3.load(fref3);
-
- sad1.addSumAbsDiff(m1, n1);
- sad2.addSumAbsDiff(m1, n2);
- sad3.addSumAbsDiff(m1, n3);
-
- fenc += FENC_STRIDE;
- fref1 += frefstride;
- fref2 += frefstride;
- fref3 += frefstride;
+ res[0] = res[1] = res[2] = 0;
+ for (int i = 0; i < ly; i += 4)
+ {
+ T00 = _mm_loadl_epi64((__m128i*)(fenc + (i + 0) * FENC_STRIDE));
+ T01 = _mm_loadl_epi64((__m128i*)(fenc + (i + 1) * FENC_STRIDE));
+ T01 = _mm_unpacklo_epi64(T00, T01);
+ T02 = _mm_loadl_epi64((__m128i*)(fenc + (i + 2) * FENC_STRIDE));
+ T03 = _mm_loadl_epi64((__m128i*)(fenc + (i + 3) * FENC_STRIDE));
+ T03 = _mm_unpacklo_epi64(T02, T03);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref1 + (i + 3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[0] = res[0] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref2 + (i + 3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[1] = res[1] + _mm_cvtsi128_si32(sum0);
+
+ T10 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 0) * frefstride));
+ T11 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 1) * frefstride));
+ T11 = _mm_unpacklo_epi64(T10, T11);
+ T12 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 2) * frefstride));
+ T13 = _mm_loadl_epi64((__m128i*)(fref3 + (i + 3) * frefstride));
+ T13 = _mm_unpacklo_epi64(T12, T13);
+
+ T20 = _mm_sad_epu8(T01, T11);
+ T21 = _mm_sad_epu8(T03, T13);
+ T21 = _mm_add_epi32(T20, T21);
+ sum0 = _mm_shuffle_epi32(T21, 2);
+ sum0 = _mm_add_epi32(sum0, T21);
+ res[2] = res[2] + _mm_cvtsi128_si32(sum0);
+ }
}
-
- sum1 += extend_low(sad1);
- sum2 += extend_low(sad2);
- sum3 += extend_low(sad3);
-
- res[0] = horizontal_add(sum1);
- res[1] = horizontal_add(sum2);
- res[2] = horizontal_add(sum3);
}
/* For performance - This function assumes that the *last load* can access 16 elements. */
More information about the x265-devel
mailing list