[x265] [PATCH] intrinsic: Added dct16 sse3 intrinsic, 55288.92 -> 45139.28
dtyx265 at gmail.com
dtyx265 at gmail.com
Wed Feb 11 18:28:08 CET 2015
# HG changeset patch
# User David T Yuen <dtyx265 at gmail.com>
# Date 1423675655 28800
# Node ID 5b1611feda7249643af85e595098bc95e9e4eee2
# Parent f52d723d8c46e9bca77bb8f3113004d449456559
intrinsic: Added dct16 sse3 intrinsic, 55288.92 -> 45139.28
This is backported from the ssse3 dct16 intrinsic
diff -r f52d723d8c46 -r 5b1611feda72 source/common/vec/dct-sse3.cpp
--- a/source/common/vec/dct-sse3.cpp Mon Feb 09 17:07:02 2015 +0530
+++ b/source/common/vec/dct-sse3.cpp Wed Feb 11 09:27:35 2015 -0800
@@ -36,6 +36,536 @@
using namespace x265;
namespace {
+ALIGN_VAR_32(static const int16_t, tab_dct_8[][8]) =
+{
+ { 0x0100, 0x0F0E, 0x0706, 0x0908, 0x0302, 0x0D0C, 0x0504, 0x0B0A },
+
+ { 64, 64, 64, 64, 64, 64, 64, 64 },
+ { 64, -64, 64, -64, 64, -64, 64, -64 },
+ { 83, 36, 83, 36, 83, 36, 83, 36 },
+ { 36, -83, 36, -83, 36, -83, 36, -83 },
+ { 89, 18, 75, 50, 89, 18, 75, 50 },
+ { 75, -50, -18, -89, 75, -50, -18, -89 },
+ { 50, 75, -89, 18, 50, 75, -89, 18 },
+ { 18, -89, -50, 75, 18, -89, -50, 75 },
+
+ { 83, 83, -83, -83, 36, 36, -36, -36 },
+ { 36, 36, -36, -36, -83, -83, 83, 83 },
+ { 89, -89, 18, -18, 75, -75, 50, -50 },
+ { 75, -75, -50, 50, -18, 18, -89, 89 },
+ { 50, -50, 75, -75, -89, 89, 18, -18 },
+ { 18, -18, -89, 89, -50, 50, 75, -75 },
+};
+
+ALIGN_VAR_32(static const int16_t, tab_dct_16_1[][8]) =
+{
+ { 90, 87, 80, 70, 57, 43, 25, 9 }, // 0
+ { 87, 57, 9, -43, -80, -90, -70, -25 }, // 1
+ { 80, 9, -70, -87, -25, 57, 90, 43 }, // 2
+ { 70, -43, -87, 9, 90, 25, -80, -57 }, // 3
+ { 57, -80, -25, 90, -9, -87, 43, 70 }, // 4
+ { 43, -90, 57, 25, -87, 70, 9, -80 }, // 5
+ { 25, -70, 90, -80, 43, 9, -57, 87 }, // 6
+ { 9, -25, 43, -57, 70, -80, 87, -90 }, // 7
+ { 83, 83, -83, -83, 36, 36, -36, -36 }, // 8
+ { 36, 36, -36, -36, -83, -83, 83, 83 }, // 9
+ { 89, 89, 18, 18, 75, 75, 50, 50 }, // 10
+ { 75, 75, -50, -50, -18, -18, -89, -89 }, // 11
+ { 50, 50, 75, 75, -89, -89, 18, 18 }, // 12
+ { 18, 18, -89, -89, -50, -50, 75, 75 }, // 13
+
+#define MAKE_COEF(a0, a1, a2, a3, a4, a5, a6, a7) \
+ { (a0), -(a0), (a3), -(a3), (a1), -(a1), (a2), -(a2) \
+ }, \
+ { (a7), -(a7), (a4), -(a4), (a6), -(a6), (a5), -(a5) },
+
+ MAKE_COEF(90, 87, 80, 70, 57, 43, 25, 9)
+ MAKE_COEF(87, 57, 9, -43, -80, -90, -70, -25)
+ MAKE_COEF(80, 9, -70, -87, -25, 57, 90, 43)
+ MAKE_COEF(70, -43, -87, 9, 90, 25, -80, -57)
+ MAKE_COEF(57, -80, -25, 90, -9, -87, 43, 70)
+ MAKE_COEF(43, -90, 57, 25, -87, 70, 9, -80)
+ MAKE_COEF(25, -70, 90, -80, 43, 9, -57, 87)
+ MAKE_COEF(9, -25, 43, -57, 70, -80, 87, -90)
+#undef MAKE_COEF
+};
+
+void dct16(const int16_t *src, int16_t *dst, intptr_t stride)
+{
+#if HIGH_BIT_DEPTH
+#define SHIFT1 5
+#define ADD1 16
+#else
+#define SHIFT1 3
+#define ADD1 4
+#endif
+
+#define SHIFT2 10
+#define ADD2 512
+
+ // Const
+ __m128i c_4 = _mm_set1_epi32(ADD1);
+ __m128i c_512 = _mm_set1_epi32(ADD2);
+
+ ALIGN_VAR_32(int16_t, tmp[16 * 16]);
+
+ __m128i T00A, T01A, T02A, T03A;
+ __m128i T00B, T01B, T02B, T03B, T04B, T05B, T06B, T07B;
+ __m128i T10, T11, T12, T13, T14, T15, T16, T17;
+ __m128i T20, T21, T22, T23, T24, T25, T26, T27;
+ __m128i T30, T31, T32, T33, T34, T35, T36, T37;
+ __m128i T40, T41, T42, T43, T44, T45, T46, T47;
+ __m128i T50, T51, T52, T53;
+ __m128i T60, T61, T62, T63, T64, T65, T66, T67;
+ __m128i T70;
+
+ // DCT1
+ {
+#define part 0
+#define SHUFFLE1(R1, SRC) \
+ R1 = _mm_shuffle_epi32(SRC, 0x1B); \
+ R1 = _mm_shufflehi_epi16(R1, 0xB1); \
+ R1 = _mm_shufflelo_epi16(R1, 0xB1);
+ SHUFFLE1(T00B, _mm_load_si128((__m128i*)&src[(part + 0) * stride + 8]))
+ SHUFFLE1(T01B, _mm_load_si128((__m128i*)&src[(part + 1) * stride + 8]))
+ SHUFFLE1(T02B, _mm_load_si128((__m128i*)&src[(part + 2) * stride + 8]))
+ SHUFFLE1(T03B, _mm_load_si128((__m128i*)&src[(part + 3) * stride + 8]))
+ SHUFFLE1(T04B, _mm_load_si128((__m128i*)&src[(part + 4) * stride + 8]))
+ SHUFFLE1(T05B, _mm_load_si128((__m128i*)&src[(part + 5) * stride + 8]))
+ SHUFFLE1(T06B, _mm_load_si128((__m128i*)&src[(part + 6) * stride + 8]))
+ SHUFFLE1(T07B, _mm_load_si128((__m128i*)&src[(part + 7) * stride + 8]))
+
+#define READ_ADD_SUB(INDEX) \
+ T10 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 0) * stride]), T00B); \
+ T11 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 1) * stride]), T01B); \
+ T12 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 2) * stride]), T02B); \
+ T13 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 3) * stride]), T03B); \
+ T14 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 4) * stride]), T04B); \
+ T15 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 5) * stride]), T05B); \
+ T16 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 6) * stride]), T06B); \
+ T17 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 7) * stride]), T07B); \
+ \
+ T20 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 0) * stride]), T00B); \
+ T21 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 1) * stride]), T01B); \
+ T22 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 2) * stride]), T02B); \
+ T23 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 3) * stride]), T03B); \
+ T24 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 4) * stride]), T04B); \
+ T25 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 5) * stride]), T05B); \
+ T26 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 6) * stride]), T06B); \
+ T27 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 7) * stride]), T07B);
+ READ_ADD_SUB(part)
+
+#define SHUFFLE2(R1, R2, R3, R4) \
+ R1 = _mm_shufflehi_epi16(R1, 0x63); \
+ R2 = _mm_shufflehi_epi16(R2, 0x63); \
+ R1 = _mm_shufflelo_epi16(R1, 0x9C); \
+ R2 = _mm_shufflelo_epi16(R2, 0x9C); \
+ R3 = _mm_unpacklo_epi64(R1, R2); \
+ R4 = _mm_unpackhi_epi64(R1, R2);
+
+ SHUFFLE2(T10, T11, T30, T31)
+ SHUFFLE2(T12, T13, T32, T33)
+ SHUFFLE2(T14, T15, T34, T35)
+ SHUFFLE2(T16, T17, T36, T37)
+
+ T40 = _mm_add_epi16(T30, T31);
+ T41 = _mm_add_epi16(T32, T33);
+ T42 = _mm_add_epi16(T34, T35);
+ T43 = _mm_add_epi16(T36, T37);
+ T44 = _mm_sub_epi16(T30, T31);
+ T45 = _mm_sub_epi16(T32, T33);
+ T46 = _mm_sub_epi16(T34, T35);
+ T47 = _mm_sub_epi16(T36, T37);
+
+#define PHADDW_PHSUBW(R1, R2, R3, R4) \
+ R1 = _mm_shufflehi_epi16(R1, 0xD8); \
+ R1 = _mm_shufflelo_epi16(R1, 0xD8); \
+ R1 = _mm_shuffle_epi32(R1, 0xD8); \
+ R2 = _mm_shufflehi_epi16(R2, 0xD8); \
+ R2 = _mm_shufflelo_epi16(R2, 0xD8); \
+ R2 = _mm_shuffle_epi32(R2, 0xD8); \
+ R4 = _mm_unpacklo_epi64(R1, R2); \
+ R1 = _mm_unpackhi_epi64(R1, R2); \
+ R3 = _mm_add_epi16(R4, R1); \
+ R4 = _mm_sub_epi16(R4, R1);
+
+ PHADDW_PHSUBW(T40, T41, T50, T52)
+ PHADDW_PHSUBW(T42, T43, T51, T53)
+
+#define MADD_SHIFT(R1, R2, tab, index) \
+ T60 = _mm_madd_epi16(R1, _mm_load_si128((__m128i*)tab_dct_8[tab])); \
+ T61 = _mm_madd_epi16(R2, _mm_load_si128((__m128i*)tab_dct_8[tab])); \
+ T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
+ T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
+ T70 = _mm_packs_epi32(T60, T61); \
+ _mm_store_si128((__m128i*)&tmp[index * 16 + part], T70);
+
+ MADD_SHIFT(T50, T51, 1, 0)
+ MADD_SHIFT(T50, T51, 2, 8)
+ MADD_SHIFT(T52, T53, 3, 4)
+ MADD_SHIFT(T52, T53, 4, 12)
+
+#define PHADDD(R1, R2, R3) \
+ R2 = _mm_shuffle_epi32(R2, 0xD8); \
+ R3 = _mm_shuffle_epi32(R3, 0xD8); \
+ T00A = _mm_unpacklo_epi64(R2, R3); \
+ T00B = _mm_unpackhi_epi64(R2, R3); \
+ R1 = _mm_add_epi32(T00A, T00B);
+
+#define PHADDD_SHIFT(TAB, ROW) \
+ T60 = _mm_madd_epi16(T44, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
+ T61 = _mm_madd_epi16(T45, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
+ T62 = _mm_madd_epi16(T46, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
+ T63 = _mm_madd_epi16(T47, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
+ PHADDD(T60, T60, T61) \
+ PHADDD(T61, T62, T63) \
+ T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
+ T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
+ T70 = _mm_packs_epi32(T60, T61); \
+ _mm_store_si128((__m128i*)&tmp[ROW * 16 + part], T70);
+
+ PHADDD_SHIFT(5, 2)
+ PHADDD_SHIFT(6, 6)
+ PHADDD_SHIFT(7, 10)
+ PHADDD_SHIFT(8, 14)
+
+#define MAKE_ODD1(tab, dstPos) \
+ T60 = _mm_madd_epi16(T20, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T61 = _mm_madd_epi16(T21, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T62 = _mm_madd_epi16(T22, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T63 = _mm_madd_epi16(T23, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T64 = _mm_madd_epi16(T24, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T65 = _mm_madd_epi16(T25, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T66 = _mm_madd_epi16(T26, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T67 = _mm_madd_epi16(T27, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ PHADDD(T60, T60, T61) \
+ PHADDD(T61, T62, T63) \
+ PHADDD(T62, T64, T65) \
+ PHADDD(T63, T66, T67) \
+ PHADDD(T60, T60, T61) \
+ PHADDD(T61, T62, T63) \
+ T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
+ T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
+ T70 = _mm_packs_epi32(T60, T61); \
+ _mm_store_si128((__m128i*)&tmp[(dstPos) * 16 + part], T70);
+
+ MAKE_ODD1(0, 1);
+ MAKE_ODD1(1, 3);
+ MAKE_ODD1(2, 5);
+ MAKE_ODD1(3, 7);
+ MAKE_ODD1(4, 9);
+ MAKE_ODD1(5, 11);
+ MAKE_ODD1(6, 13);
+ MAKE_ODD1(7, 15);
+#undef part
+ }
+ {
+#define part 8
+ SHUFFLE1(T00B, _mm_load_si128((__m128i*)&src[(part + 0) * stride + 8]))
+ SHUFFLE1(T01B, _mm_load_si128((__m128i*)&src[(part + 1) * stride + 8]))
+ SHUFFLE1(T02B, _mm_load_si128((__m128i*)&src[(part + 2) * stride + 8]))
+ SHUFFLE1(T03B, _mm_load_si128((__m128i*)&src[(part + 3) * stride + 8]))
+ SHUFFLE1(T04B, _mm_load_si128((__m128i*)&src[(part + 4) * stride + 8]))
+ SHUFFLE1(T05B, _mm_load_si128((__m128i*)&src[(part + 5) * stride + 8]))
+ SHUFFLE1(T06B, _mm_load_si128((__m128i*)&src[(part + 6) * stride + 8]))
+ SHUFFLE1(T07B, _mm_load_si128((__m128i*)&src[(part + 7) * stride + 8]))
+
+ READ_ADD_SUB(part)
+
+ SHUFFLE2(T10, T11, T30, T31)
+ SHUFFLE2(T12, T13, T32, T33)
+ SHUFFLE2(T14, T15, T34, T35)
+ SHUFFLE2(T16, T17, T36, T37)
+
+ T40 = _mm_add_epi16(T30, T31);
+ T41 = _mm_add_epi16(T32, T33);
+ T42 = _mm_add_epi16(T34, T35);
+ T43 = _mm_add_epi16(T36, T37);
+ T44 = _mm_sub_epi16(T30, T31);
+ T45 = _mm_sub_epi16(T32, T33);
+ T46 = _mm_sub_epi16(T34, T35);
+ T47 = _mm_sub_epi16(T36, T37);
+
+ PHADDW_PHSUBW(T40, T41, T50, T52)
+ PHADDW_PHSUBW(T42, T43, T51, T53)
+
+ MADD_SHIFT(T50, T51, 1, 0)
+ MADD_SHIFT(T50, T51, 2, 8)
+ MADD_SHIFT(T52, T53, 3, 4)
+ MADD_SHIFT(T52, T53, 4, 12)
+
+ PHADDD_SHIFT(5, 2)
+ PHADDD_SHIFT(6, 6)
+ PHADDD_SHIFT(7, 10)
+ PHADDD_SHIFT(8, 14)
+ MAKE_ODD1(0, 1);
+ MAKE_ODD1(1, 3);
+ MAKE_ODD1(2, 5);
+ MAKE_ODD1(3, 7);
+ MAKE_ODD1(4, 9);
+ MAKE_ODD1(5, 11);
+ MAKE_ODD1(6, 13);
+ MAKE_ODD1(7, 15);
+#undef part
+ }
+ // DCT2
+ {
+#define part 0
+#define SHUFFLE3(R1, SRC) \
+ R1 = _mm_shufflehi_epi16(SRC, 0x63); \
+ R1 = _mm_shufflelo_epi16(R1, 0x9C);
+ SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
+ SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
+ SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
+ SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
+
+#define SHUFFLE4(R1, SRC) \
+ R1 = _mm_shuffle_epi32(SRC, 0x4E); \
+ R1 = _mm_shufflehi_epi16(R1, 0x9C); \
+ R1 = _mm_shufflelo_epi16(R1, 0x63);
+ SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
+ SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
+ SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
+ SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
+
+#define UNPCK_MADD(INDEX) \
+ T10 = _mm_unpacklo_epi16(T00A, T00B); \
+ T11 = _mm_unpackhi_epi16(T00A, T00B); \
+ T12 = _mm_unpacklo_epi16(T01A, T01B); \
+ T13 = _mm_unpackhi_epi16(T01A, T01B); \
+ T14 = _mm_unpacklo_epi16(T02A, T02B); \
+ T15 = _mm_unpackhi_epi16(T02A, T02B); \
+ T16 = _mm_unpacklo_epi16(T03A, T03B); \
+ T17 = _mm_unpackhi_epi16(T03A, T03B); \
+ \
+ T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_8[1])); \
+ T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_8[1])); \
+ T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_8[1])); \
+ T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_8[1])); \
+ T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_8[1])); \
+ T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_8[1])); \
+ T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_8[1])); \
+ T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_8[1])); \
+ \
+ T30 = _mm_add_epi32(T20, T21); \
+ T31 = _mm_add_epi32(T22, T23); \
+ T32 = _mm_add_epi32(T24, T25); \
+ T33 = _mm_add_epi32(T26, T27); \
+ \
+ PHADDD(T30, T30, T31) \
+ PHADDD(T31, T32, T33) \
+ \
+ PHADDD(T40, T30, T31) \
+ T41 = _mm_sub_epi32(T00A, T00B); \
+ T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
+ T41 = _mm_srai_epi32(_mm_add_epi32(T41, c_512), SHIFT2); \
+ T40 = _mm_packs_epi32(T40, T40); \
+ T41 = _mm_packs_epi32(T41, T41); \
+ _mm_storel_epi64((__m128i*)&dst[0 * 16 + INDEX], T40); \
+ _mm_storel_epi64((__m128i*)&dst[8 * 16 + INDEX], T41);
+
+ UNPCK_MADD(part)
+#define MADD_PHADDD(TAB, ROW, INDEX) \
+ T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ \
+ T30 = _mm_add_epi32(T20, T21); \
+ T31 = _mm_add_epi32(T22, T23); \
+ T32 = _mm_add_epi32(T24, T25); \
+ T33 = _mm_add_epi32(T26, T27); \
+ \
+ PHADDD(T30, T30, T31) \
+ PHADDD(T31, T32, T33) \
+ \
+ PHADDD(T40, T30, T31) \
+ T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
+ T40 = _mm_packs_epi32(T40, T40); \
+ _mm_storel_epi64((__m128i*)&dst[ROW * 16 + INDEX], T40);
+
+ MADD_PHADDD(8, 4, part)
+ MADD_PHADDD(9, 12, part)
+
+#define SUB_PHADDD(TAB, ROW, INDEX) \
+ T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
+ \
+ T30 = _mm_sub_epi32(T20, T21); \
+ T31 = _mm_sub_epi32(T22, T23); \
+ T32 = _mm_sub_epi32(T24, T25); \
+ T33 = _mm_sub_epi32(T26, T27); \
+ \
+ PHADDD(T30, T30, T31) \
+ PHADDD(T31, T32, T33) \
+ \
+ PHADDD(T40, T30, T31) \
+ T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
+ T40 = _mm_packs_epi32(T40, T40); \
+ _mm_storel_epi64((__m128i*)&dst[ROW * 16 + INDEX], T40);
+
+ SUB_PHADDD(10, 2, part)
+ SUB_PHADDD(11, 6, part)
+ SUB_PHADDD(12, 10, part)
+ SUB_PHADDD(13, 14, part)
+
+#define MAKE_ODD2(tab, dstPos, INDEX) \
+ T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); /* [*O2_0 *O1_0 *O3_0 *O0_0] */ \
+ T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); /* [*O5_0 *O6_0 *O4_0 *O7_0] */ \
+ T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
+ T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
+ T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
+ T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
+ \
+ T30 = _mm_add_epi32(T20, T21); \
+ T31 = _mm_add_epi32(T22, T23); \
+ T32 = _mm_add_epi32(T24, T25); \
+ T33 = _mm_add_epi32(T26, T27); \
+ \
+ PHADDD(T30, T30, T31) \
+ PHADDD(T31, T32, T33) \
+ \
+ PHADDD(T40, T30, T31); \
+ T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
+ T40 = _mm_packs_epi32(T40, T40); \
+ _mm_storel_epi64((__m128i*)&dst[(dstPos) * 16 + INDEX], T40);
+
+ MAKE_ODD2(14, 1, part);
+ MAKE_ODD2(16, 3, part);
+ MAKE_ODD2(18, 5, part);
+ MAKE_ODD2(20, 7, part);
+ MAKE_ODD2(22, 9, part);
+ MAKE_ODD2(24, 11, part);
+ MAKE_ODD2(26, 13, part);
+ MAKE_ODD2(28, 15, part);
+#undef part
+ }
+ {
+#define part 4
+ SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
+ SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
+ SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
+ SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
+
+ SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
+ SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
+ SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
+ SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
+
+ UNPCK_MADD(part)
+
+ MADD_PHADDD(8, 4, part)
+ MADD_PHADDD(9, 12, part)
+
+ SUB_PHADDD(10, 2, part)
+ SUB_PHADDD(11, 6, part)
+ SUB_PHADDD(12, 10, part)
+ SUB_PHADDD(13, 14, part)
+
+ MAKE_ODD2(14, 1, part);
+ MAKE_ODD2(16, 3, part);
+ MAKE_ODD2(18, 5, part);
+ MAKE_ODD2(20, 7, part);
+ MAKE_ODD2(22, 9, part);
+ MAKE_ODD2(24, 11, part);
+ MAKE_ODD2(26, 13, part);
+ MAKE_ODD2(28, 15, part);
+#undef part
+ }
+ {
+#define part 8
+ SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
+ SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
+ SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
+ SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
+
+ SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
+ SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
+ SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
+ SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
+
+ UNPCK_MADD(part)
+
+ MADD_PHADDD(8, 4, part)
+ MADD_PHADDD(9, 12, part)
+
+ SUB_PHADDD(10, 2, part)
+ SUB_PHADDD(11, 6, part)
+ SUB_PHADDD(12, 10, part)
+ SUB_PHADDD(13, 14, part)
+
+ MAKE_ODD2(14, 1, part);
+ MAKE_ODD2(16, 3, part);
+ MAKE_ODD2(18, 5, part);
+ MAKE_ODD2(20, 7, part);
+ MAKE_ODD2(22, 9, part);
+ MAKE_ODD2(24, 11, part);
+ MAKE_ODD2(26, 13, part);
+ MAKE_ODD2(28, 15, part);
+#undef part
+ }
+ {
+#define part 12
+ SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
+ SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
+ SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
+ SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
+
+ SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
+ SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
+ SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
+ SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
+
+ UNPCK_MADD(part)
+
+ MADD_PHADDD(8, 4, part)
+ MADD_PHADDD(9, 12, part)
+
+ SUB_PHADDD(10, 2, part)
+ SUB_PHADDD(11, 6, part)
+ SUB_PHADDD(12, 10, part)
+ SUB_PHADDD(13, 14, part)
+
+ MAKE_ODD2(14, 1, part);
+ MAKE_ODD2(16, 3, part);
+ MAKE_ODD2(18, 5, part);
+ MAKE_ODD2(20, 7, part);
+ MAKE_ODD2(22, 9, part);
+ MAKE_ODD2(24, 11, part);
+ MAKE_ODD2(26, 13, part);
+ MAKE_ODD2(28, 15, part);
+#undef part
+ }
+#undef SHIFT1
+#undef ADD1
+#undef SHIFT2
+#undef ADD2
+#undef SHUFFLE1
+#undef SHUFFLE2
+#undef PHADDW_PHSUBW
+#undef MAKE_ODD1
+#undef MAKE_ODD2
+#undef PHADDD
+#undef SHUFFLE3
+#undef SHUFFLE4
+#undef PHADDD_SHIFT
+#undef MADD_SHIFT
+#undef MADD_PHADDD
+#undef SUB_PHADDD
+}
+
#define SHIFT1 7
#define ADD1 64
@@ -1426,6 +1956,7 @@
/* Note: We have AVX2 assembly for these functions, but since AVX2 is still
* somewhat rare on end-user PCs we still compile and link these SSE3
* intrinsic SIMD functions */
+ p.cu[BLOCK_16x16].dct = dct16;
p.cu[BLOCK_8x8].idct = idct8;
p.cu[BLOCK_16x16].idct = idct16;
p.cu[BLOCK_32x32].idct = idct32;
More information about the x265-devel
mailing list