[x265] [PATCH] intrinsic: Added dct16 sse3 intrinsic, 55288.92 -> 45139.28
dave
dtyx265 at gmail.com
Wed Feb 11 18:41:37 CET 2015
This didn't provide the improvement that I was hoping for as it's only
about 1.2x faster than the c code and all the other transforms are well
over 2x faster. I will be looking into further improving dct16 and dct8
for sse3 unless there is absolutely no need for them.
By the way, sse3 consists primarily of floating point enhancements and
since the transforms all use integers they are effectively sse2.
On 02/11/2015 09:28 AM, dtyx265 at gmail.com wrote:
> # HG changeset patch
> # User David T Yuen <dtyx265 at gmail.com>
> # Date 1423675655 28800
> # Node ID 5b1611feda7249643af85e595098bc95e9e4eee2
> # Parent f52d723d8c46e9bca77bb8f3113004d449456559
> intrinsic: Added dct16 sse3 intrinsic, 55288.92 -> 45139.28
>
> This is backported from the ssse3 dct16 intrinsic
>
> diff -r f52d723d8c46 -r 5b1611feda72 source/common/vec/dct-sse3.cpp
> --- a/source/common/vec/dct-sse3.cpp Mon Feb 09 17:07:02 2015 +0530
> +++ b/source/common/vec/dct-sse3.cpp Wed Feb 11 09:27:35 2015 -0800
> @@ -36,6 +36,536 @@
> using namespace x265;
>
> namespace {
> +ALIGN_VAR_32(static const int16_t, tab_dct_8[][8]) =
> +{
> + { 0x0100, 0x0F0E, 0x0706, 0x0908, 0x0302, 0x0D0C, 0x0504, 0x0B0A },
> +
> + { 64, 64, 64, 64, 64, 64, 64, 64 },
> + { 64, -64, 64, -64, 64, -64, 64, -64 },
> + { 83, 36, 83, 36, 83, 36, 83, 36 },
> + { 36, -83, 36, -83, 36, -83, 36, -83 },
> + { 89, 18, 75, 50, 89, 18, 75, 50 },
> + { 75, -50, -18, -89, 75, -50, -18, -89 },
> + { 50, 75, -89, 18, 50, 75, -89, 18 },
> + { 18, -89, -50, 75, 18, -89, -50, 75 },
> +
> + { 83, 83, -83, -83, 36, 36, -36, -36 },
> + { 36, 36, -36, -36, -83, -83, 83, 83 },
> + { 89, -89, 18, -18, 75, -75, 50, -50 },
> + { 75, -75, -50, 50, -18, 18, -89, 89 },
> + { 50, -50, 75, -75, -89, 89, 18, -18 },
> + { 18, -18, -89, 89, -50, 50, 75, -75 },
> +};
> +
> +ALIGN_VAR_32(static const int16_t, tab_dct_16_1[][8]) =
> +{
> + { 90, 87, 80, 70, 57, 43, 25, 9 }, // 0
> + { 87, 57, 9, -43, -80, -90, -70, -25 }, // 1
> + { 80, 9, -70, -87, -25, 57, 90, 43 }, // 2
> + { 70, -43, -87, 9, 90, 25, -80, -57 }, // 3
> + { 57, -80, -25, 90, -9, -87, 43, 70 }, // 4
> + { 43, -90, 57, 25, -87, 70, 9, -80 }, // 5
> + { 25, -70, 90, -80, 43, 9, -57, 87 }, // 6
> + { 9, -25, 43, -57, 70, -80, 87, -90 }, // 7
> + { 83, 83, -83, -83, 36, 36, -36, -36 }, // 8
> + { 36, 36, -36, -36, -83, -83, 83, 83 }, // 9
> + { 89, 89, 18, 18, 75, 75, 50, 50 }, // 10
> + { 75, 75, -50, -50, -18, -18, -89, -89 }, // 11
> + { 50, 50, 75, 75, -89, -89, 18, 18 }, // 12
> + { 18, 18, -89, -89, -50, -50, 75, 75 }, // 13
> +
> +#define MAKE_COEF(a0, a1, a2, a3, a4, a5, a6, a7) \
> + { (a0), -(a0), (a3), -(a3), (a1), -(a1), (a2), -(a2) \
> + }, \
> + { (a7), -(a7), (a4), -(a4), (a6), -(a6), (a5), -(a5) },
> +
> + MAKE_COEF(90, 87, 80, 70, 57, 43, 25, 9)
> + MAKE_COEF(87, 57, 9, -43, -80, -90, -70, -25)
> + MAKE_COEF(80, 9, -70, -87, -25, 57, 90, 43)
> + MAKE_COEF(70, -43, -87, 9, 90, 25, -80, -57)
> + MAKE_COEF(57, -80, -25, 90, -9, -87, 43, 70)
> + MAKE_COEF(43, -90, 57, 25, -87, 70, 9, -80)
> + MAKE_COEF(25, -70, 90, -80, 43, 9, -57, 87)
> + MAKE_COEF(9, -25, 43, -57, 70, -80, 87, -90)
> +#undef MAKE_COEF
> +};
> +
> +void dct16(const int16_t *src, int16_t *dst, intptr_t stride)
> +{
> +#if HIGH_BIT_DEPTH
> +#define SHIFT1 5
> +#define ADD1 16
> +#else
> +#define SHIFT1 3
> +#define ADD1 4
> +#endif
> +
> +#define SHIFT2 10
> +#define ADD2 512
> +
> + // Const
> + __m128i c_4 = _mm_set1_epi32(ADD1);
> + __m128i c_512 = _mm_set1_epi32(ADD2);
> +
> + ALIGN_VAR_32(int16_t, tmp[16 * 16]);
> +
> + __m128i T00A, T01A, T02A, T03A;
> + __m128i T00B, T01B, T02B, T03B, T04B, T05B, T06B, T07B;
> + __m128i T10, T11, T12, T13, T14, T15, T16, T17;
> + __m128i T20, T21, T22, T23, T24, T25, T26, T27;
> + __m128i T30, T31, T32, T33, T34, T35, T36, T37;
> + __m128i T40, T41, T42, T43, T44, T45, T46, T47;
> + __m128i T50, T51, T52, T53;
> + __m128i T60, T61, T62, T63, T64, T65, T66, T67;
> + __m128i T70;
> +
> + // DCT1
> + {
> +#define part 0
> +#define SHUFFLE1(R1, SRC) \
> + R1 = _mm_shuffle_epi32(SRC, 0x1B); \
> + R1 = _mm_shufflehi_epi16(R1, 0xB1); \
> + R1 = _mm_shufflelo_epi16(R1, 0xB1);
> + SHUFFLE1(T00B, _mm_load_si128((__m128i*)&src[(part + 0) * stride + 8]))
> + SHUFFLE1(T01B, _mm_load_si128((__m128i*)&src[(part + 1) * stride + 8]))
> + SHUFFLE1(T02B, _mm_load_si128((__m128i*)&src[(part + 2) * stride + 8]))
> + SHUFFLE1(T03B, _mm_load_si128((__m128i*)&src[(part + 3) * stride + 8]))
> + SHUFFLE1(T04B, _mm_load_si128((__m128i*)&src[(part + 4) * stride + 8]))
> + SHUFFLE1(T05B, _mm_load_si128((__m128i*)&src[(part + 5) * stride + 8]))
> + SHUFFLE1(T06B, _mm_load_si128((__m128i*)&src[(part + 6) * stride + 8]))
> + SHUFFLE1(T07B, _mm_load_si128((__m128i*)&src[(part + 7) * stride + 8]))
> +
> +#define READ_ADD_SUB(INDEX) \
> + T10 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 0) * stride]), T00B); \
> + T11 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 1) * stride]), T01B); \
> + T12 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 2) * stride]), T02B); \
> + T13 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 3) * stride]), T03B); \
> + T14 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 4) * stride]), T04B); \
> + T15 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 5) * stride]), T05B); \
> + T16 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 6) * stride]), T06B); \
> + T17 = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 7) * stride]), T07B); \
> + \
> + T20 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 0) * stride]), T00B); \
> + T21 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 1) * stride]), T01B); \
> + T22 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 2) * stride]), T02B); \
> + T23 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 3) * stride]), T03B); \
> + T24 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 4) * stride]), T04B); \
> + T25 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 5) * stride]), T05B); \
> + T26 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 6) * stride]), T06B); \
> + T27 = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 7) * stride]), T07B);
> + READ_ADD_SUB(part)
> +
> +#define SHUFFLE2(R1, R2, R3, R4) \
> + R1 = _mm_shufflehi_epi16(R1, 0x63); \
> + R2 = _mm_shufflehi_epi16(R2, 0x63); \
> + R1 = _mm_shufflelo_epi16(R1, 0x9C); \
> + R2 = _mm_shufflelo_epi16(R2, 0x9C); \
> + R3 = _mm_unpacklo_epi64(R1, R2); \
> + R4 = _mm_unpackhi_epi64(R1, R2);
> +
> + SHUFFLE2(T10, T11, T30, T31)
> + SHUFFLE2(T12, T13, T32, T33)
> + SHUFFLE2(T14, T15, T34, T35)
> + SHUFFLE2(T16, T17, T36, T37)
> +
> + T40 = _mm_add_epi16(T30, T31);
> + T41 = _mm_add_epi16(T32, T33);
> + T42 = _mm_add_epi16(T34, T35);
> + T43 = _mm_add_epi16(T36, T37);
> + T44 = _mm_sub_epi16(T30, T31);
> + T45 = _mm_sub_epi16(T32, T33);
> + T46 = _mm_sub_epi16(T34, T35);
> + T47 = _mm_sub_epi16(T36, T37);
> +
> +#define PHADDW_PHSUBW(R1, R2, R3, R4) \
> + R1 = _mm_shufflehi_epi16(R1, 0xD8); \
> + R1 = _mm_shufflelo_epi16(R1, 0xD8); \
> + R1 = _mm_shuffle_epi32(R1, 0xD8); \
> + R2 = _mm_shufflehi_epi16(R2, 0xD8); \
> + R2 = _mm_shufflelo_epi16(R2, 0xD8); \
> + R2 = _mm_shuffle_epi32(R2, 0xD8); \
> + R4 = _mm_unpacklo_epi64(R1, R2); \
> + R1 = _mm_unpackhi_epi64(R1, R2); \
> + R3 = _mm_add_epi16(R4, R1); \
> + R4 = _mm_sub_epi16(R4, R1);
> +
> + PHADDW_PHSUBW(T40, T41, T50, T52)
> + PHADDW_PHSUBW(T42, T43, T51, T53)
> +
> +#define MADD_SHIFT(R1, R2, tab, index) \
> + T60 = _mm_madd_epi16(R1, _mm_load_si128((__m128i*)tab_dct_8[tab])); \
> + T61 = _mm_madd_epi16(R2, _mm_load_si128((__m128i*)tab_dct_8[tab])); \
> + T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
> + T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
> + T70 = _mm_packs_epi32(T60, T61); \
> + _mm_store_si128((__m128i*)&tmp[index * 16 + part], T70);
> +
> + MADD_SHIFT(T50, T51, 1, 0)
> + MADD_SHIFT(T50, T51, 2, 8)
> + MADD_SHIFT(T52, T53, 3, 4)
> + MADD_SHIFT(T52, T53, 4, 12)
> +
> +#define PHADDD(R1, R2, R3) \
> + R2 = _mm_shuffle_epi32(R2, 0xD8); \
> + R3 = _mm_shuffle_epi32(R3, 0xD8); \
> + T00A = _mm_unpacklo_epi64(R2, R3); \
> + T00B = _mm_unpackhi_epi64(R2, R3); \
> + R1 = _mm_add_epi32(T00A, T00B);
> +
> +#define PHADDD_SHIFT(TAB, ROW) \
> + T60 = _mm_madd_epi16(T44, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
> + T61 = _mm_madd_epi16(T45, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
> + T62 = _mm_madd_epi16(T46, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
> + T63 = _mm_madd_epi16(T47, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
> + PHADDD(T60, T60, T61) \
> + PHADDD(T61, T62, T63) \
> + T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
> + T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
> + T70 = _mm_packs_epi32(T60, T61); \
> + _mm_store_si128((__m128i*)&tmp[ROW * 16 + part], T70);
> +
> + PHADDD_SHIFT(5, 2)
> + PHADDD_SHIFT(6, 6)
> + PHADDD_SHIFT(7, 10)
> + PHADDD_SHIFT(8, 14)
> +
> +#define MAKE_ODD1(tab, dstPos) \
> + T60 = _mm_madd_epi16(T20, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T61 = _mm_madd_epi16(T21, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T62 = _mm_madd_epi16(T22, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T63 = _mm_madd_epi16(T23, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T64 = _mm_madd_epi16(T24, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T65 = _mm_madd_epi16(T25, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T66 = _mm_madd_epi16(T26, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T67 = _mm_madd_epi16(T27, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + PHADDD(T60, T60, T61) \
> + PHADDD(T61, T62, T63) \
> + PHADDD(T62, T64, T65) \
> + PHADDD(T63, T66, T67) \
> + PHADDD(T60, T60, T61) \
> + PHADDD(T61, T62, T63) \
> + T60 = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
> + T61 = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
> + T70 = _mm_packs_epi32(T60, T61); \
> + _mm_store_si128((__m128i*)&tmp[(dstPos) * 16 + part], T70);
> +
> + MAKE_ODD1(0, 1);
> + MAKE_ODD1(1, 3);
> + MAKE_ODD1(2, 5);
> + MAKE_ODD1(3, 7);
> + MAKE_ODD1(4, 9);
> + MAKE_ODD1(5, 11);
> + MAKE_ODD1(6, 13);
> + MAKE_ODD1(7, 15);
> +#undef part
> + }
> + {
> +#define part 8
> + SHUFFLE1(T00B, _mm_load_si128((__m128i*)&src[(part + 0) * stride + 8]))
> + SHUFFLE1(T01B, _mm_load_si128((__m128i*)&src[(part + 1) * stride + 8]))
> + SHUFFLE1(T02B, _mm_load_si128((__m128i*)&src[(part + 2) * stride + 8]))
> + SHUFFLE1(T03B, _mm_load_si128((__m128i*)&src[(part + 3) * stride + 8]))
> + SHUFFLE1(T04B, _mm_load_si128((__m128i*)&src[(part + 4) * stride + 8]))
> + SHUFFLE1(T05B, _mm_load_si128((__m128i*)&src[(part + 5) * stride + 8]))
> + SHUFFLE1(T06B, _mm_load_si128((__m128i*)&src[(part + 6) * stride + 8]))
> + SHUFFLE1(T07B, _mm_load_si128((__m128i*)&src[(part + 7) * stride + 8]))
> +
> + READ_ADD_SUB(part)
> +
> + SHUFFLE2(T10, T11, T30, T31)
> + SHUFFLE2(T12, T13, T32, T33)
> + SHUFFLE2(T14, T15, T34, T35)
> + SHUFFLE2(T16, T17, T36, T37)
> +
> + T40 = _mm_add_epi16(T30, T31);
> + T41 = _mm_add_epi16(T32, T33);
> + T42 = _mm_add_epi16(T34, T35);
> + T43 = _mm_add_epi16(T36, T37);
> + T44 = _mm_sub_epi16(T30, T31);
> + T45 = _mm_sub_epi16(T32, T33);
> + T46 = _mm_sub_epi16(T34, T35);
> + T47 = _mm_sub_epi16(T36, T37);
> +
> + PHADDW_PHSUBW(T40, T41, T50, T52)
> + PHADDW_PHSUBW(T42, T43, T51, T53)
> +
> + MADD_SHIFT(T50, T51, 1, 0)
> + MADD_SHIFT(T50, T51, 2, 8)
> + MADD_SHIFT(T52, T53, 3, 4)
> + MADD_SHIFT(T52, T53, 4, 12)
> +
> + PHADDD_SHIFT(5, 2)
> + PHADDD_SHIFT(6, 6)
> + PHADDD_SHIFT(7, 10)
> + PHADDD_SHIFT(8, 14)
> + MAKE_ODD1(0, 1);
> + MAKE_ODD1(1, 3);
> + MAKE_ODD1(2, 5);
> + MAKE_ODD1(3, 7);
> + MAKE_ODD1(4, 9);
> + MAKE_ODD1(5, 11);
> + MAKE_ODD1(6, 13);
> + MAKE_ODD1(7, 15);
> +#undef part
> + }
> + // DCT2
> + {
> +#define part 0
> +#define SHUFFLE3(R1, SRC) \
> + R1 = _mm_shufflehi_epi16(SRC, 0x63); \
> + R1 = _mm_shufflelo_epi16(R1, 0x9C);
> + SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
> + SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
> + SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
> + SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
> +
> +#define SHUFFLE4(R1, SRC) \
> + R1 = _mm_shuffle_epi32(SRC, 0x4E); \
> + R1 = _mm_shufflehi_epi16(R1, 0x9C); \
> + R1 = _mm_shufflelo_epi16(R1, 0x63);
> + SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
> + SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
> + SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
> + SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
> +
> +#define UNPCK_MADD(INDEX) \
> + T10 = _mm_unpacklo_epi16(T00A, T00B); \
> + T11 = _mm_unpackhi_epi16(T00A, T00B); \
> + T12 = _mm_unpacklo_epi16(T01A, T01B); \
> + T13 = _mm_unpackhi_epi16(T01A, T01B); \
> + T14 = _mm_unpacklo_epi16(T02A, T02B); \
> + T15 = _mm_unpackhi_epi16(T02A, T02B); \
> + T16 = _mm_unpacklo_epi16(T03A, T03B); \
> + T17 = _mm_unpackhi_epi16(T03A, T03B); \
> + \
> + T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> + T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> + T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> + T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> + T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> + T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> + T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> + T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> + \
> + T30 = _mm_add_epi32(T20, T21); \
> + T31 = _mm_add_epi32(T22, T23); \
> + T32 = _mm_add_epi32(T24, T25); \
> + T33 = _mm_add_epi32(T26, T27); \
> + \
> + PHADDD(T30, T30, T31) \
> + PHADDD(T31, T32, T33) \
> + \
> + PHADDD(T40, T30, T31) \
> + T41 = _mm_sub_epi32(T00A, T00B); \
> + T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
> + T41 = _mm_srai_epi32(_mm_add_epi32(T41, c_512), SHIFT2); \
> + T40 = _mm_packs_epi32(T40, T40); \
> + T41 = _mm_packs_epi32(T41, T41); \
> + _mm_storel_epi64((__m128i*)&dst[0 * 16 + INDEX], T40); \
> + _mm_storel_epi64((__m128i*)&dst[8 * 16 + INDEX], T41);
> +
> + UNPCK_MADD(part)
> +#define MADD_PHADDD(TAB, ROW, INDEX) \
> + T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + \
> + T30 = _mm_add_epi32(T20, T21); \
> + T31 = _mm_add_epi32(T22, T23); \
> + T32 = _mm_add_epi32(T24, T25); \
> + T33 = _mm_add_epi32(T26, T27); \
> + \
> + PHADDD(T30, T30, T31) \
> + PHADDD(T31, T32, T33) \
> + \
> + PHADDD(T40, T30, T31) \
> + T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
> + T40 = _mm_packs_epi32(T40, T40); \
> + _mm_storel_epi64((__m128i*)&dst[ROW * 16 + INDEX], T40);
> +
> + MADD_PHADDD(8, 4, part)
> + MADD_PHADDD(9, 12, part)
> +
> +#define SUB_PHADDD(TAB, ROW, INDEX) \
> + T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> + \
> + T30 = _mm_sub_epi32(T20, T21); \
> + T31 = _mm_sub_epi32(T22, T23); \
> + T32 = _mm_sub_epi32(T24, T25); \
> + T33 = _mm_sub_epi32(T26, T27); \
> + \
> + PHADDD(T30, T30, T31) \
> + PHADDD(T31, T32, T33) \
> + \
> + PHADDD(T40, T30, T31) \
> + T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
> + T40 = _mm_packs_epi32(T40, T40); \
> + _mm_storel_epi64((__m128i*)&dst[ROW * 16 + INDEX], T40);
> +
> + SUB_PHADDD(10, 2, part)
> + SUB_PHADDD(11, 6, part)
> + SUB_PHADDD(12, 10, part)
> + SUB_PHADDD(13, 14, part)
> +
> +#define MAKE_ODD2(tab, dstPos, INDEX) \
> + T20 = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); /* [*O2_0 *O1_0 *O3_0 *O0_0] */ \
> + T21 = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); /* [*O5_0 *O6_0 *O4_0 *O7_0] */ \
> + T22 = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T23 = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
> + T24 = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T25 = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
> + T26 = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> + T27 = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
> + \
> + T30 = _mm_add_epi32(T20, T21); \
> + T31 = _mm_add_epi32(T22, T23); \
> + T32 = _mm_add_epi32(T24, T25); \
> + T33 = _mm_add_epi32(T26, T27); \
> + \
> + PHADDD(T30, T30, T31) \
> + PHADDD(T31, T32, T33) \
> + \
> + PHADDD(T40, T30, T31); \
> + T40 = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
> + T40 = _mm_packs_epi32(T40, T40); \
> + _mm_storel_epi64((__m128i*)&dst[(dstPos) * 16 + INDEX], T40);
> +
> + MAKE_ODD2(14, 1, part);
> + MAKE_ODD2(16, 3, part);
> + MAKE_ODD2(18, 5, part);
> + MAKE_ODD2(20, 7, part);
> + MAKE_ODD2(22, 9, part);
> + MAKE_ODD2(24, 11, part);
> + MAKE_ODD2(26, 13, part);
> + MAKE_ODD2(28, 15, part);
> +#undef part
> + }
> + {
> +#define part 4
> + SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
> + SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
> + SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
> + SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
> +
> + SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
> + SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
> + SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
> + SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
> +
> + UNPCK_MADD(part)
> +
> + MADD_PHADDD(8, 4, part)
> + MADD_PHADDD(9, 12, part)
> +
> + SUB_PHADDD(10, 2, part)
> + SUB_PHADDD(11, 6, part)
> + SUB_PHADDD(12, 10, part)
> + SUB_PHADDD(13, 14, part)
> +
> + MAKE_ODD2(14, 1, part);
> + MAKE_ODD2(16, 3, part);
> + MAKE_ODD2(18, 5, part);
> + MAKE_ODD2(20, 7, part);
> + MAKE_ODD2(22, 9, part);
> + MAKE_ODD2(24, 11, part);
> + MAKE_ODD2(26, 13, part);
> + MAKE_ODD2(28, 15, part);
> +#undef part
> + }
> + {
> +#define part 8
> + SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
> + SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
> + SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
> + SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
> +
> + SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
> + SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
> + SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
> + SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
> +
> + UNPCK_MADD(part)
> +
> + MADD_PHADDD(8, 4, part)
> + MADD_PHADDD(9, 12, part)
> +
> + SUB_PHADDD(10, 2, part)
> + SUB_PHADDD(11, 6, part)
> + SUB_PHADDD(12, 10, part)
> + SUB_PHADDD(13, 14, part)
> +
> + MAKE_ODD2(14, 1, part);
> + MAKE_ODD2(16, 3, part);
> + MAKE_ODD2(18, 5, part);
> + MAKE_ODD2(20, 7, part);
> + MAKE_ODD2(22, 9, part);
> + MAKE_ODD2(24, 11, part);
> + MAKE_ODD2(26, 13, part);
> + MAKE_ODD2(28, 15, part);
> +#undef part
> + }
> + {
> +#define part 12
> + SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
> + SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
> + SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
> + SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
> +
> + SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
> + SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
> + SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
> + SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
> +
> + UNPCK_MADD(part)
> +
> + MADD_PHADDD(8, 4, part)
> + MADD_PHADDD(9, 12, part)
> +
> + SUB_PHADDD(10, 2, part)
> + SUB_PHADDD(11, 6, part)
> + SUB_PHADDD(12, 10, part)
> + SUB_PHADDD(13, 14, part)
> +
> + MAKE_ODD2(14, 1, part);
> + MAKE_ODD2(16, 3, part);
> + MAKE_ODD2(18, 5, part);
> + MAKE_ODD2(20, 7, part);
> + MAKE_ODD2(22, 9, part);
> + MAKE_ODD2(24, 11, part);
> + MAKE_ODD2(26, 13, part);
> + MAKE_ODD2(28, 15, part);
> +#undef part
> + }
> +#undef SHIFT1
> +#undef ADD1
> +#undef SHIFT2
> +#undef ADD2
> +#undef SHUFFLE1
> +#undef SHUFFLE2
> +#undef PHADDW_PHSUBW
> +#undef MAKE_ODD1
> +#undef MAKE_ODD2
> +#undef PHADDD
> +#undef SHUFFLE3
> +#undef SHUFFLE4
> +#undef PHADDD_SHIFT
> +#undef MADD_SHIFT
> +#undef MADD_PHADDD
> +#undef SUB_PHADDD
> +}
> +
> #define SHIFT1 7
> #define ADD1 64
>
> @@ -1426,6 +1956,7 @@
> /* Note: We have AVX2 assembly for these functions, but since AVX2 is still
> * somewhat rare on end-user PCs we still compile and link these SSE3
> * intrinsic SIMD functions */
> + p.cu[BLOCK_16x16].dct = dct16;
> p.cu[BLOCK_8x8].idct = idct8;
> p.cu[BLOCK_16x16].idct = idct16;
> p.cu[BLOCK_32x32].idct = idct32;
More information about the x265-devel
mailing list