[x265] [PATCH] intrinsic: Added dct16 sse3 intrinsic, 55288.92 -> 45139.28

Steve Borho steve at borho.org
Wed Feb 11 18:56:04 CET 2015


On 02/11, dtyx265 at gmail.com wrote:
> # HG changeset patch
> # User David T Yuen <dtyx265 at gmail.com>
> # Date 1423675655 28800
> # Node ID 5b1611feda7249643af85e595098bc95e9e4eee2
> # Parent  f52d723d8c46e9bca77bb8f3113004d449456559
> intrinsic: Added dct16 sse3 intrinsic, 55288.92 -> 45139.28

which two numbers are being compared there? which is 55288.92 and which
is 45139.28?  If we have sse3 and ssse3 functions available, most modern
CPUs will only use the ssse3 function.

> This is backported from the ssse3 dct16 intrinsic
> 
> diff -r f52d723d8c46 -r 5b1611feda72 source/common/vec/dct-sse3.cpp
> --- a/source/common/vec/dct-sse3.cpp	Mon Feb 09 17:07:02 2015 +0530
> +++ b/source/common/vec/dct-sse3.cpp	Wed Feb 11 09:27:35 2015 -0800
> @@ -36,6 +36,536 @@
>  using namespace x265;
>  
>  namespace {
> +ALIGN_VAR_32(static const int16_t, tab_dct_8[][8]) =
> +{
> +    { 0x0100, 0x0F0E, 0x0706, 0x0908, 0x0302, 0x0D0C, 0x0504, 0x0B0A },
> +
> +    { 64, 64, 64, 64, 64, 64, 64, 64 },
> +    { 64, -64, 64, -64, 64, -64, 64, -64 },
> +    { 83, 36, 83, 36, 83, 36, 83, 36 },
> +    { 36, -83, 36, -83, 36, -83, 36, -83 },
> +    { 89, 18, 75, 50, 89, 18, 75, 50 },
> +    { 75, -50, -18, -89, 75, -50, -18, -89 },
> +    { 50, 75, -89, 18, 50, 75, -89, 18 },
> +    { 18, -89, -50, 75, 18, -89, -50, 75 },
> +
> +    { 83, 83, -83, -83, 36, 36, -36, -36 },
> +    { 36, 36, -36, -36, -83, -83, 83, 83 },
> +    { 89, -89, 18, -18, 75, -75, 50, -50 },
> +    { 75, -75, -50, 50, -18, 18, -89, 89 },
> +    { 50, -50, 75, -75, -89, 89, 18, -18 },
> +    { 18, -18, -89, 89, -50, 50, 75, -75 },
> +};
> +
> +ALIGN_VAR_32(static const int16_t, tab_dct_16_1[][8]) =
> +{
> +    { 90, 87, 80, 70, 57, 43, 25,  9 },  //  0
> +    { 87, 57,  9, -43, -80, -90, -70, -25 },  //  1
> +    { 80,  9, -70, -87, -25, 57, 90, 43 },  //  2
> +    { 70, -43, -87,  9, 90, 25, -80, -57 },  //  3
> +    { 57, -80, -25, 90, -9, -87, 43, 70 },  //  4
> +    { 43, -90, 57, 25, -87, 70,  9, -80 },  //  5
> +    { 25, -70, 90, -80, 43,  9, -57, 87 },  //  6
> +    {  9, -25, 43, -57, 70, -80, 87, -90 },  //  7
> +    { 83, 83, -83, -83, 36, 36, -36, -36 },  //  8
> +    { 36, 36, -36, -36, -83, -83, 83, 83 },  //  9
> +    { 89, 89, 18, 18, 75, 75, 50, 50 },  // 10
> +    { 75, 75, -50, -50, -18, -18, -89, -89 },  // 11
> +    { 50, 50, 75, 75, -89, -89, 18, 18 },  // 12
> +    { 18, 18, -89, -89, -50, -50, 75, 75 },  // 13
> +
> +#define MAKE_COEF(a0, a1, a2, a3, a4, a5, a6, a7) \
> +    { (a0), -(a0), (a3), -(a3), (a1), -(a1), (a2), -(a2) \
> +    }, \
> +    { (a7), -(a7), (a4), -(a4), (a6), -(a6), (a5), -(a5) },
> +
> +    MAKE_COEF(90, 87, 80, 70, 57, 43, 25,  9)
> +    MAKE_COEF(87, 57,  9, -43, -80, -90, -70, -25)
> +    MAKE_COEF(80,  9, -70, -87, -25, 57, 90, 43)
> +    MAKE_COEF(70, -43, -87,  9, 90, 25, -80, -57)
> +    MAKE_COEF(57, -80, -25, 90, -9, -87, 43, 70)
> +    MAKE_COEF(43, -90, 57, 25, -87, 70,  9, -80)
> +    MAKE_COEF(25, -70, 90, -80, 43,  9, -57, 87)
> +    MAKE_COEF(9, -25, 43, -57, 70, -80, 87, -90)
> +#undef MAKE_COEF
> +};
> +
> +void dct16(const int16_t *src, int16_t *dst, intptr_t stride)
> +{
> +#if HIGH_BIT_DEPTH
> +#define SHIFT1  5
> +#define ADD1    16
> +#else
> +#define SHIFT1  3
> +#define ADD1    4
> +#endif
> +
> +#define SHIFT2  10
> +#define ADD2    512
> +
> +    // Const
> +    __m128i c_4     = _mm_set1_epi32(ADD1);
> +    __m128i c_512   = _mm_set1_epi32(ADD2);
> +
> +    ALIGN_VAR_32(int16_t, tmp[16 * 16]);
> +
> +    __m128i T00A, T01A, T02A, T03A;
> +    __m128i T00B, T01B, T02B, T03B, T04B, T05B, T06B, T07B;
> +    __m128i T10, T11, T12, T13, T14, T15, T16, T17;
> +    __m128i T20, T21, T22, T23, T24, T25, T26, T27;
> +    __m128i T30, T31, T32, T33, T34, T35, T36, T37;
> +    __m128i T40, T41, T42, T43, T44, T45, T46, T47;
> +    __m128i T50, T51, T52, T53;
> +    __m128i T60, T61, T62, T63, T64, T65, T66, T67;
> +    __m128i T70;
> +
> +    // DCT1
> +    {
> +#define part 0
> +#define SHUFFLE1(R1, SRC)  \
> +    R1 = _mm_shuffle_epi32(SRC, 0x1B); \
> +    R1 = _mm_shufflehi_epi16(R1, 0xB1); \
> +    R1 = _mm_shufflelo_epi16(R1, 0xB1);
> +        SHUFFLE1(T00B, _mm_load_si128((__m128i*)&src[(part + 0) * stride + 8]))
> +        SHUFFLE1(T01B, _mm_load_si128((__m128i*)&src[(part + 1) * stride + 8]))
> +        SHUFFLE1(T02B, _mm_load_si128((__m128i*)&src[(part + 2) * stride + 8]))
> +        SHUFFLE1(T03B, _mm_load_si128((__m128i*)&src[(part + 3) * stride + 8]))
> +        SHUFFLE1(T04B, _mm_load_si128((__m128i*)&src[(part + 4) * stride + 8]))
> +        SHUFFLE1(T05B, _mm_load_si128((__m128i*)&src[(part + 5) * stride + 8]))
> +        SHUFFLE1(T06B, _mm_load_si128((__m128i*)&src[(part + 6) * stride + 8]))
> +        SHUFFLE1(T07B, _mm_load_si128((__m128i*)&src[(part + 7) * stride + 8]))
> +
> +#define READ_ADD_SUB(INDEX) \
> +    T10  = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 0) * stride]), T00B); \
> +    T11  = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 1) * stride]), T01B); \
> +    T12  = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 2) * stride]), T02B); \
> +    T13  = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 3) * stride]), T03B); \
> +    T14  = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 4) * stride]), T04B); \
> +    T15  = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 5) * stride]), T05B); \
> +    T16  = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 6) * stride]), T06B); \
> +    T17  = _mm_add_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 7) * stride]), T07B); \
> +    \
> +    T20  = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 0) * stride]), T00B); \
> +    T21  = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 1) * stride]), T01B); \
> +    T22  = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 2) * stride]), T02B); \
> +    T23  = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 3) * stride]), T03B); \
> +    T24  = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 4) * stride]), T04B); \
> +    T25  = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 5) * stride]), T05B); \
> +    T26  = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 6) * stride]), T06B); \
> +    T27  = _mm_sub_epi16(_mm_load_si128((__m128i*)&src[(INDEX + 7) * stride]), T07B);
> +        READ_ADD_SUB(part)
> +
> +#define SHUFFLE2(R1, R2, R3, R4) \
> +    R1 = _mm_shufflehi_epi16(R1, 0x63); \
> +    R2 = _mm_shufflehi_epi16(R2, 0x63); \
> +    R1 = _mm_shufflelo_epi16(R1, 0x9C); \
> +    R2 = _mm_shufflelo_epi16(R2, 0x9C); \
> +    R3 = _mm_unpacklo_epi64(R1, R2); \
> +    R4 = _mm_unpackhi_epi64(R1, R2);
> +
> +        SHUFFLE2(T10, T11, T30, T31)
> +        SHUFFLE2(T12, T13, T32, T33)
> +        SHUFFLE2(T14, T15, T34, T35)
> +        SHUFFLE2(T16, T17, T36, T37)
> +
> +        T40 = _mm_add_epi16(T30, T31);
> +        T41 = _mm_add_epi16(T32, T33);
> +        T42 = _mm_add_epi16(T34, T35);
> +        T43 = _mm_add_epi16(T36, T37);
> +        T44 = _mm_sub_epi16(T30, T31);
> +        T45 = _mm_sub_epi16(T32, T33);
> +        T46 = _mm_sub_epi16(T34, T35);
> +        T47 = _mm_sub_epi16(T36, T37);
> +
> +#define PHADDW_PHSUBW(R1, R2, R3, R4) \
> +    R1 = _mm_shufflehi_epi16(R1, 0xD8); \
> +    R1 = _mm_shufflelo_epi16(R1, 0xD8); \
> +    R1 = _mm_shuffle_epi32(R1, 0xD8); \
> +    R2 = _mm_shufflehi_epi16(R2, 0xD8); \
> +    R2 = _mm_shufflelo_epi16(R2, 0xD8); \
> +    R2 = _mm_shuffle_epi32(R2, 0xD8); \
> +    R4 = _mm_unpacklo_epi64(R1, R2); \
> +    R1 = _mm_unpackhi_epi64(R1, R2); \
> +    R3 = _mm_add_epi16(R4, R1); \
> +    R4 = _mm_sub_epi16(R4, R1);
> +
> +        PHADDW_PHSUBW(T40, T41, T50, T52)
> +        PHADDW_PHSUBW(T42, T43, T51, T53)
> +
> +#define MADD_SHIFT(R1, R2, tab, index) \
> +    T60  = _mm_madd_epi16(R1, _mm_load_si128((__m128i*)tab_dct_8[tab])); \
> +    T61  = _mm_madd_epi16(R2, _mm_load_si128((__m128i*)tab_dct_8[tab])); \
> +    T60  = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
> +    T61  = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
> +    T70  = _mm_packs_epi32(T60, T61); \
> +    _mm_store_si128((__m128i*)&tmp[index * 16 + part], T70);
> +
> +        MADD_SHIFT(T50, T51, 1, 0)
> +        MADD_SHIFT(T50, T51, 2, 8)
> +        MADD_SHIFT(T52, T53, 3, 4)
> +        MADD_SHIFT(T52, T53, 4, 12)
> +
> +#define PHADDD(R1, R2, R3) \
> +    R2 = _mm_shuffle_epi32(R2, 0xD8); \
> +    R3 = _mm_shuffle_epi32(R3, 0xD8); \
> +    T00A = _mm_unpacklo_epi64(R2, R3); \
> +    T00B = _mm_unpackhi_epi64(R2, R3); \
> +    R1 = _mm_add_epi32(T00A, T00B);
> +
> +#define PHADDD_SHIFT(TAB, ROW) \
> +    T60  = _mm_madd_epi16(T44, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
> +    T61  = _mm_madd_epi16(T45, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
> +    T62  = _mm_madd_epi16(T46, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
> +    T63  = _mm_madd_epi16(T47, _mm_load_si128((__m128i*)tab_dct_8[TAB])); \
> +    PHADDD(T60, T60, T61) \
> +    PHADDD(T61, T62, T63) \
> +    T60  = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
> +    T61  = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
> +    T70  = _mm_packs_epi32(T60, T61); \
> +    _mm_store_si128((__m128i*)&tmp[ROW * 16 + part], T70);
> +
> +        PHADDD_SHIFT(5, 2)
> +        PHADDD_SHIFT(6, 6)
> +        PHADDD_SHIFT(7, 10)
> +        PHADDD_SHIFT(8, 14)
> +
> +#define MAKE_ODD1(tab, dstPos) \
> +    T60  = _mm_madd_epi16(T20, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T61  = _mm_madd_epi16(T21, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T62  = _mm_madd_epi16(T22, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T63  = _mm_madd_epi16(T23, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T64  = _mm_madd_epi16(T24, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T65  = _mm_madd_epi16(T25, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T66  = _mm_madd_epi16(T26, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T67  = _mm_madd_epi16(T27, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    PHADDD(T60, T60, T61) \
> +    PHADDD(T61, T62, T63) \
> +    PHADDD(T62, T64, T65) \
> +    PHADDD(T63, T66, T67) \
> +    PHADDD(T60, T60, T61) \
> +    PHADDD(T61, T62, T63) \
> +    T60  = _mm_srai_epi32(_mm_add_epi32(T60, c_4), SHIFT1); \
> +    T61  = _mm_srai_epi32(_mm_add_epi32(T61, c_4), SHIFT1); \
> +    T70  = _mm_packs_epi32(T60, T61); \
> +    _mm_store_si128((__m128i*)&tmp[(dstPos) * 16 + part], T70);
> +
> +        MAKE_ODD1(0, 1);
> +        MAKE_ODD1(1, 3);
> +        MAKE_ODD1(2, 5);
> +        MAKE_ODD1(3, 7);
> +        MAKE_ODD1(4, 9);
> +        MAKE_ODD1(5, 11);
> +        MAKE_ODD1(6, 13);
> +        MAKE_ODD1(7, 15);
> +#undef part
> +    }
> +    {
> +#define part 8
> +        SHUFFLE1(T00B, _mm_load_si128((__m128i*)&src[(part + 0) * stride + 8]))
> +        SHUFFLE1(T01B, _mm_load_si128((__m128i*)&src[(part + 1) * stride + 8]))
> +        SHUFFLE1(T02B, _mm_load_si128((__m128i*)&src[(part + 2) * stride + 8]))
> +        SHUFFLE1(T03B, _mm_load_si128((__m128i*)&src[(part + 3) * stride + 8]))
> +        SHUFFLE1(T04B, _mm_load_si128((__m128i*)&src[(part + 4) * stride + 8]))
> +        SHUFFLE1(T05B, _mm_load_si128((__m128i*)&src[(part + 5) * stride + 8]))
> +        SHUFFLE1(T06B, _mm_load_si128((__m128i*)&src[(part + 6) * stride + 8]))
> +        SHUFFLE1(T07B, _mm_load_si128((__m128i*)&src[(part + 7) * stride + 8]))
> +
> +        READ_ADD_SUB(part)
> +
> +        SHUFFLE2(T10, T11, T30, T31)
> +        SHUFFLE2(T12, T13, T32, T33)
> +        SHUFFLE2(T14, T15, T34, T35)
> +        SHUFFLE2(T16, T17, T36, T37)
> +
> +        T40 = _mm_add_epi16(T30, T31);
> +        T41 = _mm_add_epi16(T32, T33);
> +        T42 = _mm_add_epi16(T34, T35);
> +        T43 = _mm_add_epi16(T36, T37);
> +        T44 = _mm_sub_epi16(T30, T31);
> +        T45 = _mm_sub_epi16(T32, T33);
> +        T46 = _mm_sub_epi16(T34, T35);
> +        T47 = _mm_sub_epi16(T36, T37);
> +
> +        PHADDW_PHSUBW(T40, T41, T50, T52)
> +        PHADDW_PHSUBW(T42, T43, T51, T53)
> +
> +        MADD_SHIFT(T50, T51, 1, 0)
> +        MADD_SHIFT(T50, T51, 2, 8)
> +        MADD_SHIFT(T52, T53, 3, 4)
> +        MADD_SHIFT(T52, T53, 4, 12)
> +
> +        PHADDD_SHIFT(5, 2)
> +        PHADDD_SHIFT(6, 6)
> +        PHADDD_SHIFT(7, 10)
> +        PHADDD_SHIFT(8, 14)
> +        MAKE_ODD1(0, 1);
> +        MAKE_ODD1(1, 3);
> +        MAKE_ODD1(2, 5);
> +        MAKE_ODD1(3, 7);
> +        MAKE_ODD1(4, 9);
> +        MAKE_ODD1(5, 11);
> +        MAKE_ODD1(6, 13);
> +        MAKE_ODD1(7, 15);
> +#undef part
> +    }
> +    // DCT2
> +    {
> +#define part 0
> +#define SHUFFLE3(R1, SRC)  \
> +    R1 = _mm_shufflehi_epi16(SRC, 0x63); \
> +    R1 = _mm_shufflelo_epi16(R1, 0x9C);
> +        SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
> +        SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
> +        SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
> +        SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
> +
> +#define SHUFFLE4(R1, SRC)  \
> +    R1 = _mm_shuffle_epi32(SRC, 0x4E); \
> +    R1 = _mm_shufflehi_epi16(R1, 0x9C); \
> +    R1 = _mm_shufflelo_epi16(R1, 0x63);
> +        SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
> +        SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
> +        SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
> +        SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
> +
> +#define UNPCK_MADD(INDEX) \
> +        T10  = _mm_unpacklo_epi16(T00A, T00B); \
> +        T11  = _mm_unpackhi_epi16(T00A, T00B); \
> +        T12  = _mm_unpacklo_epi16(T01A, T01B); \
> +        T13  = _mm_unpackhi_epi16(T01A, T01B); \
> +        T14  = _mm_unpacklo_epi16(T02A, T02B); \
> +        T15  = _mm_unpackhi_epi16(T02A, T02B); \
> +        T16  = _mm_unpacklo_epi16(T03A, T03B); \
> +        T17  = _mm_unpackhi_epi16(T03A, T03B); \
> +        \
> +        T20  = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> +        T21  = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> +        T22  = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> +        T23  = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> +        T24  = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> +        T25  = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> +        T26  = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> +        T27  = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_8[1])); \
> +        \
> +        T30  = _mm_add_epi32(T20, T21); \
> +        T31  = _mm_add_epi32(T22, T23); \
> +        T32  = _mm_add_epi32(T24, T25); \
> +        T33  = _mm_add_epi32(T26, T27); \
> +        \
> +        PHADDD(T30, T30, T31) \
> +        PHADDD(T31, T32, T33) \
> +        \
> +        PHADDD(T40, T30, T31) \
> +        T41 = _mm_sub_epi32(T00A, T00B); \
> +        T40  = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
> +        T41  = _mm_srai_epi32(_mm_add_epi32(T41, c_512), SHIFT2); \
> +        T40  = _mm_packs_epi32(T40, T40); \
> +        T41  = _mm_packs_epi32(T41, T41); \
> +        _mm_storel_epi64((__m128i*)&dst[0 * 16 + INDEX], T40); \
> +        _mm_storel_epi64((__m128i*)&dst[8 * 16 + INDEX], T41);
> +
> +        UNPCK_MADD(part)
> +#define MADD_PHADDD(TAB, ROW, INDEX) \
> +    T20  = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T21  = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T22  = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T23  = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T24  = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T25  = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T26  = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T27  = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    \
> +    T30  = _mm_add_epi32(T20, T21); \
> +    T31  = _mm_add_epi32(T22, T23); \
> +    T32  = _mm_add_epi32(T24, T25); \
> +    T33  = _mm_add_epi32(T26, T27); \
> +    \
> +    PHADDD(T30, T30, T31) \
> +    PHADDD(T31, T32, T33) \
> +    \
> +    PHADDD(T40, T30, T31) \
> +    T40  = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
> +    T40  = _mm_packs_epi32(T40, T40); \
> +    _mm_storel_epi64((__m128i*)&dst[ROW * 16 + INDEX], T40);
> +
> +        MADD_PHADDD(8, 4, part)
> +        MADD_PHADDD(9, 12, part)
> +
> +#define SUB_PHADDD(TAB, ROW, INDEX) \
> +    T20  = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T21  = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T22  = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T23  = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T24  = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T25  = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T26  = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    T27  = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[TAB])); \
> +    \
> +    T30  = _mm_sub_epi32(T20, T21); \
> +    T31  = _mm_sub_epi32(T22, T23); \
> +    T32  = _mm_sub_epi32(T24, T25); \
> +    T33  = _mm_sub_epi32(T26, T27); \
> +    \
> +    PHADDD(T30, T30, T31) \
> +    PHADDD(T31, T32, T33) \
> +    \
> +    PHADDD(T40, T30, T31) \
> +    T40  = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
> +    T40  = _mm_packs_epi32(T40, T40); \
> +    _mm_storel_epi64((__m128i*)&dst[ROW * 16 + INDEX], T40);
> +
> +        SUB_PHADDD(10, 2, part)
> +        SUB_PHADDD(11, 6, part)
> +        SUB_PHADDD(12, 10, part)
> +        SUB_PHADDD(13, 14, part)
> +
> +#define MAKE_ODD2(tab, dstPos, INDEX) \
> +    T20  = _mm_madd_epi16(T10, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)]));       /* [*O2_0 *O1_0 *O3_0 *O0_0] */ \
> +    T21  = _mm_madd_epi16(T11, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1]));   /* [*O5_0 *O6_0 *O4_0 *O7_0] */ \
> +    T22  = _mm_madd_epi16(T12, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T23  = _mm_madd_epi16(T13, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
> +    T24  = _mm_madd_epi16(T14, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T25  = _mm_madd_epi16(T15, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
> +    T26  = _mm_madd_epi16(T16, _mm_load_si128((__m128i*)tab_dct_16_1[(tab)])); \
> +    T27  = _mm_madd_epi16(T17, _mm_load_si128((__m128i*)tab_dct_16_1[(tab) + 1])); \
> +    \
> +    T30  = _mm_add_epi32(T20, T21); \
> +    T31  = _mm_add_epi32(T22, T23); \
> +    T32  = _mm_add_epi32(T24, T25); \
> +    T33  = _mm_add_epi32(T26, T27); \
> +    \
> +    PHADDD(T30, T30, T31) \
> +    PHADDD(T31, T32, T33) \
> +    \
> +    PHADDD(T40, T30, T31); \
> +    T40  = _mm_srai_epi32(_mm_add_epi32(T40, c_512), SHIFT2); \
> +    T40  = _mm_packs_epi32(T40, T40); \
> +    _mm_storel_epi64((__m128i*)&dst[(dstPos) * 16 + INDEX], T40);
> +
> +        MAKE_ODD2(14,  1, part);
> +        MAKE_ODD2(16,  3, part);
> +        MAKE_ODD2(18,  5, part);
> +        MAKE_ODD2(20,  7, part);
> +        MAKE_ODD2(22,  9, part);
> +        MAKE_ODD2(24, 11, part);
> +        MAKE_ODD2(26, 13, part);
> +        MAKE_ODD2(28, 15, part);
> +#undef part
> +    }
> +    {
> +#define part 4
> +        SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
> +        SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
> +        SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
> +        SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
> +
> +        SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
> +        SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
> +        SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
> +        SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
> +
> +        UNPCK_MADD(part)
> +
> +        MADD_PHADDD(8, 4, part)
> +        MADD_PHADDD(9, 12, part)
> +
> +        SUB_PHADDD(10, 2, part)
> +        SUB_PHADDD(11, 6, part)
> +        SUB_PHADDD(12, 10, part)
> +        SUB_PHADDD(13, 14, part)
> +
> +        MAKE_ODD2(14,  1, part);
> +        MAKE_ODD2(16,  3, part);
> +        MAKE_ODD2(18,  5, part);
> +        MAKE_ODD2(20,  7, part);
> +        MAKE_ODD2(22,  9, part);
> +        MAKE_ODD2(24, 11, part);
> +        MAKE_ODD2(26, 13, part);
> +        MAKE_ODD2(28, 15, part);
> +#undef part
> +    }
> +    {
> +#define part 8
> +        SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
> +        SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
> +        SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
> +        SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
> +
> +        SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
> +        SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
> +        SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
> +        SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
> +
> +        UNPCK_MADD(part)
> +
> +        MADD_PHADDD(8, 4, part)
> +        MADD_PHADDD(9, 12, part)
> +
> +        SUB_PHADDD(10, 2, part)
> +        SUB_PHADDD(11, 6, part)
> +        SUB_PHADDD(12, 10, part)
> +        SUB_PHADDD(13, 14, part)
> +
> +        MAKE_ODD2(14,  1, part);
> +        MAKE_ODD2(16,  3, part);
> +        MAKE_ODD2(18,  5, part);
> +        MAKE_ODD2(20,  7, part);
> +        MAKE_ODD2(22,  9, part);
> +        MAKE_ODD2(24, 11, part);
> +        MAKE_ODD2(26, 13, part);
> +        MAKE_ODD2(28, 15, part);
> +#undef part
> +    }
> +    {
> +#define part 12
> +        SHUFFLE3(T00A, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 0]))
> +        SHUFFLE3(T01A, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 0]))
> +        SHUFFLE3(T02A, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 0]))
> +        SHUFFLE3(T03A, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 0]))
> +
> +        SHUFFLE4(T00B, _mm_load_si128((__m128i*)&tmp[(part + 0) * 16 + 8]))
> +        SHUFFLE4(T01B, _mm_load_si128((__m128i*)&tmp[(part + 1) * 16 + 8]))
> +        SHUFFLE4(T02B, _mm_load_si128((__m128i*)&tmp[(part + 2) * 16 + 8]))
> +        SHUFFLE4(T03B, _mm_load_si128((__m128i*)&tmp[(part + 3) * 16 + 8]))
> +
> +        UNPCK_MADD(part)
> +
> +        MADD_PHADDD(8, 4, part)
> +        MADD_PHADDD(9, 12, part)
> +
> +        SUB_PHADDD(10, 2, part)
> +        SUB_PHADDD(11, 6, part)
> +        SUB_PHADDD(12, 10, part)
> +        SUB_PHADDD(13, 14, part)
> +
> +        MAKE_ODD2(14,  1, part);
> +        MAKE_ODD2(16,  3, part);
> +        MAKE_ODD2(18,  5, part);
> +        MAKE_ODD2(20,  7, part);
> +        MAKE_ODD2(22,  9, part);
> +        MAKE_ODD2(24, 11, part);
> +        MAKE_ODD2(26, 13, part);
> +        MAKE_ODD2(28, 15, part);
> +#undef part
> +    }
> +#undef SHIFT1
> +#undef ADD1
> +#undef SHIFT2
> +#undef ADD2
> +#undef SHUFFLE1
> +#undef SHUFFLE2
> +#undef PHADDW_PHSUBW
> +#undef MAKE_ODD1
> +#undef MAKE_ODD2
> +#undef PHADDD
> +#undef SHUFFLE3
> +#undef SHUFFLE4
> +#undef PHADDD_SHIFT
> +#undef MADD_SHIFT
> +#undef MADD_PHADDD
> +#undef SUB_PHADDD
> +}
> +
>  #define SHIFT1  7
>  #define ADD1    64
>  
> @@ -1426,6 +1956,7 @@
>      /* Note: We have AVX2 assembly for these functions, but since AVX2 is still
>       * somewhat rare on end-user PCs we still compile and link these SSE3
>       * intrinsic SIMD functions */
> +    p.cu[BLOCK_16x16].dct   = dct16;
>      p.cu[BLOCK_8x8].idct   = idct8;
>      p.cu[BLOCK_16x16].idct = idct16;
>      p.cu[BLOCK_32x32].idct = idct32;
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel

-- 
Steve Borho


More information about the x265-devel mailing list