[x265] [PATCH] dct: Replaced partialButterfly32 vector class function to intrinsic

yuvaraj at multicorewareinc.com yuvaraj at multicorewareinc.com
Fri Oct 11 13:25:26 CEST 2013


# HG changeset patch
# User Yuvaraj Venkatesh <yuvaraj at multicorewareinc.com>
# Date 1381490578 -19800
#      Fri Oct 11 16:52:58 2013 +0530
# Node ID f1e462de1341b222ceb4295129426b93b3afdeca
# Parent  46b954edb1c52a557b9d94c4ed380ea0578c1949
dct: Replaced partialButterfly32 vector class function to intrinsic

diff -r 46b954edb1c5 -r f1e462de1341 source/common/vec/dct-sse3.cpp
--- a/source/common/vec/dct-sse3.cpp	Fri Oct 11 14:09:28 2013 +0530
+++ b/source/common/vec/dct-sse3.cpp	Fri Oct 11 16:52:58 2013 +0530
@@ -1914,209 +1914,212 @@
     int j;
     int add = 1 << (shift - 1);
 
-    Vec4i zero_row_first_two(64, 64, 0, 0);
-    Vec4i eight_row_first_two(83, 36, 0, 0);
-    Vec4i sixten_row_first_two(64, -64, 0, 0);
-    Vec4i twentyfour_row_first_two(36, -83, 0, 0);
+    __m128i zero_row_first_two = _mm_setr_epi32(64, 64, 0, 0);
+    __m128i eight_row_first_two = _mm_setr_epi32(83, 36, 0, 0);
+    __m128i sixten_row_first_two = _mm_setr_epi32(64, -64, 0, 0);
+    __m128i twentyfour_row_first_two = _mm_setr_epi32(36, -83, 0, 0);
 
-    Vec4i four_row_first_four(89, 75, 50, 18);
-    Vec4i twelve_row_first_four(75, -18, -89, -50);
-    Vec4i twenty_row_first_four(50, -89, 18, 75);
-    Vec4i twentyeight_row_first_four(18, -50, 75, -89);
+    __m128i four_row_first_four = _mm_setr_epi32(89, 75, 50, 18);
+    __m128i twelve_row_first_four = _mm_setr_epi32(75, -18, -89, -50);
+    __m128i twenty_row_first_four = _mm_setr_epi32(50, -89, 18, 75);
+    __m128i twentyeight_row_first_four = _mm_setr_epi32(18, -50, 75, -89);
 
-    Vec4i two_row_first_four(90, 87, 80, 70);
-    Vec4i two_row_second_four(57, 43, 25,  9);
-    Vec4i six_row_first_four(87, 57,  9, -43);
-    Vec4i six_row_second_four(-80, -90, -70, -25);
-    Vec4i ten_row_first_four(80,  9, -70, -87);
-    Vec4i ten_row_second_four(-25, 57, 90, 43);
-    Vec4i fourteen_row_first_four(70, -43, -87,  9);
-    Vec4i fourteen_row_second_four(90, 25, -80, -57);
-    Vec4i eighteen_row_first_four(57, -80, -25, 90);
-    Vec4i eighteen_row_second_four(-9, -87, 43, 70);
-    Vec4i twentytwo_row_first_four(43, -90, 57, 25);
-    Vec4i twentytwo_row_second_four(-87, 70,  9, -80);
-    Vec4i twentysix_row_first_four(25, -70, 90, -80);
-    Vec4i twentysix_row_second_four(43,  9, -57, 87);
-    Vec4i thirty_row_first_four(9, -25, 43, -57);
-    Vec4i thirty_row_second_four(70, -80, 87, -90);
+    __m128i two_row_first_four = _mm_setr_epi32(90, 87, 80, 70);
+    __m128i two_row_second_four = _mm_setr_epi32(57, 43, 25,  9);
+    __m128i six_row_first_four = _mm_setr_epi32(87, 57,  9, -43);
+    __m128i six_row_second_four = _mm_setr_epi32(-80, -90, -70, -25);
+    __m128i ten_row_first_four = _mm_setr_epi32(80,  9, -70, -87);
+    __m128i ten_row_second_four = _mm_setr_epi32(-25, 57, 90, 43);
+    __m128i fourteen_row_first_four = _mm_setr_epi32(70, -43, -87,  9);
+    __m128i fourteen_row_second_four = _mm_setr_epi32(90, 25, -80, -57);
+    __m128i eighteen_row_first_four = _mm_setr_epi32(57, -80, -25, 90);
+    __m128i eighteen_row_second_four = _mm_setr_epi32(-9, -87, 43, 70);
+    __m128i twentytwo_row_first_four = _mm_setr_epi32(43, -90, 57, 25);
+    __m128i twentytwo_row_second_four = _mm_setr_epi32(-87, 70,  9, -80);
+    __m128i twentysix_row_first_four = _mm_setr_epi32(25, -70, 90, -80);
+    __m128i twentysix_row_second_four = _mm_setr_epi32(43,  9, -57, 87);
+    __m128i thirty_row_first_four = _mm_setr_epi32(9, -25, 43, -57);
+    __m128i thirty_row_second_four = _mm_setr_epi32(70, -80, 87, -90);
 
-    Vec4i one_row_first_four(90, 90, 88, 85);
-    Vec4i one_row_second_four(82, 78, 73, 67);
-    Vec4i one_row_third_four(61, 54, 46, 38);
-    Vec4i one_row_fourth_four(31, 22, 13,  4);
+    __m128i one_row_first_four = _mm_setr_epi32(90, 90, 88, 85);
+    __m128i one_row_second_four = _mm_setr_epi32(82, 78, 73, 67);
+    __m128i one_row_third_four = _mm_setr_epi32(61, 54, 46, 38);
+    __m128i one_row_fourth_four = _mm_setr_epi32(31, 22, 13,  4);
 
-    Vec4i three_row_first_four(90, 82, 67, 46);
-    Vec4i three_row_second_four(22, -4, -31, -54);
-    Vec4i three_row_third_four(-73, -85, -90, -88);
-    Vec4i three_row_fourth_four(-78, -61, -38, -13);
+    __m128i three_row_first_four = _mm_setr_epi32(90, 82, 67, 46);
+    __m128i three_row_second_four = _mm_setr_epi32(22, -4, -31, -54);
+    __m128i three_row_third_four = _mm_setr_epi32(-73, -85, -90, -88);
+    __m128i three_row_fourth_four = _mm_setr_epi32(-78, -61, -38, -13);
 
-    Vec4i five_row_first_four(88, 67, 31, -13);
-    Vec4i five_row_second_four(-54, -82, -90, -78);
-    Vec4i five_row_third_four(-46, -4, 38, 73);
-    Vec4i five_row_fourth_four(90, 85, 61, 22);
+    __m128i five_row_first_four = _mm_setr_epi32(88, 67, 31, -13);
+    __m128i five_row_second_four = _mm_setr_epi32(-54, -82, -90, -78);
+    __m128i five_row_third_four = _mm_setr_epi32(-46, -4, 38, 73);
+    __m128i five_row_fourth_four = _mm_setr_epi32(90, 85, 61, 22);
 
-    Vec4i seven_row_first_four(85, 46, -13, -67);
-    Vec4i seven_row_second_four(-90, -73, -22, 38);
-    Vec4i seven_row_third_four(82, 88, 54, -4);
-    Vec4i seven_row_fourth_four(-61, -90, -78, -31);
+    __m128i seven_row_first_four = _mm_setr_epi32(85, 46, -13, -67);
+    __m128i seven_row_second_four = _mm_setr_epi32(-90, -73, -22, 38);
+    __m128i seven_row_third_four = _mm_setr_epi32(82, 88, 54, -4);
+    __m128i seven_row_fourth_four = _mm_setr_epi32(-61, -90, -78, -31);
 
-    Vec4i nine_row_first_four(82, 22, -54, -90);
-    Vec4i nine_row_second_four(-61, 13, 78, 85);
-    Vec4i nine_row_third_four(31, -46, -90, -67);
-    Vec4i nine_row_fourth_four(4, 73, 88, 38);
+    __m128i nine_row_first_four = _mm_setr_epi32(82, 22, -54, -90);
+    __m128i nine_row_second_four = _mm_setr_epi32(-61, 13, 78, 85);
+    __m128i nine_row_third_four = _mm_setr_epi32(31, -46, -90, -67);
+    __m128i nine_row_fourth_four = _mm_setr_epi32(4, 73, 88, 38);
 
-    Vec4i eleven_row_first_four(78, -4, -82, -73);
-    Vec4i eleven_row_second_four(13, 85, 67, -22);
-    Vec4i eleven_row_third_four(-88, -61, 31, 90);
-    Vec4i eleven_row_fourth_four(54, -38, -90, -46);
+    __m128i eleven_row_first_four = _mm_setr_epi32(78, -4, -82, -73);
+    __m128i eleven_row_second_four = _mm_setr_epi32(13, 85, 67, -22);
+    __m128i eleven_row_third_four = _mm_setr_epi32(-88, -61, 31, 90);
+    __m128i eleven_row_fourth_four = _mm_setr_epi32(54, -38, -90, -46);
 
-    Vec4i thirteen_row_first_four(73, -31, -90, -22);
-    Vec4i thirteen_row_second_four(78, 67, -38, -90);
-    Vec4i thirteen_row_third_four(-13, 82, 61, -46);
-    Vec4i thirteen_row_fourth_four(-88, -4, 85, 54);
+    __m128i thirteen_row_first_four = _mm_setr_epi32(73, -31, -90, -22);
+    __m128i thirteen_row_second_four = _mm_setr_epi32(78, 67, -38, -90);
+    __m128i thirteen_row_third_four = _mm_setr_epi32(-13, 82, 61, -46);
+    __m128i thirteen_row_fourth_four = _mm_setr_epi32(-88, -4, 85, 54);
 
-    Vec4i fifteen_row_first_four(67, -54, -78, 38);
-    Vec4i fifteen_row_second_four(85, -22, -90,  4);
-    Vec4i fifteen_row_third_four(90, 13, -88, -31);
-    Vec4i fifteen_row_fourth_four(82, 46, -73, -61);
+    __m128i fifteen_row_first_four = _mm_setr_epi32(67, -54, -78, 38);
+    __m128i fifteen_row_second_four = _mm_setr_epi32(85, -22, -90,  4);
+    __m128i fifteen_row_third_four = _mm_setr_epi32(90, 13, -88, -31);
+    __m128i fifteen_row_fourth_four = _mm_setr_epi32(82, 46, -73, -61);
 
-    Vec4i seventeen_row_first_four(61, -73, -46, 82);
-    Vec4i seventeen_row_second_four(31, -88, -13, 90);
-    Vec4i seventeen_row_third_four(-4, -90, 22, 85);
-    Vec4i seventeen_row_fourth_four(-38, -78, 54, 67);
+    __m128i seventeen_row_first_four = _mm_setr_epi32(61, -73, -46, 82);
+    __m128i seventeen_row_second_four = _mm_setr_epi32(31, -88, -13, 90);
+    __m128i seventeen_row_third_four = _mm_setr_epi32(-4, -90, 22, 85);
+    __m128i seventeen_row_fourth_four = _mm_setr_epi32(-38, -78, 54, 67);
 
-    Vec4i nineteen_row_first_four(54, -85, -4, 88);
-    Vec4i nineteen_row_second_four(-46, -61, 82, 13);
-    Vec4i nineteen_row_third_four(-90, 38, 67, -78);
-    Vec4i nineteen_row_fourth_four(-22, 90, -31, -73);
+    __m128i nineteen_row_first_four = _mm_setr_epi32(54, -85, -4, 88);
+    __m128i nineteen_row_second_four = _mm_setr_epi32(-46, -61, 82, 13);
+    __m128i nineteen_row_third_four = _mm_setr_epi32(-90, 38, 67, -78);
+    __m128i nineteen_row_fourth_four = _mm_setr_epi32(-22, 90, -31, -73);
 
-    Vec4i twentyone_row_first_four(46, -90, 38, 54);
-    Vec4i twentyone_row_second_four(-90, 31, 61, -88);
-    Vec4i twentyone_row_third_four(22, 67, -85, 13);
-    Vec4i twentyone_row_fourth_four(73, -82,  4, 78);
+    __m128i twentyone_row_first_four = _mm_setr_epi32(46, -90, 38, 54);
+    __m128i twentyone_row_second_four = _mm_setr_epi32(-90, 31, 61, -88);
+    __m128i twentyone_row_third_four = _mm_setr_epi32(22, 67, -85, 13);
+    __m128i twentyone_row_fourth_four = _mm_setr_epi32(73, -82,  4, 78);
 
-    Vec4i twentythree_row_first_four(38, -88, 73, -4);
-    Vec4i twentythree_row_second_four(-67, 90, -46, -31);
-    Vec4i twentythree_row_third_four(85, -78, 13, 61);
-    Vec4i twentythree_row_fourth_four(-90, 54, 22, -82);
+    __m128i twentythree_row_first_four = _mm_setr_epi32(38, -88, 73, -4);
+    __m128i twentythree_row_second_four = _mm_setr_epi32(-67, 90, -46, -31);
+    __m128i twentythree_row_third_four = _mm_setr_epi32(85, -78, 13, 61);
+    __m128i twentythree_row_fourth_four = _mm_setr_epi32(-90, 54, 22, -82);
 
-    Vec4i twentyfive_row_first_four(31, -78, 90, -61);
-    Vec4i twentyfive_row_second_four(4, 54, -88, 82);
-    Vec4i twentyfive_row_third_four(-38, -22, 73, -90);
-    Vec4i twentyfive_row_fourth_four(67, -13, -46, 85);
+    __m128i twentyfive_row_first_four = _mm_setr_epi32(31, -78, 90, -61);
+    __m128i twentyfive_row_second_four = _mm_setr_epi32(4, 54, -88, 82);
+    __m128i twentyfive_row_third_four = _mm_setr_epi32(-38, -22, 73, -90);
+    __m128i twentyfive_row_fourth_four = _mm_setr_epi32(67, -13, -46, 85);
 
-    Vec4i twentyseven_row_first_four(22, -61, 85, -90);
-    Vec4i twentyseven_row_second_four(73, -38, -4, 46);
-    Vec4i twentyseven_row_third_four(-78, 90, -82, 54);
-    Vec4i twentyseven_row_fourth_four(-13, -31, 67, -88);
+    __m128i twentyseven_row_first_four = _mm_setr_epi32(22, -61, 85, -90);
+    __m128i twentyseven_row_second_four = _mm_setr_epi32(73, -38, -4, 46);
+    __m128i twentyseven_row_third_four = _mm_setr_epi32(-78, 90, -82, 54);
+    __m128i twentyseven_row_fourth_four = _mm_setr_epi32(-13, -31, 67, -88);
 
-    Vec4i twentynine_row_first_four(13, -38, 61, -78);
-    Vec4i twentynine_row_second_four(88, -90, 85, -73);
-    Vec4i twentynine_row_third_four(54, -31,  4, 22);
-    Vec4i twentynine_row_fourth_four(-46, 67, -82, 90);
+    __m128i twentynine_row_first_four = _mm_setr_epi32(13, -38, 61, -78);
+    __m128i twentynine_row_second_four = _mm_setr_epi32(88, -90, 85, -73);
+    __m128i twentynine_row_third_four = _mm_setr_epi32(54, -31,  4, 22);
+    __m128i twentynine_row_fourth_four = _mm_setr_epi32(-46, 67, -82, 90);
 
-    Vec4i thirtyone_row_first_four(4, -13, 22, -31);
-    Vec4i thirtyone_row_second_four(38, -46, 54, -61);
-    Vec4i thirtyone_row_third_four(67, -73, 78, -82);
-    Vec4i thirtyone_row_fourth_four(85, -88, 90, -90);
+    __m128i thirtyone_row_first_four = _mm_setr_epi32(4, -13, 22, -31);
+    __m128i thirtyone_row_second_four = _mm_setr_epi32(38, -46, 54, -61);
+    __m128i thirtyone_row_third_four = _mm_setr_epi32(67, -73, 78, -82);
+    __m128i thirtyone_row_fourth_four = _mm_setr_epi32(85, -88, 90, -90);
 
     for (j = 0; j < line; j++)
     {
-        Vec8s tmp1, tmp2, tmp3, tmp4;
+        __m128i tmp1, tmp2, tmp3, tmp4;
 
-        tmp1.load(src);
-        Vec4i tmp1_first_half = extend_low(tmp1);
-        Vec4i tmp1_second_half = extend_high(tmp1);
+        tmp1 = _mm_loadu_si128((__m128i*)(src));
 
-        tmp2.load(src + 8);
-        Vec4i tmp2_first_half = extend_low(tmp2);
-        Vec4i tmp2_second_half = extend_high(tmp2);
+        __m128i sign = _mm_srai_epi16(tmp1, 15);
+        __m128i tmp1_first_half = _mm_unpacklo_epi16(tmp1, sign);
+        __m128i tmp1_second_half = _mm_unpackhi_epi16(tmp1, sign);
 
-        tmp3.load(src + 16);
-        Vec4i tmp3_first_half_tmp = extend_low(tmp3);
-        Vec4i tmp3_second_half_tmp = extend_high(tmp3);
-        Vec4i tmp3_first_half = permute4i<3, 2, 1, 0>(tmp3_first_half_tmp);
-        Vec4i tmp3_second_half = permute4i<3, 2, 1, 0>(tmp3_second_half_tmp);
+        tmp2 = _mm_loadu_si128((__m128i*)(src + 8));
+        sign = _mm_srai_epi16(tmp2, 15);
+        __m128i tmp2_first_half = _mm_unpacklo_epi16(tmp2, sign);
+        __m128i tmp2_second_half = _mm_unpackhi_epi16(tmp2, sign);
 
-        tmp4.load(src + 24);
-        Vec4i tmp4_first_half_tmp = extend_low(tmp4);
-        Vec4i tmp4_second_half_tmp = extend_high(tmp4);
-        Vec4i tmp4_first_half = permute4i<3, 2, 1, 0>(tmp4_first_half_tmp);
-        Vec4i tmp4_second_half = permute4i<3, 2, 1, 0>(tmp4_second_half_tmp);
+        tmp3 = _mm_loadu_si128((__m128i*)(src + 16));
+        sign = _mm_srai_epi16(tmp3, 15);
+        __m128i tmp3_first_half_tmp = _mm_unpacklo_epi16(tmp3, sign);
+        __m128i tmp3_second_half_tmp = _mm_unpackhi_epi16(tmp3, sign);
+        __m128i tmp3_first_half = _mm_shuffle_epi32(tmp3_first_half_tmp, 27);
+        __m128i tmp3_second_half = _mm_shuffle_epi32(tmp3_second_half_tmp, 27);
 
-        Vec4i E_first_four =  tmp1_first_half + tmp4_second_half;
-        Vec4i E_second_four = tmp1_second_half + tmp4_first_half;
-        Vec4i E_third_four = tmp2_first_half + tmp3_second_half;
-        Vec4i E_last_four = tmp2_second_half + tmp3_first_half;
+        tmp4 = _mm_loadu_si128((__m128i*)(src + 24));
+        sign = _mm_srai_epi16(tmp4, 15);
+        __m128i tmp4_first_half_tmp = _mm_unpacklo_epi16(tmp4, sign);
+        __m128i tmp4_second_half_tmp = _mm_unpackhi_epi16(tmp4, sign);
+        __m128i tmp4_first_half = _mm_shuffle_epi32(tmp4_first_half_tmp, 27);
+        __m128i tmp4_second_half = _mm_shuffle_epi32(tmp4_second_half_tmp, 27);
 
-        Vec4i O_first_four =  tmp1_first_half - tmp4_second_half;
-        Vec4i O_second_four = tmp1_second_half - tmp4_first_half;
-        Vec4i O_third_four = tmp2_first_half - tmp3_second_half;
-        Vec4i O_last_four = tmp2_second_half - tmp3_first_half;
+        __m128i E_first_four = _mm_add_epi32(tmp1_first_half, tmp4_second_half);
+        __m128i E_second_four = _mm_add_epi32(tmp1_second_half, tmp4_first_half);
+        __m128i E_third_four = _mm_add_epi32(tmp2_first_half, tmp3_second_half);
+        __m128i E_last_four = _mm_add_epi32(tmp2_second_half, tmp3_first_half);
 
-        Vec4i E_last_four_rev = permute4i<3, 2, 1, 0>(E_last_four);
-        Vec4i E_third_four_rev = permute4i<3, 2, 1, 0>(E_third_four);
+        __m128i O_first_four = _mm_sub_epi32(tmp1_first_half, tmp4_second_half);
+        __m128i O_second_four = _mm_sub_epi32(tmp1_second_half, tmp4_first_half);
+        __m128i O_third_four = _mm_sub_epi32(tmp2_first_half, tmp3_second_half);
+        __m128i O_last_four = _mm_sub_epi32(tmp2_second_half, tmp3_first_half);
 
-        Vec4i EE_first_four = E_first_four + E_last_four_rev;
-        Vec4i EE_last_four = E_second_four + E_third_four_rev;
-        Vec4i EO_first_four = E_first_four - E_last_four_rev;
-        Vec4i EO_last_four = E_second_four - E_third_four_rev;
+        __m128i E_last_four_rev = _mm_shuffle_epi32(E_last_four, 27);
+        __m128i E_third_four_rev = _mm_shuffle_epi32(E_third_four, 27);
 
-        Vec4i EE_last_four_rev = permute4i<3, 2, 1, 0>(EE_last_four);
+        __m128i EE_first_four = _mm_add_epi32(E_first_four, E_last_four_rev);
+        __m128i EE_last_four = _mm_add_epi32(E_second_four, E_third_four_rev);
+        __m128i EO_first_four = _mm_sub_epi32(E_first_four, E_last_four_rev);
+        __m128i EO_last_four = _mm_sub_epi32(E_second_four, E_third_four_rev);
 
-        Vec4i EEE = EE_first_four + EE_last_four_rev;
-        Vec4i EEO = EE_first_four - EE_last_four_rev;
+        __m128i EE_last_four_rev = _mm_shuffle_epi32(EE_last_four, 27);
 
-        Vec4i EEEE_first_half = permute4i<0, 1, -1, -1>(EEE);
-        Vec4i EEEE_second_half = permute4i<3, 2, -1, -1>(EEE);
-        Vec4i EEEE = EEEE_first_half + EEEE_second_half;
-        Vec4i EEEO = EEEE_first_half - EEEE_second_half;
+        __m128i EEE = _mm_add_epi32(EE_first_four, EE_last_four_rev);
+        __m128i EEO = _mm_sub_epi32(EE_first_four, EE_last_four_rev);
 
-        int dst0_hresult = (horizontal_add(zero_row_first_two * EEEE) + add) >> shift;
-        int dst8_hresult = (horizontal_add(eight_row_first_two * EEEO) + add) >> shift;
-        int dst16_hresult = (horizontal_add(sixten_row_first_two * EEEE) + add) >> shift;
-        int dst24_hresult = (horizontal_add(twentyfour_row_first_two * EEEO) + add) >> shift;
+        __m128i EEEE_first_half = _mm_shuffle_epi32(EEE, 4);
+        __m128i EEEE_second_half = _mm_shuffle_epi32(EEE, 11);
+        __m128i EEEE = _mm_add_epi32(EEEE_first_half, EEEE_second_half);
+        __m128i EEEO = _mm_sub_epi32(EEEE_first_half, EEEE_second_half);
+
+        int dst0_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(_mm_mullo_epi32(zero_row_first_two, EEEE), _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        int dst8_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(_mm_mullo_epi32(eight_row_first_two, EEEO), _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        int dst16_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(_mm_mullo_epi32(sixten_row_first_two, EEEE), _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        int dst24_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(_mm_mullo_epi32(twentyfour_row_first_two, EEEO), _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
 
         dst[0] = dst0_hresult;
         dst[8 * line] = dst8_hresult;
         dst[16 * line] = dst16_hresult;
         dst[24 * line] = dst24_hresult;
 
-        int dst4_hresult = (horizontal_add(four_row_first_four * EEO) + add) >> shift;
-        int dst12_hresult = (horizontal_add(twelve_row_first_four * EEO) + add) >> shift;
-        int dst20_hresult = (horizontal_add(twenty_row_first_four * EEO) + add) >> shift;
-        int dst28_hresult = (horizontal_add(twentyeight_row_first_four * EEO) + add) >> shift;
+        int dst4_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(_mm_mullo_epi32(four_row_first_four, EEO), _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        int dst12_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(_mm_mullo_epi32(twelve_row_first_four, EEO), _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        int dst20_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(_mm_mullo_epi32(twenty_row_first_four, EEO), _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        int dst28_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(_mm_mullo_epi32(twentyeight_row_first_four, EEO), _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
 
         dst[4 * line] = dst4_hresult;
         dst[12 * line] = dst12_hresult;
         dst[20 * line] = dst20_hresult;
         dst[28 * line] = dst28_hresult;
+        __m128i tmp = _mm_add_epi32(_mm_mullo_epi32(two_row_first_four, EO_first_four), _mm_mullo_epi32(two_row_second_four, EO_last_four));
+        int dst2_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(tmp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
 
-        int dst2_hresult =
-            (horizontal_add((two_row_first_four *
-                             EO_first_four) + (two_row_second_four * EO_last_four)) + add) >> shift;
-        int dst6_hresult =
-            (horizontal_add((six_row_first_four *
-                             EO_first_four) + (six_row_second_four * EO_last_four)) + add) >> shift;
-        int dst10_hresult =
-            (horizontal_add((ten_row_first_four *
-                             EO_first_four) + (ten_row_second_four * EO_last_four)) + add) >> shift;
-        int dst14_hresult =
-            (horizontal_add((fourteen_row_first_four *
-                             EO_first_four) + (fourteen_row_second_four * EO_last_four)) + add) >> shift;
-        int dst18_hresult =
-            (horizontal_add((eighteen_row_first_four *
-                             EO_first_four) + (eighteen_row_second_four * EO_last_four)) + add) >> shift;
-        int dst22_hresult =
-            (horizontal_add((twentytwo_row_first_four *
-                             EO_first_four) + (twentytwo_row_second_four * EO_last_four)) + add) >> shift;
-        int dst26_hresult =
-            (horizontal_add((twentysix_row_first_four *
-                             EO_first_four) + (twentysix_row_second_four * EO_last_four)) + add) >> shift;
-        int dst30_hresult =
-            (horizontal_add((thirty_row_first_four *
-                             EO_first_four) + (thirty_row_second_four * EO_last_four)) + add) >> shift;
+        tmp = _mm_add_epi32(_mm_mullo_epi32(six_row_first_four, EO_first_four), _mm_mullo_epi32(six_row_second_four, EO_last_four));
+        int dst6_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(tmp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+
+        tmp = _mm_add_epi32(_mm_mullo_epi32(ten_row_first_four, EO_first_four), _mm_mullo_epi32(ten_row_second_four, EO_last_four));
+        int dst10_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(tmp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+
+        tmp = _mm_add_epi32(_mm_mullo_epi32(fourteen_row_first_four, EO_first_four), _mm_mullo_epi32(fourteen_row_second_four, EO_last_four));
+        int dst14_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(tmp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+
+        tmp = _mm_add_epi32(_mm_mullo_epi32(eighteen_row_first_four, EO_first_four), _mm_mullo_epi32(eighteen_row_second_four, EO_last_four));
+        int dst18_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(tmp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+
+        tmp = _mm_add_epi32(_mm_mullo_epi32(twentytwo_row_first_four, EO_first_four), _mm_mullo_epi32(twentytwo_row_second_four, EO_last_four));
+        int dst22_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(tmp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+
+        tmp = _mm_add_epi32(_mm_mullo_epi32(twentysix_row_first_four, EO_first_four), _mm_mullo_epi32(twentysix_row_second_four, EO_last_four));
+        int dst26_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(tmp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+
+        tmp = _mm_add_epi32(_mm_mullo_epi32(thirty_row_first_four, EO_first_four), _mm_mullo_epi32(thirty_row_second_four, EO_last_four));
+        int dst30_hresult = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(tmp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
 
         dst[2 * line] = dst2_hresult;
         dst[6 * line] = dst6_hresult;
@@ -2127,59 +2130,86 @@
         dst[26 * line] = dst26_hresult;
         dst[30 * line] = dst30_hresult;
 
-        Vec4i dst1_temp = (one_row_first_four * O_first_four) + (one_row_second_four * O_second_four) +
-            (one_row_third_four * O_third_four) + (one_row_fourth_four * O_last_four);
-        Vec4i dst3_temp = (three_row_first_four * O_first_four) + (three_row_second_four * O_second_four) +
-            (three_row_third_four * O_third_four) + (three_row_fourth_four * O_last_four);
-        Vec4i dst5_temp = (five_row_first_four * O_first_four) + (five_row_second_four * O_second_four) +
-            (five_row_third_four * O_third_four) + (five_row_fourth_four * O_last_four);
-        Vec4i dst7_temp = (seven_row_first_four * O_first_four) + (seven_row_second_four * O_second_four) +
-            (seven_row_third_four * O_third_four) + (seven_row_fourth_four * O_last_four);
-        Vec4i dst9_temp = (nine_row_first_four * O_first_four) + (nine_row_second_four * O_second_four) +
-            (nine_row_third_four * O_third_four) + (nine_row_fourth_four * O_last_four);
-        Vec4i dst11_temp = (eleven_row_first_four * O_first_four) + (eleven_row_second_four * O_second_four) +
-            (eleven_row_third_four * O_third_four) + (eleven_row_fourth_four * O_last_four);
-        Vec4i dst13_temp = (thirteen_row_first_four * O_first_four) + (thirteen_row_second_four * O_second_four) +
-            (thirteen_row_third_four * O_third_four) + (thirteen_row_fourth_four * O_last_four);
-        Vec4i dst15_temp = (fifteen_row_first_four * O_first_four) + (fifteen_row_second_four * O_second_four) +
-            (fifteen_row_third_four * O_third_four) + (fifteen_row_fourth_four * O_last_four);
-        Vec4i dst17_temp = (seventeen_row_first_four * O_first_four) + (seventeen_row_second_four * O_second_four) +
-            (seventeen_row_third_four * O_third_four) + (seventeen_row_fourth_four * O_last_four);
-        Vec4i dst19_temp = (nineteen_row_first_four * O_first_four) + (nineteen_row_second_four * O_second_four) +
-            (nineteen_row_third_four * O_third_four) + (nineteen_row_fourth_four * O_last_four);
-        Vec4i dst21_temp = (twentyone_row_first_four * O_first_four) + (twentyone_row_second_four * O_second_four) +
-            (twentyone_row_third_four * O_third_four) + (twentyone_row_fourth_four * O_last_four);
-        Vec4i dst23_temp =
-            (twentythree_row_first_four * O_first_four) + (twentythree_row_second_four * O_second_four) +
-            (twentythree_row_third_four * O_third_four) + (twentythree_row_fourth_four * O_last_four);
-        Vec4i dst25_temp =
-            (twentyfive_row_first_four * O_first_four) + (twentyfive_row_second_four * O_second_four) +
-            (twentyfive_row_third_four * O_third_four) + (twentyfive_row_fourth_four * O_last_four);
-        Vec4i dst27_temp =
-            (twentyseven_row_first_four * O_first_four) + (twentyseven_row_second_four * O_second_four) +
-            (twentyseven_row_third_four * O_third_four) + (twentyseven_row_fourth_four * O_last_four);
-        Vec4i dst29_temp =
-            (twentynine_row_first_four * O_first_four) + (twentynine_row_second_four * O_second_four) +
-            (twentynine_row_third_four * O_third_four) + (twentynine_row_fourth_four * O_last_four);
-        Vec4i dst31_temp = (thirtyone_row_first_four * O_first_four) + (thirtyone_row_second_four * O_second_four) +
-            (thirtyone_row_third_four * O_third_four) + (thirtyone_row_fourth_four * O_last_four);
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(one_row_first_four, O_first_four), _mm_mullo_epi32(one_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(one_row_third_four, O_third_four), _mm_mullo_epi32(one_row_fourth_four, O_last_four));
+        __m128i dst1_temp = _mm_add_epi32(tmp1, tmp2);
 
-        dst[1 * line] = (horizontal_add(dst1_temp) + add) >> shift;
-        dst[3 * line] = (horizontal_add(dst3_temp) + add) >> shift;
-        dst[5 * line] = (horizontal_add(dst5_temp) + add) >> shift;
-        dst[7 * line] = (horizontal_add(dst7_temp) + add) >> shift;
-        dst[9 * line] = (horizontal_add(dst9_temp) + add) >> shift;
-        dst[11 * line] = (horizontal_add(dst11_temp) + add) >> shift;
-        dst[13 * line] = (horizontal_add(dst13_temp) + add) >> shift;
-        dst[15 * line] = (horizontal_add(dst15_temp) + add) >> shift;
-        dst[17 * line] = (horizontal_add(dst17_temp) + add) >> shift;
-        dst[19 * line] = (horizontal_add(dst19_temp) + add) >> shift;
-        dst[21 * line] = (horizontal_add(dst21_temp) + add) >> shift;
-        dst[23 * line] = (horizontal_add(dst23_temp) + add) >> shift;
-        dst[25 * line] = (horizontal_add(dst25_temp) + add) >> shift;
-        dst[27 * line] = (horizontal_add(dst27_temp) + add) >> shift;
-        dst[29 * line] = (horizontal_add(dst29_temp) + add) >> shift;
-        dst[31 * line] = (horizontal_add(dst31_temp) + add) >> shift;
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(three_row_first_four, O_first_four), _mm_mullo_epi32(three_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(three_row_third_four, O_third_four), _mm_mullo_epi32(three_row_fourth_four, O_last_four));
+        __m128i dst3_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(five_row_first_four, O_first_four), _mm_mullo_epi32(five_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(five_row_third_four, O_third_four), _mm_mullo_epi32(five_row_fourth_four, O_last_four));
+        __m128i dst5_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(seven_row_first_four, O_first_four), _mm_mullo_epi32(seven_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(seven_row_third_four, O_third_four), _mm_mullo_epi32(seven_row_fourth_four, O_last_four));
+        __m128i dst7_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(nine_row_first_four, O_first_four), _mm_mullo_epi32(nine_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(nine_row_third_four, O_third_four), _mm_mullo_epi32(nine_row_fourth_four, O_last_four));
+        __m128i dst9_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(eleven_row_first_four, O_first_four), _mm_mullo_epi32(eleven_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(eleven_row_third_four, O_third_four), _mm_mullo_epi32(eleven_row_fourth_four, O_last_four));
+        __m128i dst11_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(thirteen_row_first_four, O_first_four), _mm_mullo_epi32(thirteen_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(thirteen_row_third_four, O_third_four), _mm_mullo_epi32(thirteen_row_fourth_four, O_last_four));
+        __m128i dst13_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(fifteen_row_first_four, O_first_four), _mm_mullo_epi32(fifteen_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(fifteen_row_third_four, O_third_four), _mm_mullo_epi32(fifteen_row_fourth_four, O_last_four));
+        __m128i dst15_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(seventeen_row_first_four, O_first_four), _mm_mullo_epi32(seventeen_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(seventeen_row_third_four, O_third_four), _mm_mullo_epi32(seventeen_row_fourth_four, O_last_four));
+        __m128i dst17_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(nineteen_row_first_four, O_first_four), _mm_mullo_epi32(nineteen_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(nineteen_row_third_four, O_third_four), _mm_mullo_epi32(nineteen_row_fourth_four, O_last_four));
+        __m128i dst19_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(twentyone_row_first_four, O_first_four), _mm_mullo_epi32(twentyone_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(twentyone_row_third_four, O_third_four), _mm_mullo_epi32(twentyone_row_fourth_four, O_last_four));
+        __m128i dst21_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(twentythree_row_first_four, O_first_four), _mm_mullo_epi32(twentythree_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(twentythree_row_third_four, O_third_four), _mm_mullo_epi32(twentythree_row_fourth_four, O_last_four));
+        __m128i dst23_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(twentyfive_row_first_four, O_first_four), _mm_mullo_epi32(twentyfive_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(twentyfive_row_third_four, O_third_four), _mm_mullo_epi32(twentyfive_row_fourth_four, O_last_four));
+        __m128i dst25_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(twentyseven_row_first_four, O_first_four), _mm_mullo_epi32(twentyseven_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(twentyseven_row_third_four, O_third_four), _mm_mullo_epi32(twentyseven_row_fourth_four, O_last_four));
+        __m128i dst27_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(twentynine_row_first_four, O_first_four), _mm_mullo_epi32(twentynine_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(twentynine_row_third_four, O_third_four), _mm_mullo_epi32(twentynine_row_fourth_four, O_last_four));
+        __m128i dst29_temp = _mm_add_epi32(tmp1, tmp2);
+
+        tmp1 = _mm_add_epi32(_mm_mullo_epi32(thirtyone_row_first_four, O_first_four), _mm_mullo_epi32(thirtyone_row_second_four, O_second_four));
+        tmp2 = _mm_add_epi32(_mm_mullo_epi32(thirtyone_row_third_four, O_third_four), _mm_mullo_epi32(thirtyone_row_fourth_four, O_last_four));
+        __m128i dst31_temp = _mm_add_epi32(tmp1, tmp2);
+
+        dst[1 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst1_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[3 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst3_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[5 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst5_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[7 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst7_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[9 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst9_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[11 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst11_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[13 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst13_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[15 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst15_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[17 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst17_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[19 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst19_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[21 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst21_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[23 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst23_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[25 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst25_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[27 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst27_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[29 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst29_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
+        dst[31 * line] = (_mm_cvtsi128_si32(_mm_hadd_epi32(_mm_hadd_epi32(dst31_temp, _mm_setzero_si128()), _mm_setzero_si128())) + add) >> shift;
 
         src += 32;
         dst++;
@@ -2202,15 +2232,9 @@
     partialButterfly32(block, coef, shift_1st, 32);
     partialButterfly32(coef, block, shift_2nd, 32);
 
-    /* TODO: inline cvt16to32 once it is intrinsic based */
 #define N (32)
-    for (int i = 0; i < N; i++)
-    {
-        for (int j = 0; j < N; j++)
-        {
-            dst[i * N + j] = block[i * N + j];
-        }
-    }
+
+    convert16to32(block, dst, N*N);
 
 #undef N
 }


More information about the x265-devel mailing list