[x265] [PATCH] asm: avx2 asm code for idct32x32
Steve Borho
steve at borho.org
Thu Sep 25 21:34:55 CEST 2014
On 09/25, murugan at multicorewareinc.com wrote:
> # HG changeset patch
> # User zibiah
Your commit username needs to have a full name and email in it
> # Date 1411652280 -19800
> # Thu Sep 25 19:08:00 2014 +0530
> # Node ID 47cfab9cbd76bc4feb076e514afe4542dcca912e
> # Parent e47e127da779d23314a402fa74723b0e82a2c75d
> asm: avx2 asm code for idct32x32
>
> diff -r e47e127da779 -r 47cfab9cbd76 source/common/x86/asm-primitives.cpp
> --- a/source/common/x86/asm-primitives.cpp Wed Sep 24 21:51:12 2014 -0500
> +++ b/source/common/x86/asm-primitives.cpp Thu Sep 25 19:08:00 2014 +0530
> @@ -1449,6 +1449,7 @@
> p.dct[DCT_16x16] = x265_dct16_avx2;
> p.dct[DCT_32x32] = x265_dct32_avx2;
> p.idct[IDCT_16x16] = x265_idct16_avx2;
> + p.idct[IDCT_32x32] = x265_idct32_avx2;
> #endif
> }
> /* at HIGH_BIT_DEPTH, pixel == short so we can reuse a number of primitives */
> @@ -1784,6 +1785,7 @@
> p.dct[DCT_16x16] = x265_dct16_avx2;
> p.dct[DCT_32x32] = x265_dct32_avx2;
> p.idct[IDCT_16x16] = x265_idct16_avx2;
> + p.idct[IDCT_32x32] = x265_idct32_avx2;
> #endif
> }
> #endif // if HIGH_BIT_DEPTH
> diff -r e47e127da779 -r 47cfab9cbd76 source/common/x86/dct8.asm
> --- a/source/common/x86/dct8.asm Wed Sep 24 21:51:12 2014 -0500
> +++ b/source/common/x86/dct8.asm Thu Sep 25 19:08:00 2014 +0530
> @@ -167,6 +167,60 @@
>
> idct16_shuff1: dd 2, 6, 0, 4, 3, 7, 1, 5
>
> +tab_idct32_1: dw 90 ,90 ,88 ,85, 82, 78, 73, 67, 61, 54, 46, 38, 31, 22, 13, 4
> + dw 90, 82, 67, 46, 22, -4, -31, -54, -73, -85, -90, -88, -78, -61, -38, -13
> + dw 88, 67, 31, -13, -54, -82, -90, -78, -46, -4, 38, 73, 90, 85, 61, 22
> + dw 85, 46, -13, -67, -90, -73, -22, 38, 82, 88, 54, -4, -61, -90, -78, -31
> + dw 82, 22, -54, -90, -61, 13, 78, 85, 31, -46, -90, -67, 4, 73, 88, 38
> + dw 78, -4, -82, -73, 13, 85, 67, -22, -88, -61, 31, 90, 54, -38, -90, -46
> + dw 73, -31, -90, -22, 78, 67, -38, -90, -13, 82, 61, -46, -88, -4, 85, 54
> + dw 67, -54, -78, 38, 85, -22, -90, 4, 90, 13, -88, -31, 82, 46, -73, -61
> + dw 61, -73, -46, 82, 31, -88, -13, 90, -4, -90, 22, 85, -38, -78, 54, 67
> + dw 54, -85, -4, 88, -46, -61, 82, 13, -90, 38, 67, -78, -22, 90, -31, -73
> + dw 46, -90, 38, 54, -90, 31, 61, -88, 22, 67, -85, 13, 73, -82, 4, 78
> + dw 38, -88, 73, -4, -67, 90, -46, -31, 85, -78, 13, 61, -90, 54, 22, -82
> + dw 31, -78, 90, -61, 4, 54, -88, 82, -38, -22, 73, -90, 67, -13, -46, 85
> + dw 22, -61, 85, -90, 73, -38, -4, 46, -78, 90, -82, 54, -13, -31, 67, -88
> + dw 13, -38, 61, -78, 88, -90, 85, -73, 54, -31, 4, 22, -46, 67, -82, 90
> + dw 4, -13, 22, -31, 38, -46, 54, -61, 67, -73, 78, -82, 85, -88, 90, -90
> +
> +
> +tab_idct32_2: dw 64, 89, 83, 75, 64, 50, 36, 18
> + dw 64, 75, 36, -18, -64, -89, -83, -50
> + dw 64, 50, -36, -89, -64, 18, 83, 75
> + dw 64, 18, -83, -50, 64, 75, -36, -89
> + dw 64, -18, -83, 50, 64, -75, -36, 89
> + dw 64, -50, -36, 89, -64, -18, 83, -75
> + dw 64, -75, 36, 18, -64, 89, -83, 50
> + dw 64, -89, 83, -75, 64, -50, 36, -18
> +
> +
> +tab_idct32_3: dw 90, 87, 80, 70, 57, 43, 25, 9
> + dw 87, 57, 9, -43, -80, -90, -70, -25
> + dw 80, 9, -70, -87, -25, 57, 90, 43
> + dw 70, -43, -87, 9, 90, 25, -80, -57
> + dw 57, -80, -25, 90, -9, -87, 43, 70
> + dw 43, -90, 57, 25, -87, 70, 9, -80
> + dw 25, -70, 90, -80, 43, 9, -57, 87
> + dw 9, -25, 43, -57, 70, -80, 87, -90
> +
> +tab_idct32_4: dw 64, 90, 89, 87, 83, 80, 75, 70, 64, 57, 50, 43, 36, 25, 18, 9
> + dw 64, 87, 75, 57, 36, 9, -18, -43, -64, -80, -89, -90, -83, -70, -50, -25
> + dw 64, 80, 50, 9, -36, -70, -89, -87, -64, -25, 18, 57, 83, 90, 75, 43
> + dw 64, 70, 18, -43, -83, -87, -50, 9, 64, 90, 75, 25, -36, -80, -89, -57
> + dw 64, 57, -18, -80, -83, -25, 50, 90, 64, -9, -75, -87, -36, 43, 89, 70
> + dw 64, 43, -50, -90, -36, 57, 89, 25, -64, -87, -18, 70, 83, 9, -75, -80
> + dw 64, 25, -75, -70, 36, 90, 18, -80, -64, 43, 89, 9, -83, -57, 50, 87
> + dw 64, 9, -89, -25, 83, 43, -75, -57, 64, 70, -50, -80, 36, 87, -18, -90
> + dw 64, -9, -89, 25, 83, -43, -75, 57, 64, -70, -50, 80, 36, -87, -18, 90
> + dw 64, -25, -75, 70, 36, -90, 18, 80, -64, -43, 89, -9, -83, 57, 50, -87
> + dw 64, -43, -50, 90, -36, -57, 89, -25, -64, 87, -18, -70, 83, -9, -75, 80
> + dw 64, -57, -18, 80, -83, 25, 50, -90, 64, 9, -75, 87, -36, -43, 89, -70
> + dw 64, -70, 18, 43, -83, 87, -50, -9, 64, -90, 75, -25, -36, 80, -89, 57
> + dw 64, -80, 50, -9, -36, 70, -89, 87, -64, 25, 18, -57, 83, -90, 75, -43
> + dw 64, -87, 75, -57, 36, -9, -18, 43, -64, 80, -89, 90, -83, 70, -50, 25
> + dw 64, -90, 89, -87, 83, -80, 75, -70, 64, -57, 50, -43, 36, -25, 18, -9
> +
> avx2_dct4: dw 64, 64, 64, 64, 64, 64, 64, 64, 64, -64, 64, -64, 64, -64, 64, -64
> dw 83, 36, 83, 36, 83, 36, 83, 36, 36, -83, 36, -83, 36, -83, 36, -83
>
> @@ -2007,4 +2061,350 @@
> dec r4d
> jnz .pass2
> RET
> +
> +%macro IDCT32_PASS1 1
> + vbroadcasti128 m3, [tab_idct32_1 + %1 * 32]
> + vbroadcasti128 m13, [tab_idct32_1 + %1 * 32 + 16]
> + pmaddwd m9, m4, m3
> + pmaddwd m10, m8, m13
> + phaddd m9, m10
> +
> + pmaddwd m10, m2, m3
> + pmaddwd m11, m1, m13
> + phaddd m10, m11
> +
> + phaddd m9, m10
> +
> + vbroadcasti128 m3, [tab_idct32_1 + (15 - %1) * 32]
> + vbroadcasti128 m13, [tab_idct32_1 + (15- %1) * 32 + 16]
> + pmaddwd m10, m4, m3
> + pmaddwd m11, m8, m13
> + phaddd m10, m11
> +
> + pmaddwd m11, m2, m3
> + pmaddwd m12, m1, m13
> + phaddd m11, m12
> +
> + phaddd m10, m11
> + phaddd m9, m10 ;[row0s0 row2s0 row0s15 row2s15 row1s0 row3s0 row1s15 row3s15]
> +
> + vbroadcasti128 m3, [tab_idct32_2 + %1 * 16]
> + pmaddwd m10, m0, m3
> + pmaddwd m11, m7, m3
> + phaddd m10, m11
> + phaddd m10, m10
> +
> + vbroadcasti128 m3, [tab_idct32_3 + %1 * 16]
> + pmaddwd m11, m5, m3
> + pmaddwd m12, m6, m3
> + phaddd m11, m12
> + phaddd m11, m11
> +
> + paddd m12, m10, m11 ;[row0a0 row2a0 NIL NIL row1sa0 row3a0 NIL NIL]
> + psubd m10, m11 ;[row0a15 row2a15 NIL NIL row1a15 row3a15 NIL NIL]
> +
> + punpcklqdq m12, m10 ;[row0a0 row2a0 row0a15 row2a15 row1a0 row3a0 row1a15 row3a15]
> + paddd m10, m9, m12
> + paddd m10, m15
> + psrad m10, IDCT_SHIFT1
> +
> + psubd m12, m9
> + paddd m12, m15
> + psrad m12, IDCT_SHIFT1
> +
> + packssdw m10, m12
> + vextracti128 xm12, m10, 1
> + movd [r3 + %1 * 64], xm10
> + movd [r3 + 32 + %1 * 64], xm12
> + pextrd [r4 - %1 * 64], xm10, 1
> + pextrd [r4+ 32 - %1 * 64], xm12, 1
> + pextrd [r3 + 16 * 64 + %1 *64], xm10, 3
> + pextrd [r3 + 16 * 64 + 32 + %1 * 64], xm12, 3
> + pextrd [r4 + 16 * 64 - %1 * 64], xm10, 2
> + pextrd [r4 + 16 * 64 + 32 - %1 * 64], xm12, 2
> +%endmacro
> +
> +;-------------------------------------------------------
> +; void idct32(int32_t *src, int16_t *dst, intptr_t stride)
> +;-------------------------------------------------------
> +
> +; TODO: Reduce PHADDD instruction by PADDD
> +
> +INIT_YMM avx2
> +cglobal idct32, 3, 6, 16, 0-32*64
> +
> +%define IDCT_SHIFT1 7
> +
> + vbroadcasti128 m15, [pd_64]
> +
> + mov r3, rsp
> + lea r4, [r3 + 15 * 64]
> + mov r5d, 8
> +
> +.pass1:
> + movu xm0, [r0 + 2 * 128]
> + movu xm1, [r0 + 18 * 128]
> + vinserti128 m0, m0, [r0 + 0 * 128], 1
> + vinserti128 m1, m1, [r0 + 16 * 128], 1
> +
> + packssdw m0, m1 ;[2 18 0 16]
> +
> + movu xm1, [r0 + 1 * 128]
> + movu xm2, [r0 + 9 * 128]
> + vinserti128 m1, m1, [r0 + 17 * 128], 1
> + vinserti128 m2, m2, [r0 + 25 * 128], 1
> + packssdw m1, m2 ;[1 9 17 25]
> +
> + movu xm2, [r0 + 6 * 128]
> + movu xm3, [r0 + 22 * 128]
> + vinserti128 m2, m2, [r0 + 4 * 128], 1
> + vinserti128 m3, m3, [r0 + 20 * 128], 1
> + packssdw m2, m3 ;[6 22 4 20]
> +
> + movu xm3, [r0 + 3 * 128]
> + movu xm4, [r0 + 11 * 128]
> + vinserti128 m3, m3, [r0 + 19 * 128], 1
> + vinserti128 m4, m4, [r0 + 27 * 128], 1
> + packssdw m3, m4 ;[3 11 19 27]
> +
> + movu xm4, [r0 + 10 * 128]
> + movu xm5, [r0 + 26 * 128]
> + vinserti128 m4, m4, [r0 + 8 * 128], 1
> + vinserti128 m5, m5, [r0 + 24 * 128], 1
> + packssdw m4, m5 ;[10 26 8 24]
> +
> + movu xm5, [r0 + 5 * 128]
> + movu xm6, [r0 + 13 * 128]
> + vinserti128 m5, m5, [r0 + 21 * 128], 1
> + vinserti128 m6, m6, [r0 + 29 * 128], 1
> + packssdw m5, m6 ;[5 13 21 29]
> +
> + movu xm6, [r0 + 14 * 128]
> + movu xm7, [r0 + 30 * 128]
> + vinserti128 m6, m6, [r0 + 12 * 128], 1
> + vinserti128 m7, m7, [r0 + 28 * 128], 1
> + packssdw m6, m7 ;[14 30 12 28]
> +
> + movu xm7, [r0 + 7 * 128]
> + movu xm8, [r0 + 15 * 128]
> + vinserti128 m7, m7, [r0 + 23 * 128], 1
> + vinserti128 m8, m8, [r0 + 31 * 128], 1
> + packssdw m7, m8 ;[7 15 23 31]
> +
> + punpckhwd m8, m0, m2 ;[18 22 16 20]
> + punpcklwd m0, m2 ;[2 6 0 4]
> +
> + punpckhwd m2, m1, m3 ;[9 11 25 27]
> + punpcklwd m1, m3 ;[1 3 17 19]
> +
> + punpckhwd m3, m4, m6 ;[26 30 24 28]
> + punpcklwd m4, m6 ;[10 14 8 12]
> +
> + punpckhwd m6, m5, m7 ;[13 15 29 31]
> + punpcklwd m5, m7 ;[5 7 21 23]
> +
> + punpckhdq m7, m0, m4 ;[22 62 102 142 23 63 103 143 02 42 82 122 03 43 83 123]
> + punpckldq m0, m4 ;[20 60 100 140 21 61 101 141 00 40 80 120 01 41 81 121]
> +
> + punpckhdq m4, m8, m3 ;[182 222 262 302 183 223 263 303 162 202 242 282 163 203 243 283]
> + punpckldq m8, m3 ;[180 220 260 300 181 221 261 301 160 200 240 280 161 201 241 281]
> +
> + punpckhdq m3, m1, m5 ;[12 32 52 72 13 33 53 73 172 192 212 232 173 193 213 233]
> + punpckldq m1, m5 ;[10 30 50 70 11 31 51 71 170 190 210 230 171 191 211 231]
> +
> + punpckhdq m5, m2, m6 ;[92 112 132 152 93 113 133 153 252 272 292 312 253 273 293 313]
> + punpckldq m2, m6 ;[90 110 130 150 91 111 131 151 250 270 290 310 251 271 291 311]
> +
> + punpckhqdq m6, m0, m8 ;[21 61 101 141 181 221 261 301 01 41 81 121 161 201 241 281]
> + punpcklqdq m0, m8 ;[20 60 100 140 180 220 260 300 00 40 80 120 160 200 240 280]
> +
> + punpckhqdq m8, m7, m4 ;[23 63 103 143 183 223 263 303 03 43 83 123 163 203 243 283]
> + punpcklqdq m7, m4 ;[22 62 102 142 182 222 262 302 02 42 82 122 162 202 242 282]
> +
> + punpckhqdq m4, m1, m2 ;[11 31 51 71 91 111 131 151 171 191 211 231 251 271 291 311]
> + punpcklqdq m1, m2 ;[10 30 50 70 90 110 130 150 170 190 210 230 250 270 290 310]
> +
> + punpckhqdq m2, m3, m5 ;[13 33 53 73 93 113 133 153 173 193 213 233 253 273 293 313]
> + punpcklqdq m3, m5 ;[12 32 52 72 92 112 132 152 172 192 212 232 252 272 292 312]
> +
> + vperm2i128 m5, m0, m6, 0x20 ;[20 60 100 140 180 220 260 300 21 61 101 141 181 221 261 301]
> + vperm2i128 m0, m0, m6, 0x31 ;[00 40 80 120 160 200 240 280 01 41 81 121 161 201 241 281]
> +
> + vperm2i128 m6, m7, m8, 0x20 ;[22 62 102 142 182 222 262 302 23 63 103 143 183 223 263 303]
> + vperm2i128 m7, m7, m8, 0x31 ;[02 42 82 122 162 202 242 282 03 43 83 123 163 203 243 283]
> +
> + vperm2i128 m8, m1, m4, 0x31 ;[170 190 210 230 250 270 290 310 171 191 211 231 251 271 291 311]
> + vperm2i128 m4, m1, m4, 0x20 ;[10 30 50 70 90 110 130 150 11 31 51 71 91 111 131 151]
> +
> + vperm2i128 m1, m3, m2, 0x31 ;[172 192 212 232 252 272 292 312 173 193 213 233 253 273 293 313]
> + vperm2i128 m2, m3, m2, 0x20 ;[12 32 52 72 92 112 132 152 13 33 53 73 93 113 133 153]
> +
> + IDCT32_PASS1 0
> + IDCT32_PASS1 1
> + IDCT32_PASS1 2
> + IDCT32_PASS1 3
> + IDCT32_PASS1 4
> + IDCT32_PASS1 5
> + IDCT32_PASS1 6
> + IDCT32_PASS1 7
> +
> + add r0, 16
> + add r3, 4
> + add r4, 4
> + dec r5d
> + jnz .pass1
> +
> +%if BIT_DEPTH == 10
> + %define IDCT_SHIFT2 10
> + vpbroadcastd m15, [pd_512]
> +%elif BIT_DEPTH == 8
> + %define IDCT_SHIFT2 12
> + vpbroadcastd m15, [pd_2048]
> +%else
> + %error Unsupported BIT_DEPTH!
> %endif
> +
> + mov r3, rsp
> + add r2d, r2d
> + mov r4d, 32
> +
> + mova m7, [tab_idct32_4]
> + mova m8, [tab_idct32_4 + 32]
> + mova m9, [tab_idct32_4 + 64]
> + mova m10, [tab_idct32_4 + 96]
> + mova m11, [tab_idct32_4 + 128]
> + mova m12, [tab_idct32_4 + 160]
> + mova m13, [tab_idct32_4 + 192]
> + mova m14, [tab_idct32_4 + 224]
> +.pass2:
> + movu m0, [r3]
> + movu m1, [r3 + 32]
> +
> + pmaddwd m2, m0, m7
> + pmaddwd m3, m0, m8
> + phaddd m2, m3
> +
> + pmaddwd m3, m0, m9
> + pmaddwd m4, m0, m10
> + phaddd m3, m4
> +
> + phaddd m2, m3
> +
> + pmaddwd m3, m0, m11
> + pmaddwd m4, m0, m12
> + phaddd m3, m4
> +
> + pmaddwd m4, m0, m13
> + pmaddwd m5, m0, m14
> + phaddd m4, m5
> +
> + phaddd m3, m4
> +
> + vperm2i128 m4, m2, m3, 0x31
> + vperm2i128 m2, m2, m3, 0x20
> + paddd m2, m4
> +
> + pmaddwd m3, m0, [tab_idct32_4 + 256]
> + pmaddwd m4, m0, [tab_idct32_4 + 288]
> + phaddd m3, m4
> +
> + pmaddwd m4, m0, [tab_idct32_4 + 320]
> + pmaddwd m5, m0, [tab_idct32_4 + 352]
> + phaddd m4, m5
> +
> + phaddd m3, m4
> +
> + pmaddwd m4, m0, [tab_idct32_4 + 384]
> + pmaddwd m5, m0, [tab_idct32_4 + 416]
> + phaddd m4, m5
> +
> + pmaddwd m5, m0, [tab_idct32_4 + 448]
> + pmaddwd m0, [tab_idct32_4 + 480]
> + phaddd m5, m0
> +
> + phaddd m4, m5
> +
> + vperm2i128 m0, m3, m4, 0x31
> + vperm2i128 m3, m3, m4, 0x20
> + paddd m3, m0
> +
> + pmaddwd m4, m1, [tab_idct32_1]
> + pmaddwd m0, m1, [tab_idct32_1 + 32]
> + phaddd m4, m0
> +
> + pmaddwd m5, m1, [tab_idct32_1 + 64]
> + pmaddwd m0, m1, [tab_idct32_1 + 96]
> + phaddd m5, m0
> +
> + phaddd m4, m5
> +
> + pmaddwd m5, m1, [tab_idct32_1 + 128]
> + pmaddwd m0, m1, [tab_idct32_1 + 160]
> + phaddd m5, m0
> +
> + pmaddwd m6, m1, [tab_idct32_1 + 192]
> + pmaddwd m0, m1, [tab_idct32_1 + 224]
> + phaddd m6, m0
> +
> + phaddd m5, m6
> +
> + vperm2i128 m0, m4, m5, 0x31
> + vperm2i128 m4, m4, m5, 0x20
> + paddd m4, m0
> +
> + pmaddwd m5, m1, [tab_idct32_1 + 256]
> + pmaddwd m0, m1, [tab_idct32_1 + 288]
> + phaddd m5, m0
> +
> + pmaddwd m6, m1, [tab_idct32_1 + 320]
> + pmaddwd m0, m1, [tab_idct32_1 + 352]
> + phaddd m6, m0
> +
> + phaddd m5, m6
> +
> + pmaddwd m6, m1, [tab_idct32_1 + 384]
> + pmaddwd m0, m1, [tab_idct32_1 + 416]
> + phaddd m6, m0
> +
> + pmaddwd m0, m1, [tab_idct32_1 + 448]
> + pmaddwd m1, [tab_idct32_1 + 480]
> + phaddd m0, m1
> +
> + phaddd m6, m0
> +
> + vperm2i128 m0, m5, m6, 0x31
> + vperm2i128 m5, m5, m6, 0x20
> + paddd m5, m0
> +
> + paddd m6, m2, m4
> + paddd m6, m15
> + psrad m6, IDCT_SHIFT2
> +
> + psubd m2, m4
> + paddd m2, m15
> + psrad m2, IDCT_SHIFT2
> +
> + paddd m4, m3, m5
> + paddd m4, m15
> + psrad m4, IDCT_SHIFT2
> +
> + psubd m3, m5
> + paddd m3, m15
> + psrad m3, IDCT_SHIFT2
> +
> + packssdw m6, m4
> + packssdw m2, m3
> +
> + vpermq m6, m6, 0xD8
> + vpermq m2, m2, 0x8D
> + pshufb m2, [dct16_shuf1]
> +
> + mova [r1], m6
> + mova [r1 + 32], m2
> +
> + add r1, r2
> + add r3, 64
> + dec r4d
> + jnz .pass2
> + RET
> +%endif
> diff -r e47e127da779 -r 47cfab9cbd76 source/common/x86/dct8.h
> --- a/source/common/x86/dct8.h Wed Sep 24 21:51:12 2014 -0500
> +++ b/source/common/x86/dct8.h Thu Sep 25 19:08:00 2014 +0530
> @@ -34,6 +34,7 @@
> void x265_dct8_avx2(int16_t *src, int32_t *dst, intptr_t stride);
> void x265_dct16_avx2(int16_t *src, int32_t *dst, intptr_t stride);
> void x265_dct32_avx2(int16_t *src, int32_t *dst, intptr_t stride);
> +void x265_idct32_avx2(int32_t *src, int16_t *dst, intptr_t stride);
>
> void x265_denoise_dct_sse4(int32_t *dct, uint32_t *sum, uint16_t *offset, int size);
> void x265_denoise_dct_avx2(int32_t *dct, uint32_t *sum, uint16_t *offset, int size);
> _______________________________________________
> x265-devel mailing list
> x265-devel at videolan.org
> https://mailman.videolan.org/listinfo/x265-devel
--
Steve Borho
More information about the x265-devel
mailing list