[x265] [PATCH] asm: avx2 asm code for idct32x32

murugan at multicorewareinc.com murugan at multicorewareinc.com
Thu Sep 25 15:38:57 CEST 2014


# HG changeset patch
# User zibiah
# Date 1411652280 -19800
#      Thu Sep 25 19:08:00 2014 +0530
# Node ID 47cfab9cbd76bc4feb076e514afe4542dcca912e
# Parent  e47e127da779d23314a402fa74723b0e82a2c75d
asm: avx2 asm code for idct32x32

diff -r e47e127da779 -r 47cfab9cbd76 source/common/x86/asm-primitives.cpp
--- a/source/common/x86/asm-primitives.cpp	Wed Sep 24 21:51:12 2014 -0500
+++ b/source/common/x86/asm-primitives.cpp	Thu Sep 25 19:08:00 2014 +0530
@@ -1449,6 +1449,7 @@
         p.dct[DCT_16x16] = x265_dct16_avx2;
         p.dct[DCT_32x32] = x265_dct32_avx2;
         p.idct[IDCT_16x16] = x265_idct16_avx2;
+        p.idct[IDCT_32x32] = x265_idct32_avx2;
 #endif
     }
     /* at HIGH_BIT_DEPTH, pixel == short so we can reuse a number of primitives */
@@ -1784,6 +1785,7 @@
         p.dct[DCT_16x16] = x265_dct16_avx2;
         p.dct[DCT_32x32] = x265_dct32_avx2;
         p.idct[IDCT_16x16] = x265_idct16_avx2;
+        p.idct[IDCT_32x32] = x265_idct32_avx2;
 #endif
     }
 #endif // if HIGH_BIT_DEPTH
diff -r e47e127da779 -r 47cfab9cbd76 source/common/x86/dct8.asm
--- a/source/common/x86/dct8.asm	Wed Sep 24 21:51:12 2014 -0500
+++ b/source/common/x86/dct8.asm	Thu Sep 25 19:08:00 2014 +0530
@@ -167,6 +167,60 @@
 
 idct16_shuff1:  dd 2, 6, 0, 4, 3, 7, 1, 5
 
+tab_idct32_1:   dw 90 ,90 ,88 ,85, 82, 78, 73, 67, 61, 54, 46, 38, 31, 22, 13, 4
+                dw 90, 82, 67, 46, 22, -4, -31, -54, -73, -85, -90, -88, -78, -61, -38, -13
+                dw 88, 67, 31, -13, -54, -82, -90, -78, -46, -4, 38, 73, 90, 85, 61, 22
+                dw 85, 46, -13, -67, -90, -73, -22, 38, 82, 88, 54, -4, -61, -90, -78, -31
+                dw 82, 22, -54, -90, -61, 13, 78, 85, 31, -46, -90, -67, 4, 73, 88, 38
+                dw 78, -4, -82, -73, 13, 85, 67, -22, -88, -61, 31, 90, 54, -38, -90, -46
+                dw 73, -31, -90, -22, 78, 67, -38, -90, -13, 82, 61, -46, -88, -4, 85, 54
+                dw 67, -54, -78, 38, 85, -22, -90, 4, 90, 13, -88, -31, 82, 46, -73, -61
+                dw 61, -73, -46, 82, 31, -88, -13, 90, -4, -90, 22, 85, -38, -78, 54, 67
+                dw 54, -85, -4, 88, -46, -61, 82, 13, -90, 38, 67, -78, -22, 90, -31, -73
+                dw 46, -90, 38, 54, -90, 31, 61, -88, 22, 67, -85, 13, 73, -82, 4, 78
+                dw 38, -88, 73, -4, -67, 90, -46, -31, 85, -78, 13, 61, -90, 54, 22, -82
+                dw 31, -78, 90, -61, 4, 54, -88, 82, -38, -22, 73, -90, 67, -13, -46, 85
+                dw 22, -61, 85, -90, 73, -38, -4, 46, -78, 90, -82, 54, -13, -31, 67, -88
+                dw 13, -38, 61, -78, 88, -90, 85, -73, 54, -31, 4, 22, -46, 67, -82, 90
+                dw 4, -13, 22, -31, 38, -46, 54, -61, 67, -73, 78, -82, 85, -88, 90, -90
+
+
+tab_idct32_2:   dw 64, 89, 83, 75, 64, 50, 36, 18
+                dw 64, 75, 36, -18, -64, -89, -83, -50
+                dw 64, 50, -36, -89, -64, 18, 83, 75
+                dw 64, 18, -83, -50, 64, 75, -36, -89
+                dw 64, -18, -83, 50, 64, -75, -36, 89
+                dw 64, -50, -36, 89, -64, -18, 83, -75
+                dw 64, -75, 36, 18, -64, 89, -83, 50
+                dw 64, -89, 83, -75, 64, -50, 36, -18
+
+
+tab_idct32_3:   dw 90, 87, 80, 70, 57, 43, 25, 9
+                dw 87, 57, 9, -43, -80, -90, -70, -25
+                dw 80, 9, -70, -87, -25, 57, 90, 43
+                dw 70, -43, -87, 9, 90, 25, -80, -57
+                dw 57, -80, -25, 90, -9, -87, 43, 70
+                dw 43, -90, 57, 25, -87, 70, 9, -80
+                dw 25, -70, 90, -80, 43, 9, -57, 87
+                dw 9, -25, 43, -57, 70, -80, 87, -90
+
+tab_idct32_4:   dw 64, 90, 89, 87, 83, 80, 75, 70, 64, 57, 50, 43, 36, 25, 18, 9
+                dw 64, 87, 75, 57, 36, 9, -18, -43, -64, -80, -89, -90, -83, -70, -50, -25
+                dw 64, 80, 50, 9, -36, -70, -89, -87, -64, -25, 18, 57, 83, 90, 75, 43
+                dw 64, 70, 18, -43, -83, -87, -50, 9, 64, 90, 75, 25, -36, -80, -89, -57
+                dw 64, 57, -18, -80, -83, -25, 50, 90, 64, -9, -75, -87, -36, 43, 89, 70
+                dw 64, 43, -50, -90, -36, 57, 89, 25, -64, -87, -18, 70, 83, 9, -75, -80
+                dw 64, 25, -75, -70, 36, 90, 18, -80, -64, 43, 89, 9, -83, -57, 50, 87
+                dw 64, 9, -89, -25, 83, 43, -75, -57, 64, 70, -50, -80, 36, 87, -18, -90
+                dw 64, -9, -89, 25, 83, -43, -75, 57, 64, -70, -50, 80, 36, -87, -18, 90
+                dw 64, -25, -75, 70, 36, -90, 18, 80, -64, -43, 89, -9, -83, 57, 50, -87
+                dw 64, -43, -50, 90, -36, -57, 89, -25, -64, 87, -18, -70, 83, -9, -75, 80
+                dw 64, -57, -18, 80, -83, 25, 50, -90, 64, 9, -75, 87, -36, -43, 89, -70
+                dw 64, -70, 18, 43, -83, 87, -50, -9, 64, -90, 75, -25, -36, 80, -89, 57
+                dw 64, -80, 50, -9, -36, 70, -89, 87, -64, 25, 18, -57, 83, -90, 75, -43
+                dw 64, -87, 75, -57, 36, -9, -18, 43, -64, 80, -89, 90, -83, 70, -50, 25
+                dw 64, -90, 89, -87, 83, -80, 75, -70, 64, -57, 50, -43, 36, -25, 18, -9
+
 avx2_dct4:      dw 64, 64, 64, 64, 64, 64, 64, 64, 64, -64, 64, -64, 64, -64, 64, -64
                 dw 83, 36, 83, 36, 83, 36, 83, 36, 36, -83, 36, -83, 36, -83, 36, -83
 
@@ -2007,4 +2061,350 @@
     dec             r4d
     jnz             .pass2
     RET
+
+%macro IDCT32_PASS1 1
+    vbroadcasti128  m3, [tab_idct32_1 + %1 * 32]
+    vbroadcasti128  m13, [tab_idct32_1 + %1 * 32 + 16]
+    pmaddwd         m9, m4, m3
+    pmaddwd         m10, m8, m13
+    phaddd          m9, m10
+
+    pmaddwd         m10, m2, m3
+    pmaddwd         m11, m1, m13
+    phaddd          m10, m11
+
+    phaddd          m9, m10
+
+    vbroadcasti128  m3, [tab_idct32_1 + (15 - %1) * 32]
+    vbroadcasti128  m13, [tab_idct32_1 + (15- %1) * 32 + 16]
+    pmaddwd         m10, m4, m3
+    pmaddwd         m11, m8, m13
+    phaddd          m10, m11
+
+    pmaddwd         m11, m2, m3
+    pmaddwd         m12, m1, m13
+    phaddd          m11, m12
+
+    phaddd          m10, m11
+    phaddd          m9, m10                       ;[row0s0 row2s0 row0s15 row2s15 row1s0 row3s0 row1s15 row3s15]
+
+    vbroadcasti128  m3, [tab_idct32_2 + %1 * 16]
+    pmaddwd         m10, m0, m3
+    pmaddwd         m11, m7, m3
+    phaddd          m10, m11
+    phaddd          m10, m10
+
+    vbroadcasti128  m3, [tab_idct32_3 + %1 * 16]
+    pmaddwd         m11, m5, m3
+    pmaddwd         m12, m6, m3
+    phaddd          m11, m12
+    phaddd          m11, m11
+
+    paddd           m12, m10, m11                 ;[row0a0 row2a0 NIL NIL row1sa0 row3a0 NIL NIL]
+    psubd           m10, m11                      ;[row0a15 row2a15 NIL NIL row1a15 row3a15 NIL NIL]
+
+    punpcklqdq      m12, m10                      ;[row0a0 row2a0 row0a15 row2a15 row1a0 row3a0 row1a15 row3a15]
+    paddd           m10, m9, m12
+    paddd           m10, m15
+    psrad           m10, IDCT_SHIFT1
+
+    psubd           m12, m9
+    paddd           m12, m15
+    psrad           m12, IDCT_SHIFT1
+
+    packssdw        m10, m12
+    vextracti128    xm12, m10, 1
+    movd            [r3 + %1 * 64], xm10
+    movd            [r3 + 32 + %1 * 64], xm12
+    pextrd          [r4 - %1 * 64], xm10, 1
+    pextrd          [r4+ 32 - %1 * 64], xm12, 1
+    pextrd          [r3 + 16 * 64 + %1 *64], xm10, 3
+    pextrd          [r3 + 16 * 64 + 32 + %1 * 64], xm12, 3
+    pextrd          [r4 + 16 * 64 - %1 * 64], xm10, 2
+    pextrd          [r4 + 16 * 64 + 32 - %1 * 64], xm12, 2
+%endmacro
+
+;-------------------------------------------------------
+; void idct32(int32_t *src, int16_t *dst, intptr_t stride)
+;-------------------------------------------------------
+
+; TODO: Reduce PHADDD instruction by PADDD
+
+INIT_YMM avx2
+cglobal idct32, 3, 6, 16, 0-32*64
+
+%define             IDCT_SHIFT1         7
+
+    vbroadcasti128  m15, [pd_64]
+
+    mov             r3, rsp
+    lea             r4, [r3 + 15 * 64]
+    mov             r5d, 8
+
+.pass1:
+    movu            xm0,    [r0 +  2 * 128]
+    movu            xm1,    [r0 + 18 * 128]
+    vinserti128     m0, m0, [r0 +  0 * 128], 1
+    vinserti128     m1, m1, [r0 + 16 * 128], 1
+
+    packssdw        m0, m1                      ;[2 18 0 16]
+
+    movu            xm1,    [r0 +  1 * 128]
+    movu            xm2,    [r0 +  9 * 128]
+    vinserti128     m1, m1, [r0 + 17 * 128], 1
+    vinserti128     m2, m2, [r0 + 25 * 128], 1
+    packssdw        m1, m2                      ;[1 9 17 25]
+
+    movu            xm2,    [r0 +  6 * 128]
+    movu            xm3,    [r0 + 22 * 128]
+    vinserti128     m2, m2, [r0 +  4 * 128], 1
+    vinserti128     m3, m3, [r0 + 20 * 128], 1
+    packssdw        m2, m3                      ;[6 22 4 20]
+
+    movu            xm3,    [r0 +  3 * 128]
+    movu            xm4,    [r0 + 11 * 128]
+    vinserti128     m3, m3, [r0 + 19 * 128], 1
+    vinserti128     m4, m4, [r0 + 27 * 128], 1
+    packssdw        m3, m4                      ;[3 11 19 27]
+
+    movu            xm4,    [r0 + 10 * 128]
+    movu            xm5,    [r0 + 26 * 128]
+    vinserti128     m4, m4, [r0 +  8 * 128], 1
+    vinserti128     m5, m5, [r0 + 24 * 128], 1
+    packssdw        m4, m5                      ;[10 26 8 24]
+
+    movu            xm5,    [r0 +  5 * 128]
+    movu            xm6,    [r0 + 13 * 128]
+    vinserti128     m5, m5, [r0 + 21 * 128], 1
+    vinserti128     m6, m6, [r0 + 29 * 128], 1
+    packssdw        m5, m6                      ;[5 13 21 29]
+
+    movu            xm6,    [r0 + 14 * 128]
+    movu            xm7,    [r0 + 30 * 128]
+    vinserti128     m6, m6, [r0 + 12 * 128], 1
+    vinserti128     m7, m7, [r0 + 28 * 128], 1
+    packssdw        m6, m7                      ;[14 30 12 28]
+
+    movu            xm7,    [r0 +  7 * 128]
+    movu            xm8,    [r0 + 15 * 128]
+    vinserti128     m7, m7, [r0 + 23 * 128], 1
+    vinserti128     m8, m8, [r0 + 31 * 128], 1
+    packssdw        m7, m8                      ;[7 15 23 31]
+
+    punpckhwd       m8, m0, m2                  ;[18 22 16 20]
+    punpcklwd       m0, m2                      ;[2 6 0 4]
+
+    punpckhwd       m2, m1, m3                  ;[9 11 25 27]
+    punpcklwd       m1, m3                      ;[1 3 17 19]
+
+    punpckhwd       m3, m4, m6                  ;[26 30 24 28]
+    punpcklwd       m4, m6                      ;[10 14 8 12]
+
+    punpckhwd       m6, m5, m7                  ;[13 15 29 31]
+    punpcklwd       m5, m7                      ;[5 7 21 23]
+
+    punpckhdq       m7, m0, m4                  ;[22 62 102 142 23 63 103 143 02 42 82 122 03 43 83 123]
+    punpckldq       m0, m4                      ;[20 60 100 140 21 61 101 141 00 40 80 120 01 41 81 121]
+
+    punpckhdq       m4, m8, m3                  ;[182 222 262 302 183 223 263 303 162 202 242 282 163 203 243 283]
+    punpckldq       m8, m3                      ;[180 220 260 300 181 221 261 301 160 200 240 280 161 201 241 281]
+
+    punpckhdq       m3, m1, m5                  ;[12 32 52 72 13 33 53 73 172 192 212 232 173 193 213 233]
+    punpckldq       m1, m5                      ;[10 30 50 70 11 31 51 71 170 190 210 230 171 191 211 231]
+
+    punpckhdq       m5, m2, m6                  ;[92 112 132 152 93 113 133 153 252 272 292 312 253 273 293 313]
+    punpckldq       m2, m6                      ;[90 110 130 150 91 111 131 151 250 270 290 310 251 271 291 311]
+
+    punpckhqdq      m6, m0, m8                  ;[21 61 101 141 181 221 261 301 01 41 81 121 161 201 241 281]
+    punpcklqdq      m0, m8                      ;[20 60 100 140 180 220 260 300 00 40 80 120 160 200 240 280]
+
+    punpckhqdq      m8, m7, m4                  ;[23 63 103 143 183 223 263 303 03 43 83 123 163 203 243 283]
+    punpcklqdq      m7, m4                      ;[22 62 102 142 182 222 262 302 02 42 82 122 162 202 242 282]
+
+    punpckhqdq      m4, m1, m2                  ;[11 31 51 71 91 111 131 151 171 191 211 231 251 271 291 311]
+    punpcklqdq      m1, m2                      ;[10 30 50 70 90 110 130 150 170 190 210 230 250 270 290 310]
+
+    punpckhqdq      m2, m3, m5                  ;[13 33 53 73 93 113 133 153 173 193 213 233 253 273 293 313]
+    punpcklqdq      m3, m5                      ;[12 32 52 72 92 112 132 152 172 192 212 232 252 272 292 312]
+
+    vperm2i128      m5, m0, m6, 0x20            ;[20 60 100 140 180 220 260 300 21 61 101 141 181 221 261 301]
+    vperm2i128      m0, m0, m6, 0x31            ;[00 40 80 120 160 200 240 280 01 41 81 121 161 201 241 281]
+
+    vperm2i128      m6, m7, m8, 0x20            ;[22 62 102 142 182 222 262 302 23 63 103 143 183 223 263 303]
+    vperm2i128      m7, m7, m8, 0x31            ;[02 42 82 122 162 202 242 282 03 43 83 123 163 203 243 283]
+
+    vperm2i128      m8, m1, m4, 0x31            ;[170 190 210 230 250 270 290 310 171 191 211 231 251 271 291 311]
+    vperm2i128      m4, m1, m4, 0x20            ;[10 30 50 70 90 110 130 150 11 31 51 71 91 111 131 151]
+
+    vperm2i128      m1, m3, m2, 0x31            ;[172 192 212 232 252 272 292 312 173 193 213 233 253 273 293 313]
+    vperm2i128      m2, m3, m2, 0x20            ;[12 32 52 72 92 112 132 152 13 33 53 73 93 113 133 153]
+
+    IDCT32_PASS1 0
+    IDCT32_PASS1 1
+    IDCT32_PASS1 2
+    IDCT32_PASS1 3
+    IDCT32_PASS1 4
+    IDCT32_PASS1 5
+    IDCT32_PASS1 6
+    IDCT32_PASS1 7
+
+    add             r0, 16
+    add             r3, 4
+    add             r4, 4
+    dec             r5d
+    jnz             .pass1
+
+%if BIT_DEPTH == 10
+    %define         IDCT_SHIFT2        10
+    vpbroadcastd    m15,                [pd_512]
+%elif BIT_DEPTH == 8
+    %define         IDCT_SHIFT2        12
+    vpbroadcastd    m15,                [pd_2048]
+%else
+    %error Unsupported BIT_DEPTH!
 %endif
+
+    mov             r3, rsp
+    add             r2d, r2d
+    mov             r4d, 32
+
+    mova            m7,  [tab_idct32_4]
+    mova            m8,  [tab_idct32_4 + 32]
+    mova            m9,  [tab_idct32_4 + 64]
+    mova            m10, [tab_idct32_4 + 96]
+    mova            m11, [tab_idct32_4 + 128]
+    mova            m12, [tab_idct32_4 + 160]
+    mova            m13, [tab_idct32_4 + 192]
+    mova            m14, [tab_idct32_4 + 224]
+.pass2:
+    movu            m0, [r3]
+    movu            m1, [r3 + 32]
+
+    pmaddwd         m2, m0, m7
+    pmaddwd         m3, m0, m8
+    phaddd          m2, m3
+
+    pmaddwd         m3, m0, m9
+    pmaddwd         m4, m0, m10
+    phaddd          m3, m4
+
+    phaddd          m2, m3
+
+    pmaddwd         m3, m0, m11
+    pmaddwd         m4, m0, m12
+    phaddd          m3, m4
+
+    pmaddwd         m4, m0, m13
+    pmaddwd         m5, m0, m14
+    phaddd          m4, m5
+
+    phaddd          m3, m4
+
+    vperm2i128      m4, m2, m3, 0x31
+    vperm2i128      m2, m2, m3, 0x20
+    paddd           m2, m4
+
+    pmaddwd         m3, m0, [tab_idct32_4 + 256]
+    pmaddwd         m4, m0, [tab_idct32_4 + 288]
+    phaddd          m3, m4
+
+    pmaddwd         m4, m0, [tab_idct32_4 + 320]
+    pmaddwd         m5, m0, [tab_idct32_4 + 352]
+    phaddd          m4, m5
+
+    phaddd          m3, m4
+
+    pmaddwd         m4, m0, [tab_idct32_4 + 384]
+    pmaddwd         m5, m0, [tab_idct32_4 + 416]
+    phaddd          m4, m5
+
+    pmaddwd         m5, m0, [tab_idct32_4 + 448]
+    pmaddwd         m0,     [tab_idct32_4 + 480]
+    phaddd          m5, m0
+
+    phaddd          m4, m5
+
+    vperm2i128      m0, m3, m4, 0x31
+    vperm2i128      m3, m3, m4, 0x20
+    paddd           m3, m0
+
+    pmaddwd         m4, m1, [tab_idct32_1]
+    pmaddwd         m0, m1, [tab_idct32_1 + 32]
+    phaddd          m4, m0
+
+    pmaddwd         m5, m1, [tab_idct32_1 + 64]
+    pmaddwd         m0, m1, [tab_idct32_1 + 96]
+    phaddd          m5, m0
+
+    phaddd          m4, m5
+
+    pmaddwd         m5, m1, [tab_idct32_1 + 128]
+    pmaddwd         m0, m1, [tab_idct32_1 + 160]
+    phaddd          m5, m0
+
+    pmaddwd         m6, m1, [tab_idct32_1 + 192]
+    pmaddwd         m0, m1, [tab_idct32_1 + 224]
+    phaddd          m6, m0
+
+    phaddd          m5, m6
+
+    vperm2i128      m0, m4, m5, 0x31
+    vperm2i128      m4, m4, m5, 0x20
+    paddd           m4, m0
+
+    pmaddwd         m5, m1, [tab_idct32_1 + 256]
+    pmaddwd         m0, m1, [tab_idct32_1 + 288]
+    phaddd          m5, m0
+
+    pmaddwd         m6, m1, [tab_idct32_1 + 320]
+    pmaddwd         m0, m1, [tab_idct32_1 + 352]
+    phaddd          m6, m0
+
+    phaddd          m5, m6
+
+    pmaddwd         m6, m1, [tab_idct32_1 + 384]
+    pmaddwd         m0, m1, [tab_idct32_1 + 416]
+    phaddd          m6, m0
+
+    pmaddwd         m0, m1, [tab_idct32_1 + 448]
+    pmaddwd         m1,     [tab_idct32_1 + 480]
+    phaddd          m0, m1
+
+    phaddd          m6, m0
+
+    vperm2i128      m0, m5, m6, 0x31
+    vperm2i128      m5, m5, m6, 0x20
+    paddd           m5, m0
+
+    paddd           m6, m2, m4
+    paddd           m6, m15
+    psrad           m6, IDCT_SHIFT2
+
+    psubd           m2, m4
+    paddd           m2, m15
+    psrad           m2, IDCT_SHIFT2
+
+    paddd           m4, m3, m5
+    paddd           m4, m15
+    psrad           m4, IDCT_SHIFT2
+
+    psubd           m3, m5
+    paddd           m3, m15
+    psrad           m3, IDCT_SHIFT2
+
+    packssdw        m6, m4
+    packssdw        m2, m3
+
+    vpermq          m6, m6, 0xD8
+    vpermq          m2, m2, 0x8D
+    pshufb          m2, [dct16_shuf1]
+
+    mova            [r1], m6
+    mova            [r1 + 32], m2
+
+    add             r1, r2
+    add             r3, 64
+    dec             r4d
+    jnz             .pass2
+    RET
+%endif
diff -r e47e127da779 -r 47cfab9cbd76 source/common/x86/dct8.h
--- a/source/common/x86/dct8.h	Wed Sep 24 21:51:12 2014 -0500
+++ b/source/common/x86/dct8.h	Thu Sep 25 19:08:00 2014 +0530
@@ -34,6 +34,7 @@
 void x265_dct8_avx2(int16_t *src, int32_t *dst, intptr_t stride);
 void x265_dct16_avx2(int16_t *src, int32_t *dst, intptr_t stride);
 void x265_dct32_avx2(int16_t *src, int32_t *dst, intptr_t stride);
+void x265_idct32_avx2(int32_t *src, int16_t *dst, intptr_t stride);
 
 void x265_denoise_dct_sse4(int32_t *dct, uint32_t *sum, uint16_t *offset, int size);
 void x265_denoise_dct_avx2(int32_t *dct, uint32_t *sum, uint16_t *offset, int size);


More information about the x265-devel mailing list