[x265] [PATCH] asm: intra_pred_ang16 code for all remaing modes
murugan at multicorewareinc.com
murugan at multicorewareinc.com
Fri Feb 7 11:32:24 CET 2014
# HG changeset patch
# User Murugan Vairavel <murugan at multicorewareinc.com>
# Date 1391769131 -19800
# Fri Feb 07 16:02:11 2014 +0530
# Node ID 3981a03c969f475538616fdf826864324cae5f6a
# Parent c1cea0534e6b6ca359a7fc2a665c3ef6909dd041
asm: intra_pred_ang16 code for all remaing modes
diff -r c1cea0534e6b -r 3981a03c969f source/common/x86/asm-primitives.cpp
--- a/source/common/x86/asm-primitives.cpp Fri Feb 07 13:14:45 2014 +0530
+++ b/source/common/x86/asm-primitives.cpp Fri Feb 07 16:02:11 2014 +0530
@@ -1041,8 +1041,35 @@
SETUP_INTRA_ANG16(3, 3, sse4);
SETUP_INTRA_ANG16(4, 4, sse4);
+ SETUP_INTRA_ANG16(5, 5, sse4);
+ SETUP_INTRA_ANG16(6, 6, sse4);
+ SETUP_INTRA_ANG16(7, 7, sse4);
+ SETUP_INTRA_ANG16(8, 8, sse4);
+ SETUP_INTRA_ANG16(9, 9, sse4);
+ SETUP_INTRA_ANG16(10, 10, sse4);
+ SETUP_INTRA_ANG16(11, 11, sse4);
+ SETUP_INTRA_ANG16(12, 12, sse4);
+ SETUP_INTRA_ANG16(13, 13, sse4);
+ SETUP_INTRA_ANG16(14, 14, sse4);
+ SETUP_INTRA_ANG16(15, 15, sse4);
+ SETUP_INTRA_ANG16(16, 16, sse4);
+ SETUP_INTRA_ANG16(17, 17, sse4);
+ SETUP_INTRA_ANG16(18, 18, sse4);
+ SETUP_INTRA_ANG16(19, 19, sse4);
+ SETUP_INTRA_ANG16(20, 20, sse4);
+ SETUP_INTRA_ANG16(21, 21, sse4);
+ SETUP_INTRA_ANG16(22, 22, sse4);
+ SETUP_INTRA_ANG16(23, 23, sse4);
+ SETUP_INTRA_ANG16(24, 24, sse4);
+ SETUP_INTRA_ANG16(25, 25, sse4);
+ SETUP_INTRA_ANG16(26, 26, sse4);
+ SETUP_INTRA_ANG16(27, 27, sse4);
+ SETUP_INTRA_ANG16(28, 28, sse4);
+ SETUP_INTRA_ANG16(29, 29, sse4);
+ SETUP_INTRA_ANG16(30, 30, sse4);
+ SETUP_INTRA_ANG16(31, 31, sse4);
+ SETUP_INTRA_ANG16(32, 32, sse4);
SETUP_INTRA_ANG16(33, 33, sse4);
- SETUP_INTRA_ANG16(32, 32, sse4);
SETUP_INTRA_ANG8(3, 3, sse4);
SETUP_INTRA_ANG8(4, 4, sse4);
diff -r c1cea0534e6b -r 3981a03c969f source/common/x86/intrapred.h
--- a/source/common/x86/intrapred.h Fri Feb 07 13:14:45 2014 +0530
+++ b/source/common/x86/intrapred.h Fri Feb 07 16:02:11 2014 +0530
@@ -93,9 +93,21 @@
DECL_ANG(16, 16, sse4);
DECL_ANG(16, 17, sse4);
DECL_ANG(16, 18, sse4);
+DECL_ANG(16, 19, sse4);
+DECL_ANG(16, 20, sse4);
+DECL_ANG(16, 21, sse4);
+DECL_ANG(16, 22, sse4);
+DECL_ANG(16, 23, sse4);
+DECL_ANG(16, 24, sse4);
+DECL_ANG(16, 25, sse4);
DECL_ANG(16, 26, sse4);
+DECL_ANG(16, 27, sse4);
+DECL_ANG(16, 28, sse4);
+DECL_ANG(16, 29, sse4);
+DECL_ANG(16, 30, sse4);
+DECL_ANG(16, 31, sse4);
+DECL_ANG(16, 32, sse4);
DECL_ANG(16, 33, sse4);
-DECL_ANG(16, 32, sse4);
DECL_ANG(32, 2, ssse3);
DECL_ANG(32, 3, sse4);
diff -r c1cea0534e6b -r 3981a03c969f source/common/x86/intrapred8.asm
--- a/source/common/x86/intrapred8.asm Fri Feb 07 13:14:45 2014 +0530
+++ b/source/common/x86/intrapred8.asm Fri Feb 07 16:02:11 2014 +0530
@@ -45,9 +45,13 @@
c_deinterval8: db 0, 8, 1, 9, 2, 10, 3, 11, 4, 12, 5, 13, 6, 14, 7, 15
tab_S1: db 15, 14, 12, 11, 10, 9, 7, 6, 5, 4, 2, 1, 0, 0, 0, 0
pb_unpackbq: db 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1
+c_mode16_12: db 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 6
+c_mode16_13: db 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 11, 7, 4
+c_mode16_14: db 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 12, 10, 7, 5, 2
c_mode16_15: db 0, 0, 0, 0, 0, 0, 0, 0, 15, 13, 11, 9, 8, 6, 4, 2
c_mode16_16: db 8, 6, 5, 3, 2, 0, 15, 14, 12, 11, 9, 8, 6, 5, 3, 2
c_mode16_17: db 4, 2, 1, 0, 15, 14, 12, 11, 10, 9, 7, 6, 5, 4, 2, 1
+c_mode16_18: db 0, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1
const ang_table
%assign x 0
@@ -2478,6 +2482,3318 @@
RET
+INIT_XMM sse4
+cglobal intra_pred_ang16_5, 3,7,8
+
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ movu m1, [r2 + 2] ;[17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpckhbw m2, m3, m1 ;[17 16 16 15 15 14 14 13 13 12 12 11 11 10 10 9]
+ punpcklbw m3, m1 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ palignr m5, m2, m3, 2
+
+ pmaddubsw m4, m3, [r3 + 16] ; [17]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m5, [r3 - 14 * 16] ; [2]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ palignr m6, m2, m3, 4
+
+ pmaddubsw m5, [r3 + 3 * 16] ; [19]
+ pmulhrsw m5, m7
+ pmaddubsw m1, m6, [r3 - 12 * 16] ; [4]
+ pmulhrsw m1, m7
+ packuswb m5, m1
+
+ palignr m1, m2, m3, 6
+
+ pmaddubsw m6, [r3 + 5 * 16] ; [21]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m1, [r3 - 10 * 16] ; [6]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m0, m2, m3, 8
+
+ pmaddubsw m1, [r3 + 7 * 16] ; [23]
+ pmulhrsw m1, m7
+ pmaddubsw m0, [r3 - 8 * 16] ; [8]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ palignr m4, m2, m3, 8
+ palignr m5, m2, m3, 10
+
+ pmaddubsw m4, [r3 + 9 * 16] ; [25]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m5, [r3 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ palignr m6, m2, m3, 12
+
+ pmaddubsw m5, [r3 + 11 * 16] ; [27]
+ pmulhrsw m5, m7
+ pmaddubsw m1, m6, [r3 - 4 * 16] ; [12]
+ pmulhrsw m1, m7
+ packuswb m5, m1
+
+ palignr m1, m2, m3, 14
+
+ pmaddubsw m6, [r3 + 13 * 16] ; [29]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m1, [r3 - 2 * 16] ; [14]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, [r3 + 15 * 16] ; [31]
+ pmulhrsw m1, m7
+ pmaddubsw m2, [r3] ; [16]
+ pmulhrsw m2, m7
+ packuswb m1, m2
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_31, 3,7,8
+ mov r2, r3mp
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ movu m1, [r2 + 2] ;[17 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpckhbw m2, m3, m1 ;[17 16 16 15 15 14 14 13 13 12 12 11 11 10 10 9]
+ punpcklbw m3, m1 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ palignr m5, m2, m3, 2
+
+ pmaddubsw m4, m3, [r3 + 16] ; [17]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m5, [r3 - 14 * 16] ; [2]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ palignr m6, m2, m3, 4
+
+ pmaddubsw m5, [r3 + 3 * 16] ; [19]
+ pmulhrsw m5, m7
+ pmaddubsw m1, m6, [r3 - 12 * 16] ; [4]
+ pmulhrsw m1, m7
+ packuswb m5, m1
+
+ palignr m1, m2, m3, 6
+
+ pmaddubsw m6, [r3 + 5 * 16] ; [21]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m1, [r3 - 10 * 16] ; [6]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m0, m2, m3, 8
+
+ pmaddubsw m1, [r3 + 7 * 16] ; [23]
+ pmulhrsw m1, m7
+ pmaddubsw m0, [r3 - 8 * 16] ; [8]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ palignr m4, m2, m3, 8
+ palignr m5, m2, m3, 10
+
+ pmaddubsw m4, [r3 + 9 * 16] ; [25]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m5, [r3 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ palignr m6, m2, m3, 12
+
+ pmaddubsw m5, [r3 + 11 * 16] ; [27]
+ pmulhrsw m5, m7
+ pmaddubsw m1, m6, [r3 - 4 * 16] ; [12]
+ pmulhrsw m1, m7
+ packuswb m5, m1
+
+ palignr m1, m2, m3, 14
+
+ pmaddubsw m6, [r3 + 13 * 16] ; [29]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m1, [r3 - 2 * 16] ; [14]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, [r3 + 15 * 16] ; [31]
+ pmulhrsw m1, m7
+ pmaddubsw m2, [r3] ; [16]
+ pmulhrsw m2, m7
+ packuswb m1, m2
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_6, 3,7,8
+
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ palignr m1, m3, 1 ;[x 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpckhbw m2, m3, m1 ;[x 16 16 15 15 14 14 13 13 12 12 11 11 10 10 9]
+ punpcklbw m3, m1 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ pmaddubsw m4, m3, [r3 - 3 * 16] ; [13]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m3, [r3 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ palignr m6, m2, m3, 2
+
+ pmaddubsw m5, m6, [r3 - 9 * 16] ; [7]
+ pmulhrsw m5, m7
+ pmaddubsw m6, [r3 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ palignr m1, m2, m3, 4
+
+ pmaddubsw m6, m1, [r3 - 15 * 16] ; [1]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m1, [r3 - 2 * 16] ; [14]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m0, m2, m3, 6
+
+ pmaddubsw m1, [r3 + 11 * 16] ; [27]
+ pmulhrsw m1, m7
+ pmaddubsw m0, [r3 - 8 * 16] ; [8]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ palignr m4, m2, m3, 6
+ palignr m6, m2, m3, 8
+
+ pmaddubsw m4, [r3 + 5 * 16] ; [21]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m6, [r3 - 14 * 16] ; [2]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ pmaddubsw m5, m6, [r3 - 16] ; [15]
+ pmulhrsw m5, m7
+ pmaddubsw m6, [r3 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ palignr m0, m2, m3, 10
+
+ pmaddubsw m6, m0, [r3 - 7 * 16] ; [9]
+ pmulhrsw m6, m7
+ pmaddubsw m0, [r3 + 6 * 16] ; [22]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m2, m3, 12
+
+ pmaddubsw m1, m2, [r3 - 13 * 16] ; [3]
+ pmulhrsw m1, m7
+ pmaddubsw m2, [r3] ; [16]
+ pmulhrsw m2, m7
+ packuswb m1, m2
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_30, 3,7,8
+ mov r2, r3mp
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ palignr m1, m3, 1 ;[x 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpckhbw m2, m3, m1 ;[x 16 16 15 15 14 14 13 13 12 12 11 11 10 10 9]
+ punpcklbw m3, m1 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ pmaddubsw m4, m3, [r3 - 3 * 16] ; [13]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m3, [r3 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ palignr m6, m2, m3, 2
+
+ pmaddubsw m5, m6, [r3 - 9 * 16] ; [7]
+ pmulhrsw m5, m7
+ pmaddubsw m6, [r3 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ palignr m1, m2, m3, 4
+
+ pmaddubsw m6, m1, [r3 - 15 * 16] ; [1]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m1, [r3 - 2 * 16] ; [14]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m0, m2, m3, 6
+
+ pmaddubsw m1, [r3 + 11 * 16] ; [27]
+ pmulhrsw m1, m7
+ pmaddubsw m0, [r3 - 8 * 16] ; [8]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ palignr m4, m2, m3, 6
+ palignr m6, m2, m3, 8
+
+ pmaddubsw m4, [r3 + 5 * 16] ; [21]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m6, [r3 - 14 * 16] ; [2]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ pmaddubsw m5, m6, [r3 - 16] ; [15]
+ pmulhrsw m5, m7
+ pmaddubsw m6, [r3 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ palignr m0, m2, m3, 10
+
+ pmaddubsw m6, m0, [r3 - 7 * 16] ; [9]
+ pmulhrsw m6, m7
+ pmaddubsw m0, [r3 + 6 * 16] ; [22]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m2, m3, 12
+
+ pmaddubsw m1, m2, [r3 - 13 * 16] ; [3]
+ pmulhrsw m1, m7
+ pmaddubsw m2, [r3] ; [16]
+ pmulhrsw m2, m7
+ packuswb m1, m2
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_7, 3,7,8
+
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ palignr m1, m3, 1 ;[x 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpckhbw m2, m3, m1 ;[x 16 16 15 15 14 14 13 13 12 12 11 11 10 10 9]
+ punpcklbw m3, m1 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ pmaddubsw m4, m3, [r3 - 7 * 16] ; [9]
+ pmulhrsw m4, m7
+ pmaddubsw m0, m3, [r3 + 2 * 16] ; [18]
+ pmulhrsw m0, m7
+ packuswb m4, m0
+
+ palignr m1, m2, m3, 2
+
+ pmaddubsw m5, m3, [r3 + 11 * 16] ; [27]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m1, [r3 - 12 * 16] ; [4]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m1, [r3 - 3 * 16] ; [13]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m1, [r3 + 6 * 16] ; [22]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m0, m2, m3, 4
+
+ pmaddubsw m1, [r3 + 15 * 16] ; [31]
+ pmulhrsw m1, m7
+ pmaddubsw m0, [r3 - 8 * 16] ; [8]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ palignr m1, m2, m3, 4
+
+ pmaddubsw m4, m1, [r3 + 16] ; [17]
+ pmulhrsw m4, m7
+ pmaddubsw m1, [r3 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ palignr m0, m2, m3, 6
+
+ pmaddubsw m5, m0, [r3 - 13 * 16] ; [03]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m0, [r3 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m0, [r3 + 5 * 16] ; [21]
+ pmulhrsw m6, m7
+ pmaddubsw m0, [r3 + 14 * 16] ; [30]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m2, m3, 8
+
+ pmaddubsw m1, m2, [r3 - 9 * 16] ; [07]
+ pmulhrsw m1, m7
+ pmaddubsw m2, [r3] ; [16]
+ pmulhrsw m2, m7
+ packuswb m1, m2
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_29, 3,7,8
+ mov r2, r3mp
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ palignr m1, m3, 1 ;[x 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpckhbw m2, m3, m1 ;[x 16 16 15 15 14 14 13 13 12 12 11 11 10 10 9]
+ punpcklbw m3, m1 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ pmaddubsw m4, m3, [r3 - 7 * 16] ; [9]
+ pmulhrsw m4, m7
+ pmaddubsw m0, m3, [r3 + 2 * 16] ; [18]
+ pmulhrsw m0, m7
+ packuswb m4, m0
+
+ palignr m1, m2, m3, 2
+
+ pmaddubsw m5, m3, [r3 + 11 * 16] ; [27]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m1, [r3 - 12 * 16] ; [4]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m1, [r3 - 3 * 16] ; [13]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m1, [r3 + 6 * 16] ; [22]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m0, m2, m3, 4
+
+ pmaddubsw m1, [r3 + 15 * 16] ; [31]
+ pmulhrsw m1, m7
+ pmaddubsw m0, [r3 - 8 * 16] ; [8]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ palignr m1, m2, m3, 4
+
+ pmaddubsw m4, m1, [r3 + 16] ; [17]
+ pmulhrsw m4, m7
+ pmaddubsw m1, [r3 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ palignr m0, m2, m3, 6
+
+ pmaddubsw m5, m0, [r3 - 13 * 16] ; [03]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m0, [r3 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m0, [r3 + 5 * 16] ; [21]
+ pmulhrsw m6, m7
+ pmaddubsw m0, [r3 + 14 * 16] ; [30]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m2, m3, 8
+
+ pmaddubsw m1, m2, [r3 - 9 * 16] ; [07]
+ pmulhrsw m1, m7
+ pmaddubsw m2, [r3] ; [16]
+ pmulhrsw m2, m7
+ packuswb m1, m2
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_8, 3,7,8
+
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+.loop:
+ movu m1, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ palignr m3, m1, 1 ;[x 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpckhbw m0, m1, m3 ;[x 16 16 15 15 14 14 13 13 12 12 11 11 10 10 9]
+ punpcklbw m1, m3 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ pmaddubsw m4, m1, [r3 - 11 * 16] ; [5]
+ pmulhrsw m4, m7
+ pmaddubsw m2, m1, [r3 - 6 * 16] ; [10]
+ pmulhrsw m2, m7
+ packuswb m4, m2
+
+ pmaddubsw m5, m1, [r3 - 1 * 16] ; [15]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m1, [r3 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m1, [r3 + 9 * 16] ; [25]
+ pmulhrsw m6, m7
+ pmaddubsw m2, m1, [r3 + 14 * 16] ; [30]
+ pmulhrsw m2, m7
+ packuswb m6, m2
+
+ palignr m2, m0, m1, 2
+ palignr m3, m0, m1, 4
+
+ pmaddubsw m1, m2, [r3 - 13 * 16] ; [3]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m2, [r3 - 8 * 16] ; [8]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m2, [r3 - 3 * 16] ; [13]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m2, [r3 + 2 * 16] ; [18]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m2, [r3 + 7 * 16] ; [23]
+ pmulhrsw m5, m7
+ pmaddubsw m2, [r3 + 12 * 16] ; [28]
+ pmulhrsw m2, m7
+ packuswb m5, m2
+
+ pmaddubsw m6, m3, [r3 - 15 * 16] ; [01]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r3 - 10 * 16] ; [06]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r3 - 5 * 16] ; [11]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r3] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_28, 3,7,8
+ mov r2, r3mp
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+.loop:
+ movu m1, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ palignr m3, m1, 1 ;[x 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpckhbw m0, m1, m3 ;[x 16 16 15 15 14 14 13 13 12 12 11 11 10 10 9]
+ punpcklbw m1, m3 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ pmaddubsw m4, m1, [r3 - 11 * 16] ; [5]
+ pmulhrsw m4, m7
+ pmaddubsw m2, m1, [r3 - 6 * 16] ; [10]
+ pmulhrsw m2, m7
+ packuswb m4, m2
+
+ pmaddubsw m5, m1, [r3 - 1 * 16] ; [15]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m1, [r3 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m1, [r3 + 9 * 16] ; [25]
+ pmulhrsw m6, m7
+ pmaddubsw m2, m1, [r3 + 14 * 16] ; [30]
+ pmulhrsw m2, m7
+ packuswb m6, m2
+
+ palignr m2, m0, m1, 2
+ palignr m3, m0, m1, 4
+
+ pmaddubsw m1, m2, [r3 - 13 * 16] ; [3]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m2, [r3 - 8 * 16] ; [8]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m2, [r3 - 3 * 16] ; [13]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m2, [r3 + 2 * 16] ; [18]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m2, [r3 + 7 * 16] ; [23]
+ pmulhrsw m5, m7
+ pmaddubsw m2, [r3 + 12 * 16] ; [28]
+ pmulhrsw m2, m7
+ packuswb m5, m2
+
+ pmaddubsw m6, m3, [r3 - 15 * 16] ; [01]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r3 - 10 * 16] ; [06]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r3 - 5 * 16] ; [11]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r3] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_9, 3,7,8
+
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+.loop:
+ movu m2, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ palignr m3, m2, 1 ;[x 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpcklbw m2, m3 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ pmaddubsw m4, m2, [r3 - 14 * 16] ; [2]
+ pmulhrsw m4, m7
+ pmaddubsw m0, m2, [r3 - 12 * 16] ; [4]
+ pmulhrsw m0, m7
+ packuswb m4, m0
+
+ pmaddubsw m5, m2, [r3 - 10 * 16] ; [6]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m2, [r3 - 8 * 16] ; [8]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m2, [r3 - 6 * 16] ; [10]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m2, [r3 - 4 * 16] ; [12]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m2, [r3 - 2 * 16] ; [14]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m2, [r3] ; [16]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m2, [r3 + 2 * 16] ; [18]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m2, [r3 + 4 * 16] ; [20]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m2, [r3 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m2, [r3 + 8 * 16] ; [24]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m2, [r3 + 10 * 16] ; [26]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m2, [r3 + 12 * 16] ; [28]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m2, [r3 + 14 * 16] ; [30]
+ pmulhrsw m1, m7
+ packuswb m1, m1
+
+ punpcklqdq m1, m3 ; [00]
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_27, 3,7,8
+ mov r2, r3mp
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2 + 1] ;[16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ palignr m2, m3, 1 ;[x 16 15 14 13 12 11 10 9 8 7 6 5 4 3 2]
+ punpcklbw m3, m2 ;[9 8 8 7 7 6 6 5 5 4 4 3 3 2 2 1]
+
+ pmaddubsw m4, m3, [r3 - 14 * 16] ; [2]
+ pmulhrsw m4, m7
+ pmaddubsw m0, m3, [r3 - 12 * 16] ; [4]
+ pmulhrsw m0, m7
+ packuswb m4, m0
+
+ pmaddubsw m5, m3, [r3 - 10 * 16] ; [6]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r3 - 8 * 16] ; [8]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r3 - 6 * 16] ; [10]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r3 - 4 * 16] ; [12]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r3 - 2 * 16] ; [14]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m3, [r3] ; [16]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r3 + 2 * 16] ; [18]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r3 + 4 * 16] ; [20]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r3 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r3 + 8 * 16] ; [24]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r3 + 10 * 16] ; [26]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r3 + 12 * 16] ; [28]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r3 + 14 * 16] ; [30]
+ pmulhrsw m1, m7
+ packuswb m1, m1
+
+ movh [r0 ], m4
+ movhps [r0 + r1 ], m4
+ movh [r0 + r1 * 2], m5
+ movhps [r0 + r5 ], m5
+ lea r0, [r0 + r1 * 4]
+ movh [r0 ], m6
+ movhps [r0 + r1 ], m6
+ movh [r0 + r1 * 2], m1
+ movh [r0 + r5 ], m2
+
+ lea r0, [r6 + 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_10, 6,6,8
+ lea r4, [r1 * 3]
+ pxor m7, m7
+
+ movu m0, [r2 + 1]
+ palignr m1, m0, 1
+ pshufb m1, m7
+ palignr m2, m0, 2
+ pshufb m2, m7
+ palignr m3, m0, 3
+ pshufb m3, m7
+ palignr m4, m0, 4
+ pshufb m4, m7
+ palignr m5, m0, 5
+ pshufb m5, m7
+ palignr m6, m0, 6
+ pshufb m6, m7
+
+ movu [r0 + r1], m1
+ movu [r0 + r1 * 2], m2
+ movu [r0 + r4], m3
+ lea r2, [r0 + r1 * 4]
+ movu [r2], m4
+ movu [r2 + r1], m5
+ movu [r2 + r1 * 2], m6
+
+ palignr m1, m0, 7
+ pshufb m1, m7
+ movhlps m2, m0
+ pshufb m2, m7
+ palignr m3, m0, 9
+ pshufb m3, m7
+ palignr m4, m0, 10
+ pshufb m4, m7
+ palignr m5, m0, 11
+ pshufb m5, m7
+ palignr m6, m0, 12
+ pshufb m6, m7
+
+ movu [r2 + r4], m1
+ lea r2, [r2 + r1 * 4]
+ movu [r2], m2
+ movu [r2 + r1], m3
+ movu [r2 + r1 * 2], m4
+ movu [r2 + r4], m5
+ lea r2, [r2 + r1 * 4]
+ movu [r2], m6
+
+ palignr m1, m0, 13
+ pshufb m1, m7
+ palignr m2, m0, 14
+ pshufb m2, m7
+ palignr m3, m0, 15
+ pshufb m3, m7
+ pshufb m0, m7
+
+ movu [r2 + r1], m1
+ movu [r2 + r1 * 2], m2
+ movu [r2 + r4], m3
+
+; filter
+ cmp r5b, byte 0
+ jz .quit
+ pmovzxbw m0, m0
+ mova m1, m0
+ movu m2, [r3]
+ movu m3, [r3 + 1]
+
+ pshufb m2, m7
+ pmovzxbw m2, m2
+ movhlps m4, m3
+ pmovzxbw m3, m3
+ pmovzxbw m4, m4
+ psubw m3, m2
+ psubw m4, m2
+ psraw m3, 1
+ psraw m4, 1
+ paddw m0, m3
+ paddw m1, m4
+ packuswb m0, m1
+
+.quit:
+ movu [r0], m0
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_26, 6,7,5
+
+ movu m0, [r3 + 1]
+
+ lea r4, [r1 * 3]
+ lea r3, [r0 + r1 * 4]
+ lea r6, [r3 + r1 * 4]
+
+ movu [r0], m0
+ movu [r0 + r1], m0
+ movu [r0 + r1 * 2], m0
+ movu [r0 + r4], m0
+ movu [r3], m0
+ movu [r3 + r1], m0
+ movu [r3 + r1 * 2], m0
+ movu [r3 + r4], m0
+ movu [r6], m0
+ movu [r6 + r1], m0
+ movu [r6 + r1 * 2], m0
+ movu [r6 + r4], m0
+ lea r6, [r6 + r1 * 4]
+ movu [r6], m0
+ movu [r6 + r1], m0
+ movu [r6 + r1 * 2], m0
+ movu [r6 + r4], m0
+
+; filter
+ cmp r5b, byte 0
+ jz .quit
+ lea r5, [r3 + r1 * 4]
+ pxor m4, m4
+ pshufb m0, m4
+ pmovzxbw m0, m0
+ mova m1, m0
+ movu m2, [r2]
+ movu m3, [r2 + 1]
+
+ pshufb m2, m4
+ pmovzxbw m2, m2
+ movhlps m4, m3
+ pmovzxbw m3, m3
+ pmovzxbw m4, m4
+ psubw m3, m2
+ psubw m4, m2
+ psraw m3, 1
+ psraw m4, 1
+ paddw m0, m3
+ paddw m1, m4
+ packuswb m0, m1
+
+ pextrb [r0], m0, 0
+ pextrb [r0 + r1], m0, 1
+ pextrb [r0 + r1 * 2], m0, 2
+ pextrb [r0 + r4], m0, 3
+ pextrb [r3], m0, 4
+ pextrb [r3 + r1], m0, 5
+ pextrb [r3 + r1 * 2], m0, 6
+ pextrb [r3 + r4], m0, 7
+ pextrb [r5], m0, 8
+ pextrb [r5 + r1], m0, 9
+ pextrb [r5 + r1 * 2], m0, 10
+ pextrb [r5 + r4], m0, 11
+ pextrb [r6], m0, 12
+ pextrb [r6 + r1], m0, 13
+ pextrb [r6 + r1 * 2], m0, 14
+ pextrb [r6 + r4], m0, 15
+
+.quit:
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_11, 3,7,8
+
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2] ;[15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ mova m2, m3
+ palignr m1, m3, 1 ;[15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ punpcklbw m3, m1 ;[8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, m3, [r3 + 14 * 16] ; [30]
+ pmulhrsw m4, m7
+ pmaddubsw m0, m3, [r3 + 12 * 16] ; [28]
+ pmulhrsw m0, m7
+ packuswb m4, m0
+
+ pmaddubsw m5, m3, [r3 + 10 * 16] ; [26]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r3 + 8 * 16] ; [24]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r3 + 6 * 16] ; [22]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r3 + 4 * 16] ; [20]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r3 + 2 * 16] ; [18]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m3, [r3] ; [16]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r3 - 2 * 16] ; [14]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r3 - 4 * 16] ; [12]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r3 - 6 * 16] ; [10]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r3 - 8 * 16] ; [08]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r3 - 10 * 16] ; [06]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r3 - 12 * 16] ; [04]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r3 - 14 * 16] ; [02]
+ pmulhrsw m1, m7
+ packuswb m1, m1
+ punpcklqdq m1, m2 ;[00]
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_25, 3,7,8
+ mov r2, r3mp
+ lea r3, [ang_table + 16 * 16]
+ mov r4d, 2
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+.loop:
+ movu m3, [r2] ;[15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ mova m2, m3
+ palignr m1, m3, 1 ;[15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ punpcklbw m3, m1 ;[8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, m3, [r3 + 14 * 16] ; [30]
+ pmulhrsw m4, m7
+ pmaddubsw m0, m3, [r3 + 12 * 16] ; [28]
+ pmulhrsw m0, m7
+ packuswb m4, m0
+
+ pmaddubsw m5, m3, [r3 + 10 * 16] ; [26]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r3 + 8 * 16] ; [24]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r3 + 6 * 16] ; [22]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r3 + 4 * 16] ; [20]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r3 + 2 * 16] ; [18]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m3, [r3] ; [16]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r3 - 2 * 16] ; [14]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r3 - 4 * 16] ; [12]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r3 - 6 * 16] ; [10]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r3 - 8 * 16] ; [08]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r3 - 10 * 16] ; [06]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r3 - 12 * 16] ; [04]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r3 - 14 * 16] ; [02]
+ pmulhrsw m1, m7
+ packuswb m1, m1
+
+ movh [r0 ], m4
+ movhps [r0 + r1 ], m4
+ movh [r0 + r1 * 2], m5
+ movhps [r0 + r5 ], m5
+ lea r0, [r0 + r1 * 4]
+ movh [r0 ], m6
+ movhps [r0 + r1 ], m6
+ movh [r0 + r1 * 2], m1
+ movh [r0 + r5 ], m2
+
+ lea r0, [r6 + 8]
+ add r2, 8
+ dec r4
+ jnz .loop
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_12, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+ movu m3, [r2] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m0, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r3]
+ pshufb m2, [c_mode16_12]
+
+ palignr m0, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, m0, [r4 + 11 * 16] ; [27]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m0, [r4 + 6 * 16] ; [22]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ pmaddubsw m5, m0, [r4 + 1 * 16] ; [17]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m0, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m0, [r4 - 9 * 16] ; [7]
+ pmulhrsw m6, m7
+ pmaddubsw m0, [r4 - 14 * 16] ; [2]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m3, m2, 15
+
+ pmaddubsw m1, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 7 * 16] ; [09]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 15 * 16] ; [31]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 + 5 * 16] ; [21]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+
+ movu m1, [r2 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 x x x x x x x]
+
+ pmaddubsw m4, m3, [r4 + 11 * 16] ; [27]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 + 1 * 16] ; [17]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 9 * 16] ; [7]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 14 * 16] ; [2]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 7 * 16] ; [09]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 15 * 16] ; [31]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 + 5 * 16] ; [21]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_24, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+ movu m3, [r3] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m0, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r2]
+ pshufb m2, [c_mode16_12]
+
+ palignr m0, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, m0, [r4 + 11 * 16] ; [27]
+ pmulhrsw m4, m7
+ pmaddubsw m1, m0, [r4 + 6 * 16] ; [22]
+ pmulhrsw m1, m7
+ packuswb m4, m1
+
+ pmaddubsw m5, m0, [r4 + 1 * 16] ; [17]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m0, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m0, [r4 - 9 * 16] ; [7]
+ pmulhrsw m6, m7
+ pmaddubsw m0, [r4 - 14 * 16] ; [2]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m3, m2, 15
+
+ pmaddubsw m1, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 7 * 16] ; [09]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 15 * 16] ; [31]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 + 5 * 16] ; [21]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+
+ movu m1, [r3 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 x x x x x x x]
+
+ pmaddubsw m4, m3, [r4 + 11 * 16] ; [27]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 + 1 * 16] ; [17]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 9 * 16] ; [7]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 14 * 16] ; [2]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m1, m7
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 7 * 16] ; [09]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 15 * 16] ; [31]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 + 5 * 16] ; [21]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_13, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+ movu m3, [r2] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m5, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r3]
+ pshufb m2, [c_mode16_13]
+
+ palignr m5, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, m5, [r4 + 7 * 16] ; [23]
+ pmulhrsw m4, m7
+ pmaddubsw m0, m5, [r4 - 2 * 16] ; [14]
+ pmulhrsw m0, m7
+ packuswb m4, m0
+
+ pmaddubsw m5, [r4 - 11 * 16] ; [05]
+ pmulhrsw m5, m7
+
+ palignr m3, m2, 15
+
+ pmaddubsw m6, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m1, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 16] ; [15]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 10 * 16] ; [06]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+
+ movu m1, [r2 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 x x x x x x x]
+
+ pmaddubsw m4, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m5, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 16] ; [15]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 10 * 16] ; [06]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_23, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+ movu m3, [r3] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m5, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r2]
+ pshufb m2, [c_mode16_13]
+
+ palignr m5, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, m5, [r4 + 7 * 16] ; [23]
+ pmulhrsw m4, m7
+ pmaddubsw m0, m5, [r4 - 2 * 16] ; [14]
+ pmulhrsw m0, m7
+ packuswb m4, m0
+
+ pmaddubsw m5, [r4 - 11 * 16] ; [05]
+ pmulhrsw m5, m7
+
+ palignr m3, m2, 15
+
+ pmaddubsw m6, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m1, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 16] ; [15]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 10 * 16] ; [06]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+
+ movu m1, [r3 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 x x x x x x x]
+
+ pmaddubsw m4, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m5, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 16] ; [15]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 10 * 16] ; [06]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_14, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+ movu m3, [r2] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m5, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r3]
+ pshufb m2, [c_mode16_14]
+
+ palignr m5, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, m5, [r4 + 3 * 16] ; [19]
+ pmulhrsw m4, m7
+ pmaddubsw m5, [r4 - 10 * 16] ; [06]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ palignr m3, m2, 15
+
+ pmaddubsw m5, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 15 * 16] ; [31]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 + 16] ; [17]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+
+ movu m1, [r2 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 x x x x x x x]
+
+ pmaddubsw m4, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 10 * 16] ; [06]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 15 * 16] ; [31]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 + 16] ; [17]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_22, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+ movu m3, [r3] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m5, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r2]
+ pshufb m2, [c_mode16_14]
+
+ palignr m5, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, m5, [r4 + 3 * 16] ; [19]
+ pmulhrsw m4, m7
+ pmaddubsw m5, [r4 - 10 * 16] ; [06]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ palignr m3, m2, 15
+
+ pmaddubsw m5, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 15 * 16] ; [31]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 + 16] ; [17]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+
+ movu m1, [r3 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 x x x x x x x]
+
+ pmaddubsw m4, m3, [r4 + 3 * 16] ; [19]
+ pmulhrsw m4, m7
+ pmaddubsw m5, m3, [r4 - 10 * 16] ; [06]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 15 * 16] ; [31]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 + 16] ; [17]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m6, m7
+ pmaddubsw m1, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 13 * 16] ; [29]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_15, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+ movu m3, [r2] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m4, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r3]
+ pshufb m2, [c_mode16_15]
+
+ palignr m4, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, [r4 - 16] ; [15]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 15
+
+ pmaddubsw m5, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 3 * 16] ; [13]
+ pmulhrsw m5, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 7 * 16] ; [09]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 9 * 16] ; [07]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 13 * 16] ; [03]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+
+ movu m1, [r2 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 0 0 0 0 0 0 0 15L]
+
+ pmaddubsw m4, m3, [r4 - 16] ; [15]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 3 * 16] ; [13]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 7 * 16] ; [09]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 9 * 16] ; [07]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 13 * 16] ; [03]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_21, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+ movu m3, [r3] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m4, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r2]
+ pshufb m2, [c_mode16_15]
+
+ palignr m4, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, [r4 - 16] ; [15]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 15
+
+ pmaddubsw m5, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 3 * 16] ; [13]
+ pmulhrsw m5, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 7 * 16] ; [09]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 9 * 16] ; [07]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 13 * 16] ; [03]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+
+ movu m1, [r3 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 0 0 0 0 0 0 0 15L]
+
+ pmaddubsw m4, m3, [r4 - 16] ; [15]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 3 * 16] ; [13]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pmaddubsw m1, m3, [r4 - 7 * 16] ; [09]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 9 * 16] ; [07]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pmaddubsw m6, m3, [r4 - 13 * 16] ; [03]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_16, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+ movu m3, [r2] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m4, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r3]
+ pshufb m2, [c_mode16_16] ; [2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8]
+ palignr m4, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, [r4 - 5 * 16] ; [11]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 15
+
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m5, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1 ; [3, 5, 6, 8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pslldq m2, 1 ; [5, 6, 8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 - 3 * 16] ; [13]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1 ; [6, 8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 13 * 16] ; [03]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1 ; [8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1 ; [9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1 ; [11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 16] ; [15]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1 ; [12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1 ; [14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+
+ movu m1, [r2 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ palignr m2, m2, 6 ; [x, x, x, x, x, x, 14, 15, 0, 2, 3, 5, 6, 8, x, x]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 0, 2, 3, 5, 6, 8, x, x]
+
+ pmaddubsw m4, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m6, m7
+
+ pmaddubsw m0, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 - 3 * 16] ; [13]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 13 * 16] ; [03]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 16] ; [15]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_20, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+ movu m3, [r3] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m4, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r2]
+ pshufb m2, [c_mode16_16] ; [2, 3, 5, 6, 8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8]
+ palignr m4, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, [r4 - 5 * 16] ; [11]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 15
+
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m5, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1 ; [3, 5, 6, 8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pslldq m2, 1 ; [5, 6, 8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 - 3 * 16] ; [13]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1 ; [6, 8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 13 * 16] ; [03]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1 ; [8, 9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1 ; [9, 11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1 ; [11, 12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 16] ; [15]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1 ; [12, 14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1 ; [14, 15, 0, 2, 3, 5, 6, 8, x, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+
+ movu m1, [r3 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ palignr m2, m2, 6 ; [x, x, x, x, x, x, 14, 15, 0, 2, 3, 5, 6, 8, x, x]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 0, 2, 3, 5, 6, 8, x, x]
+
+ pmaddubsw m4, m3, [r4 - 5 * 16] ; [11]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 15 * 16] ; [01]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 7 * 16] ; [23]
+ pmulhrsw m6, m7
+
+ pmaddubsw m0, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 - 3 * 16] ; [13]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pmaddubsw m4, m3, [r4 - 13 * 16] ; [03]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 9 * 16] ; [25]
+ pmulhrsw m5, m7
+ pmaddubsw m6, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 16] ; [15]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pmaddubsw m1, m3, [r4 - 11 * 16] ; [05]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m3, [r4] ; [16]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_17, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ lea r6, [r0 + r1 * 4] ; r6 -> 4 * stride
+ mova m7, [pw_1024]
+
+ movu m3, [r2] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m4, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r3]
+ pshufb m2, [c_mode16_17] ; [1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4]
+ palignr m4, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, [r4 - 10 * 16] ; [06]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 15
+
+ pmaddubsw m5, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1 ; [2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, x]
+ pinsrb m2, [r3 + 5], 0 ; [2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1 ; [4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pslldq m2, 1 ; [5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1 ; [6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4] ; [16]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pslldq m2, 1 ; [7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m4, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1 ; [9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1 ; [10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 8 * 16] ; [08]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1 ; [11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1 ; [12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1 ; [14, 15, 0, 1, 2, 4, 5, x, x, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4 - 16 * 16] ; [00]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ lea r0, [r6 + r1 * 4]
+ lea r6, [r6 + r1 * 8]
+
+ movu m1, [r2 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ palignr m2, m2, 6 ; [x, x, x, x, x, x, 14, 15, 0, 1, 2, 4, 5, x, x, x]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 0, 1, 2, 4, 5, x, x, x]
+
+ pmaddubsw m4, m3, [r4 - 10 * 16] ; [06]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4] ; [16]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 1, m4, m5, m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m4, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 8 * 16] ; [08]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4 - 16 * 16] ; [00]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 1, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_19, 4,7,8
+
+ lea r4, [ang_table + 16 * 16]
+ lea r5, [r1 * 3] ; r5 -> 3 * stride
+ mov r6, r0
+ mova m7, [pw_1024]
+
+ movu m3, [r3] ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0]
+ punpckhbw m4, m3, m3 ; [15 15 14 14 13 13 12 12 11 11 10 10 9 9 8 8]
+ punpcklbw m3, m3 ; [7 7 6 6 5 5 4 4 3 3 2 2 1 1 0 0]
+ movu m2, [r2]
+ pshufb m2, [c_mode16_17] ; [1, 2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4]
+ palignr m4, m3, 1 ; [8 7 7 6 6 5 5 4 4 3 3 2 2 1 1 0]
+
+ pmaddubsw m4, [r4 - 10 * 16] ; [06]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 15
+
+ pmaddubsw m5, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1 ; [2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, x]
+ pinsrb m2, [r2 + 5], 0 ; [2, 4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1 ; [4, 5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pslldq m2, 1 ; [5, 6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1 ; [6, 7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4] ; [16]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pslldq m2, 1 ; [7, 9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m4, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1 ; [9, 10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1 ; [10, 11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 8 * 16] ; [08]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1 ; [11, 12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1 ; [12, 14, 15, 0, 1, 2, 4, 5, x, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1 ; [14, 15, 0, 1, 2, 4, 5, x, x, x, x, x, x, x, x, x]
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4 - 16 * 16] ; [00]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ lea r0, [r6 + 8]
+
+ movu m1, [r3 + 1] ; [16 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1]
+ pslldq m3, m1, 1 ; [15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 x]
+ punpckhbw m3, m1 ; [16 15 15 14 14 13 13 12 12 11 11 10 10 9 9 8]
+ palignr m2, m2, 6 ; [x, x, x, x, x, 14, 15, 0, 1, 2, 4, 5, x, x, x]
+ movlhps m2, m1 ; [8 7 6 5 4 3 2 1 0, 2, 3, 5, 6, 8, x, x]
+
+ pmaddubsw m4, m3, [r4 - 10 * 16] ; [06]
+ pmulhrsw m4, m7
+
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 - 4 * 16] ; [12]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 2 * 16] ; [18]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 8 * 16] ; [24]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 + 14 * 16] ; [30]
+ pmulhrsw m6, m7
+ pmaddubsw m0, m3, [r4 - 12 * 16] ; [04]
+ pmulhrsw m0, m7
+ packuswb m6, m0
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 - 6 * 16] ; [10]
+ pmulhrsw m1, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m0, m3, [r4] ; [16]
+ pmulhrsw m0, m7
+ packuswb m1, m0
+
+ TRANSPOSE_STORE_8x8 0, 0, m4, m5, m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m4, m3, [r4 + 6 * 16] ; [22]
+ pmulhrsw m4, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m5, m3, [r4 + 12 * 16] ; [28]
+ pmulhrsw m5, m7
+ packuswb m4, m5
+
+ pmaddubsw m5, m3, [r4 - 14 * 16] ; [02]
+ pmulhrsw m5, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 8 * 16] ; [08]
+ pmulhrsw m6, m7
+ packuswb m5, m6
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m6, m3, [r4 - 2 * 16] ; [14]
+ pmulhrsw m6, m7
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 4 * 16] ; [20]
+ pmulhrsw m1, m7
+ packuswb m6, m1
+
+ pslldq m2, 1
+ palignr m3, m2, 14
+
+ pmaddubsw m1, m3, [r4 + 10 * 16] ; [26]
+ pmulhrsw m1, m7
+ pmaddubsw m3, [r4 - 16 * 16] ; [00]
+ pmulhrsw m3, m7
+ packuswb m1, m3
+
+ TRANSPOSE_STORE_8x8 1, 0, m4, m5, m6, m1
+
+ RET
+
+INIT_XMM sse4
+cglobal intra_pred_ang16_18, 4,5,3
+
+ movu m0, [r3]
+ movu m1, [r2]
+ mova m2, [c_mode16_18]
+ pshufb m1, m2
+
+ lea r2, [r1 * 2]
+ lea r3, [r1 * 3]
+ lea r4, [r1 * 4]
+ movu [r0], m0
+ palignr m2, m0, m1, 15
+ movu [r0 + r1], m2
+ palignr m2, m0, m1, 14
+ movu [r0 + r2], m2
+ palignr m2, m0, m1, 13
+ movu [r0 + r3], m2
+ lea r0, [r0 + r4]
+ palignr m2, m0, m1, 12
+ movu [r0], m2
+ palignr m2, m0, m1, 11
+ movu [r0 + r1], m2
+ palignr m2, m0, m1, 10
+ movu [r0 + r2], m2
+ palignr m2, m0, m1, 9
+ movu [r0 + r3], m2
+ lea r0, [r0 + r4]
+ palignr m2, m0, m1, 8
+ movu [r0], m2
+ palignr m2, m0, m1, 7
+ movu [r0 + r1], m2
+ palignr m2, m0, m1, 6
+ movu [r0 + r2], m2
+ palignr m2, m0, m1, 5
+ movu [r0 + r3], m2
+ lea r0, [r0 + r4]
+ palignr m2, m0, m1, 4
+ movu [r0], m2
+ palignr m2, m0, m1, 3
+ movu [r0 + r1], m2
+ palignr m2, m0, m1, 2
+ movu [r0 + r2], m2
+ palignr m2, m0, m1, 1
+ movu [r0 + r3], m2
+ RET
+
;---------------------------------------------------------------------------------------------------------------
; void intraPredAng32(pixel* dst, intptr_t dstStride, pixel *refLeft, pixel *refAbove, int dirMode, int bFilter)
;---------------------------------------------------------------------------------------------------------------
More information about the x265-devel
mailing list