Mercurial > hg > forks > libbpg
diff x265/source/common/x86/intrapred8_allangs.asm @ 0:772086c29cc7
Initial import.
| author | Matti Hamalainen <ccr@tnsp.org> |
|---|---|
| date | Wed, 16 Nov 2016 11:16:33 +0200 |
| parents | |
| children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/x265/source/common/x86/intrapred8_allangs.asm Wed Nov 16 11:16:33 2016 +0200 @@ -0,0 +1,24166 @@ +;***************************************************************************** +;* Copyright (C) 2013 x265 project +;* +;* Authors: Min Chen <chenm003@163.com> <min.chen@multicorewareinc.com> +;* Praveen Tiwari <praveen@multicorewareinc.com> +;* +;* This program is free software; you can redistribute it and/or modify +;* it under the terms of the GNU General Public License as published by +;* the Free Software Foundation; either version 2 of the License, or +;* (at your option) any later version. +;* +;* This program is distributed in the hope that it will be useful, +;* but WITHOUT ANY WARRANTY; without even the implied warranty of +;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;* GNU General Public License for more details. +;* +;* You should have received a copy of the GNU General Public License +;* along with this program; if not, write to the Free Software +;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. +;* +;* This program is also available under a commercial proprietary license. +;* For more information, contact us at license @ x265.com. +;*****************************************************************************/ + +%include "x86inc.asm" +%include "x86util.asm" + +SECTION_RODATA 32 + +all_ang4_shuff: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 + db 0, 1, 1, 2, 2, 3, 3, 4, 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6, 3, 4, 4, 5, 5, 6, 6, 7 + db 0, 1, 1, 2, 2, 3, 3, 4, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6 + db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5 + db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 1, 2, 2, 3, 3, 4, 4, 5 + db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4 + db 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 + db 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12 + db 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 4, 0, 0, 9, 9, 10, 10, 11 + db 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 2, 0, 0, 9, 9, 10, 10, 11, 2, 0, 0, 9, 9, 10, 10, 11 + db 0, 9, 9, 10, 10, 11, 11, 12, 2, 0, 0, 9, 9, 10, 10, 11, 2, 0, 0, 9, 9, 10, 10, 11, 4, 2, 2, 0, 0, 9, 9, 10 + db 0, 9, 9, 10, 10, 11, 11, 12, 2, 0, 0, 9, 9, 10, 10, 11, 2, 0, 0, 9, 9, 10, 10, 11, 3, 2, 2, 0, 0, 9, 9, 10 + db 0, 9, 9, 10, 10, 11, 11, 12, 1, 0, 0, 9, 9, 10, 10, 11, 2, 1, 1, 0, 0, 9, 9, 10, 4, 2, 2, 1, 1, 0, 0, 9 + db 0, 1, 2, 3, 9, 0, 1, 2, 10, 9, 0, 1, 11, 10, 9, 0, 0, 1, 2, 3, 9, 0, 1, 2, 10, 9, 0, 1, 11, 10, 9, 0 + db 0, 1, 1, 2, 2, 3, 3, 4, 9, 0, 0, 1, 1, 2, 2, 3, 10, 9, 9, 0, 0, 1, 1, 2, 12, 10, 10, 9, 9, 0, 0, 1 + db 0, 1, 1, 2, 2, 3, 3, 4, 10, 0, 0, 1, 1, 2, 2, 3, 10, 0, 0, 1, 1, 2, 2, 3, 11, 10, 10, 0, 0, 1, 1, 2 + db 0, 1, 1, 2, 2, 3, 3, 4, 10, 0, 0, 1, 1, 2, 2, 3, 10, 0, 0, 1, 1, 2, 2, 3, 12, 10, 10, 0, 0, 1, 1, 2 + db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 10, 0, 0, 1, 1, 2, 2, 3, 10, 0, 0, 1, 1, 2, 2, 3 + db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 12, 0, 0, 1, 1, 2, 2, 3 + db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4 + db 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4 + db 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5 + db 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6 + db 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6, 2, 3, 3, 4, 4, 5, 5, 6 + db 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6, 2, 3, 3, 4, 4, 5, 5, 6, 3, 4, 4, 5, 5, 6, 6, 7 + db 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6, 3, 4, 4, 5, 5, 6, 6, 7, 4, 5, 5, 6, 6, 7, 7, 8 + db 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8 + +all_ang4: db 6, 26, 6, 26, 6, 26, 6, 26, 12, 20, 12, 20, 12, 20, 12, 20, 18, 14, 18, 14, 18, 14, 18, 14, 24, 8, 24, 8, 24, 8, 24, 8 + db 11, 21, 11, 21, 11, 21, 11, 21, 22, 10, 22, 10, 22, 10, 22, 10, 1, 31, 1, 31, 1, 31, 1, 31, 12, 20, 12, 20, 12, 20, 12, 20 + db 15, 17, 15, 17, 15, 17, 15, 17, 30, 2, 30, 2, 30, 2, 30, 2, 13, 19, 13, 19, 13, 19, 13, 19, 28, 4, 28, 4, 28, 4, 28, 4 + db 19, 13, 19, 13, 19, 13, 19, 13, 6, 26, 6, 26, 6, 26, 6, 26, 25, 7, 25, 7, 25, 7, 25, 7, 12, 20, 12, 20, 12, 20, 12, 20 + db 23, 9, 23, 9, 23, 9, 23, 9, 14, 18, 14, 18, 14, 18, 14, 18, 5, 27, 5, 27, 5, 27, 5, 27, 28, 4, 28, 4, 28, 4, 28, 4 + db 27, 5, 27, 5, 27, 5, 27, 5, 22, 10, 22, 10, 22, 10, 22, 10, 17, 15, 17, 15, 17, 15, 17, 15, 12, 20, 12, 20, 12, 20, 12, 20 + db 30, 2, 30, 2, 30, 2, 30, 2, 28, 4, 28, 4, 28, 4, 28, 4, 26, 6, 26, 6, 26, 6, 26, 6, 24, 8, 24, 8, 24, 8, 24, 8 + db 2, 30, 2, 30, 2, 30, 2, 30, 4, 28, 4, 28, 4, 28, 4, 28, 6, 26, 6, 26, 6, 26, 6, 26, 8, 24, 8, 24, 8, 24, 8, 24 + db 5, 27, 5, 27, 5, 27, 5, 27, 10, 22, 10, 22, 10, 22, 10, 22, 15, 17, 15, 17, 15, 17, 15, 17, 20, 12, 20, 12, 20, 12, 20, 12 + db 9, 23, 9, 23, 9, 23, 9, 23, 18, 14, 18, 14, 18, 14, 18, 14, 27, 5, 27, 5, 27, 5, 27, 5, 4, 28, 4, 28, 4, 28, 4, 28 + db 13, 19, 13, 19, 13, 19, 13, 19, 26, 6, 26, 6, 26, 6, 26, 6, 7, 25, 7, 25, 7, 25, 7, 25, 20, 12, 20, 12, 20, 12, 20, 12 + db 17, 15, 17, 15, 17, 15, 17, 15, 2, 30, 2, 30, 2, 30, 2, 30, 19, 13, 19, 13, 19, 13, 19, 13, 4, 28, 4, 28, 4, 28, 4, 28 + db 21, 11, 21, 11, 21, 11, 21, 11, 10, 22, 10, 22, 10, 22, 10, 22, 31, 1, 31, 1, 31, 1, 31, 1, 20, 12, 20, 12, 20, 12, 20, 12 + db 26, 6, 26, 6, 26, 6, 26, 6, 20, 12, 20, 12, 20, 12, 20, 12, 14, 18, 14, 18, 14, 18, 14, 18, 8, 24, 8, 24, 8, 24, 8, 24 + db 26, 6, 26, 6, 26, 6, 26, 6, 20, 12, 20, 12, 20, 12, 20, 12, 14, 18, 14, 18, 14, 18, 14, 18, 8, 24, 8, 24, 8, 24, 8, 24 + db 21, 11, 21, 11, 21, 11, 21, 11, 10, 22, 10, 22, 10, 22, 10, 22, 31, 1, 31, 1, 31, 1, 31, 1, 20, 12, 20, 12, 20, 12, 20, 12 + db 17, 15, 17, 15, 17, 15, 17, 15, 2, 30, 2, 30, 2, 30, 2, 30, 19, 13, 19, 13, 19, 13, 19, 13, 4, 28, 4, 28, 4, 28, 4, 28 + db 13, 19, 13, 19, 13, 19, 13, 19, 26, 6, 26, 6, 26, 6, 26, 6, 7, 25, 7, 25, 7, 25, 7, 25, 20, 12, 20, 12, 20, 12, 20, 12 + db 9, 23, 9, 23, 9, 23, 9, 23, 18, 14, 18, 14, 18, 14, 18, 14, 27, 5, 27, 5, 27, 5, 27, 5, 4, 28, 4, 28, 4, 28, 4, 28 + db 5, 27, 5, 27, 5, 27, 5, 27, 10, 22, 10, 22, 10, 22, 10, 22, 15, 17, 15, 17, 15, 17, 15, 17, 20, 12, 20, 12, 20, 12, 20, 12 + db 2, 30, 2, 30, 2, 30, 2, 30, 4, 28, 4, 28, 4, 28, 4, 28, 6, 26, 6, 26, 6, 26, 6, 26, 8, 24, 8, 24, 8, 24, 8, 24 + db 30, 2, 30, 2, 30, 2, 30, 2, 28, 4, 28, 4, 28, 4, 28, 4, 26, 6, 26, 6, 26, 6, 26, 6, 24, 8, 24, 8, 24, 8, 24, 8 + db 27, 5, 27, 5, 27, 5, 27, 5, 22, 10, 22, 10, 22, 10, 22, 10, 17, 15, 17, 15, 17, 15, 17, 15, 12, 20, 12, 20, 12, 20, 12, 20 + db 23, 9, 23, 9, 23, 9, 23, 9, 14, 18, 14, 18, 14, 18, 14, 18, 5, 27, 5, 27, 5, 27, 5, 27, 28, 4, 28, 4, 28, 4, 28, 4 + db 19, 13, 19, 13, 19, 13, 19, 13, 6, 26, 6, 26, 6, 26, 6, 26, 25, 7, 25, 7, 25, 7, 25, 7, 12, 20, 12, 20, 12, 20, 12, 20 + db 15, 17, 15, 17, 15, 17, 15, 17, 30, 2, 30, 2, 30, 2, 30, 2, 13, 19, 13, 19, 13, 19, 13, 19, 28, 4, 28, 4, 28, 4, 28, 4 + db 11, 21, 11, 21, 11, 21, 11, 21, 22, 10, 22, 10, 22, 10, 22, 10, 1, 31, 1, 31, 1, 31, 1, 31, 12, 20, 12, 20, 12, 20, 12, 20 + db 6, 26, 6, 26, 6, 26, 6, 26, 12, 20, 12, 20, 12, 20, 12, 20, 18, 14, 18, 14, 18, 14, 18, 14, 24, 8, 24, 8, 24, 8, 24, 8 + + +SECTION .text + +; global constant +cextern pw_1024 + +; common constant with intrapred8.asm +cextern ang_table +cextern pw_ang_table +cextern tab_S1 +cextern tab_S2 +cextern tab_Si +cextern pw_16 +cextern pb_000000000000000F +cextern pb_0000000000000F0F +cextern pw_FFFFFFFFFFFFFFF0 + + +;----------------------------------------------------------------------------- +; void all_angs_pred_4x4(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) +;----------------------------------------------------------------------------- +INIT_XMM sse4 +cglobal all_angs_pred_4x4, 4, 4, 8 + +; mode 2 + +movh m0, [r1 + 10] +movd [r0], m0 + +palignr m1, m0, 1 +movd [r0 + 4], m1 + +palignr m1, m0, 2 +movd [r0 + 8], m1 + +palignr m1, m0, 3 +movd [r0 + 12], m1 + +; mode 3 + +mova m2, [pw_1024] + +pslldq m1, m0, 1 +pinsrb m1, [r1 + 9], 0 +punpcklbw m1, m0 + +lea r3, [ang_table] + +pmaddubsw m6, m1, [r3 + 26 * 16] +pmulhrsw m6, m2 +packuswb m6, m6 +movd [r0 + 16], m6 + +palignr m0, m1, 2 + +mova m7, [r3 + 20 * 16] + +pmaddubsw m3, m0, m7 +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 20], m3 + +; mode 6 [row 3] +movd [r0 + 76], m3 + +palignr m3, m1, 4 + +pmaddubsw m4, m3, [r3 + 14 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 24], m4 + +palignr m4, m1, 6 + +pmaddubsw m4, [r3 + 8 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 28], m4 + +; mode 4 + +pmaddubsw m5, m1, [r3 + 21 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 32], m5 + +pmaddubsw m5, m0, [r3 + 10 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 36], m5 + +pmaddubsw m5, m0, [r3 + 31 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 40], m5 + +pmaddubsw m4, m3, m7 +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 44], m4 + +; mode 5 + +pmaddubsw m5, m1, [r3 + 17 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 48], m5 + +pmaddubsw m5, m0, [r3 + 2 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 52], m5 + +pmaddubsw m5, m0, [r3 + 19 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 56], m5 + +pmaddubsw m4, m3, [r3 + 4 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 60], m4 + +; mode 6 + +pmaddubsw m5, m1, [r3 + 13 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 64], m5 + +movd [r0 + 68], m6 + +pmaddubsw m5, m0, [r3 + 7 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 72], m5 + +; mode 7 + +pmaddubsw m5, m1, [r3 + 9 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 80], m5 + +pmaddubsw m5, m1, [r3 + 18 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 84], m5 + +pmaddubsw m5, m1, [r3 + 27 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 88], m5 + +pmaddubsw m5, m0, [r3 + 4 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 92], m5 + +; mode 8 + +pmaddubsw m5, m1, [r3 + 5 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 96], m5 + +pmaddubsw m5, m1, [r3 + 10 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 100], m5 + +pmaddubsw m5, m1, [r3 + 15 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 104], m5 + +pmaddubsw m5, m1, [r3 + 20 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 108], m5 + +; mode 9 + +pmaddubsw m5, m1, [r3 + 2 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 112], m5 + +pmaddubsw m5, m1, [r3 + 4 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 116], m5 + +pmaddubsw m5, m1, [r3 + 6 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 120], m5 + +pmaddubsw m5, m1, [r3 + 8 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 124], m5 + +; mode 10 + +movd m3, [r1 + 9] +pshufd m4, m3, 0 +movu [r0 + 128], m4 + +pxor m5, m5 +movd m7, [r1 + 1] +pshufd m4, m7, 0 +punpcklbw m4, m5 + +pinsrb m7, [r1], 0 +pshufb m6, m7, m5 +punpcklbw m6, m5 + +psubw m4, m6 +psraw m4, 1 + +pshufb m6, m3, m5 +punpcklbw m6, m5 + +paddw m4, m6 +packuswb m4, m5 + +pextrb [r0 + 128], m4, 0 +pextrb [r0 + 132], m4, 1 +pextrb [r0 + 136], m4, 2 +pextrb [r0 + 140], m4, 3 + +; mode 11 + +pslldq m1, m1, 2 +pinsrb m1, [r1], 0 +pinsrb m1, [r1 + 9], 1 + +pmaddubsw m3, m1, [r3 + 30 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 144], m3 + +pmaddubsw m3, m1, [r3 + 28 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 148], m3 + +pmaddubsw m3, m1, [r3 + 26 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 152], m3 + +pmaddubsw m3, m1, [r3 + 24 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 156], m3 + +; mode 12 + +pmaddubsw m3, m1, [r3 + 27 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 160], m3 + +pmaddubsw m3, m1, [r3 + 22 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 164], m3 + +pmaddubsw m3, m1, [r3 + 17 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 168], m3 + +pmaddubsw m3, m1, [r3 + 12 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 172], m3 + +; mode 13 + +pmaddubsw m3, m1, [r3 + 23 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 176], m3 + +pmaddubsw m3, m1, [r3 + 14 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 180], m3 + +pmaddubsw m3, m1, [r3 + 5 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 184], m3 + +pslldq m5, m1, 2 +pinsrb m5, [r1 + 0], 1 +pinsrb m5, [r1 + 4], 0 + +pmaddubsw m4, m5, [r3 + 28 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 188], m4 + +; mode 14 + +pmaddubsw m4, m1, [r3 + 19 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 192], m4 + +pmaddubsw m7, m1, [r3 + 6 * 16] +pmulhrsw m7, m2 +packuswb m7, m7 +movd [r0 + 196], m7 + +pinsrb m5, [r1 + 2], 0 + +pmaddubsw m4, m5, [r3 + 25 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 200], m4 + +pmaddubsw m4, m5, [r3 + 12 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 204], m4 + +; mode 15 + +pmaddubsw m4, m1, [r3 + 15 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 208], m4 + +pmaddubsw m4, m5, [r3 + 30 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 212], m4 + +pmaddubsw m4, m5, [r3 + 13 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 216], m4 + +pslldq m4, m5, 2 +pinsrb m4, [r1 + 2], 1 +pinsrb m4, [r1 + 4], 0 + +pmaddubsw m6, m4, [r3 + 28 * 16] +pmulhrsw m6, m2 +packuswb m6, m6 +movd [r0 + 220], m6 + +; mode 16 + +pmaddubsw m6, m1, [r3 + 11 * 16] +pmulhrsw m6, m2 +packuswb m6, m6 +movd [r0 + 224], m6 + +pmaddubsw m6, m5, [r3 + 22 * 16] +pmulhrsw m6, m2 +packuswb m6, m6 +movd [r0 + 228], m6 + +pmaddubsw m6, m5, [r3 + 1 * 16] +pmulhrsw m6, m2 +packuswb m6, m6 +movd [r0 + 232], m6 + +pinsrb m4, [r1 + 3], 0 + +pmaddubsw m4, [r3 + 12 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 236], m4 + +; mode 17 + +movd [r0 + 240], m7 + +pslldq m1, 2 +pinsrb m1, [r1 + 1], 0 +pinsrb m1, [r1 + 0], 1 + +pmaddubsw m3, m1, [r3 + 12 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 244], m3 + +pslldq m1, 2 +pinsrb m1, [r1 + 1], 1 +pinsrb m1, [r1 + 2], 0 + +pmaddubsw m3, m1, [r3 + 18 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 248], m3 + +pslldq m1, 2 +pinsrb m1, [r1 + 2], 1 +pinsrb m1, [r1 + 4], 0 + +pmaddubsw m1, [r3 + 24 * 16] +pmulhrsw m1, m2 +packuswb m1, m1 +movd [r0 + 252], m1 + +; mode 18 + +movh m1, [r1] +movd [r0 + 256], m1 + +pslldq m3, m1, 1 +pinsrb m3, [r1 + 9], 0 +movd [r0 + 260], m3 + +pslldq m4, m3, 1 +pinsrb m4, [r1 + 10], 0 +movd [r0 + 264], m4 + +pslldq m4, 1 +pinsrb m4, [r1 + 11], 0 +movd [r0 + 268], m4 + +; mode 19 + +palignr m3, m1, 1 +punpcklbw m1, m3 + +pmaddubsw m7, m1, [r3 + 6 * 16] +pmulhrsw m7, m2 +packuswb m7, m7 +movd [r0 + 272], m7 + +pslldq m3, m1, 2 +pinsrb m3, [r1], 1 +pinsrb m3, [r1 + 9], 0 + +pmaddubsw m4, m3, [r3 + 12 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 276], m4 + +pslldq m4, m3, 2 +pinsrb m4, [r1 + 9], 1 +pinsrb m4, [r1 + 10], 0 + +pmaddubsw m5, m4, [r3 + 18 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 280], m5 + +pslldq m4, 2 +pinsrb m4, [r1 + 10], 1 +pinsrb m4, [r1 + 12], 0 + +pmaddubsw m4, [r3 + 24 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 284], m4 + +; mode 20 + +pmaddubsw m4, m1, [r3 + 11 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 288], m4 + +pinsrb m3, [r1 + 10], 0 + +pmaddubsw m4, m3, [r3 + 22 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 292], m4 + +pmaddubsw m4, m3, [r3 + 1 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 296], m4 + +pslldq m6, m3, 2 +pinsrb m6, [r1 + 10], 1 +pinsrb m6, [r1 + 11], 0 + +pmaddubsw m5, m6, [r3 + 12 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 300], m5 + +; mode 21 + +pmaddubsw m4, m1, [r3 + 15 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 304], m4 + +pmaddubsw m4, m3, [r3 + 30 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 308], m4 + +pmaddubsw m4, m3, [r3 + 13 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 312], m4 + +pinsrb m6, [r1 + 12], 0 + +pmaddubsw m6, [r3 + 28 * 16] +pmulhrsw m6, m2 +packuswb m6, m6 +movd [r0 + 316], m6 + +; mode 22 + +pmaddubsw m4, m1, [r3 + 19 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 320], m4 + +movd [r0 + 324], m7 + +pmaddubsw m4, m3, [r3 + 25 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 328], m4 + +pmaddubsw m4, m3, [r3 + 12 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 332], m4 + +; mode 23 + +pmaddubsw m4, m1, [r3 + 23 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 336], m4 + +pmaddubsw m4, m1, [r3 + 14 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 340], m4 + +pmaddubsw m4, m1, [r3 + 5 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 344], m4 + +pinsrb m3, [r1 + 12], 0 + +pmaddubsw m3, [r3 + 28 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 348], m3 + +; mode 24 + +pmaddubsw m3, m1, [r3 + 27 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 352], m3 + +pmaddubsw m3, m1, [r3 + 22 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 356], m3 + +pmaddubsw m3, m1, [r3 + 17 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 360], m3 + +pmaddubsw m3, m1, [r3 + 12 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 364], m3 + +; mode 25 + +pmaddubsw m3, m1, [r3 + 30 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 368], m3 + +pmaddubsw m3, m1, [r3 + 28 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 372], m3 + +pmaddubsw m3, m1, [r3 + 26 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 376], m3 + +pmaddubsw m1, [r3 + 24 * 16] +pmulhrsw m1, m2 +packuswb m1, m1 +movd [r0 + 380], m1 + +; mode 26 + +movh m1, [r1 + 1] +pshufd m3, m1, 0 +movu [r0 + 384], m3 + +pxor m4, m4 +movd m5, [r1 + 9] +pshufd m5, m5, 0 +punpcklbw m5, m4 + +pinsrb m6, [r1], 0 +pshufb m6, m4 +punpcklbw m6, m4 + +psubw m5, m6 +psraw m5, 1 + +pshufb m6, m1, m4 +punpcklbw m6, m4 + +paddw m5, m6 +packuswb m5, m4 + +pextrb [r0 + 384], m5, 0 +pextrb [r0 + 388], m5, 1 +pextrb [r0 + 392], m5, 2 +pextrb [r0 + 396], m5, 3 + +; mode 27 + +palignr m3, m1, 1 +punpcklbw m1, m3 + +pmaddubsw m3, m1, [r3 + 2 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 400], m3 + +pmaddubsw m3, m1, [r3 + 4 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 404], m3 + +pmaddubsw m3, m1, [r3 + 6 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 408], m3 + +pmaddubsw m3, m1, [r3 + 8 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 412], m3 + +; mode 28 + +pmaddubsw m3, m1, [r3 + 5 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 416], m3 + +pmaddubsw m3, m1, [r3 + 10 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 420], m3 + +pmaddubsw m3, m1, [r3 + 15 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 424], m3 + +pmaddubsw m3, m1, [r3 + 20 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 428], m3 + +; mode 29 + +pmaddubsw m3, m1, [r3 + 9 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 432], m3 + +pmaddubsw m3, m1, [r3 + 18 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 436], m3 + +pmaddubsw m3, m1, [r3 + 27 * 16] +pmulhrsw m3, m2 +packuswb m3, m3 +movd [r0 + 440], m3 + +palignr m3, m1, 2 + +pmaddubsw m4, m3, [r3 + 4 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 444], m4 + +; mode 30 + +pmaddubsw m4, m1, [r3 + 13 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 448], m4 + +pmaddubsw m7, m1, [r3 + 26 * 16] +pmulhrsw m7, m2 +packuswb m7, m7 +movd [r0 + 452], m7 + +pmaddubsw m5, m3, [r3 + 7 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 456], m5 + +pmaddubsw m6, m3, [r3 + 20 * 16] +pmulhrsw m6, m2 +packuswb m6, m6 +movd [r0 + 460], m6 + +; mode 31 + +pmaddubsw m4, m1, [r3 + 17 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 464], m4 + +pmaddubsw m5, m3, [r3 + 2 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 468], m5 + +pmaddubsw m5, m3, [r3 + 19 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 472], m5 + +palignr m4, m3, 2 + +pmaddubsw m5, m4, [r3 + 4 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 476], m5 + +; mode 32 + +pmaddubsw m5, m1, [r3 + 21 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 480], m5 + +pmaddubsw m5, m3, [r3 + 10 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 484], m5 + +pmaddubsw m5, m3, [r3 + 31 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 488], m5 + +pmaddubsw m5, m4, [r3 + 20 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 492], m5 + +; mode 33 + +movd [r0 + 496], m7 + +movd [r0 + 500], m6 + +pmaddubsw m5, m4, [r3 + 14 * 16] +pmulhrsw m5, m2 +packuswb m5, m5 +movd [r0 + 504], m5 + +psrldq m4, 2 + +pmaddubsw m4, [r3 + 8 * 16] +pmulhrsw m4, m2 +packuswb m4, m4 +movd [r0 + 508], m4 + +; mode 34 + +movh m7, [r1 + 2] +movd [r0 + 512], m7 + +psrldq m7, 1 +movd [r0 + 516], m7 + +psrldq m7, 1 +movd [r0 + 520], m7 + +psrldq m7, 1 +movd [r0 + 524], m7 + +RET + +;------------------------------------------------------------------------------ +; void all_angs_pred_8x8(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) +;------------------------------------------------------------------------------ +INIT_XMM sse4 +cglobal all_angs_pred_8x8, 3,4,8 + ; mode 2 + + movu m0, [r2 + 18] + palignr m1, m0, 1 + punpcklqdq m2, m0, m1 + movu [r0], m2 + + palignr m1, m0, 2 + palignr m2, m0, 3 + punpcklqdq m1, m2 + movu [r0 + 16], m1 + + palignr m1, m0, 4 + palignr m2, m0, 5 + punpcklqdq m1, m2 + movu [r0 + 32], m1 + + palignr m1, m0, 6 + palignr m2, m0, 7 + punpcklqdq m1, m2 + movu [r0 + 48], m1 + + ; mode 3 [row 0, 1] + + mova m7, [pw_1024] + lea r3, [ang_table] + + movu m0, [r1 + 17] + + palignr m1, m0, 1 + palignr m2, m0, 2 + + punpcklbw m3, m0, m1 + pmaddubsw m4, m3, [r3 + 26 * 16] + pmulhrsw m4, m7 + + punpcklbw m1, m2 + pmaddubsw m5, m1, [r3 + 20 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + + movu [r0 + 64], m4 + + ; mode 6 [row 1] + + movh [r0 + 264], m4 + + ; mode 6 [row 3] + + movhps [r0 + 280], m4 + + ; mode 4 [row 0, 1] + + pmaddubsw m4, m3, [r3 + 21 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m1, [r3 + 10 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 128], m4 + + ; mode 5 [row 0, 1] + + pmaddubsw m4, m3, [r3 + 17 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m1, [r3 + 2 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 192], m4 + + ; mode 6 [row 0] + + pmaddubsw m4, m3, [r3 + 13 * 16] + pmulhrsw m4, m7 + + pxor m5, m5 + + packuswb m4, m5 + movh [r0 + 256], m4 + + ; mode 7 [row 0, 1] + + pmaddubsw m4, m3, [r3 + 9 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 18 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 320], m4 + + ; mode 8 [row 0, 1] + + pmaddubsw m4, m3, [r3 + 5 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 10 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 384], m4 + + ; mode 8 [row 2, 3] + + pmaddubsw m4, m3, [r3 + 15 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 20 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 400], m4 + + ; mode 8 [row 4, 5] + + pmaddubsw m4, m3, [r3 + 25 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 30 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 416], m4 + + ; mode 8 [row 6, 7] + + pmaddubsw m4, m1, [r3 + 3 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m1, [r3 + 8 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 432], m4 + + ; mode 9 [row 0, 1] + + pmaddubsw m4, m3, [r3 + 2 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 4 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 448], m4 + + ; mode 9 [row 2, 3] + + pmaddubsw m4, m3, [r3 + 6 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 8 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 464], m4 + + ; mode 9 [row 4, 5] + + pmaddubsw m4, m3, [r3 + 10 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 12 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 480], m4 + + ; mode 9 [row 6, 7] + + pmaddubsw m4, m3, [r3 + 14 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 16 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 496], m4 + + ; mode 7 [row 2, 3] + + pmaddubsw m4, m3, [r3 + 27 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m1, [r3 + 4 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 336], m4 + + ; mode 7 [row 4, 5] + + pmaddubsw m4, m1, [r3 + 13 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m1, [r3 + 22 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 352], m4 + + ; mode 6 [row 2] + + pmaddubsw m4, m1, [r3 + 7 * 16] + pmulhrsw m4, m7 + + pxor m5, m5 + + packuswb m4, m5 + movh [r0 + 272], m4 + + ; mode 3 [row 2, 3] + + palignr m1, m0, 3 + palignr m3, m0, 4 + + punpcklbw m2, m1 + pmaddubsw m5, m2, [r3 + 14 * 16] + pmulhrsw m5, m7 + + punpcklbw m1, m3 + pmaddubsw m6, m1, [r3 + 8 * 16] + pmulhrsw m6, m7 + + packuswb m5, m6 + movu [r0 + 80], m5 + + ; mode 6 [row 7] + + movhps [r0 + 312], m5 + + ; mode 6 [row 5] + + movh [r0 + 296], m5 + + ; mode 4 [calculate and store row 4, 5] + + pmaddubsw m4, m1, [r3 + 9 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m1, [r3 + 30 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 160], m4 + + ; mode 5 [row 4, 5] + + pmaddubsw m4, m2, [r3 + 21 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m1, [r3 + 6 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 224], m4 + + ; mode 6 [row 4, 5] + + pmaddubsw m5, m2, [r3 + 1 * 16] + pmulhrsw m5, m7 + + pxor m6, m6 + + packuswb m5, m6 + movh [r0 + 288], m5 + + ; mode 6 [row 6, 7] + + pmaddubsw m5, m2, [r3 + 27 * 16] + pmulhrsw m5, m7 + + pxor m6, m6 + + packuswb m5, m6 + movh [r0 + 304], m5 + + ; mode 5 [calculate row 6] + + pmaddubsw m6, m1, [r3 + 23 * 16] + pmulhrsw m6, m7 + + ; mode 3 [row 4, 5] + + palignr m1, m0, 5 + + punpcklbw m3, m1 + pmaddubsw m4, m3, [r3 + 2 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 28 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 96], m4 + + ; mode 4 [calculate row 7] + + pmaddubsw m5, m3, [r3 + 19 * 16] + pmulhrsw m5, m7 + + ; mode 5 [calculate row 6] + + pmaddubsw m4, m3, [r3 + 8 * 16] + pmulhrsw m4, m7 + + packuswb m6, m4 + movu [r0 + 240], m6 + + ; mode 3 [row 6, 7] + + palignr m2, m0, 6 + palignr m3, m0, 7 + + punpcklbw m1, m2 + pmaddubsw m4, m1, [r3 + 22 * 16] + pmulhrsw m4, m7 + + punpcklbw m2, m3 + pmaddubsw m2, [r3 + 16 * 16] + pmulhrsw m2, m7 + + packuswb m4, m2 + movu [r0 + 112], m4 + + ; mode 4 [calculate row 7] + + pmaddubsw m2, m1, [r3 + 8 * 16] + pmulhrsw m2, m7 + + ; mode 4 [store row 6 and 7] + + packuswb m5, m2 + movu [r0 + 176], m5 + + ; mode 4 [row 2, 3] + + palignr m1, m0, 1 + palignr m2, m0, 2 + palignr m3, m0, 3 + + punpcklbw m1, m2 + pmaddubsw m4, m1, [r3 + 31 * 16] + pmulhrsw m4, m7 + + punpcklbw m2, m3 + pmaddubsw m5, m2, [r3 + 20 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 144], m4 + + ; mode 5 [row 2, 3] + + pmaddubsw m4, m1, [r3 + 19 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m2, [r3 + 4 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 208], m4 + + ; mode 7 [row 6, 7] + + pmaddubsw m4, m1, [r3 + 31 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m2, [r3 + 8 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 368], m4 + + ; mode 10 + + pshufb m1, m0, [tab_Si] + movu [r0 + 512], m1 + movu [r0 + 528], m1 + movu [r0 + 544], m1 + movu [r0 + 560], m1 + + pxor m0, m0 + + pshufb m1, m1, m0 + punpcklbw m1, m0 + + movu m2, [r1] + + pshufb m3, m2, m0 + punpcklbw m3, m0 + + psrldq m4, m2, 1 + punpcklbw m4, m0 + + movu m2, [r1 + 9] + punpcklbw m2, m0 + + psubw m4, m3 + psubw m2, m3 + + psraw m4, 1 + psraw m2, 1 + + paddw m4, m1 + paddw m2, m1 + + packuswb m4, m2 + + pextrb [r0 + 512], m4, 0 + pextrb [r0 + 520], m4, 1 + pextrb [r0 + 528], m4, 2 + pextrb [r0 + 536], m4, 3 + pextrb [r0 + 544], m4, 4 + pextrb [r0 + 552], m4, 5 + pextrb [r0 + 560], m4, 6 + pextrb [r0 + 568], m4, 7 + + ; mode 11 [row 0, 1] + + movu m0, [r1 + 16] + pinsrb m0, [r1], 0 + palignr m1, m0, 1 + punpcklbw m2, m0, m1 + + pmaddubsw m3, m2, [r3 + 30 * 16] + pmulhrsw m3, m7 + + pmaddubsw m4, m2, [r3 + 28 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 576], m3 + + ; mode 11 [row 2, 3] + + pmaddubsw m3, m2, [r3 + 26 * 16] + pmulhrsw m3, m7 + + pmaddubsw m4, m2, [r3 + 24 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 592], m3 + + ; mode 11 [row 4, 5] + + pmaddubsw m3, m2, [r3 + 22 * 16] + pmulhrsw m3, m7 + + pmaddubsw m4, m2, [r3 + 20 * 16] + pmulhrsw m4, m7 + + packuswb m5, m3, m4 + movu [r0 + 608], m5 + + ; mode 12 [row 0, 1] + + pmaddubsw m4, m2, [r3 + 27 * 16] + pmulhrsw m4, m7 + + packuswb m4, m3 + movu [r0 + 640], m4 + + ; mode 11 [row 6, 7] + + pmaddubsw m3, m2, [r3 + 18 * 16] + pmulhrsw m3, m7 + + pmaddubsw m4, m2, [r3 + 16 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 624], m3 + + ; mode 12 [row 2, 3] + + pmaddubsw m3, m2, [r3 + 17 * 16] + pmulhrsw m3, m7 + + pmaddubsw m4, m2, [r3 + 12 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 656], m3 + + ; mode 12 [row 4, 5] + + pmaddubsw m3, m2, [r3 + 7 * 16] + pmulhrsw m3, m7 + + pmaddubsw m4, m2, [r3 + 2 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 672], m3 + + ; mode 12 [row 6, 7] + + pslldq m3, m2, 2 + pinsrb m3, [r1 + 0], 1 + pinsrb m3, [r1 + 6], 0 + + pmaddubsw m4, m3, [r3 + 29 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 24 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 688], m4 + + ; mode 13 [row 0, 1] + + pmaddubsw m4, m2, [r3 + 23 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m2, [r3 + 14 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 704], m4 + + ; mode 13 [row 2, 3] + + pmaddubsw m4, m2, [r3 + 5 * 16] + pmulhrsw m4, m7 + + pinsrb m3, [r1 + 4], 0 + pmaddubsw m5, m3, [r3 + 28 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 720], m4 + + ; mode 13 [row 4, 5] + + pmaddubsw m4, m3, [r3 + 19 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 10 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 736], m4 + + ; mode 13 [row 6, 7] + + pmaddubsw m4, m3, [r3 + 1 * 16] + pmulhrsw m4, m7 + + pslldq m5, m3, 2 + pinsrb m5, [r1 + 4], 1 + pinsrb m5, [r1 + 7], 0 + + pmaddubsw m5, [r3 + 24 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 752], m4 + + ; mode 14 [row 0, 1] + + pmaddubsw m4, m2, [r3 + 19 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m2, [r3 + 6 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 768], m4 + + ; mode 14 [row 2, 3] + + pinsrb m3, [r1 + 2], 0 + + pmaddubsw m4, m3, [r3 + 25 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 12 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 784], m4 + + ; mode 14 [row 4, 5] + + pslldq m1, m3, 2 + pinsrb m1, [r1 + 2], 1 + pinsrb m1, [r1 + 5], 0 + + pmaddubsw m4, m1, [r3 + 31 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m1, [r3 + 18 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 800], m4 + + ; mode 14 [row 6, 7] + + pmaddubsw m4, m1, [r3 + 5 * 16] + pmulhrsw m4, m7 + + pslldq m1, 2 + pinsrb m1, [r1 + 5], 1 + pinsrb m1, [r1 + 7], 0 + + pmaddubsw m5, m1, [r3 + 24 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 816], m4 + + ; mode 15 [row 0, 1] + + pmaddubsw m4, m2, [r3 + 15 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 30 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 832], m4 + + ; mode 15 [row 2, 3] + + pmaddubsw m4, m3, [r3 + 13 * 16] + pmulhrsw m4, m7 + + pslldq m1, m3, 2 + pinsrb m1, [r1 + 2], 1 + pinsrb m1, [r1 + 4], 0 + + pmaddubsw m5, m1, [r3 + 28 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 848], m4 + + ; mode 15 [row 4, 5] + + pmaddubsw m4, m1, [r3 + 11 * 16] + pmulhrsw m4, m7 + + pslldq m1, 2 + pinsrb m1, [r1 + 4], 1 + pinsrb m1, [r1 + 6], 0 + + pmaddubsw m5, m1, [r3 + 26 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 864], m4 + + ; mode 15 [row 6, 7] + + pmaddubsw m4, m1, [r3 + 9 * 16] + pmulhrsw m4, m7 + + pslldq m1, 2 + pinsrb m1, [r1 + 6], 1 + pinsrb m1, [r1 + 8], 0 + + pmaddubsw m1, [r3 + 24 * 16] + pmulhrsw m1, m7 + + packuswb m4, m1 + movu [r0 + 880], m4 + + ; mode 16 [row 0, 1] + + pmaddubsw m4, m2, [r3 + 11 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 22 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 896], m4 + + ; mode 16 [row 2, 3] + + pmaddubsw m4, m3, [r3 + 1 * 16] + pmulhrsw m4, m7 + + pslldq m3, 2 + pinsrb m3, [r1 + 2], 1 + pinsrb m3, [r1 + 3], 0 + + pmaddubsw m5, m3, [r3 + 12 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 912], m4 + + ; mode 16 [row 4, 5] + + pslldq m3, 2 + pinsrb m3, [r1 + 3], 1 + pinsrb m3, [r1 + 5], 0 + + pmaddubsw m4, m3, [r3 + 23 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m3, [r3 + 2 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 928], m4 + + ; mode 16 [row 6, 7] + + pslldq m3, 2 + pinsrb m3, [r1 + 5], 1 + pinsrb m3, [r1 + 6], 0 + + pmaddubsw m4, m3, [r3 + 13 * 16] + pmulhrsw m4, m7 + + pslldq m3, 2 + pinsrb m3, [r1 + 6], 1 + pinsrb m3, [r1 + 8], 0 + + pmaddubsw m3, [r3 + 24 * 16] + pmulhrsw m3, m7 + + packuswb m4, m3 + movu [r0 + 944], m4 + + ; mode 17 [row 0, 1] + + pmaddubsw m4, m2, [r3 + 6 * 16] + pmulhrsw m4, m7 + + pslldq m2, 2 + pinsrb m2, [r1 + 0], 1 + pinsrb m2, [r1 + 1], 0 + + pmaddubsw m3, m2, [r3 + 12 * 16] + pmulhrsw m3, m7 + + packuswb m4, m3 + movu [r0 + 960], m4 + + ; mode 17 [row 2, 3] + + pslldq m2, 2 + pinsrb m2, [r1 + 1], 1 + pinsrb m2, [r1 + 2], 0 + + pmaddubsw m4, m2, [r3 + 18 * 16] + pmulhrsw m4, m7 + + pslldq m2, 2 + pinsrb m2, [r1 + 2], 1 + pinsrb m2, [r1 + 4], 0 + + pmaddubsw m3, m2, [r3 + 24 * 16] + pmulhrsw m3, m7 + + packuswb m4, m3 + movu [r0 + 976], m4 + + ; mode 17 [row 4, 5] + + pslldq m2, 2 + pinsrb m2, [r1 + 4], 1 + pinsrb m2, [r1 + 5], 0 + + pmaddubsw m4, m2, [r3 + 30 * 16] + pmulhrsw m4, m7 + + pmaddubsw m3, m2, [r3 + 4 * 16] + pmulhrsw m3, m7 + + packuswb m4, m3 + movu [r0 + 992], m4 + + ; mode 17 [row 6, 7] + + pslldq m2, 2 + pinsrb m2, [r1 + 5], 1 + pinsrb m2, [r1 + 6], 0 + + pmaddubsw m4, m2, [r3 + 10 * 16] + pmulhrsw m4, m7 + + pslldq m2, 2 + pinsrb m2, [r1 + 6], 1 + pinsrb m2, [r1 + 7], 0 + + pmaddubsw m3, m2, [r3 + 16 * 16] + pmulhrsw m3, m7 + + packuswb m4, m3 + movu [r0 + 1008], m4 + + ; mode 18 [row 0, 1, 2, 3, 4, 5, 6, 7] + + movh m1, [r2] + + pslldq m2, m1, 1 + pinsrb m2, [r2 + 1 + 16], 0 + punpcklqdq m1, m2 + movu [r0 + 1024], m1 + + pslldq m2, 1 + pinsrb m2, [r2 + 2 + 16], 0 + + pslldq m0, m2, 1 + pinsrb m0, [r2 + 3 + 16], 0 + punpcklqdq m2, m0 + movu [r0 + 1040], m2 + + pslldq m0, 1 + pinsrb m0, [r2 + 4 + 16], 0 + + pslldq m2, m0, 1 + pinsrb m2, [r2 + 5 + 16], 0 + punpcklqdq m0, m2 + movu [r0 + 1056], m0 + + pslldq m2, 1 + pinsrb m2, [r2 + 6 + 16], 0 + + pslldq m0, m2, 1 + pinsrb m0, [r2 + 7 + 16], 0 + punpcklqdq m2, m0 + movu [r0 + 1072], m2 + + ; mode 19 [row 0, 1] + + movu m0, [r1] + palignr m1, m0, 1 + punpcklbw m0, m1 + + pmaddubsw m1, m0, [r3 + 6 * 16] + pmulhrsw m1, m7 + + pslldq m2, m0, 2 + pinsrb m2, [r1], 1 + pinsrb m2, [r1 + 1 + 16], 0 + + pmaddubsw m3, m2, [r3 + 12 * 16] + pmulhrsw m3, m7 + + packuswb m1, m3 + movu [r0 + 1088], m1 + + ; mode 19 [row 2, 3] + + pslldq m2, 2 + pinsrb m2, [r1 + 1 + 16], 1 + pinsrb m2, [r1 + 2 + 16], 0 + + pmaddubsw m4, m2, [r3 + 18 * 16] + pmulhrsw m4, m7 + + pslldq m2, 2 + pinsrb m2, [r1 + 2 + 16], 1 + pinsrb m2, [r1 + 4 + 16], 0 + + pmaddubsw m5, m2, [r3 + 24 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 1104], m4 + + ; mode 19 [row 4, 5] + + pslldq m2, 2 + pinsrb m2, [r1 + 4 + 16], 1 + pinsrb m2, [r1 + 5 + 16], 0 + + pmaddubsw m4, m2, [r3 + 30 * 16] + pmulhrsw m4, m7 + + pmaddubsw m5, m2, [r3 + 4 * 16] + pmulhrsw m5, m7 + + packuswb m4, m5 + movu [r0 + 1120], m4 + + ; mode 19 [row 6, 7] + + pslldq m2, 2 + pinsrb m2, [r1 + 5 + 16], 1 + pinsrb m2, [r1 + 6 + 16], 0 + + pmaddubsw m4, m2, [r3 + 10 * 16] + pmulhrsw m4, m7 + + pslldq m2, 2 + pinsrb m2, [r1 + 6 + 16], 1 + pinsrb m2, [r1 + 7 + 16], 0 + + pmaddubsw m2, [r3 + 16 * 16] + pmulhrsw m2, m7 + + packuswb m4, m2 + movu [r0 + 1136], m4 + + ; mode 20 [row 0, 1] + + pmaddubsw m3, m0, [r3 + 11 * 16] + pmulhrsw m3, m7 + + pslldq m1, m0, 2 + pinsrb m1, [r1 + 0], 1 + pinsrb m1, [r1 + 2 + 16], 0 + + pmaddubsw m4, m1, [r3 + 22 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 1152], m3 + + ; mode 20 [row 2, 3] + + pmaddubsw m3, m1, [r3 + 1 * 16] + pmulhrsw m3, m7 + + pslldq m2, m1, 2 + pinsrb m2, [r1 + 2 + 16], 1 + pinsrb m2, [r1 + 3 + 16], 0 + + pmaddubsw m4, m2, [r3 + 12 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 1168], m3 + + ; mode 20 [row 4, 5] + + pslldq m2, 2 + pinsrb m2, [r1 + 3 + 16], 1 + pinsrb m2, [r1 + 5 + 16], 0 + + pmaddubsw m3, m2, [r3 + 23 * 16] + pmulhrsw m3, m7 + + pmaddubsw m4, m2, [r3 + 2 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 1184], m3 + + ; mode 20 [row 6, 7] + + pslldq m2, 2 + pinsrb m2, [r1 + 5 + 16], 1 + pinsrb m2, [r1 + 6 + 16], 0 + + pmaddubsw m3, m2, [r3 + 13 * 16] + pmulhrsw m3, m7 + + pslldq m2, 2 + pinsrb m2, [r1 + 6 + 16], 1 + pinsrb m2, [r1 + 8 + 16], 0 + + pmaddubsw m4, m2, [r3 + 24 * 16] + pmulhrsw m4, m7 + + packuswb m3, m4 + movu [r0 + 1200], m3 + + ; mode 21 [row 0, 1] + + pmaddubsw m2, m0, [r3 + 15 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m1, [r3 + 30 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1216], m2 + + ; mode 21 [row 2, 3] + + pmaddubsw m2, m1, [r3 + 13 * 16] + pmulhrsw m2, m7 + + pslldq m3, m1, 2 + pinsrb m3, [r1 + 2 + 16], 1 + pinsrb m3, [r1 + 4 + 16], 0 + + pmaddubsw m4, m3, [r3 + 28 * 16] + pmulhrsw m4, m7 + + packuswb m2, m4 + movu [r0 + 1232], m2 + + ; mode 21 [row 4, 5] + + pmaddubsw m2, m3, [r3 + 11 * 16] + pmulhrsw m2, m7 + + pslldq m3, 2 + pinsrb m3, [r1 + 4 + 16], 1 + pinsrb m3, [r1 + 6 + 16], 0 + + pmaddubsw m4, m3, [r3 + 26 * 16] + pmulhrsw m4, m7 + + packuswb m2, m4 + movu [r0 + 1248], m2 + + ; mode 21 [row 6, 7] + + pmaddubsw m2, m3, [r3 + 9 * 16] + pmulhrsw m2, m7 + + pslldq m3, 2 + pinsrb m3, [r1 + 6 + 16], 1 + pinsrb m3, [r1 + 8 + 16], 0 + + pmaddubsw m4, m3, [r3 + 24 * 16] + pmulhrsw m4, m7 + + packuswb m2, m4 + movu [r0 + 1264], m2 + + ; mode 22 [row 0, 1] + + pmaddubsw m2, m0, [r3 + 19 * 16] + pmulhrsw m2, m7 + + pmaddubsw m4, m0, [r3 + 6 * 16] + pmulhrsw m4, m7 + + packuswb m2, m4 + movu [r0 + 1280], m2 + + ; mode 22 [row 2, 3] + + pmaddubsw m2, m1, [r3 + 25 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m1, [r3 + 12 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1296], m2 + + ; mode 22 [row 4, 5] + + pslldq m1, 2 + pinsrb m1, [r1 + 5 + 16], 0 + pinsrb m1, [r1 + 2 + 16], 1 + + pmaddubsw m2, m1, [r3 + 31 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m1, [r3 + 18 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1312], m2 + + ; mode 22 [row 6, 7] + + pmaddubsw m2, m1, [r3 + 5 * 16] + pmulhrsw m2, m7 + + pslldq m1, 2 + pinsrb m1, [r1 + 5 + 16], 1 + pinsrb m1, [r1 + 7 + 16], 0 + + pmaddubsw m1, [r3 + 24 * 16] + pmulhrsw m1, m7 + + packuswb m2, m1 + movu [r0 + 1328], m2 + + ; mode 23 [row 0, 1] + + pmaddubsw m2, m0, [r3 + 23 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m0, [r3 + 14 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1344], m2 + + ; mode 23 [row 2, 3] + + pmaddubsw m2, m0, [r3 + 5 * 16] + pmulhrsw m2, m7 + + pslldq m1, m0, 2 + pinsrb m1, [r1], 1 + pinsrb m1, [r1 + 4 + 16], 0 + + pmaddubsw m3, m1, [r3 + 28 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1360], m2 + + ; mode 23 [row 4, 5] + + pmaddubsw m2, m1, [r3 + 19 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m1, [r3 + 10 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1376], m2 + + ; mode 23 [row 6, 7] + + pmaddubsw m2, m1, [r3 + 1 * 16] + pmulhrsw m2, m7 + + pslldq m3, m1, 2 + pinsrb m3, [r1 + 4 + 16], 1 + pinsrb m3, [r1 + 7 + 16], 0 + + pmaddubsw m3, [r3 + 24 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1392], m2 + + ; mode 24 [row 0, 1] + + pmaddubsw m2, m0, [r3 + 27 * 16] + pmulhrsw m2, m7 + + pmaddubsw m5, m0, [r3 + 22 * 16] + pmulhrsw m5, m7 + + packuswb m2, m5 + movu [r0 + 1408], m2 + + ; mode 24 [row 2, 3] + + pmaddubsw m2, m0, [r3 + 17 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m0, [r3 + 12 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1424], m2 + + ; mode 24 [row 4, 5] + + pmaddubsw m2, m0, [r3 + 7 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m0, [r3 + 2 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1440], m2 + + ; mode 24 [row 6, 7] + + pinsrb m1, [r1 + 6 + 16], 0 + + pmaddubsw m2, m1, [r3 + 29 * 16] + pmulhrsw m2, m7 + + pmaddubsw m1, [r3 + 24 * 16] + pmulhrsw m1, m7 + + packuswb m2, m1 + movu [r0 + 1456], m2 + + ; mode 25 [row 0, 1] + + pmaddubsw m2, m0, [r3 + 30 * 16] + pmulhrsw m2, m7 + + pmaddubsw m1, m0, [r3 + 28 * 16] + pmulhrsw m1, m7 + + packuswb m2, m1 + movu [r0 + 1472], m2 + + ; mode 25 [row 2, 3] + + pmaddubsw m2, m0, [r3 + 26 * 16] + pmulhrsw m2, m7 + + pmaddubsw m1, m0, [r3 + 24 * 16] + pmulhrsw m1, m7 + + packuswb m2, m1 + movu [r0 + 1488], m2 + + ; mode 25 [row 4, 5] + + pmaddubsw m1, m0, [r3 + 20 * 16] + pmulhrsw m1, m7 + + packuswb m5, m1 + movu [r0 + 1504], m5 + + ; mode 25 [row 6, 7] + + pmaddubsw m2, m0, [r3 + 18 * 16] + pmulhrsw m2, m7 + + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m7 + + packuswb m2, m1 + movu [r0 + 1520], m2 + + ; mode 26 + + movu m0, [r1 + 1] + + pshufb m1, m0, [tab_Si] + movu [r0 + 1536], m1 + movu [r0 + 1552], m1 + movu [r0 + 1568], m1 + movu [r0 + 1584], m1 + + pxor m5, m5 + + pshufb m1, m1, m5 + punpcklbw m1, m5 + + movu m2, [r1 + 16] + pinsrb m2, [r1], 0 + + pshufb m3, m2, m5 + punpcklbw m3, m5 + + psrldq m4, m2, 1 + punpcklbw m4, m5 + + movu m2, [r1 + 9 + 16] + punpcklbw m2, m5 + + psubw m4, m3 + psubw m2, m3 + + psraw m4, 1 + psraw m2, 1 + + paddw m4, m1 + paddw m2, m1 + + packuswb m4, m2 + + pextrb [r0 + 1536], m4, 0 + pextrb [r0 + 1544], m4, 1 + pextrb [r0 + 1552], m4, 2 + pextrb [r0 + 1560], m4, 3 + pextrb [r0 + 1568], m4, 4 + pextrb [r0 + 1576], m4, 5 + pextrb [r0 + 1584], m4, 6 + pextrb [r0 + 1592], m4, 7 + + ; mode 27 [row 0, 1] + + palignr m6, m0, 1 + punpcklbw m4, m0, m6 + + pmaddubsw m1, m4, [r3 + 2 * 16] + pmulhrsw m1, m7 + + pmaddubsw m2, m4, [r3 + 4 * 16] + pmulhrsw m2, m7 + + packuswb m1, m2 + movu [r0 + 1600], m1 + + ; mode 27 [row 2, 3] + + pmaddubsw m1, m4, [r3 + 6 * 16] + pmulhrsw m1, m7 + + pmaddubsw m2, m4, [r3 + 8 * 16] + pmulhrsw m2, m7 + + packuswb m1, m2 + movu [r0 + 1616], m1 + + ; mode 27 [row 4, 5] + + pmaddubsw m3, m4, [r3 + 10 * 16] + pmulhrsw m3, m7 + + pmaddubsw m2, m4, [r3 + 12 * 16] + pmulhrsw m2, m7 + + packuswb m1, m3, m2 + movu [r0 + 1632], m1 + + ; mode 27 [row 6, 7] + + pmaddubsw m1, m4, [r3 + 14 * 16] + pmulhrsw m1, m7 + + pmaddubsw m2, m4, [r3 + 16 * 16] + pmulhrsw m2, m7 + + packuswb m1, m2 + movu [r0 + 1648], m1 + + ; mode 28 [row 0, 1] + + pmaddubsw m1, m4, [r3 + 5 * 16] + pmulhrsw m1, m7 + + packuswb m1, m3 + movu [r0 + 1664], m1 + + ; mode 28 [row 2, 3] + + pmaddubsw m1, m4, [r3 + 15 * 16] + pmulhrsw m1, m7 + + pmaddubsw m2, m4, [r3 + 20 * 16] + pmulhrsw m2, m7 + + packuswb m1, m2 + movu [r0 + 1680], m1 + + ; mode 28 [row 4, 5] + + pmaddubsw m1, m4, [r3 + 25 * 16] + pmulhrsw m1, m7 + + pmaddubsw m2, m4, [r3 + 30 * 16] + pmulhrsw m2, m7 + + packuswb m1, m2 + movu [r0 + 1696], m1 + + ; mode 28 [row 6, 7] + + palignr m1, m0, 2 + punpcklbw m5, m6, m1 + + pmaddubsw m2, m5, [r3 + 3 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m5, [r3 + 8 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1712], m2 + + ; mode 29 [row 0, 1] + + pmaddubsw m2, m4, [r3 + 9 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m4, [r3 + 18 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1728], m2 + + ; mode 29 [row 2, 3] + + pmaddubsw m2, m4, [r3 + 27 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m5, [r3 + 4 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1744], m2 + + ; mode 29 [row 4, 5] + + pmaddubsw m2, m5, [r3 + 13 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m5, [r3 + 22 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1760], m2 + + ; mode 29 [row 6, 7] + + pmaddubsw m2, m5, [r3 + 31 * 16] + pmulhrsw m2, m7 + + palignr m6, m0, 3 + punpcklbw m1, m6 + + pmaddubsw m3, m1, [r3 + 8 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1776], m2 + + ; mode 32 [row 2] + + movh [r0 + 1936], m2 + + ; mode 30 [row 0, 1] + + pmaddubsw m2, m4, [r3 + 13 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m4, [r3 + 26 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1792], m2 + + ; mode 30 [row 2, 3] + + pmaddubsw m2, m5, [r3 + 7 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m5, [r3 + 20 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1808], m2 + + ; mode 33 [row 1] + + movhps [r0 + 1992], m2 + + ; mode 30 [row 4, 5] + + pmaddubsw m2, m1, [r3 + 1 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m1, [r3 + 14 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1824], m2 + + ; mode 33 [row 2] + + movhps [r0 + 2000], m2 + + ; mode 30 [row 6, 7] + + pmaddubsw m2, m1, [r3 + 27 * 16] + pmulhrsw m2, m7 + + psrldq m0, 4 + punpcklbw m6, m0 + + pmaddubsw m3, m6, [r3 + 8 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1840], m2 + + ; mode 33 [row 3] + + movhps [r0 + 2008], m2 + + ; mode 31 [row 0, 1] + + pmaddubsw m2, m4, [r3 + 17 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m5, [r3 + 2 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1856], m2 + + ; mode 31 [row 2, 3] + + pmaddubsw m2, m5, [r3 + 19 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m1, [r3 + 4 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1872], m2 + + ; mode 31 [row 4, 5] + + pmaddubsw m2, m1, [r3 + 21 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m6, [r3 + 6 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1888], m2 + + ; mode 31 [row 6, 7] + + pmaddubsw m2, m6, [r3 + 23 * 16] + pmulhrsw m2, m7 + + movu m3, [r1 + 6] + punpcklbw m0, m3 + + pmaddubsw m3, m0, [r3 + 8 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1904], m2 + + ; mode 32 [row 0, 1] + + pmaddubsw m2, m4, [r3 + 21 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m5, [r3 + 10 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1920], m2 + + ; mode 32 [row 3] + + pmaddubsw m2, m1, [r3 + 20 * 16] + pmulhrsw m2, m7 + + pxor m3, m3 + + packuswb m2, m3 + movh [r0 + 1944], m2 + + ; mode 32 [row 4, 5] + + pmaddubsw m2, m6, [r3 + 9 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m6, [r3 + 30 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1952], m2 + + ; mode 33 [row 4, 5] + + pmaddubsw m2, m0, [r3 + 2 * 16] + pmulhrsw m2, m7 + + pmaddubsw m3, m0, [r3 + 28 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 2016], m2 + + ; mode 32 [row 6] + + pmaddubsw m2, m0, [r3 + 19 * 16] + pmulhrsw m2, m7 + + ; mode 32 [row 7] + + movu m0, [r1 + 6] + palignr m3, m0, 1 + punpcklbw m0, m3 + + pmaddubsw m3, m0, [r3 + 8 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 1968], m2 + + ; mode 33 [row 6, 7] + + pmaddubsw m2, m0, [r3 + 22 * 16] + pmulhrsw m2, m7 + + movu m0, [r1 + 7] + palignr m3, m0, 1 + punpcklbw m0, m3 + + pmaddubsw m3, m0, [r3 + 16 * 16] + pmulhrsw m3, m7 + + packuswb m2, m3 + movu [r0 + 2032], m2 + + ; mode 33 [row 0] + + pmaddubsw m2, m4, [r3 + 26 * 16] + pmulhrsw m2, m7 + + pxor m3, m3 + + packuswb m2, m3 + movh [r0 + 1984], m2 + + ; mode 34 [row 0, 1, 2, 3, 4, 5, 6, 7] + + movu m0, [r2 + 2] + palignr m1, m0, 1 + punpcklqdq m2, m0, m1 + movu [r0 + 2048], m2 + + palignr m1, m0, 2 + palignr m2, m0, 3 + punpcklqdq m1, m2 + movu [r0 + 2064], m1 + + palignr m1, m0, 4 + palignr m2, m0, 5 + punpcklqdq m1, m2 + movu [r0 + 2080], m1 + + palignr m1, m0, 6 + palignr m2, m0, 7 + punpcklqdq m1, m2 + movu [r0 + 2096], m1 +RET + +;-------------------------------------------------------------------------------- +; void all_angs_pred_16x16(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) +;-------------------------------------------------------------------------------- +INIT_XMM sse4 +cglobal all_angs_pred_16x16, 3,4,8 + ; mode 2 + + movu m0, [r2 + 2 + 32] + movu [r0 + 0 * 16], m0 + + movu m1, m0 + + movu m6, [r2 + 18 + 32] + palignr m5, m6, m0, 1 + movu [r0 + 1 * 16], m5 + + movu m4, m5 + + palignr m5, m6, m0, 2 + movu [r0 + 2 * 16], m5 + palignr m5, m6, m0, 3 + movu [r0 + 3 * 16], m5 + palignr m5, m6, m0, 4 + movu [r0 + 4 * 16], m5 + palignr m5, m6, m0, 5 + movu [r0 + 5 * 16], m5 + palignr m5, m6, m0, 6 + movu [r0 + 6 * 16], m5 + palignr m5, m6, m0, 7 + movu [r0 + 7 * 16], m5 + + movu m7, m5 + + palignr m5, m6, m0, 8 + movu [r0 + 8 * 16], m5 + + movu m2, m5 + + palignr m5, m6, m0, 9 + movu [r0 + 9 * 16], m5 + + palignr m3, m6, m0, 10 + movu [r0 + 10 * 16], m3 + palignr m3, m6, m0, 11 + movu [r0 + 11 * 16], m3 + palignr m3, m6, m0, 12 + movu [r0 + 12 * 16], m3 + + ; mode 3 [row 15] + movu [r0 + (3-2)*16*16 + 15 * 16], m3 + + palignr m3, m6, m0, 13 + movu [r0 + 13 * 16], m3 + palignr m3, m6, m0, 14 + movu [r0 + 14 * 16], m3 + palignr m3, m6, m0, 15 + movu [r0 + 15 * 16], m3 + + ; mode 3 [row 0] + lea r3, [ang_table] + movu m3, [pw_1024] + movu m0, [r2 + 1 + 32] + punpcklbw m0, m1 + + ; mode 17 [row 8 - second half] + pmaddubsw m1, m0, [r3 + 22 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 248 * 16 + 8], m1 + ; mode 17 [row 8 - second half] end + + pmaddubsw m1, m0, [r3 + 26 * 16] + pmulhrsw m1, m3 + punpcklbw m7, m2 + pmaddubsw m2, m7, [r3 + 26 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 16 * 16], m1 + + ;mode 6 [row 1] + movu [r0 + 65 * 16], m1 + + ; mode 4 [row 0] + pmaddubsw m1, m0, [r3 + 21 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 21 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 32 * 16], m1 + + ; mode 5 [row 0] + pmaddubsw m1, m0, [r3 + 17 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 17 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 48 * 16], m1 + + ; mode 6 [row 0] + pmaddubsw m1, m0, [r3 + 13 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 13 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 64 * 16], m1 + + ; mode 7 [row 0] + pmaddubsw m1, m0, [r3 + 9 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 9 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 80 * 16], m1 + + ; mode 7 [row 1] + pmaddubsw m1, m0, [r3 + 18 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 18 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 81 * 16], m1 + + ; mode 7 [row 2] + pmaddubsw m1, m0, [r3 + 27 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 27 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 82 * 16], m1 + + ; mode 8 [row 0] + pmaddubsw m1, m0, [r3 + 5 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 5 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 96 * 16], m1 + + ; mode 8 [row 1] + pmaddubsw m1, m0, [r3 + 10 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 10 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 97 * 16], m1 + + ; mode 8 [row 2] + pmaddubsw m1, m0, [r3 + 15 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 15 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 98 * 16], m1 + + ; mode 8 [row 3] + pmaddubsw m1, m0, [r3 + 20 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 20 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 99 * 16], m1 + + ; mode 8 [row 4] + pmaddubsw m1, m0, [r3 + 25 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 25 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 100 * 16], m1 + + ; mode 8 [row 5] + pmaddubsw m1, m0, [r3 + 30 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 30 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 101 * 16], m1 + + ; mode 15 [row 13 - second half] + pmaddubsw m1, m0, [r3 + 18 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 221 * 16 + 8], m1 + ; mode 15 [row 13 - second half] end + + ; mode 15 [row 14 - second half] + pmaddubsw m1, m0, [r3 + 1 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 222 * 16 + 8], m1 + ; mode 15 [row 14 - second half] end + + ; mode 16 [row 10 - second half] + pmaddubsw m1, m0, [r3 + 25 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 234 * 16 + 8], m1 + ; mode 16 [row 10 - second half] end + + ; mode 16 [row 11 - second half] + pmaddubsw m1, m0, [r3 + 4 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 235 * 16 + 8], m1 + ; mode 16 [row 11 - second half] end + + ; mode 3 [row 1] + movu m6, [r3 + 20 * 16] + movu m0, [r2 + 2 + 32] + punpcklbw m0, m4 + + ; mode 17 [row 7 - second half] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 247 * 16 + 8], m1 + + ; mode 17 [row 7 - second half] end + pmaddubsw m1, m0, m6 + pmulhrsw m1, m3 + movu m2, [r2 + 10 + 32] + punpcklbw m2, m5 + pmaddubsw m4, m2, m6 + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 17 * 16], m1 + + ;mode 6 [row 3] + movu [r0 + 67 * 16], m1 + + ; mode 4 row [row 1] + pmaddubsw m1, m0, [r3 + 10 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 10 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 33 * 16], m1 + + ; mode 4 row [row 2] + pmaddubsw m1, m0, [r3 + 31 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 31 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 34 * 16], m1 + + ; mode 7 [row 6] + movu [r0 + 86 * 16], m1 + + ; mode 5 row [row 1] + pmaddubsw m1, m0, [r3 + 2 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 2 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 49 * 16], m1 + + ; mode 5 row [row 2] + pmaddubsw m1, m0, [r3 + 19 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 19 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 50 * 16], m1 + + ; mode 6 [row 2] + pmaddubsw m1, m0, [r3 + 7 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 7 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 66 * 16], m1 + + ; mode 7 [row 3] + pmaddubsw m1, m0, [r3 + 4 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 83 * 16], m1 + + ; mode 7 [row 4] + pmaddubsw m1, m0, [r3 + 13 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 13 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 84 * 16], m1 + + ; mode 8 [row 8] + movu [r0 + 104 * 16], m1 + + ; mode 7 [row 5] + pmaddubsw m1, m0, [r3 + 22 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 22 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 85 * 16], m1 + + ; mode 8 [row 6] + pmaddubsw m1, m0, [r3 + 3 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 3 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 102 * 16], m1 + + ; mode 8 [row 7] + pmaddubsw m1, m0, [r3 + 8 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 8 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 103 * 16], m1 + + ; mode 8 [row 9] + pmaddubsw m1, m0, [r3 + 18 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 18 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 105 * 16], m1 + + ; mode 8 [row 10] + pmaddubsw m1, m0, [r3 + 23 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 23 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 106 * 16], m1 + + ; mode 8 [row 11] + pmaddubsw m1, m0, [r3 + 28 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 28 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 107 * 16], m1 + + ; mode 3 [row 2] + movu m0, [r2 + 3 + 32] + movd m1, [r2 + 19 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + + ; mode 17 [row 6 - second half] + pmaddubsw m1, m0, [r3 + 10 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 246 * 16 + 8], m1 + ; mode 17 [row 6 - second half] end + + pmaddubsw m1, m0, [r3 + 14 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 11 + 32] + movd m4, [r2 + 27 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 14 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 18 * 16], m1 + + ; mode 6 [row 5] + movu [r0 + 69 * 16], m1 + + ; mode 4 row [row 3] + pmaddubsw m1, m0, [r3 + 20 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 20 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 35 * 16], m1 + + ; mode 5 row [row 3] + pmaddubsw m1, m0, [r3 + 4 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 51 * 16], m1 + + ; mode 5 row [row 4] + pmaddubsw m1, m0, [r3 + 21 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 21 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 52 * 16], m1 + + ; mode 6 [row 4] + pmaddubsw m1, m0, [r3 + 1 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 68 * 16], m1 + + ; mode 6 [row 6] + pmaddubsw m1, m0, [r3 + 27 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 27 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 70 * 16], m1 + + ; mode 7 [row 7] + pmaddubsw m1, m0, [r3 + 8 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 8 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 87 * 16], m1 + + ; mode 7 [row 8] + pmaddubsw m1, m0, [r3 + 17 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 17 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 88 * 16], m1 + + ; mode 7 [row 9] + pmaddubsw m1, m0, [r3 + 26 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 26 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 89 * 16], m1 + + ; mode 8 [row 12] + pmaddubsw m1, m0, [r3 + 1 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 108 * 16], m1 + + ; mode 8 [row 13] + pmaddubsw m1, m0, [r3 + 6 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 6 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 109 * 16], m1 + + ; mode 8 [row 14] + pmaddubsw m1, m0, [r3 + 11 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 11 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 110 * 16], m1 + + ; mode 8 [row 15] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 111 * 16], m1 + + ; mode 3 [row 3] + movu m0, [r2 + 4 + 32] + movd m1, [r2 + 20 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + + ; mode 17 [row 4 - second half] + pmaddubsw m1, m0, [r3 + 30 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 244 * 16 + 8], m1 + ; mode 17 [row 4 - second half] end + + ; mode 17 [row 5 - second half] + pmaddubsw m1, m0, [r3 + 4 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 245 * 16 + 8], m1 + ; mode 17 [row 5 - second half] end + + pmaddubsw m1, m0, [r3 + 8 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 12 + 32] + movd m4, [r2 + 28 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 8 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 19 * 16], m1 + + ; mode 6 [row 7] + movu [r0 + 71 * 16], m1 + + ; mode 4 row [row 4] + pmaddubsw m1, m0, [r3 + 9 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 9 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 36 * 16], m1 + + ; mode 4 row [row 5] + pmaddubsw m1, m0, [r3 + 30 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 30 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 37 * 16], m1 + + ; mode 7 row [row 13] + movu [r0 + 93 * 16], m1 + + ; mode 5 row [row 5] + pmaddubsw m1, m0, [r3 + 6 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 6 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 53 * 16], m1 + + ; mode 5 row [row 6] + pmaddubsw m1, m0, [r3 + 23 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 23 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 54 * 16], m1 + + ; mode 6 [row 8] + pmaddubsw m1, m0, [r3 + 21 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 21 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 72 * 16], m1 + + ; mode 7 [row 12] + movu [r0 + 92 * 16], m1 + + ; mode 7 [row 10] + pmaddubsw m1, m0, [r3 + 3 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 3 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 90 * 16], m1 + + ; mode 7 [row 11] + pmaddubsw m1, m0, [r3 + 12 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 12 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 91 * 16], m1 + + ; mode 3 [row 4] + movu m0, [r2 + 5 + 32] + movd m1, [r2 + 20 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + + ; mode 17 [row 3 - second half] + pmaddubsw m1, m0, [r3 + 24 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 243 * 16 + 8], m1 + + ; mode 17 [row 3 - second half] end + pmaddubsw m1, m0, [r3 + 2 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 13 + 32] + movd m4, [r2 + 29 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 2 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 20 * 16], m1 + + ;mode 6 [row 9] + movu [r0 + 73 * 16], m1 + + ; mode 4 row [row 6] + movu m6, [r3 + 19 * 16] + pmaddubsw m1, m0, m6 + pmulhrsw m1, m3 + pmaddubsw m4, m2, m6 + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 38 * 16], m1 + + ; mode 3 [row 5] + pmaddubsw m1, m0, [r3 + 28 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 28 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 21 * 16], m1 + + ;mode 6 [row 11] + movu [r0 + 75 * 16], m1 + + ; mode 5 row [row 7] + pmaddubsw m1, m0, [r3 + 8 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 8 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 55 * 16], m1 + + ; mode 5 row [row 8] + pmaddubsw m1, m0, [r3 + 25 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 25 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 56 * 16], m1 + + ; mode 6 [row 10] + pmaddubsw m1, m0, [r3 + 15 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 15 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 74 * 16], m1 + + ; mode 7 [row 14] + pmaddubsw m1, m0, [r3 + 7 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 7 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 94 * 16], m1 + + ; mode 7 [row 15] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 95 * 16], m1 + + ; mode 3 [row 6] + movu m0, [r2 + 6 + 32] + movd m1, [r2 + 22 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + + ; mode 17 [row 2 - second half] + pmaddubsw m1, m0, [r3 + 18 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 242 * 16 + 8], m1 + ; mode 17 [row 2 - second half] end + + pmaddubsw m1, m0, [r3 + 22 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 14 + 32] + movd m4, [r2 + 30 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 22 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 22 * 16], m1 + + ; mode 6 [row 13] + movu [r0 + 77 * 16], m1 + + ; mode 4 row [row 7] + pmaddubsw m1, m0, [r3 + 8 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 8 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 39 * 16], m1 + + ; mode 4 row [row 8] + pmaddubsw m1, m0, [r3 + 29 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 40 * 16], m1 + + ; mode 5 row [row 9] + pmaddubsw m1, m0, [r3 + 10 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 10 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 57 * 16], m1 + + ; mode 5 row [row 10] + pmaddubsw m1, m0, [r3 + 27 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 27 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 58 * 16], m1 + + ; mode 6 [row 12] + pmaddubsw m1, m0, [r3 + 9 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 9 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 76 * 16], m1 + + ; mode 3 [row 7] + movu m0, [r2 + 7 + 32] + movd m1, [r2 + 27 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + + ; mode 17 [row 1 - second half] + pmaddubsw m1, m0, [r3 + 12 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 241 * 16 + 8], m1 + ; mode 17 [row 1 - second half] end + + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 15 + 32] + movd m4, [r2 + 25 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 23 * 16], m1 + + ; mode 6 [row 15] + movu [r0 + 79 * 16], m1 + + ; mode 4 row [row 9] + pmaddubsw m1, m0, [r3 + 18 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 18 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 41 * 16], m1 + + ; mode 5 row [row 11] + pmaddubsw m1, m0, [r3 + 12 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 12 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 59 * 16], m1 + + ; mode 5 row [row 12] + pmaddubsw m1, m0, [r3 + 29 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 60 * 16], m1 + + ; mode 6 [row 14] + pmaddubsw m1, m0, [r3 + 3 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 3 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 78 * 16], m1 + + ; mode 3 [row 8] + movu m0, [r2 + 8 + 32] + movd m1, [r2 + 24 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 10 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 16 + 32] + psrldq m4, m2, 1 + pinsrb m4, [r2 + 32], 15 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 10 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 24 * 16], m1 + + ; mode 4 row [row 10] + pmaddubsw m1, m0, [r3 + 7 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 7 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 42 * 16], m1 + + ; mode 4 row [row 11] + pmaddubsw m1, m0, [r3 + 28 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 28 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 43 * 16], m1 + + ; mode 5 row [row 13] + pmaddubsw m1, m0, [r3 + 14 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 14 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 61 * 16], m1 + + ; mode 5 row [row 14] + pmaddubsw m1, m0, [r3 + 31 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 31 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 62 * 16], m1 + + ; mode 3 [row 9] + movu m0, [r2 + 9 + 32] + movd m1, [r2 + 16 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 4 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 17 + 32] + movd m4, [r2 + 33 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 25 * 16], m1 + + ; mode 4 row [row 12] + pmaddubsw m1, m0, [r3 + 17 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 17 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 44 * 16], m1 + + ; mode 3 [row 10] + pmaddubsw m1, m0, [r3 + 30 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 30 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 26 * 16], m1 + + ; mode 5 row [row 15] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 63 * 16], m1 + + ; mode 3 [row 11] + movu m0, [r2 + 10 + 32] + movd m1, [r2 + 26 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 24 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 18 + 32] + movd m4, [r2 + 34 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 27 * 16], m1 + + ; mode 4 row [row 13] + pmaddubsw m1, m0, [r3 + 6 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 6 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 45 * 16], m1 + + ; mode 4 row [row 14] + pmaddubsw m1, m0, [r3 + 27 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 27 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 46 * 16], m1 + + ; mode 3 [row 12] + movu m0, [r2 + 11 + 32] + movd m1, [r2 + 27 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 18 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 19 + 32] + movd m4, [r2 + 35 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 18 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 28 * 16], m1 + + ; mode 4 row [row 15] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m2, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 47 * 16], m1 + + ; mode 3 [row 13] + movu m0, [r2 + 12 + 32] + movd m1, [r2 + 28 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 12 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 20 + 32] + movd m4, [r2 + 36 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 12 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 29 * 16], m1 + + ; mode 3 [row 14] + movu m0, [r2 + 13 + 32] + movd m1, [r2 + 29 + 32] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 6 * 16] + pmulhrsw m1, m3 + movu m2, [r2 + 21 + 32] + movd m4, [r2 + 37 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m2, [r3 + 6 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 30 * 16], m1 + + ; mode 9 + movu m0, [r1 + 1 + 32] + movd m1, [r1 + 17 + 32] + palignr m1, m0, 1 + + ; mode 9 [row 15] + movu [r0 + 127 * 16], m1 + + ; mode 9 [row 0] + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 2 * 16] + pmulhrsw m1, m3 + movu m7, [r1 + 9 + 32] + movd m4, [r2 + 25 + 32] + palignr m2, m7, 1 + punpcklbw m7, m2 + pmaddubsw m2, m7, [r3 + 2 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 112 * 16], m1 + + ; mode 9 [row 1] + pmaddubsw m1, m0, [r3 + 4 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 4 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 113 * 16], m1 + + ; mode 9 [row 2] + pmaddubsw m1, m0, [r3 + 6 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 6 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 114 * 16], m1 + + ; mode 9 [row 3] + pmaddubsw m1, m0, [r3 + 8 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 8 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 115 * 16], m1 + + ; mode 9 [row 4] + pmaddubsw m1, m0, [r3 + 10 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 10 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 116 * 16], m1 + + ; mode 9 [row 5] + pmaddubsw m1, m0, [r3 + 12 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 12 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 117 * 16], m1 + + ; mode 9 [row 6] + pmaddubsw m1, m0, [r3 + 14 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 14 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 118 * 16], m1 + + ; mode 9 [row 7] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 16 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 119 * 16], m1 + + ; mode 9 [row 8] + pmaddubsw m1, m0, [r3 + 18 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 18 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 120 * 16], m1 + + ; mode 9 [row 9] + pmaddubsw m1, m0, [r3 + 20 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 20 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 121 * 16], m1 + + ; mode 9 [row 10] + pmaddubsw m1, m0, [r3 + 22 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 22 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 122 * 16], m1 + + ; mode 9 [row 11] + pmaddubsw m1, m0, [r3 + 24 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 24 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 123 * 16], m1 + + ; mode 9 [row 12] + pmaddubsw m1, m0, [r3 + 26 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 26 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 124 * 16], m1 + + ; mode 9 [row 13] + pmaddubsw m1, m0, [r3 + 28 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 28 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 125 * 16], m1 + + ; mode 9 [row 14] + pmaddubsw m1, m0, [r3 + 30 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 30 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 126 * 16], m1 + + ; mode 10 + movu m1, [r1 + 1 + 32] + movu [r0 + 128 * 16], m1 + movu [r0 + 129 * 16], m1 + movu [r0 + 130 * 16], m1 + movu [r0 + 131 * 16], m1 + movu [r0 + 132 * 16], m1 + movu [r0 + 133 * 16], m1 + movu [r0 + 134 * 16], m1 + movu [r0 + 135 * 16], m1 + movu [r0 + 136 * 16], m1 + movu [r0 + 137 * 16], m1 + movu [r0 + 138 * 16], m1 + movu [r0 + 139 * 16], m1 + movu [r0 + 140 * 16], m1 + movu [r0 + 141 * 16], m1 + movu [r0 + 142 * 16], m1 + movu [r0 + 143 * 16], m1 + + pxor m0, m0 + pshufb m1, m1, m0 + punpcklbw m1, m0 + pinsrb m2, [r1], 0 + pshufb m2, m2, m0 + punpcklbw m2, m0 + movu m4, [r1 + 1] + punpcklbw m5, m4, m0 + punpckhbw m4, m0 + psubw m5, m2 + psubw m4, m2 + psraw m5, 1 + psraw m4, 1 + paddw m5, m1 + paddw m4, m1 + packuswb m5, m4 + + pextrb [r0 + 128 * 16], m5, 0 + pextrb [r0 + 129 * 16], m5, 1 + pextrb [r0 + 130 * 16], m5, 2 + pextrb [r0 + 131 * 16], m5, 3 + pextrb [r0 + 132 * 16], m5, 4 + pextrb [r0 + 133 * 16], m5, 5 + pextrb [r0 + 134 * 16], m5, 6 + pextrb [r0 + 135 * 16], m5, 7 + pextrb [r0 + 136 * 16], m5, 8 + pextrb [r0 + 137 * 16], m5, 9 + pextrb [r0 + 138 * 16], m5, 10 + pextrb [r0 + 139 * 16], m5, 11 + pextrb [r0 + 140 * 16], m5, 12 + pextrb [r0 + 141 * 16], m5, 13 + pextrb [r0 + 142 * 16], m5, 14 + pextrb [r0 + 143 * 16], m5, 15 + + ; mode 11 + movu m0, [r1 + 32] + pinsrb m0, [r1], 0 + + ; mode 11 [row 15] + movu [r0 + 159 * 16], m0 + + ; mode 11 [row 0] + movu m1, [r1 + 1 + 32] + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 30 * 16] + pmulhrsw m1, m3 + movu m7, [r1 + 8 + 32] + movu m2, [r1 + 9 + 32] + punpcklbw m7, m2 + pmaddubsw m2, m7, [r3 + 30 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 144 * 16], m1 + + ; mode 11 [row 1] + pmaddubsw m1, m0, [r3 + 28 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 28 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 145 * 16], m1 + + ; mode 11 [row 2] + pmaddubsw m1, m0, [r3 + 26 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 26 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 146 * 16], m1 + + ; mode 11 [row 3] + pmaddubsw m1, m0, [r3 + 24 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 24 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 147 * 16], m1 + + ; mode 11 [row 4] + pmaddubsw m1, m0, [r3 + 22 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 22 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 148 * 16], m1 + + ; mode 11 [row 5] + pmaddubsw m1, m0, [r3 + 20 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 20 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 149 * 16], m1 + + ; mode 11 [row 6] + pmaddubsw m1, m0, [r3 + 18 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 18 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 150 * 16], m1 + + ; mode 11 [row 7] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 16 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 151 * 16], m1 + + ; mode 11 [row 8] + pmaddubsw m1, m0, [r3 + 14 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 14 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 152 * 16], m1 + + ; mode 11 [row 9] + pmaddubsw m1, m0, [r3 + 12 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 12 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 153 * 16], m1 + + ; mode 11 [row 10] + pmaddubsw m1, m0, [r3 + 10 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 10 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 154 * 16], m1 + + ; mode 11 [row 11] + pmaddubsw m1, m0, [r3 + 8 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 8 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 155 * 16], m1 + + ; mode 11 [row 12] + pmaddubsw m1, m0, [r3 + 6 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 6 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 156 * 16], m1 + + ; mode 11 [row 13] + pmaddubsw m1, m0, [r3 + 4 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 4 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 157 * 16], m1 + + ; mode 11 [row 14] + pmaddubsw m1, m0, [r3 + 2 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 2 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 158 * 16], m1 + + ; mode 12 [row 0] + movu m0, [r2 + 32] + pinsrb m0, [r2], 0 + movu m1, [r2 + 1 + 32] + punpcklbw m0, m1 + pmaddubsw m1, m0, [r3 + 27 * 16] + pmulhrsw m1, m3 + movu m7, [r2 + 8 + 32] + movd m2, [r2 + 24 + 32] + palignr m2, m7, 1 + punpcklbw m7, m2 + pmaddubsw m2, m7, [r3 + 27 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 160 * 16], m1 + + ; mode 12 [row 1] + pmaddubsw m1, m0, [r3 + 22 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 22 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 161 * 16], m1 + + ; mode 12 [row 2] + pmaddubsw m1, m0, [r3 + 17 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 17 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 162 * 16], m1 + + ; mode 12 [row 3] + pmaddubsw m1, m0, [r3 + 12 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 12 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 163 * 16], m1 + + ; mode 12 [row 4] + pmaddubsw m1, m0, [r3 + 7 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 7 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 164 * 16], m1 + + ; mode 12 [row 5] + pmaddubsw m1, m0, [r3 + 2 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 2 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 165 * 16], m1 + + ; mode 13 [row 0] + pmaddubsw m1, m0, [r3 + 23 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 23 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 176 * 16], m1 + + ; mode 13 [row 1] + pmaddubsw m1, m0, [r3 + 14 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 14 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 177 * 16], m1 + + ; mode 13 [row 2] + pmaddubsw m1, m0, [r3 + 5 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 5 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 178 * 16], m1 + + ; mode 14 [row 0] + pmaddubsw m1, m0, [r3 + 19 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 19 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 192 * 16], m1 + + ; mode 14 [row 1] + pmaddubsw m1, m0, [r3 + 6 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 6 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 193 * 16], m1 + + ; mode 17 [row 0] + movu [r0 + 240 * 16], m1 + + ; mode 15 [row 0] + pmaddubsw m1, m0, [r3 + 15 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 15 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 208 * 16], m1 + + ; mode 15 [row 15 - second half] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 223 * 16 + 8], m1 + ; mode 15 [row 15 - second half] end + + ; mode 16 [row 0] + pmaddubsw m1, m0, [r3 + 11 * 16] + pmulhrsw m1, m3 + pmaddubsw m2, m7, [r3 + 11 * 16] + pmulhrsw m2, m3 + packuswb m1, m2 + movu [r0 + 224 * 16], m1 + + ; mode 17 [row 9 - second half] + pmaddubsw m1, m0, [r3 + 28 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 249 * 16 + 8], m1 + ; mode 17 [row 9 - second half] end + + ; mode 17 [row 10 - second half] + pmaddubsw m1, m0, [r3 + 2 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 250 * 16 + 8], m1 + ; mode 17 [row 10 - second half] end + + ; mode 17 [row 1 - first half] + pslldq m6, m0, 2 + pinsrb m6, [r2], 1 + pinsrb m6, [r2 + 1], 0 + pmaddubsw m1, m6, [r3 + 12 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 241 * 16], m1 + + ; mode 17 [row 11 - second half] + pmaddubsw m1, m6, [r3 + 8 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 251 * 16 + 8], m1 + ; mode 17 [row 11 - second half] end + + ; mode 17 [row 2 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 1], 1 + pinsrb m6, [r2 + 2], 0 + pmaddubsw m1, m6, [r3 + 18 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 242 * 16], m1 + + ; mode 17 [row 12 - second half] + pmaddubsw m1, m6, [r3 + 14 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 252 * 16 + 8], m1 + ; mode 17 [row 12 - second half] end + + ; mode 17 [row 3 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 2], 1 + pinsrb m6, [r2 + 4], 0 + pmaddubsw m1, m6, [r3 + 24 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 243 * 16], m1 + + ; mode 17 [row 13 - first half] + pmaddubsw m1, m6, [r3 + 20 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 253 * 16 + 8], m1 + + ; mode 17 [row 4 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 4], 1 + pinsrb m6, [r2 + 5], 0 + pmaddubsw m1, m6, [r3 + 30 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 244 * 16], m1 + + ; mode 17 [row 5 - first half] + pmaddubsw m1, m6, [r3 + 4 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 245 * 16], m1 + + ; mode 17 [row 14 - second half] + pmaddubsw m1, m6, [r3 + 26 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 254 * 16 + 8], m1 + ; mode 17 [row 14 - second half] end + + ; mode 17 [row 6 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 5], 1 + pinsrb m6, [r2 + 6], 0 + pmaddubsw m1, m6, [r3 + 10 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 246 * 16], m1 + + ; mode 17 [row 7 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 6], 1 + pinsrb m6, [r2 + 7], 0 + pmaddubsw m1, m6, [r3 + 16 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 247 * 16], m1 + + ; mode 17 [row 8 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 7], 1 + pinsrb m6, [r2 + 9], 0 + pmaddubsw m1, m6, [r3 + 22 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 248 * 16], m1 + + ; mode 17 [row 9 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 9], 1 + pinsrb m6, [r2 + 10], 0 + pmaddubsw m1, m6, [r3 + 28 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 249 * 16], m1 + + ; mode 17 [row 10 - first half] + pmaddubsw m1, m6, [r3 + 2 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 250 * 16], m1 + + ; mode 17 [row 11 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 10], 1 + pinsrb m6, [r2 + 11], 0 + pmaddubsw m1, m6, [r3 + 8 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 251 * 16], m1 + + ; mode 17 [row 12 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 11], 1 + pinsrb m6, [r2 + 12], 0 + pmaddubsw m1, m6, [r3 + 14 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 252 * 16], m1 + + ; mode 17 [row 13 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 12], 1 + pinsrb m6, [r2 + 14], 0 + pmaddubsw m1, m6, [r3 + 20 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 253 * 16], m1 + + ; mode 17 [row 14 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 14], 1 + pinsrb m6, [r2 + 15], 0 + pmaddubsw m1, m6, [r3 + 26 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 254 * 16], m1 + + ; mode 16 [row 12 - second half] + pmaddubsw m1, m0, [r3 + 15 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 236 * 16 + 8], m1 + ; mode 16 [row 12 - second half] + + ; mode 12 [row 6] + pslldq m2, m0, 2 + pinsrb m2, [r2], 1 + pinsrb m2, [r2 + 6], 0 + pmaddubsw m1, m2, [r3 + 29 * 16] + pmulhrsw m1, m3 + movu m0, [r2 + 7 + 32] + psrldq m4, m0, 1 + punpcklbw m0, m4 + pmaddubsw m4, m0, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 166 * 16], m1 + + ; mode 12 [row 7] + pmaddubsw m1, m2, [r3 + 24 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 167 * 16], m1 + + ; mode 12 [row 8] + pmaddubsw m1, m2, [r3 + 19 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 19 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 168 * 16], m1 + + ; mode 12 [row 9] + pmaddubsw m1, m2, [r3 + 14 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 14 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 169 * 16], m1 + + ; mode 12 [row 10] + pmaddubsw m1, m2, [r3 + 9 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 9 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 170 * 16], m1 + + ; mode 12 [row 11] + pmaddubsw m1, m2, [r3 + 4 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 171 * 16], m1 + + ; mode 13 [row 3] + pinsrb m7, m2, [r2 + 4], 0 + pmaddubsw m1, m7, [r3 + 28 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 28 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 179 * 16], m1 + + ; mode 13 [row 4] + pmaddubsw m1, m7, [r3 + 19 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 19 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 180 * 16], m1 + + ; mode 13 [row 5] + pmaddubsw m1, m7, [r3 + 10 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 10 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 181 * 16], m1 + + ; mode 13 [row 6] + pmaddubsw m1, m7, [r3 + 1 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 182 * 16], m1 + + ; mode 14 [row 2] + pinsrb m5, m7, [r2 + 2], 0 + pmaddubsw m1, m5, [r3 + 25 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 25 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 194 * 16], m1 + + ; mode 14 [row 3] + pmaddubsw m1, m5, [r3 + 12 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 12 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 195 * 16], m1 + + ; mode 15 [row 1] + pmaddubsw m1, m5, [r3 + 30 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 30 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 209 * 16], m1 + + ; mode 15 [row 2] + pmaddubsw m1, m5, [r3 + 13 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 13 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 210 * 16], m1 + + ; mode 16 [row 1] + pmaddubsw m1, m5, [r3 + 22 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 22 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 225 * 16], m1 + + ; mode 16 [row 2] + pmaddubsw m1, m5, [r3 + 1 * 16] + pmulhrsw m1, m3 + pmaddubsw m4, m0, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m1, m4 + movu [r0 + 226 * 16], m1 + + ; mode 16 [row 13 - second half] + pmaddubsw m1, m5, [r3 + 26 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 237 * 16 + 8], m1 + ; mode 16 [row 13 - second half] + + ; mode 16 [row 14 - second half] + pmaddubsw m1, m5, [r3 + 5 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 238 * 16 + 8], m1 + ; mode 16 [row 14 - second half] + + ; mode 16 [row 3] + pslldq m6, m5, 2 + pinsrb m6, [r2 + 2], 1 + pinsrb m6, [r2 + 3], 0 + pmaddubsw m1, m6, [r3 + 12 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 227 * 16], m1 + + ; mode 16 [row 15 - second half] + pmaddubsw m1, m6, [r3 + 16 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 239 * 16 + 8], m1 + ; mode 16 [row 15 - second half] end + + ; mode 16 [row 4- first half] + pslldq m6, 2 + pinsrb m6, [r2 + 3], 1 + pinsrb m6, [r2 + 5], 0 + pmaddubsw m1, m6, [r3 + 23 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 228 * 16], m1 + + ; mode 16 [row 5- first half] + pmaddubsw m1, m6, [r3 + 2 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 229 * 16], m1 + + ; mode 16 [row 6- first half] + pslldq m6, 2 + pinsrb m6, [r2 + 5], 1 + pinsrb m6, [r2 + 6], 0 + pmaddubsw m1, m6, [r3 + 13 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 230 * 16], m1 + + ; mode 16 [row 7- first half] + pslldq m6, 2 + pinsrb m6, [r2 + 6], 1 + pinsrb m6, [r2 + 8], 0 + pmaddubsw m1, m6, [r3 + 24 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 231 * 16], m1 + + ; mode 16 [row 8- first half] + pmaddubsw m1, m6, [r3 + 3 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 232 * 16], m1 + ; mode 19 [row 0 - second half] end + + ; mode 16 [row 9- first half] + pslldq m6, 2 + pinsrb m6, [r2 + 8], 1 + pinsrb m6, [r2 + 9], 0 + pmaddubsw m1, m6, [r3 + 14 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 233 * 16], m1 + + ; mode 16 [row 10 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 9], 1 + pinsrb m6, [r2 + 11], 0 + pmaddubsw m1, m6, [r3 + 25 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 234 * 16], m1 + + ; mode 16 [row 11 - first half] + pmaddubsw m1, m6, [r3 + 4 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 235 * 16], m1 + + ; mode 16 [row 12 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 11], 1 + pinsrb m6, [r2 + 12], 0 + pmaddubsw m1, m6, [r3 + 15 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 236 * 16], m1 + + ; mode 16 [row 13 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 12], 1 + pinsrb m6, [r2 + 14], 0 + pmaddubsw m1, m6, [r3 + 26 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 237 * 16], m1 + + ; mode 16 [row 14 - first half] + pmaddubsw m1, m6, [r3 + 5 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 238 * 16], m1 + + ; mode 16 [row 15 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 14], 1 + pinsrb m6, [r2 + 15], 0 + pmaddubsw m1, m6, [r3 + 16 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 239 * 16], m1 + + ; mode 14 [row 4] + pslldq m5, 2 + pinsrb m5, [r2 + 2], 1 + pinsrb m5, [r2 + 5], 0 + movu m4, [r2 + 6 + 32] + psrldq m0, m4, 1 + punpcklbw m4, m0 + + ; mode 16 [row 3 - second half] + pmaddubsw m1, m4, [r3 + 12 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 227 * 16 + 8], m1 + + ; mode 16 [row 3 - second half] end + pmaddubsw m1, m5, [r3 + 31 * 16] + pmulhrsw m1, m3 + pmaddubsw m0, m4, [r3 + 31 * 16] + pmulhrsw m0, m3 + packuswb m1, m0 + movu [r0 + 196 * 16], m1 + + ; mode 14 [row 5] + pmaddubsw m1, m5, [r3 + 18 * 16] + pmulhrsw m1, m3 + pmaddubsw m0, m4, [r3 + 18 * 16] + pmulhrsw m0, m3 + packuswb m1, m0 + movu [r0 + 197 * 16], m1 + + ; mode 14 [row 6] + pmaddubsw m1, m5, [r3 + 5 * 16] + pmulhrsw m1, m3 + pmaddubsw m0, m4, [r3 + 5 * 16] + pmulhrsw m0, m3 + packuswb m1, m0 + movu [r0 + 198 * 16], m1 + + ; mode 15 [row 3] + movu m6, m5 + pinsrb m6, [r2 + 4], 0 + pmaddubsw m1, m6, [r3 + 28 * 16] + pmulhrsw m1, m3 + pmaddubsw m0, m4, [r3 + 28 * 16] + pmulhrsw m0, m3 + packuswb m1, m0 + movu [r0 + 211 * 16], m1 + + ; mode 15 [row 4] + pmaddubsw m1, m6, [r3 + 11 * 16] + pmulhrsw m1, m3 + pmaddubsw m0, m4, [r3 + 11 * 16] + pmulhrsw m0, m3 + packuswb m1, m0 + movu [r0 + 212 * 16], m1 + + ; mode 15 [row 5 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 4], 1 + pinsrb m6, [r2 + 6], 0 + pmaddubsw m1, m6, [r3 + 26 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 213 * 16], m1 + + ; mode 15 [row 6 - first half] + pmaddubsw m1, m6, [r3 + 9 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 214 * 16], m1 + + ; mode 15 [row 7 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 6], 1 + pinsrb m6, [r2 + 8], 0 + pmaddubsw m1, m6, [r3 + 24 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 215 * 16], m1 + + ; mode 15 [row 8 - first half] + pmaddubsw m1, m6, [r3 + 7 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 216 * 16], m1 + + ; mode 15 [row 9 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 8], 1 + pinsrb m6, [r2 + 9], 0 + pmaddubsw m1, m6, [r3 + 22 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 217 * 16], m1 + + ; mode 15 [row 10 - first half] + pmaddubsw m1, m6, [r3 + 5 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 218 * 16], m1 + + ; mode 15 [row 11 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 9], 1 + pinsrb m6, [r2 + 11], 0 + pmaddubsw m1, m6, [r3 + 20 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 219 * 16], m1 + + ; mode 15 [row 12 - first half] + pmaddubsw m1, m6, [r3 + 3 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 220 * 16], m1 + + ; mode 15 [row 13 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 11], 1 + pinsrb m6, [r2 + 13], 0 + pmaddubsw m1, m6, [r3 + 18 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 221 * 16], m1 + + ; mode 15 [row 14 - first half] + pmaddubsw m1, m6, [r3 + 1 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 222 * 16], m1 + + ; mode 15 [row 15 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 13], 1 + pinsrb m6, [r2 + 15], 0 + pmaddubsw m1, m6, [r3 + 16 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 223 * 16], m1 + + ; mode 14 [row 7] + pslldq m5, 2 + pinsrb m5, [r2 + 5], 1 + pinsrb m5, [r2 + 7], 0 + movu m0, [r2 + 5 + 32] + psrldq m6, m0, 1 + punpcklbw m0, m6 + + ; mode 15 [row 5 - second half] + pmaddubsw m1, m0, [r3 + 26 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 213 * 16 + 8], m1 + ; mode 15 [row 5 - second half] end + + ; mode 15 [row 6 - second half] + pmaddubsw m1, m0, [r3 + 9 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 214 * 16 + 8], m1 + ; mode 15 [row 6 - second half] end + + ; mode 16 [row 4 - second half] + pmaddubsw m1, m0, [r3 + 23 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 228 * 16 + 8], m1 + ; mode 16 [row 4 - second half] end + + ; mode 16 [row 5 - second half] + pmaddubsw m1, m0, [r3 + 2 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 229 * 16 + 8], m1 + + ; mode 16 [row 5 - second half] end + pmaddubsw m1, m5, [r3 + 24 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 24 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 199 * 16], m1 + + ; mode 14 [row 8] + pmaddubsw m1, m5, [r3 + 11 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 11 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 200 * 16], m1 + + ; mode 14 [row 9] + pslldq m5, 2 + pinsrb m5, [r2 + 7], 1 + pinsrb m5, [r2 + 10], 0 + movu m0, [r2 + 4 + 32] + psrldq m6, m0, 1 + punpcklbw m0, m6 + + ; mode 15 [row 7 - second half] + pmaddubsw m1, m0, [r3 + 24 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 215 * 16 + 8], m1 + ; mode 15 [row 7 - second half] end + + ; mode 15 [row 8 - second half] + pmaddubsw m1, m0, [r3 + 7 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 216 * 16 + 8], m1 + ; mode 15 [row 8 - second half] end + + ; mode 16 [row 6 - second half] + pmaddubsw m1, m0, [r3 + 13 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 230 * 16 + 8], m1 + ; mode 16 [row 6 - second half] end + + ; mode 15 [row 6 - second half] end + pmaddubsw m1, m5, [r3 + 30 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 30 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 201 * 16], m1 + + ; mode 14 [row 10] + pmaddubsw m1, m5, [r3 + 17 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 17 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 202 * 16], m1 + + ; mode 14 [row 11] + pmaddubsw m1, m5, [r3 + 4 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 4 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 203 * 16], m1 + + ; mode 14 [row 12] + pslldq m5, 2 + pinsrb m5, [r2 + 10], 1 + pinsrb m5, [r2 + 12], 0 + movu m0, [r2 + 3 + 32] + psrldq m6, m0, 1 + punpcklbw m0, m6 + + ; mode 15 [row 9 - second half] + pmaddubsw m1, m0, [r3 + 22 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 217 * 16 + 8], m1 + ; mode 15 [row 9 - second half] end + + ; mode 15 [row 10 - second half] + pmaddubsw m1, m0, [r3 + 5 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 218 * 16 + 8], m1 + ; mode 15 [row 10 - second half] end + + ; mode 16 [row 7 - second half] + pmaddubsw m1, m0, [r3 + 24 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 231 * 16 + 8], m1 + ; mode 16 [row 7 - second half] end + + ; mode 16 [row 8 - second half] + pmaddubsw m1, m0, [r3 + 3 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 232 * 16 + 8], m1 + ; mode 16 [row 8 - second half] end + + pmaddubsw m1, m5, [r3 + 23 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 23 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 204 * 16], m1 + + ; mode 14 [row 13] + pmaddubsw m1, m5, [r3 + 10 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 10 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 205 * 16], m1 + + ; mode 14 [row 14] + pslldq m5, 2 + pinsrb m5, [r2 + 12], 1 + pinsrb m5, [r2 + 15], 0 + movu m0, [r2 + 2 + 32] + psrldq m6, m0, 1 + punpcklbw m0, m6 + + ; mode 15 [row 11 - second half] + pmaddubsw m1, m0, [r3 + 20 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 219 * 16 + 8], m1 + ; mode 15 [row 11 - second half] end + + ; mode 15 [row 12 - second half] + pmaddubsw m1, m0, [r3 + 3 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 220 * 16 + 8], m1 + ; mode 15 [row 12 - second half] end + + ; mode 16 [row 9 - second half] + pmaddubsw m1, m0, [r3 + 14 * 16] + pmulhrsw m1, m3 + packuswb m1, m1 + movh [r0 + 233 * 16 + 8], m1 + + ; mode 16 [row 9 - second half] end + pmaddubsw m1, m5, [r3 + 29 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 29 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 206 * 16], m1 + + ; mode 14 [row 15] + pmaddubsw m1, m5, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m6, m0, [r3 + 16 * 16] + pmulhrsw m6, m3 + packuswb m1, m6 + movu [r0 + 207 * 16], m1 + + ; mode 12 [row 12] + pslldq m0, m2, 2 + pinsrb m0, [r2 + 6], 1 + pinsrb m0, [r2 + 13], 0 + pmaddubsw m1, m0, [r3 + 31 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 31 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 172 * 16], m1 + + ; mode 12 [row 13] + pmaddubsw m1, m0, [r3 + 26 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 26 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 173 * 16], m1 + + ; mode 12 [row 14] + pmaddubsw m1, m0, [r3 + 21 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 21 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 174 * 16], m1 + + ; mode 12 [row 15] + pmaddubsw m1, m0, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 175 * 16], m1 + + ; mode 13 [row 7] + pslldq m7, 2 + pinsrb m7, [r2 + 4], 1 + pinsrb m7, [r2 + 7], 0 + pmaddubsw m1, m7, [r3 + 24 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 24 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 183 * 16], m1 + + ; mode 13 [row 8] + pmaddubsw m1, m7, [r3 + 15 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 15 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 184 * 16], m1 + + ; mode 13 [row 9] + pmaddubsw m1, m7, [r3 + 6 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 6 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 185 * 16], m1 + + ; mode 13 [row 10] + pslldq m7, 2 + pinsrb m7, [r2 + 7], 1 + pinsrb m7, [r2 + 11], 0 + pmaddubsw m1, m7, [r3 + 29 * 16] + pmulhrsw m1, m3 + movu m4, [r2 + 5 + 32] + psrldq m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, [r3 + 29 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 186 * 16], m1 + + ; mode 13 [row 11] + pmaddubsw m1, m7, [r3 + 20 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 20 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 187 * 16], m1 + + ; mode 13 [row 12] + pmaddubsw m1, m7, [r3 + 11 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 11 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 188 * 16], m1 + + ; mode 13 [row 13] + pmaddubsw m1, m7, [r3 + 2 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 2 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 189 * 16], m1 + + ; mode 13 [row 14] + pslldq m7, 2 + pinsrb m7, [r2 + 11], 1 + pinsrb m7, [r2 + 14], 0 + pmaddubsw m1, m7, [r3 + 25 * 16] + pmulhrsw m1, m3 + movu m4, [r2 + 4 + 32] + psrldq m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, [r3 + 25 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 190 * 16], m1 + + ; mode 13 [row 15] + pmaddubsw m1, m7, [r3 + 16 * 16] + pmulhrsw m1, m3 + pmaddubsw m5, m4, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m1, m5 + movu [r0 + 191 * 16], m1 + + ; mode 17 [row 15] + movu m0, [r2] + pshufb m1, m0, [tab_S1] + movu [r0 + 255 * 16], m1 + movu m2, [r2 + 32] + pinsrb m2, [r2], 0 + movd [r0 + 255 * 16 + 12], m2 + + ; mode 18 [row 0] + movu [r0 + 256 * 16], m0 + + ; mode 18 [row 1] + pslldq m4, m0, 1 + pinsrb m4, [r2 + 1 + 32], 0 + movu [r0 + 257 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 2 + 32], 0 + movu [r0 + 258 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 3 + 32], 0 + movu [r0 + 259 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 4 + 32], 0 + movu [r0 + 260 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 5 + 32], 0 + movu [r0 + 261 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 6 + 32], 0 + movu [r0 + 262 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 7 + 32], 0 + movu [r0 + 263 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 8 + 32], 0 + movu [r0 + 264 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 9 + 32], 0 + movu [r0 + 265 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 10 + 32], 0 + movu [r0 + 266 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 11 + 32], 0 + movu [r0 + 267 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 12 + 32], 0 + movu [r0 + 268 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 13 + 32], 0 + movu [r0 + 269 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 14 + 32], 0 + movu [r0 + 270 * 16], m4 + pslldq m4, 1 + pinsrb m4, [r2 + 15 + 32], 0 + movu [r0 + 271 * 16], m4 + + ; mode 19 [row 0] + psrldq m2, m0, 1 + punpcklbw m0, m2 + movu m5, [r2 + 8] + psrldq m6, m5, 1 + punpcklbw m5, m6 + pmaddubsw m4, m0, [r3 + 6 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 6 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 272 * 16], m4 + + ; mode 20 [row 0] + pmaddubsw m4, m0, [r3 + 11 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 11 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 288 * 16], m4 + + ; mode 21 [row 0] + pmaddubsw m4, m0, [r3 + 15 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 15 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 304 * 16], m4 + + ; mode 22 [row 0] + pmaddubsw m4, m0, [r3 + 19 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 19 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 320 * 16], m4 + + ; mode 22 [row 1] + pmaddubsw m4, m0, [r3 + 6 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 6 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 321 * 16], m4 + + ; mode 23 [row 0] + pmaddubsw m4, m0, [r3 + 23 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 23 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 336 * 16], m4 + + ; mode 23 [row 1] + pmaddubsw m4, m0, [r3 + 14 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 14 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 337 * 16], m4 + + ; mode 23 [row 2] + pmaddubsw m4, m0, [r3 + 5 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 5 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 338 * 16], m4 + + ; mode 24 [row 0] + pmaddubsw m4, m0, [r3 + 27 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 27 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 352 * 16], m4 + + ; mode 24 [row 1] + pmaddubsw m4, m0, [r3 + 22 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 22 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 353 * 16], m4 + + ; mode 24 [row 2] + pmaddubsw m4, m0, [r3 + 17 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 17 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 354 * 16], m4 + + ; mode 24 [row 3] + pmaddubsw m4, m0, [r3 + 12 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 12 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 355 * 16], m4 + + ; mode 24 [row 4] + pmaddubsw m4, m0, [r3 + 7 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 7 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 356 * 16], m4 + + ; mode 24 [row 5] + pmaddubsw m4, m0, [r3 + 2 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 2 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 357 * 16], m4 + + ; mode 24 [row 6 - first half] + pslldq m7, m0, 2 + pinsrb m7, [r2 + 0], 1 + pinsrb m7, [r2 + 6 + 32], 0 + pmaddubsw m4, m7, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 358 * 16], m4 + + ; mode 24 [row 7 - first half] + pmaddubsw m4, m7, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 359 * 16], m4 + + ; mode 24 [row 8 - first half] + pmaddubsw m4, m7, [r3 + 19 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 360 * 16], m4 + + ; mode 24 [row 9 - first half] + pmaddubsw m4, m7, [r3 + 14 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 361 * 16], m4 + + ; mode 24 [row 10 - first half] + pmaddubsw m4, m7, [r3 + 9 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 362 * 16], m4 + + ; mode 24 [row 11 - first half] + pmaddubsw m4, m7, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 363 * 16], m4 + + ; mode 24 [row 12 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 6 + 32], 1 + pinsrb m7, [r2 + 13 + 32], 0 + pmaddubsw m4, m7, [r3 + 31 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 364 * 16], m4 + + ; mode 24 [row 13 - first half] + pmaddubsw m4, m7, [r3 + 26 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 365 * 16], m4 + + ; mode 24 [row 14 - first half] + pmaddubsw m4, m7, [r3 + 21 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 366 * 16], m4 + + ; mode 24 [row 15 - first half] + pmaddubsw m4, m7, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 367 * 16], m4 + + ; mode 23 [row 3 - first half] + pslldq m7, m0, 2 + pinsrb m7, [r2 + 0], 1 + pinsrb m7, [r2 + 4 + 32], 0 + pmaddubsw m4, m7, [r3 + 28 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 339 * 16], m4 + + ; mode 23 [row 4 - first half] + pmaddubsw m4, m7, [r3 + 19 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 340 * 16], m4 + + ; mode 23 [row 5 - first half] + pmaddubsw m4, m7, [r3 + 10 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 341 * 16], m4 + + ; mode 23 [row 6 - first half] + pmaddubsw m4, m7, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 342 * 16], m4 + + ; mode 23 [row 7 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 4 + 32], 1 + pinsrb m7, [r2 + 7 + 32], 0 + pmaddubsw m4, m7, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 343 * 16], m4 + + ; mode 23 [row 8 - first half] + pmaddubsw m4, m7, [r3 + 15 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 344 * 16], m4 + + ; mode 23 [row 9 - first half] + pmaddubsw m4, m7, [r3 + 6 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 345 * 16], m4 + + ; mode 23 [row 10 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 7 + 32], 1 + pinsrb m7, [r2 + 11 + 32], 0 + pmaddubsw m4, m7, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 346 * 16], m4 + + ; mode 23 [row 11 - first half] + pmaddubsw m4, m7, [r3 + 20 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 347 * 16], m4 + + ; mode 23 [row 12 - first half] + pmaddubsw m4, m7, [r3 + 11 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 348 * 16], m4 + + ; mode 23 [row 13 - first half] + pmaddubsw m4, m7, [r3 + 2 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 349 * 16], m4 + + ; mode 23 [row 14 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 11 + 32], 1 + pinsrb m7, [r2 + 14 + 32], 0 + pmaddubsw m4, m7, [r3 + 25 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 350 * 16], m4 + + ; mode 23 [row 15 - first half] + pmaddubsw m4, m7, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 351 * 16], m4 + + ; mode 21 [row 15 - first half] + pmaddubsw m4, m0, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 319 * 16 + 8], m4 + ; mode 21 [row 15 - second half] end + + ; mode 20 [row 1 - first half] + pslldq m7, m0, 2 + pinsrb m7, [r2 + 0], 1 + pinsrb m7, [r2 + 2 + 32], 0 + pmaddubsw m4, m7, [r3 + 22 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 289 * 16], m4 + + ; mode 20 [row 2 - first half] + pmaddubsw m4, m7, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 290 * 16], m4 + + ; mode 21 [row 1 - first half] + pmaddubsw m4, m7, [r3 + 30 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 305 * 16], m4 + + ; mode 21 [row 2 - first half] + pmaddubsw m4, m7, [r3 + 13 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 306 * 16], m4 + + ; mode 22 [row 2 - first half] + pmaddubsw m4, m7, [r3 + 25 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 322 * 16], m4 + + ; mode 22 [row 3 - first half] + pmaddubsw m4, m7, [r3 + 12 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 323 * 16], m4 + + ; mode 22 [row 4 - first half] + pslldq m1, m7, 2 + pinsrb m1, [r2 + 2 + 32], 1 + pinsrb m1, [r2 + 5 + 32], 0 + pmaddubsw m4, m1, [r3 + 31 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 324 * 16], m4 + + ; mode 22 [row 5 - first half] + pmaddubsw m4, m1, [r3 + 18 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 325 * 16], m4 + + ; mode 22 [row 6 - first half] + pmaddubsw m4, m1, [r3 + 5 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 326 * 16], m4 + + ; mode 22 [row 7 - first half] + pslldq m1, 2 + pinsrb m1, [r2 + 5 + 32], 1 + pinsrb m1, [r2 + 7 + 32], 0 + pmaddubsw m4, m1, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 327 * 16], m4 + + ; mode 22 [row 8 - first half] + pmaddubsw m4, m1, [r3 + 11 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 328 * 16], m4 + + ; mode 22 [row 9 - first half] + pslldq m1, 2 + pinsrb m1, [r2 + 7 + 32], 1 + pinsrb m1, [r2 + 10 + 32], 0 + pmaddubsw m4, m1, [r3 + 30 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 329 * 16], m4 + + ; mode 22 [row 10 - first half] + pmaddubsw m4, m1, [r3 + 17 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 330 * 16], m4 + + ; mode 22 [row 11 - first half] + pmaddubsw m4, m1, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 331 * 16], m4 + + ; mode 22 [row 12 - first half] + pslldq m1, 2 + pinsrb m1, [r2 + 10 + 32], 1 + pinsrb m1, [r2 + 12 + 32], 0 + pmaddubsw m4, m1, [r3 + 23 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 332 * 16], m4 + + ; mode 22 [row 13 - first half] + pmaddubsw m4, m1, [r3 + 10 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 333 * 16], m4 + + ; mode 22 [row 14 - first half] + pslldq m1, 2 + pinsrb m1, [r2 + 12 + 32], 1 + pinsrb m1, [r2 + 15 + 32], 0 + pmaddubsw m4, m1, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 334 * 16], m4 + + ; mode 22 [row 15 - first half] + pmaddubsw m4, m1, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 335 * 16], m4 + + ; mode 21 [row 3 - first half] + pslldq m6, m7, 2 + pinsrb m6, [r2 + 2 + 32], 1 + pinsrb m6, [r2 + 4 + 32], 0 + pmaddubsw m4, m6, [r3 + 28 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 307 * 16], m4 + + ; mode 21 [row 4 - first half] + pmaddubsw m4, m6, [r3 + 11 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 308 * 16], m4 + + ; mode 21 [row 5 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 4 + 32], 1 + pinsrb m6, [r2 + 6 + 32], 0 + pmaddubsw m4, m6, [r3 + 26 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 309 * 16], m4 + + ; mode 21 [row 6 - first half] + pmaddubsw m4, m6, [r3 + 9 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 310 * 16], m4 + + ; mode 21 [row 7 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 6 + 32], 1 + pinsrb m6, [r2 + 8 + 32], 0 + pmaddubsw m4, m6, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 311 * 16], m4 + + ; mode 21 [row 8 - first half] + pmaddubsw m4, m6, [r3 + 7 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 312 * 16], m4 + + ; mode 21 [row 9 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 8 + 32], 1 + pinsrb m6, [r2 + 9 + 32], 0 + pmaddubsw m4, m6, [r3 + 22 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 313 * 16], m4 + + ; mode 21 [row 10 - first half] + pmaddubsw m4, m6, [r3 + 5 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 314 * 16], m4 + + ; mode 21 [row 11 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 9 + 32], 1 + pinsrb m6, [r2 + 11 + 32], 0 + pmaddubsw m4, m6, [r3 + 20 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 315 * 16], m4 + + ; mode 21 [row 12 - first half] + pmaddubsw m4, m6, [r3 + 3 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 316 * 16], m4 + + ; mode 21 [row 13 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 11 + 32], 1 + pinsrb m6, [r2 + 13 + 32], 0 + pmaddubsw m4, m6, [r3 + 18 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 317 * 16], m4 + + ; mode 21 [row 14 - first half] + pmaddubsw m4, m6, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 318 * 16], m4 + + ; mode 21 [row 15 - first half] + pslldq m6, 2 + pinsrb m6, [r2 + 32 + 13], 1 + pinsrb m6, [r2 + 32 + 15], 0 + pmaddubsw m4, m6, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 319 * 16], m4 + + ; mode 20 [row 13 - second half] + pmaddubsw m4, m7, [r3 + 26 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 301 * 16 + 8], m4 + ; mode 20 [row 13 - second half] + + ; mode 20 [row 14 - second half] + pmaddubsw m4, m7, [r3 + 5 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 302 * 16 + 8], m4 + ; mode 20 [row 14 - second half] + + ; mode 20 [row 3 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 2], 1 + pinsrb m7, [r2 + 32 + 3], 0 + pmaddubsw m4, m7, [r3 + 12 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 291 * 16], m4 + + ; mode 20 [row 15 - second half] + pmaddubsw m4, m7, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 303 * 16 + 8], m4 + ; mode 20 [row 15 - second half] + + ; mode 20 [row 4 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 3], 1 + pinsrb m7, [r2 + 32 + 5], 0 + pmaddubsw m4, m7, [r3 + 23 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 292 * 16], m4 + + ; mode 20 [row 5 - first half] + pmaddubsw m4, m7, [r3 + 2 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 293 * 16], m4 + + ; mode 20 [row 6 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 5], 1 + pinsrb m7, [r2 + 32 + 6], 0 + pmaddubsw m4, m7, [r3 + 13 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 294 * 16], m4 + + ; mode 20 [row 7 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 6], 1 + pinsrb m7, [r2 + 32 + 8], 0 + pmaddubsw m4, m7, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 295 * 16], m4 + + ; mode 20 [row 8 - first half] + pmaddubsw m4, m7, [r3 + 3 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 296 * 16], m4 + + ; mode 20 [row 9 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 8], 1 + pinsrb m7, [r2 + 32 + 9], 0 + pmaddubsw m4, m7, [r3 + 14 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 297 * 16], m4 + + ; mode 20 [row 10 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 9], 1 + pinsrb m7, [r2 + 32 + 11], 0 + pmaddubsw m4, m7, [r3 + 25 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 298 * 16], m4 + + ; mode 20 [row 11 - first half] + pmaddubsw m4, m7, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 299 * 16], m4 + + ; mode 20 [row 12 - first half] + movu m1, [r3 + 15 * 16] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 11], 1 + pinsrb m7, [r2 + 32 + 12], 0 + pmaddubsw m4, m7, [r3 + 15 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 300 * 16], m4 + + ; mode 20 [row 13 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 12], 1 + pinsrb m7, [r2 + 32 + 14], 0 + pmaddubsw m4, m7, [r3 + 26 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 301 * 16], m4 + + ; mode 20 [row 14 - first half] + pmaddubsw m4, m7, [r3 + 5 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 302 * 16], m4 + + ; mode 20 [row 15 - first half] + pslldq m7, 2 + pinsrb m7, [r2 + 32 + 14], 1 + pinsrb m7, [r2 + 32 + 15], 0 + pmaddubsw m4, m7, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 303 * 16], m4 + + ; mode 19 [row 1] + pslldq m0, 2 + pinsrb m0, [r2], 1 + pinsrb m0, [r2 + 32 + 1], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 8], 1 + pinsrb m5, [r2 + 7], 0 + + ; mode 20 [row 1 - second half] + pmaddubsw m4, m5, [r3 + 22 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 289 * 16 + 8], m4 + ; mode 20 [row 1 - second half] end + + ; mode 20 [row 2 - second half] + pmaddubsw m4, m5, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 290 * 16 + 8], m4 + ; mode 20 [row 2 - second half] end + + ; mode 21 [row 2 - second half] + pmaddubsw m4, m5, [r3 + 30 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 305 * 16 + 8], m4 + ; mode 21 [row 2 - second half] end + + ; mode 21 [row 3 - second half] + pmaddubsw m4, m5, [r3 + 13 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 306 * 16 + 8], m4 + ; mode 21 [row 3 - second half] end + + ; mode 21 [row 4 - second half] + pmaddubsw m4, m5, [r3 + 11 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 307 * 16 + 8], m4 + ; mode 21 [row 4 - second half] end + + ; mode 22 [row 2 - second half] + pmaddubsw m4, m5, [r3 + 25 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 322 * 16 + 8], m4 + ; mode 22 [row 2 - second half] end + + ; mode 22 [row 3 - second half] + pmaddubsw m4, m5, [r3 + 12 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 323 * 16 + 8], m4 + ; mode 22 [row 3 - second half] end + + ; mode 23 [row 3 - second half] + pmaddubsw m4, m5, [r3 + 28 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 339 * 16 + 8], m4 + ; mode 23 [row 3 - second half] end + + ; mode 23 [row 4 - second half] + pmaddubsw m4, m5, [r3 + 19 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 340 * 16 + 8], m4 + ; mode 23 [row 4 - second half] end + + ; mode 23 [row 5 - second half] + pmaddubsw m4, m5, [r3 + 10 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 341 * 16 + 8], m4 + ; mode 23 [row 5 - second half] end + + ; mode 23 [row 6 - second half] + pmaddubsw m4, m5, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 342 * 16 + 8], m4 + ; mode 23 [row 6 - second half] end + + ; mode 24 [row 6 - second half] + pmaddubsw m4, m5, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 358 * 16 + 8], m4 + ; mode 24 [row 6 - second half] end + + ; mode 24 [row 7 - second half] + pmaddubsw m4, m5, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 359 * 16 + 8], m4 + ; mode 24 [row 7 - second half] end + + ; mode 24 [row 8 - second half] + pmaddubsw m4, m5, [r3 + 19 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 360 * 16 + 8], m4 + ; mode 24 [row 8 - second half] end + + ; mode 24 [row 9 - second half] + pmaddubsw m4, m5, [r3 + 14 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 361 * 16 + 8], m4 + ; mode 24 [row 9 - second half] end + + ; mode 24 [row 10 - second half] + pmaddubsw m4, m5, [r3 + 9 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 362 * 16 + 8], m4 + ; mode 24 [row 10 - second half] end + + ; mode 24 [row 11 - second half] + pmaddubsw m4, m5, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 363 * 16 + 8], m4 + ; mode 24 [row 11 - second half] end + + pmaddubsw m4, m0, [r3 + 12 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 12 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 273 * 16], m4 + + ; mode 19 [row 2] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 1], 1 + pinsrb m0, [r2 + 32 + 2], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 7], 1 + pinsrb m5, [r2 + 6], 0 + + ; mode 20 [row 3 - second half] + pmaddubsw m4, m5, [r3 + 12 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 291 * 16 + 8], m4 + ; mode 20 [row 3 - second half] end + + ; mode 21 [row 3 - second half] + pmaddubsw m4, m5, [r3 + 28 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 307 * 16 + 8], m4 + ; mode 21 [row 3 - second half] end + + ; mode 21 [row 4 - second half] + pmaddubsw m4, m5, [r3 + 11 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 308 * 16 + 8], m4 + ; mode 21 [row 4 - second half] end + + ; mode 22 [row 4 - second half] + pmaddubsw m4, m5, [r3 + 31 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 324 * 16 + 8], m4 + ; mode 22 [row 4 - second half] end + + ; mode 22 [row 5 - second half] + pmaddubsw m4, m5, [r3 + 18 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 325 * 16 + 8], m4 + ; mode 22 [row 5 - second half] end + + ; mode 22 [row 6 - second half] + pmaddubsw m4, m5, [r3 + 5 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 326 * 16 + 8], m4 + ; mode 22 [row 6 - second half] end + + ; mode 23 [row 7 - second half] + pmaddubsw m4, m5, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 343 * 16 + 8], m4 + ; mode 23 [row 7 - second half] end + + ; mode 23 [row 8 - second half] + pmaddubsw m4, m5, [r3 + 15 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 344 * 16 + 8], m4 + ; mode 23 [row 8 - second half] end + + ; mode 23 [row 9 - second half] + pmaddubsw m4, m5, [r3 + 6 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 345 * 16 + 8], m4 + ; mode 23 [row 9 - second half] end + + ; mode 24 [row 12 - second half] + pmaddubsw m4, m5, [r3 + 31 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 364 * 16 + 8], m4 + ; mode 24 [row 12 - second half] end + + ; mode 24 [row 13 - second half] + pmaddubsw m4, m5, [r3 + 26 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 365 * 16 + 8], m4 + ; mode 24 [row 13 - second half] end + + ; mode 24 [row 14 - second half] + pmaddubsw m4, m5, [r3 + 21 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 366 * 16 + 8], m4 + ; mode 24 [row 14 - second half] end + + ; mode 24 [row 15 - second half] + pmaddubsw m4, m5, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 367 * 16 + 8], m4 + ; mode 24 [row 15 - second half] end + + pmaddubsw m4, m0, [r3 + 18 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 18 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 274 * 16], m4 + + ; mode 19 [row 3] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 2], 1 + pinsrb m0, [r2 + 32 + 4], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 6], 1 + pinsrb m5, [r2 + 5], 0 + + ; mode 20 [row 4 - second half] + pmaddubsw m4, m5, [r3 + 23 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 292 * 16 + 8], m4 + ; mode 20 [row 4 - second half] end + + ; mode 20 [row 5 - second half] + pmaddubsw m4, m5, [r3 + 2 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 293 * 16 + 8], m4 + ; mode 20 [row 5 - second half] end + + ; mode 21 [row 5 - second half] + pmaddubsw m4, m5, [r3 + 26 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 309 * 16 + 8], m4 + ; mode 21 [row 5 - second half] end + + ; mode 21 [row 6 - second half] + pmaddubsw m4, m5, [r3 + 9 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 310 * 16 + 8], m4 + ; mode 21 [row 6 - second half] end + + ; mode 22 [row 7 - second half] + pmaddubsw m4, m5, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 327 * 16 + 8], m4 + ; mode 22 [row 7 - second half] end + + ; mode 22 [row 8 - second half] + pmaddubsw m4, m5, [r3 + 11 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 328 * 16 + 8], m4 + ; mode 22 [row 7 - second half] end + + ; mode 23 [row 10 - second half] + pmaddubsw m4, m5, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 346 * 16 + 8], m4 + ; mode 23 [row 10 - second half] end + + ; mode 23 [row 11 - second half] + pmaddubsw m4, m5, [r3 + 20 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 347 * 16 + 8], m4 + ; mode 23 [row 11 - second half] end + + ; mode 23 [row 12 - second half] + pmaddubsw m4, m5, [r3 + 11 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 348 * 16 + 8], m4 + ; mode 23 [row 12 - second half] end + + ; mode 23 [row 13 - second half] + pmaddubsw m4, m5, [r3 + 2 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 349 * 16 + 8], m4 + ; mode 23 [row 13 - second half] end + + pmaddubsw m4, m0, [r3 + 24 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 24 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 275 * 16], m4 + + ; mode 19 [row 4] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 4], 1 + pinsrb m0, [r2 + 32 + 5], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 5], 1 + pinsrb m5, [r2 + 4], 0 + + ; mode 20 [row 6 - second half] + pmaddubsw m4, m5, [r3 + 13 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 294 * 16 + 8], m4 + ; mode 20 [row 6 - second half] end + + ; mode 21 [row 7 - second half] + pmaddubsw m4, m5, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 311 * 16 + 8], m4 + ; mode 21 [row 7 - second half] end + + ; mode 21 [row 8 - second half] + pmaddubsw m4, m5, [r3 + 7 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 312 * 16 + 8], m4 + ; mode 21 [row 8 - second half] end + + ; mode 22 [row 9 - second half] + pmaddubsw m4, m5, [r3 + 30 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 329 * 16 + 8], m4 + ; mode 22 [row 9 - second half] end + + ; mode 22 [row 10 - second half] + pmaddubsw m4, m5, [r3 + 17 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 330 * 16 + 8], m4 + ; mode 22 [row 10 - second half] end + + ; mode 22 [row 11 - second half] + pmaddubsw m4, m5, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 331 * 16 + 8], m4 + ; mode 22 [row 11 - second half] end + + ; mode 23 [row 14 - second half] + pmaddubsw m4, m5, [r3 + 25 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 350 * 16 + 8], m4 + ; mode 23 [row 14 - second half] end + + ; mode 23 [row 15 - second half] + pmaddubsw m4, m5, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 351 * 16 + 8], m4 + + ; mode 23 [row 15 - second half] end + pmaddubsw m4, m0, [r3 + 30 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 30 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 276 * 16], m4 + + ; mode 19 [row 5] + pmaddubsw m4, m0, [r3 + 4 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 4 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 277 * 16], m4 + + ; mode 19 [row 6] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 5], 1 + pinsrb m0, [r2 + 32 + 6], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 4], 1 + pinsrb m5, [r2 + 3], 0 + + ; mode 20 [row 7 - second half] + pmaddubsw m4, m5, [r3 + 24 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 295 * 16 + 8], m4 + ; mode 20 [row 7 - second half] end + + ; mode 20 [row 8 - second half] + pmaddubsw m4, m5, [r3 + 3 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 296 * 16 + 8], m4 + ; mode 20 [row 8 - second half] end + + ; mode 21 [row 9 - second half] + pmaddubsw m4, m5, [r3 + 22 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 313 * 16 + 8], m4 + ; mode 21 [row 9 - second half] end + + ; mode 21 [row 10 - second half] + pmaddubsw m4, m5, [r3 + 5 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 314 * 16 + 8], m4 + ; mode 21 [row 10 - second half] end + + ; mode 22 [row 12 - second half] + pmaddubsw m4, m5, [r3 + 23 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 332 * 16 + 8], m4 + ; mode 22 [row 12 - second half] end + + ; mode 22 [row 12 - second half] + pmaddubsw m4, m5, [r3 + 10 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 333 * 16 + 8], m4 + ; mode 22 [row 12 - second half] end + + pmaddubsw m4, m0, [r3 + 10 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 10 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 278 * 16], m4 + + ; mode 19 [row 7] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 6], 1 + pinsrb m0, [r2 + 32 + 7], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 3], 1 + pinsrb m5, [r2 + 2], 0 + + ; mode 20 [row 9 - second half] + pmaddubsw m4, m5, [r3 + 14 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 297 * 16 + 8], m4 + ; mode 20 [row 9 - second half] + + ; mode 21 [row 11 - second half] + pmaddubsw m4, m5, [r3 + 20 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 315 * 16 + 8], m4 + ; mode 21 [row 11 - second half] end + + ; mode 21 [row 12 - second half] + pmaddubsw m4, m5, [r3 + 3 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 316 * 16 + 8], m4 + ; mode 21 [row 12 - second half] end + + ; mode 22 [row 14 - second half] + pmaddubsw m4, m5, [r3 + 29 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 334 * 16 + 8], m4 + ; mode 22 [row 14 - second half] end + + ; mode 22 [row 15 - second half] + pmaddubsw m4, m5, [r3 + 16 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 335 * 16 + 8], m4 + ; mode 22 [row 15 - second half] end + + pmaddubsw m4, m0, [r3 + 16 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 16 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 279 * 16], m4 + + ; mode 19 [row 8] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 7], 1 + pinsrb m0, [r2 + 32 + 9], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 2], 1 + pinsrb m5, [r2 + 1], 0 + + ; mode 20 [row 10 - second half] + pmaddubsw m4, m5, [r3 + 25 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 298 * 16 + 8], m4 + ; mode 20 [row 10 - second half] end + + ; mode 20 [row 11 - second half] + pmaddubsw m4, m5, [r3 + 4 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 299 * 16 + 8], m4 + ; mode 20 [row 11 - second half] end + + ; mode 21 [row 13 - second half] + pmaddubsw m4, m5, [r3 + 18 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 317 * 16 + 8], m4 + ; mode 21 [row 13 - second half] end + + ; mode 21 [row 14 - second half] + pmaddubsw m4, m5, [r3 + 1 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 318 * 16 + 8], m4 + ; mode 21 [row 14 - second half] end + + pmaddubsw m4, m0, [r3 + 22 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 22 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 280 * 16], m4 + + ; mode 19 [row 9] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 9], 1 + pinsrb m0, [r2 + 32 + 10], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 1], 1 + pinsrb m5, [r2 + 0], 0 + + ; mode 20 [row 12 - second half] + pmaddubsw m4, m5, [r3 + 15 * 16] + pmulhrsw m4, m3 + packuswb m4, m4 + movh [r0 + 300 * 16 + 8], m4 + + ; mode 20 [row 12 - second half] end + pmaddubsw m4, m0, [r3 + 28 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 28 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 281 * 16], m4 + + ; mode 19 [row 10] + pmaddubsw m4, m0, [r3 + 2 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 2 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 282 * 16], m4 + + ; mode 19 [row 11] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 10], 1 + pinsrb m0, [r2 + 32 + 11], 0 + pmaddubsw m4, m0, [r3 + 8 * 16] + pmulhrsw m4, m3 + pslldq m5, 2 + pinsrb m5, [r2], 1 + pinsrb m5, [r2 + 32 + 1], 0 + pmaddubsw m6, m5, [r3 + 8 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 283 * 16], m4 + + ; mode 19 [row 12] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 11], 1 + pinsrb m0, [r2 + 32 + 12], 0 + pslldq m5, 2 + pinsrb m5, [r2 + 32 + 1], 1 + pinsrb m5, [r2 + 32 + 2], 0 + pmaddubsw m4, m0, [r3 + 14 * 16] + pmulhrsw m4, m3 + pmaddubsw m6, m5, [r3 + 14 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 284 * 16], m4 + + ; mode 19 [row 13] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 12], 1 + pinsrb m0, [r2 + 32 + 14], 0 + pmaddubsw m4, m0, [r3 + 20 * 16] + pmulhrsw m4, m3 + pslldq m5, 2 + pinsrb m5, [r2 + 32 + 2], 1 + pinsrb m5, [r2 + 32 + 4], 0 + pmaddubsw m6, m5, [r3 + 20 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 285 * 16], m4 + + ; mode 19 [row 14] + pslldq m0, 2 + pinsrb m0, [r2 + 32 + 14], 1 + pinsrb m0, [r2 + 32 + 15], 0 + pmaddubsw m4, m0, [r3 + 26 * 16] + pmulhrsw m4, m3 + pslldq m5, 2 + pinsrb m5, [r2 + 32 + 4], 1 + pinsrb m5, [r2 + 32 + 5], 0 + pmaddubsw m6, m5, [r3 + 26 * 16] + pmulhrsw m6, m3 + packuswb m4, m6 + movu [r0 + 286 * 16], m4 + + ; mode 19 [row 15] + movu m0, [r2 + 32] + pshufb m0, [tab_S1] + movu [r0 + 287 * 16], m0 + movd m1, [r2] + movd [r0 + 287 * 16 + 12], m1 + + ; mode 25 + movu m1, [r1] + + ; mode 26 [all rows] + psrldq m6, m1, 1 + pinsrb m6, [r1 + 16], 15 + movu m7, m6 + movu [r0 + 384 * 16], m6 + movu [r0 + 385 * 16], m6 + movu [r0 + 386 * 16], m6 + movu [r0 + 387 * 16], m6 + movu [r0 + 388 * 16], m6 + movu [r0 + 389 * 16], m6 + movu [r0 + 390 * 16], m6 + movu [r0 + 391 * 16], m6 + movu [r0 + 392 * 16], m6 + movu [r0 + 393 * 16], m6 + movu [r0 + 394 * 16], m6 + movu [r0 + 395 * 16], m6 + movu [r0 + 396 * 16], m6 + movu [r0 + 397 * 16], m6 + movu [r0 + 398 * 16], m6 + movu [r0 + 399 * 16], m6 + + pxor m0, m0 + pshufb m6, m6, m0 + punpcklbw m6, m0 + pinsrb m2, [r1], 0 + pshufb m2, m2, m0 + punpcklbw m2, m0 + movu m4, [r1 + 1 + 32] + punpcklbw m5, m4, m0 + punpckhbw m4, m0 + psubw m5, m2 + psubw m4, m2 + psraw m5, 1 + psraw m4, 1 + paddw m5, m6 + paddw m4, m6 + packuswb m5, m4 + + pextrb [r0 + 384 * 16], m5, 0 + pextrb [r0 + 385 * 16], m5, 1 + pextrb [r0 + 386 * 16], m5, 2 + pextrb [r0 + 387 * 16], m5, 3 + pextrb [r0 + 388 * 16], m5, 4 + pextrb [r0 + 389 * 16], m5, 5 + pextrb [r0 + 390 * 16], m5, 6 + pextrb [r0 + 391 * 16], m5, 7 + pextrb [r0 + 392 * 16], m5, 8 + pextrb [r0 + 393 * 16], m5, 9 + pextrb [r0 + 394 * 16], m5, 10 + pextrb [r0 + 395 * 16], m5, 11 + pextrb [r0 + 396 * 16], m5, 12 + pextrb [r0 + 397 * 16], m5, 13 + pextrb [r0 + 398 * 16], m5, 14 + pextrb [r0 + 399 * 16], m5, 15 + + ; mode 25 [row 15] + movu [r0 + 383 * 16], m1 + + ; mode 25 [row 0] + psrldq m2, m1, 1 + punpcklbw m1, m2 + movu m2, [r1 + 8] + psrldq m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m4, m1, [r3 + 30 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 30 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 368 * 16], m4 + + ; mode 25 [row 1] + pmaddubsw m4, m1, [r3 + 28 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 28 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 369 * 16], m4 + + ; mode 25 [row 2] + pmaddubsw m4, m1, [r3 + 26 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 26 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 370 * 16], m4 + + ; mode 25 [row 3] + pmaddubsw m4, m1, [r3 + 24 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 24 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 371 * 16], m4 + + ; mode 25 [row 4] + pmaddubsw m4, m1, [r3 + 22 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 22 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 372 * 16], m4 + + ; mode 25 [row 5] + pmaddubsw m4, m1, [r3 + 20 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 20 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 373 * 16], m4 + + ; mode 25 [row 6] + pmaddubsw m4, m1, [r3 + 18 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 18 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 374 * 16], m4 + + ; mode 25 [row 7] + pmaddubsw m4, m1, [r3 + 16 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 375 * 16], m4 + + ; mode 25 [row 8] + pmaddubsw m4, m1, [r3 + 14 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 14 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 376 * 16], m4 + + ; mode 25 [row 9] + pmaddubsw m4, m1, [r3 + 12 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 12 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 377 * 16], m4 + + ; mode 25 [row 10] + pmaddubsw m4, m1, [r3 + 10 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 10 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 378 * 16], m4 + + ; mode 25 [row 11] + pmaddubsw m4, m1, [r3 + 8 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 8 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 379 * 16], m4 + + ; mode 25 [row 12] + pmaddubsw m4, m1, [r3 + 6 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 6 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 380 * 16], m4 + + ; mode 25 [row 13] + pmaddubsw m4, m1, [r3 + 4 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 4 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 381 * 16], m4 + + ; mode 25 [row 14] + pmaddubsw m4, m1, [r3 + 2 * 16] + pmulhrsw m4, m3 + pmaddubsw m5, m2, [r3 + 2 * 16] + pmulhrsw m5, m3 + packuswb m4, m5 + movu [r0 + 382 * 16], m4 + + ; mode 27 [row 15] + psrldq m6, m7, 1 + punpcklbw m7, m6 + pinsrb m6, [r1 + 17], 15 + movu [r0 + 415 * 16], m6 + + ; mode 27 [row 0] + movu m4, [r1 + 9] + psrldq m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m6, m7, [r3 + 2 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 2 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 400 * 16], m6 + + ; mode 27 [row 1] + pmaddubsw m6, m7, [r3 + 4 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 4 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 401 * 16], m6 + + ; mode 27 [row 2] + pmaddubsw m6, m7, [r3 + 6 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 6 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 402 * 16], m6 + + ; mode 27 [row 3] + pmaddubsw m6, m7, [r3 + 8 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 8 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 403 * 16], m6 + + ; mode 27 [row 4] + pmaddubsw m6, m7, [r3 + 10 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 10 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 404 * 16], m6 + + ; mode 27 [row 5] + pmaddubsw m6, m7, [r3 + 12 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 12 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 405 * 16], m6 + + ; mode 27 [row 6] + pmaddubsw m6, m7, [r3 + 14 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 14 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 406 * 16], m6 + + ; mode 27 [row 7] + pmaddubsw m6, m7, [r3 + 16 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 407 * 16], m6 + + ; mode 27 [row 8] + pmaddubsw m6, m7, [r3 + 18 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 18 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 408 * 16], m6 + + ; mode 27 [row 9] + pmaddubsw m6, m7, [r3 + 20 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 20 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 409 * 16], m6 + + ; mode 27 [row 10] + pmaddubsw m6, m7, [r3 + 22 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 22 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 410 * 16], m6 + + ; mode 27 [row 11] + pmaddubsw m6, m7, [r3 + 24 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 24 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 411 * 16], m6 + + ; mode 27 [row 12] + pmaddubsw m6, m7, [r3 + 26 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 26 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 412 * 16], m6 + + ; mode 27 [row 13] + pmaddubsw m6, m7, [r3 + 28 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 28 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 413 * 16], m6 + + ; mode 27 [row 14] + pmaddubsw m6, m7, [r3 + 30 * 16] + pmulhrsw m6, m3 + pmaddubsw m5, m4, [r3 + 30 * 16] + pmulhrsw m5, m3 + packuswb m6, m5 + movu [r0 + 414 * 16], m6 + + ; mode 28 [row 0] + movu m1, [r2 + 1] + psrldq m2, m1, 1 + punpcklbw m1, m2 + movu m4, [r2 + 9] + psrldq m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m2, m1, [r3 + 5 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 5 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 416 * 16], m2 + + ; mode 28 [row 0] + pmaddubsw m2, m1, [r3 + 5 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 5 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 416 * 16], m2 + + ; mode 28 [row 1] + pmaddubsw m2, m1, [r3 + 10 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 10 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 417 * 16], m2 + + ; mode 28 [row 2] + pmaddubsw m2, m1, [r3 + 15 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 15 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 418 * 16], m2 + + ; mode 28 [row 3] + pmaddubsw m2, m1, [r3 + 20 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 20 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 419 * 16], m2 + + ; mode 28 [row 4] + pmaddubsw m2, m1, [r3 + 25 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 25 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 420 * 16], m2 + + ; mode 28 [row 5] + pmaddubsw m2, m1, [r3 + 30 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 30 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 421 * 16], m2 + + ; mode 29 [row 0] + pmaddubsw m2, m1, [r3 + 9 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 9 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 432 * 16], m2 + + ; mode 29 [row 1] + pmaddubsw m2, m1, [r3 + 18 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 18 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 433 * 16], m2 + + ; mode 29 [row 2] + pmaddubsw m2, m1, [r3 + 27 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 27 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 434 * 16], m2 + + ; mode 30 [row 0] + pmaddubsw m2, m1, [r3 + 13 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 13 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 448 * 16], m2 + + ; mode 30 [row 1] + pmaddubsw m2, m1, [r3 + 26 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 26 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 449 * 16], m2 + + ; mode 33 [row 0] + movu [r0 + 496 * 16], m2 + + ; mode 31 [row 0] + pmaddubsw m2, m1, [r3 + 17 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 17 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 464 * 16], m2 + + ; mode 32 [row 0] + pmaddubsw m2, m1, [r3 + 21 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 21 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 480 * 16], m2 + + ; mode 28 [row 6] + movd m7, [r2 + 9] + palignr m7, m1, 2 + pmaddubsw m2, m7, [r3 + 3 * 16] + pmulhrsw m2, m3 + movd m6, [r2 + 17] + palignr m6, m4, 2 + pmaddubsw m5, m6, [r3 + 3 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 422 * 16], m2 + + ; mode 28 [row 7] + pmaddubsw m2, m7, [r3 + 8 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 8 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 423 * 16], m2 + + ; mode 28 [row 8] + pmaddubsw m2, m7, [r3 + 13 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 13 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 424 * 16], m2 + + ; mode 28 [row 9] + pmaddubsw m2, m7, [r3 + 18 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 18 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 425 * 16], m2 + + ; mode 28 [row 10] + pmaddubsw m2, m7, [r3 + 23 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 23 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 426 * 16], m2 + + ; mode 29 [row 3] + pmaddubsw m2, m7, [r3 + 4 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 4 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 435 * 16], m2 + + ; mode 29 [row 4] + pmaddubsw m2, m7, [r3 + 13 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 13 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 436 * 16], m2 + + ; mode 29 [row 5] + pmaddubsw m2, m7, [r3 + 22 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 22 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 437 * 16], m2 + + ; mode 29 [row 6] + pmaddubsw m2, m7, [r3 + 31 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 31 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 438 * 16], m2 + + ; mode 32 [row 2] + movu [r0 + 482 * 16], m2 + + ; mode 30 [row 2] + pmaddubsw m2, m7, [r3 + 7 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 7 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 450 * 16], m2 + + ; mode 30 [row 3] + pmaddubsw m2, m7, [r3 + 20 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 20 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 451 * 16], m2 + + ; mode 33 [row 1] + movu [r0 + 497 * 16], m2 + + ; mode 31 [row 1] + pmaddubsw m2, m7, [r3 + 2 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 2 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 465 * 16], m2 + + ; mode 31 [row 2] + pmaddubsw m2, m7, [r3 + 19 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 19 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 466 * 16], m2 + + ; mode 32 [row 1] + pmaddubsw m2, m7, [r3 + 10 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 10 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 481 * 16], m2 + + ; mode 28 [row 11] + pmaddubsw m2, m7, [r3 + 28 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 28 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 427 * 16], m2 + + ; mode 28 [row 12] + movd m1, [r2 + 10] + palignr m1, m7, 2 + pmaddubsw m2, m1, [r3 + 1 * 16] + pmulhrsw m2, m3 + movd m4, [r2 + 18] + palignr m4, m6, 2 + pmaddubsw m5, m4, [r3 + 1 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 428 * 16], m2 + + ; mode 30 [row 4] + movu [r0 + 452 * 16], m2 + + ; mode 28 [row 13] + pmaddubsw m2, m1, [r3 + 6 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 6 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 429 * 16], m2 + + ; mode 28 [row 14] + pmaddubsw m2, m1, [r3 + 11 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 11 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 430 * 16], m2 + + ; mode 28 [row 15] + pmaddubsw m2, m1, [r3 + 16 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 431 * 16], m2 + + ; mode 29 [row 7] + pmaddubsw m2, m1, [r3 + 8 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 8 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 439 * 16], m2 + + ; mode 29 [row 8] + pmaddubsw m2, m1, [r3 + 17 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 17 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 440 * 16], m2 + + ; mode 29 [row 9] + pmaddubsw m2, m1, [r3 + 26 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 26 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 441 * 16], m2 + + ; mode 30 [row 5] + pmaddubsw m2, m1, [r3 + 14 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 14 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 453 * 16], m2 + + ; mode 33 [row 2] + movu [r0 + 498 * 16], m2 + + ; mode 30 [row 6] + pmaddubsw m2, m1, [r3 + 27 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 27 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 454 * 16], m2 + + ; mode 31 [row 3] + pmaddubsw m2, m1, [r3 + 4 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 4 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 467 * 16], m2 + + ; mode 31 [row 4] + pmaddubsw m2, m1, [r3 + 21 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 21 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 468 * 16], m2 + + ; mode 32 [row 3] + pmaddubsw m2, m1, [r3 + 20 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 20 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 483 * 16], m2 + + ; mode 29 [row 10] + movd m7, [r2 + 11] + palignr m7, m1, 2 + pmaddubsw m2, m7, [r3 + 3 * 16] + pmulhrsw m2, m3 + movd m6, [r2 + 19] + palignr m6, m4, 2 + pmaddubsw m5, m6, [r3 + 3 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 442 * 16], m2 + + ; mode 29 [row 11] + pmaddubsw m2, m7, [r3 + 12 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 12 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 443 * 16], m2 + + ; mode 29 [row 12] + pmaddubsw m2, m7, [r3 + 21 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 21 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 444 * 16], m2 + + ; mode 30 [row 8] + movu [r0 + 456 * 16], m2 + + ; mode 29 [row 13] + pmaddubsw m2, m7, [r3 + 30 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 30 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 445 * 16], m2 + + ; mode 32 [row 5] + movu [r0 + 485 * 16], m2 + + ; mode 30 [row 7] + pmaddubsw m2, m7, [r3 + 8 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 8 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 455 * 16], m2 + + ; mode 33 [row 3] + movu [r0 + 499 * 16], m2 + + ; mode 31 [row 5] + pmaddubsw m2, m7, [r3 + 6 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 6 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 469 * 16], m2 + + ; mode 31 [row 6] + pmaddubsw m2, m7, [r3 + 23 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 23 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 470 * 16], m2 + + ; mode 32 [row 4] + pmaddubsw m2, m7, [r3 + 9 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 9 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 484 * 16], m2 + + movu m1, m7 + movu m4, m6 + + ; mode 29 [row 14] + movu m1, [r2 + 12] + palignr m1, m7, 2 + pmaddubsw m2, m1, [r3 + 7 * 16] + pmulhrsw m2, m3 + movd m4, [r2 + 20] + palignr m4, m6, 2 + pmaddubsw m5, m4, [r3 + 7 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 446 * 16], m2 + + ; mode 29 [row 15] + pmaddubsw m2, m1, [r3 + 16 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 447 * 16], m2 + + ; mode 30 [row 9] + pmaddubsw m2, m1, [r3 + 2 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 2 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 457 * 16], m2 + + ; mode 33 [row 4] + movu [r0 + 500 * 16], m2 + + ; mode 30 [row 10] + pmaddubsw m2, m1, [r3 + 15 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 15 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 458 * 16], m2 + + ; mode 30 [row 11] + pmaddubsw m2, m1, [r3 + 28 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 28 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 459 * 16], m2 + + ; mode 33 [row 5] + movu [r0 + 501 * 16], m2 + + ; mode 31 [row 7] + pmaddubsw m2, m1, [r3 + 8 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 8 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 471 * 16], m2 + + ; mode 31 [row 8] + pmaddubsw m2, m1, [r3 + 25 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 25 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 472 * 16], m2 + + ; mode 32 [row 6] + pmaddubsw m2, m1, [r3 + 19 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 19 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 486 * 16], m2 + + ; mode 30 [row 12] + movd m7, [r2 + 13] + palignr m7, m1, 2 + pmaddubsw m2, m7, [r3 + 9 * 16] + pmulhrsw m2, m3 + movd m6, [r2 + 21] + palignr m6, m4, 2 + pmaddubsw m5, m6, [r3 + 9 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 460 * 16], m2 + + ; mode 30 [row 13] + pmaddubsw m2, m7, [r3 + 22 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 22 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 461 * 16], m2 + + ; mode 33 [row 6] + movu [r0 + 502 * 16], m2 + + ; mode 31 [row 9] + pmaddubsw m2, m7, [r3 + 10 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 10 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 473 * 16], m2 + + ; mode 31 [row 10] + pmaddubsw m2, m7, [r3 + 27 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 27 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 474 * 16], m2 + + ; mode 32 [row 7] + pmaddubsw m2, m7, [r3 + 8 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 8 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 487 * 16], m2 + + ; mode 32 [row 8] + pmaddubsw m2, m7, [r3 + 29 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 29 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 488 * 16], m2 + + + movu m1, m7 + movu m4, m6 + + ; mode 30 [row 14] + movd m1, [r2 + 14] + palignr m1, m7, 2 + pmaddubsw m2, m1, [r3 + 3 * 16] + pmulhrsw m2, m3 + movd m4, [r2 + 22] + palignr m4, m6, 2 + pmaddubsw m5, m4, [r3 + 3 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 462 * 16], m2 + + ; mode 30 [row 15] + pmaddubsw m2, m1, [r3 + 16 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 463 * 16], m2 + + ; mode 33 [row 7] + movu [r0 + 503 * 16], m2 + + ; mode 31 [row 11] + pmaddubsw m2, m1, [r3 + 12 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 12 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 475 * 16], m2 + + ; mode 31 [row 12] + pmaddubsw m2, m1, [r3 + 29 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 29 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 476 * 16], m2 + + ; mode 32 [row 9] + pmaddubsw m2, m1, [r3 + 18 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 18 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 489 * 16], m2 + + ; mode 31 [row 13] + movd m7, [r2 + 15] + palignr m7, m1, 2 + pmaddubsw m2, m7, [r3 + 14 * 16] + pmulhrsw m2, m3 + movd m6, [r2 + 23] + palignr m6, m4, 2 + pmaddubsw m5, m6, [r3 + 14 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 477 * 16], m2 + + ; mode 31 [row 14] + pmaddubsw m2, m7, [r3 + 31 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 31 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 478 * 16], m2 + + ; mode 32 [row 10] + pmaddubsw m2, m7, [r3 + 7 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 7 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 490 * 16], m2 + + ; mode 32 [row 11] + pmaddubsw m2, m7, [r3 + 28 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 28 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 491 * 16], m2 + + ; mode 33 [row 8] + pmaddubsw m2, m7, [r3 + 10 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 10 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 504 * 16], m2 + + ; mode 31 [row 15] + movd m1, [r2 + 16] + palignr m1, m7, 2 + pmaddubsw m2, m1, [r3 + 16 * 16] + pmulhrsw m2, m3 + movd m4, [r2 + 24] + palignr m4, m6, 2 + pmaddubsw m5, m4, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 479 * 16], m2 + + ; mode 32 [row 12] + pmaddubsw m2, m1, [r3 + 17 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 17 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 492 * 16], m2 + + ; mode 33 [row 9] + pmaddubsw m2, m1, [r3 + 4 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 4 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 505 * 16], m2 + + ; mode 33 [row 10] + pmaddubsw m2, m1, [r3 + 30 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 30 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 506 * 16], m2 + + ; mode 33 [row 10] + pmaddubsw m2, m1, [r3 + 4 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 4 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 505 * 16], m2 + + ; mode 32 [row 13] + movd m7, [r2 + 17] + palignr m7, m1, 2 + pmaddubsw m2, m7, [r3 + 6 * 16] + pmulhrsw m2, m3 + + movd m6, [r2 + 25] + palignr m6, m4, 2 + pmaddubsw m5, m6, [r3 + 6 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 493 * 16], m2 + + ; mode 32 [row 14] + pmaddubsw m2, m7, [r3 + 27 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 27 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 494 * 16], m2 + + ; mode 33 [row 11] + pmaddubsw m2, m7, [r3 + 24 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m6, [r3 + 24 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 507 * 16], m2 + + ; mode 32 [row 15] + movd m1, [r2 + 18] + palignr m1, m7, 2 + pmaddubsw m2, m1, [r3 + 16 * 16] + pmulhrsw m2, m3 + psrldq m4, 2 + pinsrb m4, [r2 + 26], 14 + pinsrb m4, [r2 + 27], 15 + movd m4, [r2 + 26] + palignr m4, m6, 2 + pmaddubsw m5, m4, [r3 + 16 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 495 * 16], m2 + + ; mode 33 [row 12] + pmaddubsw m2, m1, [r3 + 18 * 16] + pmulhrsw m2, m3 + pmaddubsw m5, m4, [r3 + 18 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 508 * 16], m2 + + ; mode 33 [row 13] + movd m7, [r2 + 19] + palignr m7, m1, 2 + pmaddubsw m2, m7, [r3 + 12 * 16] + pmulhrsw m2, m3 + movd m6, [r2 + 27] + palignr m6, m4, 2 + pmaddubsw m5, m6, [r3 + 12 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 509 * 16], m2 + + ; mode 33 [row 14] + movd m1, [r2 + 20] + palignr m1, m7, 2 + pmaddubsw m2, m1, [r3 + 6 * 16] + pmulhrsw m2, m3 + movd m4, [r2 + 28] + palignr m4, m6, 2 + pmaddubsw m5, m4, [r3 + 6 * 16] + pmulhrsw m5, m3 + packuswb m2, m5 + movu [r0 + 510 * 16], m2 + + ; mode 34 [row 0] + movu m1, [r2 + 2] + movu [r0 + 512 * 16], m1 + movu m2, [r2 + 18] + palignr m3, m2, m1, 1 + movu [r0 + 513 * 16], m3 + palignr m3, m2, m1, 2 + movu [r0 + 514 * 16], m3 + palignr m3, m2, m1, 3 + movu [r0 + 515 * 16], m3 + palignr m3, m2, m1, 4 + movu [r0 + 516 * 16], m3 + palignr m3, m2, m1, 5 + movu [r0 + 517 * 16], m3 + palignr m3, m2, m1, 6 + movu [r0 + 518 * 16], m3 + palignr m3, m2, m1, 7 + movu [r0 + 519 * 16], m3 + palignr m3, m2, m1, 8 + movu [r0 + 520 * 16], m3 + palignr m3, m2, m1, 9 + movu [r0 + 521 * 16], m3 + palignr m3, m2, m1, 10 + movu [r0 + 522 * 16], m3 + palignr m3, m2, m1, 11 + movu [r0 + 523 * 16], m3 + palignr m3, m2, m1, 12 + movu [r0 + 524 * 16], m3 + + ; mode 33 [row 15] + movu [r0 + 511 * 16], m3 + + ; mode 34 + palignr m3, m2, m1, 13 + movu [r0 + 525 * 16], m3 + palignr m3, m2, m1, 14 + movu [r0 + 526 * 16], m3 + palignr m3, m2, m1, 15 + movu [r0 + 527 * 16], m3 + RET + +;-------------------------------------------------------------------------------- +; void all_angs_pred_32x32(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) +;-------------------------------------------------------------------------------- +INIT_XMM sse4 +cglobal all_angs_pred_32x32, 3,7,8, 0-4 + mov r6d, [r1 + 64] + mov r3d, [r1] + mov [rsp], r6d + mov [r1 + 64], r3b + mov r3d, [r2] + mov r6d, [r2 + 64] + mov [r2 + 64], r3b + + lea r3, [r2] + lea r4, [r2 + 64] + lea r2, [r1 + 64] + + ;mode 2[row 0] + movu m0, [r4 + 2] + movu [r0 + 0 * 16], m0 + movu m1, [r4 + 18] + movu [r0 + 1 * 16], m1 + + ;mode 9 [row 15] + movu [r0 + 478 * 16], m0 + movu [r0 + 479 * 16], m1 + + ;mode 2[row 1] + movu m2, [r4 + 34] + palignr m3, m1, m0, 1 + movu [r0 + 2 * 16], m3 + palignr m4, m2, m1, 1 + movu [r0 + 3 * 16], m4 + + ; mode 9 [row 31] + movu [r0 + 510 * 16], m3 + movu [r0 + 511 * 16], m4 + + ;mode 2[row 17] + movu [r0 + 34 * 16], m4 + movu m5, [r4 + 35] + movu [r0 + 35 * 16], m5 + + ;mode 2[row 2] + palignr m3, m1, m0, 2 + movu [r0 + 4 * 16], m3 + palignr m4, m2, m1, 2 + movu [r0 + 5 * 16], m4 + + ;mode 2[row 18] + movu [r0 + 36 * 16], m4 + movu m6, [r4 + 51] + palignr m7, m6, m5, 1 + movu [r0 + 37 * 16], m7 + + ;mode 2[row 3] + palignr m3, m1, m0, 3 + movu [r0 + 6 * 16], m3 + palignr m4, m2, m1, 3 + movu [r0 + 7 * 16], m4 + + ;mode 2[row 19] + movu [r0 + 38 * 16], m4 + palignr m7, m6, m5, 2 + movu [r0 + 39 * 16], m7 + + ;mode 2[row 4] + palignr m3, m1, m0, 4 + movu [r0 + 8 * 16], m3 + palignr m4, m2, m1, 4 + movu [r0 + 9 * 16], m4 + + ; mode 8 [row 31] + movu [r0 + 446 * 16], m3 + movu [r0 + 447 * 16], m4 + + ;mode 2[row 20] + movu [r0 + 40 * 16], m4 + palignr m7, m6, m5, 3 + movu [r0 + 41 * 16], m7 + + ; mode 4 [row 31] + movu [r0 + 190 * 16], m4 + movu [r0 + 191 * 16], m7 + + ;mode 2[row 5] + palignr m3, m1, m0, 5 + movu [r0 + 10 * 16], m3 + palignr m4, m2, m1, 5 + movu [r0 + 11 * 16], m4 + + ;mode 2[row 21] + movu [r0 + 42 * 16], m4 + palignr m7, m6, m5, 4 + movu [r0 + 43 * 16], m7 + + ;mode 2[row 6] + palignr m3, m1, m0, 6 + movu [r0 + 12 * 16], m3 + palignr m4, m2, m1, 6 + movu [r0 + 13 * 16], m4 + + ;mode 2[row 22] + movu [r0 + 44 * 16], m4 + palignr m7, m6, m5, 5 + movu [r0 + 45 * 16], m7 + + ;mode 2[row 7] + palignr m3, m1, m0, 7 + movu [r0 + 14 * 16], m3 + palignr m4, m2, m1, 7 + movu [r0 + 15 * 16], m4 + + ;mode 2[row 23] + movu [r0 + 46 * 16], m4 + palignr m7, m6, m5, 6 + movu [r0 + 47 * 16], m7 + + ;mode 2[row 8] + palignr m3, m1, m0, 8 + movu [r0 + 16 * 16], m3 + palignr m4, m2, m1, 8 + movu [r0 + 17 * 16], m4 + + ;mode 7[row 31] + movu [r0 + 382 * 16], m3 + movu [r0 + 383 * 16], m4 + + ;mode 2[row 24] + movu [r0 + 48 * 16], m4 + palignr m7, m6, m5, 7 + movu [r0 + 49 * 16], m7 + + ;mode 2[row 9] + palignr m3, m1, m0, 9 + movu [r0 + 18 * 16], m3 + palignr m4, m2, m1, 9 + movu [r0 + 19 * 16], m4 + + ;mode 2[row 25] + movu [r0 + 50 * 16], m4 + palignr m7, m6, m5, 8 + movu [r0 + 51 * 16], m7 + + ; mode 3 [row 31] + movu [r0 + 126 * 16], m4 + movu [r0 + 127 * 16], m7 + + ;mode 2[row 10] + palignr m3, m1, m0, 10 + movu [r0 + 20 * 16], m3 + palignr m4, m2, m1, 10 + movu [r0 + 21 * 16], m4 + + ;mode 2[row 26] + movu [r0 + 52 * 16], m4 + palignr m7, m6, m5, 9 + movu [r0 + 53 * 16], m7 + + ;mode 2[row 11] + palignr m3, m1, m0, 11 + movu [r0 + 22 * 16], m3 + palignr m4, m2, m1, 11 + movu [r0 + 23 * 16], m4 + + ;mode 2[row 27] + movu [r0 + 54 * 16], m4 + palignr m7, m6, m5, 10 + movu [r0 + 55 * 16], m7 + + ;mode 2[row 12] + palignr m3, m1, m0, 12 + movu [r0 + 24 * 16], m3 + palignr m4, m2, m1, 12 + movu [r0 + 25 * 16], m4 + + ; mode 6 [row 31] + movu [r0 + 318 * 16], m3 + movu [r0 + 319 * 16], m4 + + ; mode 3 [row 15] + movu [r0 + 94 * 16], m3 + movu [r0 + 95 * 16], m4 + + ;mode 2[row 28] + movu [r0 + 56 * 16], m4 + palignr m7, m6, m5, 11 + movu [r0 + 57 * 16], m7 + + ;mode 2[row 13] + palignr m3, m1, m0, 13 + movu [r0 + 26 * 16], m3 + palignr m4, m2, m1, 13 + movu [r0 + 27 * 16], m4 + + ;mode 2[row 29] + movu [r0 + 58 * 16], m4 + palignr m7, m6, m5, 12 + movu [r0 + 59 * 16], m7 + + ;mode 2[row 14] + palignr m3, m1, m0, 14 + movu [r0 + 28 * 16], m3 + palignr m4, m2, m1, 14 + movu [r0 + 29 * 16], m4 + + ;mode 2[row 30] + movu [r0 + 60 * 16], m4 + palignr m7, m6, m5, 13 + movu [r0 + 61 * 16], m7 + + ;mode 2[row 15] + palignr m3, m1, m0, 15 + movu [r0 + 30 * 16], m3 + palignr m4, m2, m1, 15 + movu [r0 + 31 * 16], m4 + + ;mode 2[row 31] + movu [r0 + 62 * 16], m4 + palignr m7, m6, m5, 14 + movu [r0 + 63 * 16], m7 + + ;mode 2[row 16] + movu [r0 + 32 * 16], m1 + movu [r0 + 33 * 16], m2 + + ; mode 5[row 31] + movu [r0 + 254 * 16], m1 + movu [r0 + 255 * 16], m2 + + ; mode 3 [row 0] + lea r5, [ang_table] + movu m6, [r5 + 26 * 16] + movu m7, [pw_1024 ] + movu m1, [r4 + 1 ] + punpcklbw m1, m0 + pmaddubsw m0, m1, m6 + pmulhrsw m0, m7 + movu m2, [r4 + 9] + movd m3, [r4 + 10] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m0, m3 + movu [r0 + 64 * 16], m0 + + ; mode 6 [row 1 - first half] + movu [r0 + 258 * 16], m0 + + ; mode 9 [row 12 - first half] + movu [r0 + 472 * 16], m0 + + movu m0, [r4 + 17] + movd m3, [r4 + 18] + palignr m3, m0, 1 + punpcklbw m0, m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 25] + movd m5, [r4 + 26] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 65 * 16], m3 + + ; mode 6 [row 1 - second half] + movu [r0 + 259 * 16], m3 + + ; mode 9 [row 12 - second half] + movu [r0 + 473 * 16], m3 + + ; mode 4 [row 0] + movu m6, [r5 + 21 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 128 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 129 * 16], m3 + + ; mode 5 [row 0] + movu m6, [r5 + 17 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 192 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 193 * 16], m3 + + ; mode 6 [row 0] + movu m6, [r5 + 13 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 256 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 257 * 16], m3 + + ; mode 7 [row 0] + movu m6, [r5 + 9 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 320 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 321 * 16], m3 + + ; mode 7 [row 1] + movu m6, [r5 + 18 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 322 * 16], m3 + + ; mode 9 [row 8 - first half] + movu [r0 + 464 * 16], m3 + + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 323 * 16], m3 + + ; mode 9 [row 8 - second half] + movu [r0 + 465 * 16], m3 + + ; mode 7 [row 2] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 324 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 325 * 16], m3 + + ; mode 8 [row 0] + movu m6, [r5 + 5 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 384 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 385 * 16], m3 + + ; mode 8 [row 1] + movu m6, [r5 + 10 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 386 * 16], m3 + + ; mode 9 [row 4 - first half] + movu [r0 + 456 * 16], m3 + + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 387 * 16], m3 + + ; mode 9 [row 4 - second half] + movu [r0 + 457 * 16], m3 + + ; mode 8 [row 2] + movu m6, [r5 + 15 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 388 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 389 * 16], m3 + + ; mode 8 [row 3] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 390 * 16], m3 + + ; mode 9 [row 9 - first half] + movu [r0 + 466 * 16], m3 + + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 391 * 16], m3 + + ; mode 9 [row 9 - second half] + movu [r0 + 467 * 16], m3 + + ; mode 8 [row 4] + movu m6, [r5 + 25 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 392 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 393 * 16], m3 + + ; mode 8 [row 5] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 394 * 16], m3 + + ; mode 9 [row 14 - first half] + movu [r0 + 476 * 16], m3 + + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 395 * 16], m3 + + ; mode 9 [row 14 - second half] + movu [r0 + 477 * 16], m3 + + ; mode 9 [row 0] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 448 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 449 * 16], m3 + + ; mode 9 [row 1] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 450 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 451 * 16], m3 + + ; mode 9 [row 2] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 452 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 453 * 16], m3 + + ; mode 9 [row 3] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 454 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 455 * 16], m3 + + ; mode 9 [row 5] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 458 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 459 * 16], m3 + + ; mode 9 [row 6] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 460 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 461 * 16], m3 + + ; mode 9 [row 7] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 462 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 463 * 16], m3 + + ; mode 9 [row 10] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 468 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 469 * 16], m3 + + ; mode 9 [row 11] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 470 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 471 * 16], m3 + + ; mode 9 [row 13] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 474 * 16], m3 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 475 * 16], m3 + + ; mode 3 [row 1] + movu m6, [r5 + 20 * 16] + movu m0, [r4 + 2] + movd m1, [r4 + 3] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 10] + movd m3, [r4 + 11] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 66 * 16], m1 + + ; mode 6 [row 3 - first half] + movu [r0 + 262 * 16], m1 + + ; mode 9 [row 25 - first half] + movu [r0 + 498 * 16], m1 + + movu m1, [r4 + 18] + movd m3, [r4 + 19] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 26] + movd m5, [r4 + 27] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 67 * 16], m3 + + ; mode 6 [row 3 - second half] + movu [r0 + 263 * 16], m3 + + ; mode 9 [row 25 - second half] + movu [r0 + 499 * 16], m3 + + ; mode 4 [row 1] + movu m6, [r5 + 10 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 130 * 16], m3 + + ; mode 9 [row 20 - first half] + movu [r0 + 488 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 131 * 16], m3 + + ; mode 9 [row 20 - second half] + movu [r0 + 489 * 16], m3 + + ; mode 4 [row 2] + movu m6, [r5 + 31 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 132 * 16], m3 + + ; mode 7 [row 6 - first half] + movu [r0 + 332 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 133 * 16], m3 + + ; mode 7 [row 6 - second half] + movu [r0 + 333 * 16], m3 + + ; mode 5 [row 1] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 194 * 16], m3 + + ; mode 5 [row 1 - first half] + movu [r0 + 480 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 195 * 16], m3 + + ; mode 5 [row 1 - second half] + movu [r0 + 481 * 16], m3 + + ; mode 5 [row 2] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 196 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 197 * 16], m3 + + ; mode 6 [row 2] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 260 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 261 * 16], m3 + + ; mode 7 [row 3] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 326 * 16], m3 + + ; mode 9 [row 17 - first half] + movu [r0 + 482 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 327 * 16], m3 + + ; mode 9 [row 17 - second half] + movu [r0 + 483 * 16], m3 + + ; mode 7 [row 4] + movu m6, [r5 + 13 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 328 * 16], m3 + + ; mode 8 [row 8 - first half] + movu [r0 + 400 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 329 * 16], m3 + + ; mode 8 [row 8 - second half] + movu [r0 + 401 * 16], m3 + + ; mode 7 [row 5] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 330 * 16], m3 + + ; mode 9 [row 26 - first half] + movu [r0 + 500 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 331 * 16], m3 + + ; mode 9 [row 26 - second half] + movu [r0 + 501 * 16], m3 + + ; mode 8 [row 6] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 396 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 397 * 16], m3 + + ; mode 9 [row 18] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 484 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 485 * 16], m3 + + ; mode 9 [row 21] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 490 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 491 * 16], m3 + + ; mode 9 [row 22] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 492 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 493 * 16], m3 + + ; mode 9 [row 23] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 494 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 495 * 16], m3 + + ; mode 9 [row 27] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 502 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 503 * 16], m3 + + ; mode 9 [row 28] + movu m6, [r5 + 26 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 504 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 505 * 16], m3 + + ; mode 9 [row 30] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 508 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 509 * 16], m3 + + ; mode 8 [row 7] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 398 * 16], m3 + + ; mode 9 [row 19 - first half] + movu [r0 + 486 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 399 * 16], m3 + + ; mode 9 [row 19 - second half] + movu [r0 + 487 * 16], m3 + + ; mode 8 [row 9] + movu m6, [r5 + 18 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 402 * 16], m3 + + ; mode 9 [row 24 - first half] + movu [r0 + 496 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 403 * 16], m3 + + ; mode 9 [row 24 - second half] + movu [r0 + 497 * 16], m3 + + ; mode 8 [row 10] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 404 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 405 * 16], m3 + + ; mode 8 [row 11] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 406 * 16], m3 + + ; mode 9 [row 29 - first half] + movu [r0 + 506 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 407 * 16], m3 + + ; mode 9 [row 29 - second half] + movu [r0 + 507 * 16], m3 + + ; mode 3 [row 2] + movu m6, [r5 + 14 * 16] + movu m0, [r4 + 3] + movd m1, [r4 + 4] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 11] + movd m3, [r4 + 12] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 68 * 16], m1 + + ; mode 3 [row 2 - first half] + movu [r0 + 266 * 16], m1 + + movu m1, [r4 + 19] + movd m3, [r4 + 20] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 27] + movd m5, [r4 + 28] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 69 * 16], m3 + + ; mode 3 [row 2 - second half] + movu [r0 + 267 * 16], m3 + + ; mode 4 [row 3] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 134 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 135 * 16], m3 + + ; mode 5 [row 3] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 198 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 199 * 16], m3 + + ; mode 5 [row 4] + movu m6, [r5 + 21 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 200 * 16], m3 + + ; mode 8 [row 16 - first half] + movu [r0 + 416 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 201 * 16], m3 + + ; mode 8 [row 16 - second half] + movu [r0 + 417 * 16], m3 + + ; mode 6 [row 4] + movu m6, [r5 + 1 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 264 * 16], m3 + + ; mode 6 [row 4 - first half] + movu [r0 + 408 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 265 * 16], m3 + + ; mode 6 [row 4 - second half] + movu [r0 + 409 * 16], m3 + + ; mode 6 [row 6] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 268 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 269 * 16], m3 + + ; mode 7 [row 7] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 334 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 335 * 16], m3 + + ; mode 7 [row 8] + movu m6, [r5 + 17 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 336 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 337 * 16], m3 + + ; mode 7 [row 9] + movu m6, [r5 + 26 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 338 * 16], m3 + + ; mode 8 [row 17 - first half] + movu [r0 + 418 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 339 * 16], m3 + + ; mode 8 [row 17 - second half] + movu [r0 + 419 * 16], m3 + + ; mode 8 [row 13] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 410 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 411 * 16], m3 + + ; mode 8 [row 14] + movu m6, [r5 + 11 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 412 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 413 * 16], m3 + + ; mode 8 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 414 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 415 * 16], m3 + + ; mode 8 [row 18] + movu m6, [r5 + 31 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 420 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 421 * 16], m3 + + ; mode 3 [row 3] + movu m6, [r5 + 8 * 16] + movu m0, [r4 + 4] + movd m1, [r4 + 5] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 12] + movd m3, [r4 + 13] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 70 * 16], m1 + + ; mode 6 [row 7 - first half] + movu [r0 + 270 * 16], m1 + + movu m1, [r4 + 20] + movd m3, [r4 + 21] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 28] + movd m5, [r4 + 29] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 71 * 16], m3 + + ; mode 6 [row 7 - second half] + movu [r0 + 271 * 16], m3 + + ; mode 4 [row 4] + movu m6, [r5 + 9 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 136 * 16], m3 + + ; mode 4 [row 4 - first half] + movu [r0 + 424 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 137 * 16], m3 + + ; mode 4 [row 4 - second half] + movu [r0 + 425 * 16], m3 + + ; mode 4 [row 5] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 138 * 16], m3 + + ; mode 7 [row 13 - first half] + movu [r0 + 346 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 139 * 16], m3 + + ; mode 7 [row 13 - second half] + movu [r0 + 347 * 16], m3 + + ; mode 5 [row 5] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 202 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 203 * 16], m3 + + ; mode 5 [row 6] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 204 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 205 * 16], m3 + + ; mode 6 [row 8] + movu m6, [r5 + 21 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 272 * 16], m3 + + ; mode 7 [row 12 - first half] + movu [r0 + 344 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 273 * 16], m3 + + ; mode 7 [row 12 - second half] + movu [r0 + 345 * 16], m3 + + ; mode 7 [row 10] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 340 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 341 * 16], m3 + + ; mode 7 [row 11] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 342 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 343 * 16], m3 + + ; mode 8 [row 19] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 422 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 423 * 16], m3 + + ; mode 8 [row 21] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 426 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 427 * 16], m3 + + ; mode 8 [row 22] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 428 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 429 * 16], m3 + + ; mode 8 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 430 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 431 * 16], m3 + + ; mode 8 [row 24] + movu m6, [r5 + 29 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 432 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 433 * 16], m3 + + ; mode 3 [row 4] + movu m6, [r5 + 2 * 16] + movu m0, [r4 + 5] + movd m1, [r4 + 6] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 13] + movd m3, [r4 + 14] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 72 * 16], m1 + + ; mode 3 [row 4 - first half] + movu [r0 + 274 * 16], m1 + + ; mode 8 [row 25 - first half] + movu [r0 + 434 * 16], m1 + + movu m1, [r4 + 21] + movd m3, [r4 + 22] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 29] + movd m5, [r4 + 30] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 73 * 16], m3 + + ; mode 3 [row 4 - second half] + movu [r0 + 275 * 16], m3 + + ; mode 8 [row 25 - second half] + movu [r0 + 435 * 16], m3 + + ; mode 3 [row 5] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 74 * 16], m3 + + ; mode 3 [row 5 - first half] + movu [r0 + 278 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 75 * 16], m3 + + ; mode 3 [row 5 - second half] + movu [r0 + 279 * 16], m3 + + ; mode 4 [row 6] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 140 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 141 * 16], m3 + + ; mode 5 [row 7] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 206 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 207 * 16], m3 + + ; mode 5 [row 8] + movu m6, [r5 + 25 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 208 * 16], m3 + + ; mode 7 [row 16 - first half] + movu [r0 + 352 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 209 * 16], m3 + + ; mode 7 [row 16 - second half] + movu [r0 + 353 * 16], m3 + + ; mode 6 [row 10] + movu m6, [r5 + 15 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 276 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 277 * 16], m3 + + ; mode 7 [row 14] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 348 * 16], m3 + + ; mode 8 [row 26 - first half] + movu [r0 + 436 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 349 * 16], m3 + + ; mode 8 [row 26 - second half] + movu [r0 + 437 * 16], m3 + + ; mode 7 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 350 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 351 * 16], m3 + + ; mode 8 [row 27] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 438 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 439 * 16], m3 + + ; mode 8 [row 28] + movu m6, [r5 + 17 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 440 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 441 * 16], m3 + + ; mode 8 [row 29] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 442 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 443 * 16], m3 + + ; mode 8 [row 30] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 444 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 445 * 16], m3 + + ; mode 3 [row 6] + movu m6, [r5 + 22 * 16] + movu m0, [r4 + 6] + movd m1, [r4 + 7] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 14] + movd m3, [r4 + 15] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 76 * 16], m1 + + ; mode 6 [row 13 - first half] + movu [r0 + 282 * 16], m1 + + movu m1, [r4 + 22] + movd m3, [r4 + 23] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 30] + movd m5, [r4 + 31] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 77 * 16], m3 + + ; mode 6 [row 13 - second half] + movu [r0 + 283 * 16], m3 + + ; mode 4 [row 7] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 142 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 143 * 16], m3 + + ; mode 4 [row 8] + movu m6, [r5 + 29 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 144 * 16], m3 + + ; mode 4 [row 8 - first half] + movu [r0 + 360 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 145 * 16], m3 + + ; mode 4 [row 8 - second half] + movu [r0 + 361 * 16], m3 + + ; mode 5 [row 9] + movu m6, [r5 + 10 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 210 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 211 * 16], m3 + + ; mode 5 [row 10] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 212 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 213 * 16], m3 + + ; mode 7 [row 17] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 354 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 355 * 16], m3 + + ; mode 7 [row 18] + movu m6, [r5 + 11 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 356 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 357 * 16], m3 + + ; mode 7 [row 19] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 358 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 359 * 16], m3 + + ; mode 6 [row 12] + movu m6, [r5 + 9 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 280 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 281 * 16], m3 + + ; mode 3 [row 7] + movu m6, [r5 + 16 * 16] + movu m0, [r4 + 7] + movd m1, [r4 + 8] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 15] + movd m3, [r4 + 16] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 78 * 16], m1 + + ; mode 6 [row 15 - first half] + movu [r0 + 286 * 16], m1 + + movu m1, [r4 + 23] + movd m3, [r4 + 24] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 31] + movd m5, [r4 + 32] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 79 * 16], m3 + + ; mode 6 [row 15 - second half] + movu [r0 + 287 * 16], m3 + + ; mode 4 [row 9] + movu m6, [r5 + 18 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 146 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 147 * 16], m3 + + ; mode 5 [row 11] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 214 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 215 * 16], m3 + + ; mode 5 [row 12] + movu m6, [r5 + 29 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 216 * 16], m3 + + ; mode 6 [row 16 - first half] + movu [r0 + 288 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 217 * 16], m3 + + ; mode 6 [row 16 - second half] + movu [r0 + 289 * 16], m3 + + ; mode 6 [row 14] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 284 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 285 * 16], m3 + + ; mode 7 [row 21] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 362 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 363 * 16], m3 + + ; mode 7 [row 22] + movu m6, [r5 + 15 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 364 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 365 * 16], m3 + + ; mode 7 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 366 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 367 * 16], m3 + + ; mode 3 [row 8] + movu m6, [r5 + 10 * 16] + movu m0, [r4 + 8] + movd m1, [r4 + 9] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 16] + movd m3, [r4 + 17] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 80 * 16], m1 + + ; mode 7 [row 25 - first half] + movu [r0 + 290 * 16], m1 + + ; mode 6 [row 17 - first half] + movu [r0 + 370 * 16], m1 + + movu m1, [r4 + 24] + movd m3, [r4 + 25] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 32] + movd m5, [r4 + 33] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 81 * 16], m3 + + ; mode 7 [row 25 - second half] + movu [r0 + 291 * 16], m3 + + ; mode 6 [row 17 - second half] + movu [r0 + 371 * 16], m3 + + ; mode 4 [row 10] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 148 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 149 * 16], m3 + + ; mode 4 [row 11] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 150 * 16], m3 + + ; mode 7 [row 27 - first half] + movu [r0 + 374 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 151 * 16], m3 + + ; mode 7 [row 27 - second half] + movu [r0 + 375 * 16], m3 + + ; mode 5 [row 13] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 218 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 219 * 16], m3 + + ; mode 5 [row 14] + movu m6, [r5 + 31 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 220 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 221 * 16], m3 + + ; mode 6 [row 18] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 292 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 293 * 16], m3 + + ; mode 7 [row 24] + movu m6, [r5 + 1 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 368 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 369 * 16], m3 + + ; mode 7 [row 26] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 372 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 373 * 16], m3 + + ; mode 3 [row 9] + movu m6, [r5 + 4 * 16] + movu m0, [r4 + 9] + movd m1, [r4 + 10] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 17] + movd m3, [r4 + 18] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 82 * 16], m1 + + ; mode 6 [row 19 - first half] + movu [r0 + 294 * 16], m1 + + movu m1, [r4 + 25] + movd m3, [r4 + 26] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 33] + movd m5, [r4 + 34] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 83 * 16], m3 + + ; mode 6 [row 19 - second half] + movu [r0 + 295 * 16], m3 + + ; mode 4 [row 12] + movu m6, [r5 + 17 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 152 * 16], m3 + + ; mode 4 [row 12 - first half] + movu [r0 + 296 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 153 * 16], m3 + + ; mode 4 [row 12 - second half] + movu [r0 + 297 * 16], m3 + + ; mode 3 [row 10] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 84 * 16], m3 + + ; mode 6 [row 21 - first half] + movu [r0 + 298 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 85 * 16], m3 + + ; mode 6 [row 21 - second half] + movu [r0 + 299 * 16], m3 + + ; mode 5 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 222 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 223 * 16], m3 + + ; mode 7 [row 28] + movu m6, [r5 + 5 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 376 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 377 * 16], m3 + + ; mode 7 [row 29] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 378 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 379 * 16], m3 + + ; mode 7 [row 30] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 380 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 381 * 16], m3 + + ; mode 3 [row 11] + movu m6, [r5 + 24 * 16] + movu m0, [r4 + 10] + movd m1, [r4 + 11] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 18] + movd m3, [r4 + 19] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 86 * 16], m1 + + ; mode 6 [row 23 - first half] + movu [r0 + 302 * 16], m1 + + movu m1, [r4 + 26] + movd m3, [r4 + 27] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 34] + movd m5, [r4 + 35] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 87 * 16], m3 + + ; mode 6 [row 23 - second half] + movu [r0 + 303 * 16], m3 + + ; mode 4 [row 13] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 154 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 155 * 16], m3 + + ; mode 4 [row 14] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 156 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 157 * 16], m3 + + ; mode 5 [row 16] + movu m6, [r5 + 1 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 224 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 225 * 16], m3 + + ; mode 5 [row 17] + movu m6, [r5 + 18 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 226 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 227 * 16], m3 + + ; mode 6 [row 22] + movu m6, [r5 + 11 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 300 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 301 * 16], m3 + + ; mode 3 [row 12] + movu m6, [r5 + 18 * 16] + movu m0, [r4 + 11] + movd m1, [r4 + 12] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 19] + movd m3, [r4 + 20] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 88 * 16], m1 + + ; mode 6 [row 25 - first half] + movu [r0 + 306 * 16], m1 + + movu m1, [r4 + 27] + movd m3, [r4 + 28] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 35] + movd m5, [r4 + 36] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 89 * 16], m3 + + ; mode 6 [row 25 - second half] + movu [r0 + 307 * 16], m3 + + ; mode 4 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 158 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 159 * 16], m3 + + ; mode 5 [row 18] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 228 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 229 * 16], m3 + + ; mode 5 [row 19] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 230 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 231 * 16], m3 + + ; mode 6 [row 24] + movu m6, [r5 + 5 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 304 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 305 * 16], m3 + + ; mode 6 [row 26] + movu m6, [r5 + 31 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 308 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 309 * 16], m3 + + ; mode 3 [row 13] + movu m6, [r5 + 12 * 16] + movu m0, [r4 + 12] + movd m1, [r4 + 13] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 20] + movd m3, [r4 + 21] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 90 * 16], m1 + + movu m1, [r4 + 28] + movd m3, [r4 + 29] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 36] + movd m5, [r4 + 37] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 91 * 16], m3 + + ; mode 4 [row 16] + movu m6, [r5 + 5 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 160 * 16], m3 + + ; mode 5 [row 20 - first half] + movu [r0 + 232 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 161 * 16], m3 + + ; mode 5 [row 20 - second half] + movu [r0 + 233 * 16], m3 + + ; mode 4 [row 17] + movu m6, [r5 + 26 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 162 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 163 * 16], m3 + + ; mode 5 [row 21] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 234 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 235 * 16], m3 + + ; mode 6 [row 27] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 310 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 311 * 16], m3 + + ; mode 6 [row 28] + movu m6, [r5 + 25 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 312 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 313 * 16], m3 + + ; mode 3 [row 14] + movu m6, [r5 + 6 * 16] + movu m0, [r4 + 13] + movd m1, [r4 + 14] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 21] + movd m3, [r4 + 22] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 92 * 16], m1 + + ; mode 6 [row 29 - first half] + movu [r0 + 314 * 16], m1 + + movu m1, [r4 + 29] + movd m3, [r4 + 30] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 37] + movd m5, [r4 + 38] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 93 * 16], m3 + + ; mode 6 [row 29 - second half] + movu [r0 + 315 * 16], m3 + + ; mode 4 [row 18] + movu m6, [r5 + 15 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 164 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 165 * 16], m3 + + ; mode 5 [row 22] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 236 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 237 * 16], m3 + + ; mode 5 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 238 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 239 * 16], m3 + + ; mode 6 [row 30] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 316 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 317 * 16], m3 + + ; mode 3 [row 16] + movu m6, [r5 + 26 * 16] + movu m0, [r4 + 14] + movd m1, [r4 + 15] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 22] + movd m3, [r4 + 23] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 96 * 16], m1 + + ; mode 5 [row 25 - first half] + movu [r0 + 242 * 16], m1 + + movu m1, [r4 + 30] + movd m3, [r4 + 31] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 38] + movd m5, [r4 + 39] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 97 * 16], m3 + + ; mode 5 [row 25 - second half] + movu [r0 + 243 * 16], m3 + + ; mode 4 [row 19] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 166 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 167 * 16], m3 + + ; mode 4 [row 20] + movu m6, [r5 + 25 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 168 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 169 * 16], m3 + + ; mode 5 [row 24] + movu m6, [r5 + 9 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 240 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 241 * 16], m3 + + ; mode 3 [row 17] + movu m6, [r5 + 20 * 16] + movu m0, [r4 + 15] + movd m1, [r4 + 16] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 23] + movd m3, [r4 + 24] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 98 * 16], m1 + + movu m1, [r4 + 31] + movd m3, [r4 + 32] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 39] + movd m5, [r4 + 40] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 99 * 16], m3 + + ; mode 4 [row 21] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 170 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 171 * 16], m3 + + ; mode 5 [row 26] + movu m6, [r5 + 11 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 244 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 245 * 16], m3 + + ; mode 5 [row 27] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 246 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 247 * 16], m3 + + ; mode 3 [row 18] + movu m6, [r5 + 14 * 16] + movu m0, [r4 + 16] + movd m1, [r4 + 17] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 24] + movd m3, [r4 + 25] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 100 * 16], m1 + + movu m1, [r4 + 32] + movd m3, [r4 + 33] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 40] + movd m5, [r4 + 41] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 101 * 16], m3 + + ; mode 4 [row 22] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 172 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 173 * 16], m3 + + ; mode 4 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 174 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 175 * 16], m3 + + ; mode 5 [row 28] + movu m6, [r5 + 13 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 248 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 249 * 16], m3 + + ; mode 5 [row 29] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 250 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 251 * 16], m3 + + ; mode 3 [row 19] + movu m6, [r5 + 8 * 16] + movu m0, [r4 + 17] + movd m1, [r4 + 18] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 25] + movd m3, [r4 + 26] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 102 * 16], m1 + + movu m1, [r4 + 33] + movd m3, [r4 + 34] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 41] + movd m5, [r4 + 42] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 103 * 16], m3 + + ; mode 4 [row 24] + movu m6, [r5 + 13 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 176 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 177 * 16], m3 + + ; mode 5 [row 30] + movu m6, [r5 + 15 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 252 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 253 * 16], m3 + + ; mode 3 [row 20] + movu m6, [r5 + 2 * 16] + movu m0, [r4 + 18] + movd m1, [r4 + 19] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 26] + movd m3, [r4 + 27] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 104 * 16], m1 + + movu m1, [r4 + 34] + movd m3, [r4 + 35] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 42] + movd m5, [r4 + 43] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 105 * 16], m3 + + ; mode 4 [row 25] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 178 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 179 * 16], m3 + + ; mode 4 [row 26] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 180 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 181 * 16], m3 + + ; mode 3 [row 21] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 106 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 107 * 16], m3 + + ; mode 3 [row 22] + movu m6, [r5 + 22 * 16] + movu m0, [r4 + 19] + movd m1, [r4 + 20] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 27] + movd m3, [r4 + 28] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 108 * 16], m1 + + movu m1, [r4 + 35] + movd m3, [r4 + 36] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 43] + movd m5, [r4 + 44] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 109 * 16], m3 + + ; mode 4 [row 27] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 182 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 183 * 16], m3 + + ; mode 3 [row 23] + movu m6, [r5 + 16 * 16] + movu m0, [r4 + 20] + movd m1, [r4 + 21] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 28] + movd m3, [r4 + 29] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 110 * 16], m1 + + movu m1, [r4 + 36] + movd m3, [r4 + 37] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 44] + movd m5, [r4 + 45] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 111 * 16], m3 + + ; mode 4 [row 28] + movu m6, [r5 + 1 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 184 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 185 * 16], m3 + + ; mode 4 [row 29] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 186 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 187 * 16], m3 + + ; mode 3 [row 24] + movu m6, [r5 + 10 * 16] + movu m0, [r4 + 21] + movd m1, [r4 + 22] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 29] + movd m3, [r4 + 30] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 112 * 16], m1 + + movu m1, [r4 + 37] + movd m3, [r4 + 38] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 45] + movd m5, [r4 + 46] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 113 * 16], m3 + + ; mode 4 [row 30] + movu m6, [r5 + 11 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 188 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 189 * 16], m3 + + ; mode 3 [row 25] + movu m6, [r5 + 4 * 16] + movu m0, [r4 + 22] + movd m1, [r4 + 23] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 30] + movd m3, [r4 + 31] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 114 * 16], m1 + + movu m1, [r4 + 38] + movd m3, [r4 + 39] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 46] + movd m5, [r4 + 47] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 115 * 16], m3 + + ; mode 3 [row 26] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 116 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 117 * 16], m3 + + ; mode 3 [row 27] + movu m6, [r5 + 24 * 16] + movu m0, [r4 + 23] + movd m1, [r4 + 24] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 31] + movd m3, [r4 + 32] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 118 * 16], m1 + + movu m1, [r4 + 39] + movd m3, [r4 + 40] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 47] + movd m5, [r4 + 48] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 119 * 16], m3 + + ; mode 3 [row 28] + movu m6, [r5 + 18 * 16] + movu m0, [r4 + 24] + movd m1, [r4 + 25] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 32] + movd m3, [r4 + 33] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 120 * 16], m1 + + movu m1, [r4 + 40] + movd m3, [r4 + 41] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 48] + movd m5, [r4 + 49] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 121 * 16], m3 + + ; mode 3 [row 29] + movu m6, [r5 + 12 * 16] + movu m0, [r4 + 25] + movd m1, [r4 + 26] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 33] + movd m3, [r4 + 34] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 122 * 16], m1 + + movu m1, [r4 + 41] + movd m3, [r4 + 42] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 49] + movd m5, [r4 + 50] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 123 * 16], m3 + + ; mode 3 [row 30] + movu m6, [r5 + 6 * 16] + movu m0, [r4 + 26] + movd m1, [r4 + 27] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r4 + 34] + movd m3, [r4 + 35] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 124 * 16], m1 + + movu m1, [r4 + 42] + movd m3, [r4 + 43] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r4 + 50] + movd m5, [r4 + 51] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 125 * 16], m3 + + ; mode 10 + movu m1, [r2 + 1] + movu m2, [r2 + 17] + movu [r0 + 512 * 16], m1 + movu [r0 + 513 * 16], m2 + movu [r0 + 514 * 16], m1 + movu [r0 + 515 * 16], m2 + movu [r0 + 516 * 16], m1 + movu [r0 + 517 * 16], m2 + movu [r0 + 518 * 16], m1 + movu [r0 + 519 * 16], m2 + movu [r0 + 520 * 16], m1 + movu [r0 + 521 * 16], m2 + movu [r0 + 522 * 16], m1 + movu [r0 + 523 * 16], m2 + movu [r0 + 524 * 16], m1 + movu [r0 + 525 * 16], m2 + movu [r0 + 526 * 16], m1 + movu [r0 + 527 * 16], m2 + + movu [r0 + 528 * 16], m1 + movu [r0 + 529 * 16], m2 + movu [r0 + 530 * 16], m1 + movu [r0 + 531 * 16], m2 + movu [r0 + 532 * 16], m1 + movu [r0 + 533 * 16], m2 + movu [r0 + 534 * 16], m1 + movu [r0 + 535 * 16], m2 + movu [r0 + 536 * 16], m1 + movu [r0 + 537 * 16], m2 + movu [r0 + 538 * 16], m1 + movu [r0 + 539 * 16], m2 + movu [r0 + 540 * 16], m1 + movu [r0 + 541 * 16], m2 + movu [r0 + 542 * 16], m1 + movu [r0 + 543 * 16], m2 + + movu [r0 + 544 * 16], m1 + movu [r0 + 545 * 16], m2 + movu [r0 + 546 * 16], m1 + movu [r0 + 547 * 16], m2 + movu [r0 + 548 * 16], m1 + movu [r0 + 549 * 16], m2 + movu [r0 + 550 * 16], m1 + movu [r0 + 551 * 16], m2 + movu [r0 + 552 * 16], m1 + movu [r0 + 553 * 16], m2 + movu [r0 + 554 * 16], m1 + movu [r0 + 555 * 16], m2 + movu [r0 + 556 * 16], m1 + movu [r0 + 557 * 16], m2 + movu [r0 + 558 * 16], m1 + movu [r0 + 559 * 16], m2 + + movu [r0 + 560 * 16], m1 + movu [r0 + 561 * 16], m2 + movu [r0 + 562 * 16], m1 + movu [r0 + 563 * 16], m2 + movu [r0 + 564 * 16], m1 + movu [r0 + 565 * 16], m2 + movu [r0 + 566 * 16], m1 + movu [r0 + 567 * 16], m2 + movu [r0 + 568 * 16], m1 + movu [r0 + 569 * 16], m2 + movu [r0 + 570 * 16], m1 + movu [r0 + 571 * 16], m2 + movu [r0 + 572 * 16], m1 + movu [r0 + 573 * 16], m2 + movu [r0 + 574 * 16], m1 + movu [r0 + 575 * 16], m2 + + ; mode 11 [row 0] + movu m0, [r4] + + ; mode 11 [row 15 - first half] + movu [r0 + 606 * 16], m0 + + movu [r0 + 606 * 16], m0 + + ; mode 12 [row 31] + pslldq m6, m0, 4 + pinsrb m6, [r3 + 26], 0 + pinsrb m6, [r3 + 19], 1 + pinsrb m6, [r3 + 13], 2 + pinsrb m6, [r3 + 6], 3 + movu [r0 + 702 * 16], m6 + movu m6, [r4 + 12] + movu [r0 + 703 * 16], m6 + + ; mode 11 [row 31] + pslldq m6, m0, 1 + pinsrb m6, [r3 + 16], 0 + movu [r0 + 638 * 16], m6 + movu m6, [r4 + 15] + movu [r0 + 639 * 16], m6 + + movd m1, [r4 + 1] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m1, m0, [r5 + 30 * 16] + pmulhrsw m1, m7 + movu m2, [r4 + 8] + movd m3, [r4 + 9] + palignr m3, m2, 1 + punpcklbw m2, m3 + pmaddubsw m3, m2, [r5 + 30 * 16] + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 576 * 16], m1 + + movu m1, [r4 + 16] + + ; mode 11 [row 15 - second half] + movu [r0 + 607 * 16], m1 + + movd m3, [r4 + 17] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, [r5 + 30 * 16] + pmulhrsw m3, m7 + movu m4, [r4 + 24] + movd m5, [r4 + 25] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, [r5 + 30 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 577 * 16], m3 + + ; mode 11 [row 1] + pmaddubsw m3, m0, [r5 + 28 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 578 * 16], m3 + pmaddubsw m3, m1, [r5 + 28 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 579 * 16], m3 + + ; mode 11 [row 2] + pmaddubsw m3, m0, [r5 + 26 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 26 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 580 * 16], m3 + pmaddubsw m3, m1, [r5 + 26 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 26 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 581 * 16], m3 + + ; mode 11 [row 3] + pmaddubsw m3, m0, [r5 + 24 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 582 * 16], m3 + pmaddubsw m3, m1, [r5 + 24 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 583 * 16], m3 + + ; mode 11 [row 4] + pmaddubsw m3, m0, [r5 + 22 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 22 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 584 * 16], m3 + + ; mode 12 [row 1 - first half] + movu [r0 + 642 * 16], m3 + + pmaddubsw m3, m1, [r5 + 22 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 22 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 585 * 16], m3 + + ; mode 12 [row 1 - second half] + movu [r0 + 643 * 16], m3 + + ; mode 11 [row 5] + pmaddubsw m3, m0, [r5 + 20 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 20 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 586 * 16], m3 + pmaddubsw m3, m1, [r5 + 20 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 20 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 587 * 16], m3 + + ; mode 11 [row 6] + pmaddubsw m3, m0, [r5 + 18 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 18 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 588 * 16], m3 + pmaddubsw m3, m1, [r5 + 18 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 18 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 589 * 16], m3 + + ; mode 11 [row 7] + pmaddubsw m3, m0, [r5 + 16 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 16 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 590 * 16], m3 + pmaddubsw m3, m1, [r5 + 16 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 16 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 591 * 16], m3 + + ; mode 11 [row 8] + pmaddubsw m3, m0, [r5 + 14 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 14 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 592 * 16], m3 + + ; mode 13 [row 1 - first half] + movu [r0 + 706 * 16], m3 + + pmaddubsw m3, m1, [r5 + 14 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 14 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 593 * 16], m3 + + ; mode 13 [row 1 - second half] + movu [r0 + 707 * 16], m3 + + ; mode 11 [row 9] + pmaddubsw m3, m0, [r5 + 12 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 12 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 594 * 16], m3 + + ; mode 12 [row 3 - first half] + movu [r0 + 646 * 16], m3 + + pmaddubsw m3, m1, [r5 + 12 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 12 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 595 * 16], m3 + + ; mode 12 [row 3 - second half] + movu [r0 + 647 * 16], m3 + + ; mode 11 [row 10] + pmaddubsw m3, m0, [r5 + 10 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 10 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 596 * 16], m3 + pmaddubsw m3, m1, [r5 + 10 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 10 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 597 * 16], m3 + + ; mode 11 [row 11] + pmaddubsw m3, m0, [r5 + 8 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 8 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 598 * 16], m3 + pmaddubsw m3, m1, [r5 + 8 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 8 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 599 * 16], m3 + + ; mode 11 [row 12] + pmaddubsw m3, m0, [r5 + 6 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 6 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 600 * 16], m3 + + ; mode 14 [row 1 - first half] + movu [r0 + 770 * 16], m3 + + pmaddubsw m3, m1, [r5 + 6 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 6 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 601 * 16], m3 + + ; mode 14 [row 1 - second half] + movu [r0 + 771 * 16], m3 + + ; mode 11 [row 13] + pmaddubsw m3, m0, [r5 + 4 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 4 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 602 * 16], m3 + pmaddubsw m3, m1, [r5 + 4 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 4 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 603 * 16], m3 + + ; mode 11 [row 14] + pmaddubsw m3, m0, [r5 + 2 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 2 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 604 * 16], m3 + + ; mode 13 [row 5 - first half] + movu [r0 + 650 * 16], m3 + + pmaddubsw m3, m1, [r5 + 2 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 2 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 605 * 16], m3 + + ; mode 13 [row 5 - second half] + movu [r0 + 651 * 16], m3 + + ; mode 12 [row 0] + pmaddubsw m3, m0, [r5 + 27 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 27 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 640 * 16], m3 + pmaddubsw m3, m1, [r5 + 27 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 27 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 641 * 16], m3 + + ; mode 12 [row 2] + pmaddubsw m3, m0, [r5 + 17 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 17 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 644 * 16], m3 + pmaddubsw m3, m1, [r5 + 17 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 17 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 645 * 16], m3 + + ; mode 12 [row 4] + pmaddubsw m3, m0, [r5 + 7 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 7 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 648 * 16], m3 + pmaddubsw m3, m1, [r5 + 7 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 7 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 649 * 16], m3 + + ; mode 13 [row 0] + pmaddubsw m3, m0, [r5 + 23 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 23 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 704 * 16], m3 + pmaddubsw m3, m1, [r5 + 23 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 23 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 705 * 16], m3 + + ; mode 13 [row 2] + pmaddubsw m3, m0, [r5 + 5 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 5 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 708 * 16], m3 + pmaddubsw m3, m1, [r5 + 5 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 5 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 709 * 16], m3 + + ; mode 14 [row 0] + pmaddubsw m3, m0, [r5 + 19 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 19 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 768 * 16], m3 + pmaddubsw m3, m1, [r5 + 19 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 19 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 769 * 16], m3 + + ; mode 15 [row 0] + pmaddubsw m3, m0, [r5 + 15 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 15 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 832 * 16], m3 + pmaddubsw m3, m1, [r5 + 15 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 15 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 833 * 16], m3 + + ; mode 11 [row 16] + pslldq m0, 2 + pinsrb m0, [r4 + 0], 1 + pinsrb m0, [r3 + 16], 0 + pmaddubsw m3, m0, [r5 + 30 * 16] + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 8], 1 + pinsrb m2, [r4 + 7], 0 + pmaddubsw m5, m2, [r5 + 30 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 608 * 16], m3 + pslldq m1, 2 + pinsrb m1, [r4 + 16], 1 + pinsrb m1, [r4 + 15], 0 + pmaddubsw m3, m1, [r5 + 30 * 16] + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrb m4, [r4 + 24], 1 + pinsrb m4, [r4 + 23], 0 + pmaddubsw m5, m4, [r5 + 30 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 609 * 16], m3 + + ; mode 11 [row 17] + pmaddubsw m3, m0, [r5 + 28 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 610 * 16], m3 + pmaddubsw m3, m1, [r5 + 28 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 611 * 16], m3 + + ; mode 11 [row 18] + pmaddubsw m3, m0, [r5 + 26 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 26 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 612 * 16], m3 + pmaddubsw m3, m1, [r5 + 26 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 26 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 613 * 16], m3 + + ; mode 11 [row 19] + pmaddubsw m3, m0, [r5 + 24 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 614 * 16], m3 + pmaddubsw m3, m1, [r5 + 24 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 615 * 16], m3 + + ; mode 11 [row 20] + pmaddubsw m3, m0, [r5 + 22 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 22 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 616 * 16], m3 + pmaddubsw m3, m1, [r5 + 22 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 22 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 617 * 16], m3 + + ; mode 11 [row 21] + pmaddubsw m3, m0, [r5 + 20 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 20 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 618 * 16], m3 + pmaddubsw m3, m1, [r5 + 20 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 20 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 619 * 16], m3 + + ; mode 11 [row 22] + pmaddubsw m3, m0, [r5 + 18 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 18 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 620 * 16], m3 + pmaddubsw m3, m1, [r5 + 18 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 18 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 621 * 16], m3 + + ; mode 11 [row 23] + pmaddubsw m3, m0, [r5 + 16 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 16 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 622 * 16], m3 + pmaddubsw m3, m1, [r5 + 16 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 16 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 623 * 16], m3 + + ; mode 11 [row 24] + pmaddubsw m3, m0, [r5 + 14 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 14 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 624 * 16], m3 + pmaddubsw m3, m1, [r5 + 14 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 14 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 625 * 16], m3 + + ; mode 11 [row 25] + pmaddubsw m3, m0, [r5 + 12 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 12 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 626 * 16], m3 + pmaddubsw m3, m1, [r5 + 12 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 12 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 627 * 16], m3 + + ; mode 11 [row 26] + pmaddubsw m3, m0, [r5 + 10 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 10 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 628 * 16], m3 + pmaddubsw m3, m1, [r5 + 10 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 10 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 629 * 16], m3 + + ; mode 11 [row 27] + pmaddubsw m3, m0, [r5 + 8 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 8 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 630 * 16], m3 + pmaddubsw m3, m1, [r5 + 8 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 8 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 631 * 16], m3 + + ; mode 11 [row 28] + pmaddubsw m3, m0, [r5 + 6 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 6 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 632 * 16], m3 + pmaddubsw m3, m1, [r5 + 6 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 6 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 633 * 16], m3 + + ; mode 11 [row 29] + pmaddubsw m3, m0, [r5 + 4 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 4 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 634 * 16], m3 + pmaddubsw m3, m1, [r5 + 4 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 4 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 635 * 16], m3 + + ; mode 11 [row 30] + pmaddubsw m3, m0, [r5 + 2 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 2 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 636 * 16], m3 + pmaddubsw m3, m1, [r5 + 2 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 2 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 637 * 16], m3 + + ; mode 12 [row 6] + pinsrb m0, [r3 + 6], 0 + pmaddubsw m3, m0, [r5 + 29 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 29 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 652 * 16], m3 + pmaddubsw m3, m1, [r5 + 29 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 29 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 653 * 16], m3 + + ; mode 12 [row 7] + pmaddubsw m3, m0, [r5 + 24 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 654 * 16], m3 + pmaddubsw m3, m1, [r5 + 24 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 655 * 16], m3 + + ; mode 12 [row 8] + pmaddubsw m3, m0, [r5 + 19 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 19 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 656 * 16], m3 + pmaddubsw m3, m1, [r5 + 19 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 19 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 657 * 16], m3 + + ; mode 12 [row 9] + pmaddubsw m3, m0, [r5 + 14 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 14 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 658 * 16], m3 + pmaddubsw m3, m1, [r5 + 14 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 14 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 659 * 16], m3 + + ; mode 12 [row 10] + pmaddubsw m3, m0, [r5 + 9 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 9 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 660 * 16], m3 + pmaddubsw m3, m1, [r5 + 9 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 9 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 661 * 16], m3 + + ; mode 12 [row 11] + pmaddubsw m3, m0, [r5 + 4 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 4 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 662 * 16], m3 + pmaddubsw m3, m1, [r5 + 4 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 4 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 663 * 16], m3 + + ; mode 13 [row 3] + movu m6, m0 + pinsrb m6, [r3 + 4], 0 + pmaddubsw m3, m6, [r5 + 28 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 710 * 16], m3 + pmaddubsw m3, m1, [r5 + 28 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 711 * 16], m3 + + ; mode 13 [row 4] + pmaddubsw m3, m6, [r5 + 19 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 19 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 712 * 16], m3 + pmaddubsw m3, m1, [r5 + 19 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 19 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 713 * 16], m3 + + ; mode 13 [row 5] + pmaddubsw m3, m6, [r5 + 10 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 10 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 714 * 16], m3 + pmaddubsw m3, m1, [r5 + 10 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 10 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 715 * 16], m3 + + ; mode 13 [row 6] + pmaddubsw m3, m6, [r5 + 1 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 1 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 716 * 16], m3 + pmaddubsw m3, m1, [r5 + 1 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 1 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 717 * 16], m3 + + ; mode 14 [row 2] + movu m6, m0 + pinsrb m6, [r4 + 0], 1 + pinsrb m6, [r3 + 2], 0 + pmaddubsw m3, m6, [r5 + 25 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 25 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 772 * 16], m3 + pmaddubsw m3, m1, [r5 + 25 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 25 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 773 * 16], m3 + + ; mode 14 [row 3] + pmaddubsw m3, m6, [r5 + 12 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 12 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 774 * 16], m3 + pmaddubsw m3, m1, [r5 + 12 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 12 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 775 * 16], m3 + + ; mode 15 [row 1] + pmaddubsw m3, m6, [r5 + 30 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 30 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 834 * 16], m3 + pmaddubsw m3, m1, [r5 + 30 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 30 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 835 * 16], m3 + + ; mode 15 [row 2] + pmaddubsw m3, m6, [r5 + 13 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 13 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 836 * 16], m3 + pmaddubsw m3, m1, [r5 + 13 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 13 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 837 * 16], m3 + + ; mode 15 [row 3] + pslldq m6, 2 + pinsrb m6, [r3 + 2], 1 + pinsrb m6, [r3 + 4], 0 + pmaddubsw m3, m6, [r5 + 28 * 16] + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 7], 1 + pinsrb m2, [r4 + 6], 0 + pmaddubsw m5, m2, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 838 * 16], m3 + pslldq m1, 2 + pinsrb m1, [r4 + 15], 1 + pinsrb m1, [r4 + 14], 0 + pmaddubsw m3, m1, [r5 + 28 * 16] + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrb m4, [r4 + 23], 1 + pinsrb m4, [r4 + 22], 0 + pmaddubsw m5, m4, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 839 * 16], m3 + + ; mode 15 [row 4] + pmaddubsw m3, m6, [r5 + 11 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 11 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 840 * 16], m3 + pmaddubsw m3, m1, [r5 + 11 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 11 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 841 * 16], m3 + + ; mode 15 [row 5, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 4], 1 + pinsrb m6, [r3 + 6], 0 + pmaddubsw m3, m6, [r5 + 26 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 842 * 16], m3 + + ; mode 15 [row 6, 0-7] + pmaddubsw m3, m6, [r5 + 9 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 844 * 16], m3 + + ; mode 15 [row 7, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 6], 1 + pinsrb m6, [r3 + 8], 0 + pmaddubsw m3, m6, [r5 + 24 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 846 * 16], m3 + + ; mode 15 [row 8, 0-7] + pmaddubsw m3, m6, [r5 + 7 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 848 * 16], m3 + + ; mode 15 [row 9, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 8], 1 + pinsrb m6, [r3 + 9], 0 + pmaddubsw m3, m6, [r5 + 22 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 850 * 16], m3 + + ; mode 15 [row 10, 0-7] + pmaddubsw m3, m6, [r5 + 5 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 852 * 16], m3 + + ; mode 15 [row 11, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 9], 1 + pinsrb m6, [r3 + 11], 0 + pmaddubsw m3, m6, [r5 + 20 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 854 * 16], m3 + + ; mode 15 [row 12, 0-7] + pmaddubsw m3, m6, [r5 + 3 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 856 * 16], m3 + + ; mode 15 [row 13, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 11], 1 + pinsrb m6, [r3 + 13], 0 + pmaddubsw m3, m6, [r5 + 18 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 858 * 16], m3 + + ; mode 15 [row 14, 0-7] + pmaddubsw m3, m6, [r5 + 1 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 860 * 16], m3 + + ; mode 15 [row 15, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 13], 1 + pinsrb m6, [r3 + 15], 0 + pmaddubsw m3, m6, [r5 + 16 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 862 * 16], m3 + + ; mode 15 [row 16, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 15], 1 + pinsrb m6, [r3 + 17], 0 + pmaddubsw m3, m6, [r5 + 31 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 864 * 16], m3 + + ; mode 15 [row 17, 0-7] + pmaddubsw m3, m6, [r5 + 14 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 866 * 16], m3 + + ; mode 15 [row 18, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 17], 1 + pinsrb m6, [r3 + 19], 0 + pmaddubsw m3, m6, [r5 + 29 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 868 * 16], m3 + + ; mode 15 [row 19, 0-7] + pmaddubsw m3, m6, [r5 + 12 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 870 * 16], m3 + + ; mode 15 [row 20, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 19], 1 + pinsrb m6, [r3 + 21], 0 + pmaddubsw m3, m6, [r5 + 27 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 872 * 16], m3 + + ; mode 15 [row 21, 0-7] + pmaddubsw m3, m6, [r5 + 10 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 874 * 16], m3 + + ; mode 15 [row 22, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 21], 1 + pinsrb m6, [r3 + 23], 0 + pmaddubsw m3, m6, [r5 + 25 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 876 * 16], m3 + + ; mode 15 [row 23, 0-7] + pmaddubsw m3, m6, [r5 + 8 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 878 * 16], m3 + + ; mode 15 [row 24, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 23], 1 + pinsrb m6, [r3 + 24], 0 + pmaddubsw m3, m6, [r5 + 23 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 880 * 16], m3 + + ; mode 15 [row 25, 0-7] + pmaddubsw m3, m6, [r5 + 6 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 882 * 16], m3 + + ; mode 15 [row 26, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 24], 1 + pinsrb m6, [r3 + 26], 0 + pmaddubsw m3, m6, [r5 + 21 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 884 * 16], m3 + + ; mode 15 [row 27, 0-7] + pmaddubsw m3, m6, [r5 + 4 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 886 * 16], m3 + + ; mode 15 [row 28, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 26], 1 + pinsrb m6, [r3 + 28], 0 + pmaddubsw m3, m6, [r5 + 19 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 888 * 16], m3 + + ; mode 15 [row 29, 0-7] + pmaddubsw m3, m6, [r5 + 2 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 890 * 16], m3 + + ; mode 15 [row 30, 0-7] + pslldq m6, 2 + pinsrb m6, [r3 + 28], 1 + pinsrb m6, [r3 + 30], 0 + pmaddubsw m3, m6, [r5 + 17 * 16] + pmulhrsw m3, m7 + packuswb m3, m3 + movh [r0 + 892 * 16], m3 + + ; mode 15 [row 31, 0-7] + pshufb m3, m6, [tab_S2] + movh [r0 + 894 * 16], m3 + + ; mode 12 [row 12] + pslldq m0, 2 + pinsrb m0, [r3 + 6], 1 + pinsrb m0, [r3 + 13], 0 + pmaddubsw m3, m0, [r5 + 31 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 31 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 664 * 16], m3 + pmaddubsw m3, m1, [r5 + 31 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 31 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 665 * 16], m3 + + ; mode 12 [row 13] + pmaddubsw m3, m0, [r5 + 26 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 26 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 666 * 16], m3 + pmaddubsw m3, m1, [r5 + 26 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 26 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 667 * 16], m3 + + ; mode 12 [row 14] + pmaddubsw m3, m0, [r5 + 21 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 21 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 668 * 16], m3 + pmaddubsw m3, m1, [r5 + 21 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 21 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 669 * 16], m3 + + ; mode 12 [row 15] + pmaddubsw m3, m0, [r5 + 16 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 16 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 670 * 16], m3 + pmaddubsw m3, m1, [r5 + 16 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 16 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 671 * 16], m3 + + ; mode 12 [row 16] + pmaddubsw m3, m0, [r5 + 11 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 11 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 672 * 16], m3 + pmaddubsw m3, m1, [r5 + 11 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 11 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 673 * 16], m3 + + ; mode 12 [row 17] + pmaddubsw m3, m0, [r5 + 6 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 6 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 674 * 16], m3 + pmaddubsw m3, m1, [r5 + 6 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 6 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 675 * 16], m3 + + ; mode 12 [row 18] + pmaddubsw m3, m0, [r5 + 1 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 1 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 676 * 16], m3 + pmaddubsw m3, m1, [r5 + 1 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 1 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 677 * 16], m3 + + ; mode 13 [row 7] + movu m6, m0 + pinsrb m6, [r3 + 4], 2 + pinsrb m6, [r3 + 4], 1 + pinsrb m6, [r3 + 7], 0 + pmaddubsw m3, m6, [r5 + 24 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 718 * 16], m3 + pmaddubsw m3, m1, [r5 + 24 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 719 * 16], m3 + + ; mode 13 [row 8] + pmaddubsw m3, m6, [r5 + 15 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 15 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 720 * 16], m3 + pmaddubsw m3, m1, [r5 + 15 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 15 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 721 * 16], m3 + + ; mode 13 [row 9] + pmaddubsw m3, m6, [r5 + 6 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 6 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 722 * 16], m3 + pmaddubsw m3, m1, [r5 + 6 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 6 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 723 * 16], m3 + + ; mode 14 [row 4] + pinsrb m6, [r3 + 2], 2 + pinsrb m6, [r3 + 2], 1 + pinsrb m6, [r3 + 5], 0 + pmaddubsw m3, m6, [r5 + 31 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 31 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 776 * 16], m3 + pmaddubsw m3, m1, [r5 + 31 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 31 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 777 * 16], m3 + + ; mode 14 [row 5] + pmaddubsw m3, m6, [r5 + 18 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 18 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 778 * 16], m3 + pmaddubsw m3, m1, [r5 + 18 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 18 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 779 * 16], m3 + + ; mode 14 [row 6] + pmaddubsw m3, m6, [r5 + 5 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 5 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 780 * 16], m3 + pmaddubsw m3, m1, [r5 + 5 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 5 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 781 * 16], m3 + + ; mode 14 [row 7] + pslldq m6, 2 + pinsrb m6, [r3 + 5], 1 + pinsrb m6, [r3 + 7], 0 + pmaddubsw m3, m6, [r5 + 24 * 16] + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 5], 0 + pmaddubsw m5, m2, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 782 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 13], 0 + pmaddubsw m3, m1, [r5 + 24 * 16] + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 21], 0 + pmaddubsw m5, m4, [r5 + 24 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 783 * 16], m3 + + ; mode 14 [row 8] + pmaddubsw m3, m6, [r5 + 11 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 11 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 784 * 16], m3 + pmaddubsw m3, m1, [r5 + 11 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 11 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 785 * 16], m3 + + ; mode 15 [row 5, 8-31] + pmaddubsw m5, m2, [r5 + 26 * 16] + pmulhrsw m5, m7 + packuswb m5, m5 + movh [r0 + 842 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 26 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 26 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 843 * 16], m3 + + ; mode 15 [row 6, 8-31] + pmaddubsw m5, m2, [r5 + 9 * 16] + pmulhrsw m5, m7 + packuswb m5, m5 + movh [r0 + 844 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 9 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 9 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 845 * 16], m3 + + ; mode 12 [row 19] + pslldq m0, 2 + pinsrb m0, [r3 + 13], 1 + pinsrb m0, [r3 + 19], 0 + pmaddubsw m3, m0, [r5 + 28 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 678 * 16], m3 + pmaddubsw m3, m1, [r5 + 28 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 28 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 679 * 16], m3 + + ; mode 12 [row 20] + pmaddubsw m3, m0, [r5 + 23 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 23 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 680 * 16], m3 + pmaddubsw m3, m1, [r5 + 23 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 23 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 681 * 16], m3 + + ; mode 12 [row 21] + pmaddubsw m3, m0, [r5 + 18 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 18 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 682 * 16], m3 + pmaddubsw m3, m1, [r5 + 18 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 18 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 683 * 16], m3 + + ; mode 12 [row 22] + pmaddubsw m3, m0, [r5 + 13 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 13 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 684 * 16], m3 + pmaddubsw m3, m1, [r5 + 13 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 13 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 685 * 16], m3 + + ; mode 12 [row 23] + pmaddubsw m3, m0, [r5 + 8 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 8 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 686 * 16], m3 + pmaddubsw m3, m1, [r5 + 8 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 8 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 687 * 16], m3 + + ; mode 12 [row 24] + pmaddubsw m3, m0, [r5 + 3 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m2, [r5 + 3 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 688 * 16], m3 + pmaddubsw m3, m1, [r5 + 3 * 16] + pmulhrsw m3, m7 + pmaddubsw m5, m4, [r5 + 3 * 16] + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 689 * 16], m3 + + ; mode 13 [row 10] + movu m7, m6 + movu m6, m0 + pinsrb m6, [r3 + 4], 4 + pinsrb m6, [r3 + 4], 3 + pinsrb m6, [r3 + 7], 2 + pinsrb m6, [r3 + 7], 1 + pinsrb m6, [r3 + 11], 0 + pmaddubsw m3, m6, [r5 + 29 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 29 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 724 * 16], m3 + pmaddubsw m3, m1, [r5 + 29 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 29 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 725 * 16], m3 + + ; mode 13 [row 11] + pmaddubsw m3, m6, [r5 + 20 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 20 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 726 * 16], m3 + pmaddubsw m3, m1, [r5 + 20 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 20 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 727 * 16], m3 + + ; mode 13 [row 12] + pmaddubsw m3, m6, [r5 + 11 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 11 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 728 * 16], m3 + pmaddubsw m3, m1, [r5 + 11 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 11 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 729 * 16], m3 + + ; mode 13 [row 13] + pmaddubsw m3, m6, [r5 + 2 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 2 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 730 * 16], m3 + pmaddubsw m3, m1, [r5 + 2 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 2 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 731 * 16], m3 + + ; mode 14 [row 9] + pslldq m7, 2 + pinsrb m7, [r3 + 7], 1 + pinsrb m7, [r3 + 10], 0 + pmaddubsw m3, m7, [r5 + 30 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrw m2, [r4 + 4], 0 + pmaddubsw m5, m2, [r5 + 30 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 786 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 12], 0 + pmaddubsw m3, m1, [r5 + 30 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrb m4, [r4 + 21], 1 + pinsrb m4, [r4 + 20], 0 + pmaddubsw m5, m4, [r5 + 30 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 787 * 16], m3 + + ; mode 14 [row 10] + pmaddubsw m3, m7, [r5 + 17 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 17 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 788 * 16], m3 + pmaddubsw m3, m1, [r5 + 17 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 17 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 789 * 16], m3 + + ; mode 14 [row 11] + pmaddubsw m3, m7, [r5 + 4 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 4 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 790 * 16], m3 + pmaddubsw m3, m1, [r5 + 4 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 4 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 791 * 16], m3 + + movu m6, [pw_1024] + + ; mode 15 [row 7, 8-31] + pmaddubsw m5, m2, [r5 + 24 * 16] + pmulhrsw m5, m6 + packuswb m5, m5 + movh [r0 + 846 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 24 * 16] + pmulhrsw m3, m6 + pmaddubsw m5, m4, [r5 + 24 * 16] + pmulhrsw m5, m6 + packuswb m3, m5 + movu [r0 + 847 * 16], m3 + + ; mode 15 [row 8, 8-31] + pmaddubsw m5, m2, [r5 + 7 * 16] + pmulhrsw m5, m6 + packuswb m5, m5 + movh [r0 + 848 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 7 * 16] + pmulhrsw m3, m6 + pmaddubsw m5, m4, [r5 + 7 * 16] + pmulhrsw m5, m6 + packuswb m3, m5 + movu [r0 + 849 * 16], m3 + + ; mode 12 [row 25] + pslldq m0, 2 + pinsrb m0, [r3 + 19], 1 + pinsrb m0, [r3 + 26], 0 + pmaddubsw m3, m0, [r5 + 30 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 30 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 690 * 16], m3 + pmaddubsw m3, m1, [r5 + 30 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 30 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 691 * 16], m3 + + ; mode 12 [row 26] + pmaddubsw m3, m0, [r5 + 25 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 25 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 692 * 16], m3 + pmaddubsw m3, m1, [r5 + 25 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 25 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 693 * 16], m3 + + ; mode 12 [row 27] + pmaddubsw m3, m0, [r5 + 20 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 20 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 694 * 16], m3 + pmaddubsw m3, m1, [r5 + 20 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 20 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 695 * 16], m3 + + ; mode 12 [row 28] + pmaddubsw m3, m0, [r5 + 15 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 15 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 696 * 16], m3 + pmaddubsw m3, m1, [r5 + 15 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 15 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 697 * 16], m3 + + ; mode 12 [row 29] + pmaddubsw m3, m0, [r5 + 10 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 10 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 698 * 16], m3 + pmaddubsw m3, m1, [r5 + 10 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 10 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 699 * 16], m3 + + ; mode 12 [row 30] + pmaddubsw m3, m0, [r5 + 5 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 5 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 700 * 16], m3 + pmaddubsw m3, m1, [r5 + 5 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 5 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 701 * 16], m3 + + ; mode 13 [row 14] + movu m6, m0 + pinsrb m6, [r3 + 4], 6 + pinsrb m6, [r3 + 4], 5 + pinsrb m6, [r3 + 7], 4 + pinsrb m6, [r3 + 7], 3 + pinsrb m6, [r3 + 11], 2 + pinsrb m6, [r3 + 11], 1 + pinsrb m6, [r3 + 14], 0 + pmaddubsw m3, m6, [r5 + 25 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 25 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 732 * 16], m3 + pmaddubsw m3, m1, [r5 + 25 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 25 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 733 * 16], m3 + + ; mode 13 [row 15] + pmaddubsw m3, m6, [r5 + 16 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 16 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 734 * 16], m3 + pmaddubsw m3, m1, [r5 + 16 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 16 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 735 * 16], m3 + + ; mode 13 [row 16] + pmaddubsw m3, m6, [r5 + 7 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 7 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 736 * 16], m3 + pmaddubsw m3, m1, [r5 + 7 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 7 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 737 * 16], m3 + + ; mode 13 [row 17] + pslldq m6, 2 + pinsrb m6, [r3 + 14], 1 + pinsrb m6, [r3 + 18], 0 + pmaddubsw m3, m6, [r5 + 30 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrw m2, [r4 + 3], 0 + pmaddubsw m5, m2, [r5 + 30 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 738 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 11], 0 + pmaddubsw m3, m1, [r5 + 30 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 19], 0 + pmaddubsw m5, m4, [r5 + 30 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 739 * 16], m3 + + ; mode 13 [row 18] + pmaddubsw m3, m6, [r5 + 21 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 21 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 740 * 16], m3 + pmaddubsw m3, m1, [r5 + 21 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 21 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 741 * 16], m3 + + ; mode 13 [row 19] + pmaddubsw m3, m6, [r5 + 12 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 12 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 742 * 16], m3 + pmaddubsw m3, m1, [r5 + 12 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 12 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 743 * 16], m3 + + ; mode 13 [row 20] + pmaddubsw m3, m6, [r5 + 3 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 3 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 744 * 16], m3 + pmaddubsw m3, m1, [r5 + 3 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 3 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 745 * 16], m3 + + ; mode 14 [row 12] + pslldq m7, 2 + pinsrb m7, [r3 + 10], 1 + pinsrb m7, [r3 + 12], 0 + pmaddubsw m3, m7, [r5 + 23 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 23 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 792 * 16], m3 + pmaddubsw m3, m1, [r5 + 23 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 23 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 793 * 16], m3 + + ; mode 14 [row 13] + pmaddubsw m3, m7, [r5 + 10 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 10 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 794 * 16], m3 + pmaddubsw m3, m1, [r5 + 10 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 10 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 795 * 16], m3 + + ; mode 15 [row 9] + pmaddubsw m5, m2, [r5 + 22 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movu [r0 + 850 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 22 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 22 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 851 * 16], m3 + + ; mode 15 [row 10] + pmaddubsw m5, m2, [r5 + 5 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movu [r0 + 852 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 5 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 5 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 853 * 16], m3 + + ; mode 13 [row 21] + pslldq m6, 2 + pinsrb m6, [r3 + 18], 1 + pinsrb m6, [r3 + 21], 0 + pmaddubsw m3, m6, [r5 + 26 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrw m2, [r4 + 2], 0 + pmaddubsw m5, m2, [r5 + 26 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 746 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 10], 0 + pmaddubsw m3, m1, [r5 + 26 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 18], 0 + pmaddubsw m5, m4, [r5 + 26 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 747 * 16], m3 + + ; mode 13 [row 22] + pmaddubsw m3, m6, [r5 + 17 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 17 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 748 * 16], m3 + pmaddubsw m3, m1, [r5 + 17 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 17 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 749 * 16], m3 + + ; mode 13 [row 23] + pmaddubsw m3, m6, [r5 + 8 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 8 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 750 * 16], m3 + pmaddubsw m3, m1, [r5 + 8 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 8 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 751 * 16], m3 + + ; mode 14 [row 14] + pslldq m7, 2 + pinsrb m7, [r3 + 12], 1 + pinsrb m7, [r3 + 15], 0 + pmaddubsw m3, m7, [r5 + 29 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 29 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 796 * 16], m3 + pmaddubsw m3, m1, [r5 + 29 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 29 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 797 * 16], m3 + + ; mode 14 [row 15] + pmaddubsw m3, m7, [r5 + 16 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 16 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 798 * 16], m3 + pmaddubsw m3, m1, [r5 + 16 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 16 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 799 * 16], m3 + + ; mode 14 [row 16] + pmaddubsw m3, m7, [r5 + 3 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 3 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 800 * 16], m3 + pmaddubsw m3, m1, [r5 + 3 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 3 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 801 * 16], m3 + + ; mode 15 [row 11] + pmaddubsw m5, m2, [r5 + 20 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 854 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 20 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 20 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 855 * 16], m3 + + ; mode 15 [row 12] + pmaddubsw m5, m2, [r5 + 3 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 856 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 3 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 3 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 857 * 16], m3 + + ; mode 13 [row 24] + pslldq m6, 2 + pinsrb m6, [r3 + 21], 1 + pinsrb m6, [r3 + 25], 0 + pmaddubsw m3, m6, [r5 + 31 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrw m2, [r4 + 1], 0 + pmaddubsw m5, m2, [r5 + 31 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 752 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 9], 0 + pmaddubsw m3, m1, [r5 + 31 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 17], 0 + pmaddubsw m5, m4, [r5 + 31 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 753 * 16], m3 + + ; mode 13 [row 25] + pmaddubsw m3, m6, [r5 + 22 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 22 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 754 * 16], m3 + pmaddubsw m3, m1, [r5 + 22 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 22 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 755 * 16], m3 + + ; mode 13 [row 26] + pmaddubsw m3, m6, [r5 + 13 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 13 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 756 * 16], m3 + pmaddubsw m3, m1, [r5 + 13 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 13 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 757 * 16], m3 + + ; mode 13 [row 27] + pmaddubsw m3, m6, [r5 + 4 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 4 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 758 * 16], m3 + pmaddubsw m3, m1, [r5 + 4 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 4 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 759 * 16], m3 + + ; mode 14 [row 17] + pslldq m7, 2 + pinsrb m7, [r3 + 15], 1 + pinsrb m7, [r3 + 17], 0 + pmaddubsw m3, m7, [r5 + 22 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 22 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 802 * 16], m3 + pmaddubsw m3, m1, [r5 + 22 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 22 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 803 * 16], m3 + + ; mode 14 [row 18] + pmaddubsw m3, m7, [r5 + 9 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 9 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 804 * 16], m3 + pmaddubsw m3, m1, [r5 + 9 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 9 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 805 * 16], m3 + + ; mode 15 [row 13] + pmaddubsw m5, m2, [r5 + 18 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 858 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 18 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 18 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 859 * 16], m3 + + ; mode 15 [row 14] + pmaddubsw m5, m2, [r5 + 1 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 860 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 1 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 1 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 861 * 16], m3 + + ; mode 13 [row 28] + pslldq m6, 2 + pinsrb m6, [r3 + 25], 1 + pinsrb m6, [r3 + 28], 0 + pmaddubsw m3, m6, [r5 + 27 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrw m2, [r4 + 0], 0 + pmaddubsw m5, m2, [r5 + 27 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 760 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 8], 0 + pmaddubsw m3, m1, [r5 + 27 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 16], 0 + pmaddubsw m5, m4, [r5 + 27 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 761 * 16], m3 + + ; mode 13 [row 29] + pmaddubsw m3, m6, [r5 + 18 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 18 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 762 * 16], m3 + pmaddubsw m3, m1, [r5 + 18 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 18 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 763 * 16], m3 + + ; mode 13 [row 30] + pmaddubsw m3, m6, [r5 + 9 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 9 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 764 * 16], m3 + pmaddubsw m3, m1, [r5 + 9 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 9 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 765 * 16], m3 + + ; mode 14 [row 19] + pslldq m7, 2 + pinsrb m7, [r3 + 17], 1 + pinsrb m7, [r3 + 20], 0 + pmaddubsw m3, m7, [r5 + 28 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 28 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 806 * 16], m3 + pmaddubsw m3, m1, [r5 + 28 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 28 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 807 * 16], m3 + + ; mode 14 [row 20] + pmaddubsw m3, m7, [r5 + 15 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 15 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 808 * 16], m3 + pmaddubsw m3, m1, [r5 + 15 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 15 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 809 * 16], m3 + + ; mode 14 [row 21] + pmaddubsw m3, m7, [r5 + 2 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 2 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 810 * 16], m3 + pmaddubsw m3, m1, [r5 + 2 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 2 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 811 * 16], m3 + + ; mode 15 [row 15] + pmaddubsw m5, m2, [r5 + 16 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 862 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 16 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 16 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 863 * 16], m3 + + ; mode 14 [row 22] + pslldq m7, 2 + pinsrb m7, [r3 + 20], 1 + pinsrb m7, [r3 + 22], 0 + pmaddubsw m3, m7, [r5 + 21 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrb m2, [r4 + 0], 1 + pinsrb m2, [r3 + 2], 0 + pmaddubsw m5, m2, [r5 + 21 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 812 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 7], 0 + pmaddubsw m3, m1, [r5 + 21 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 15], 0 + pmaddubsw m5, m4, [r5 + 21 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 813 * 16], m3 + + ; mode 14 [row 23] + pmaddubsw m3, m7, [r5 + 8 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 8 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 814 * 16], m3 + pmaddubsw m3, m1, [r5 + 8 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 8 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 815 * 16], m3 + + ; mode 15 [row 16] + pmaddubsw m5, m2, [r5 + 31 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 864 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 31 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 31 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 865 * 16], m3 + + ; mode 15 [row 17] + pmaddubsw m5, m2, [r5 + 14 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 866 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 14 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 14 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 867 * 16], m3 + + ; mode 14 [row 24] + pslldq m7, 2 + pinsrb m7, [r3 + 22], 1 + pinsrb m7, [r3 + 25], 0 + pmaddubsw m3, m7, [r5 + 27 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrb m2, [r3 + 2], 1 + pinsrb m2, [r3 + 5], 0 + pmaddubsw m5, m2, [r5 + 27 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 816 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 6], 0 + pmaddubsw m3, m1, [r5 + 27 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 14], 0 + pmaddubsw m5, m4, [r5 + 27 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 817 * 16], m3 + + ; mode 14 [row 25] + pmaddubsw m3, m7, [r5 + 14 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 14 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 818 * 16], m3 + pmaddubsw m3, m1, [r5 + 14 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 14 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 819 * 16], m3 + + ; mode 14 [row 26] + pmaddubsw m3, m7, [r5 + 1 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 1 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 820 * 16], m3 + pmaddubsw m3, m1, [r5 + 1 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 1 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 821 * 16], m3 + + ; mode 15 [row 18] + pinsrb m2, [r3 + 4], 0 + pmaddubsw m5, m2, [r5 + 29 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 868 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 29 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 29 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 869 * 16], m3 + + ; mode 15 [row 19] + pmaddubsw m5, m2, [r5 + 12 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 870 * 16 + 8], m5 + pmaddubsw m3, m1, [r5 + 12 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 12 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 871 * 16], m3 + + ; mode 15 [row 20 - 8 to 15] + pslldq m3, m2, 2 + pinsrb m3, [r3 + 4], 1 + pinsrb m3, [r3 + 6], 0 + pmaddubsw m5, m3, [r5 + 27 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 872 * 16 + 8], m5 + + ; mode 15 [row 21 - 8 to 15] + pmaddubsw m5, m3, [r5 + 10 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 874 * 16 + 8], m5 + + ; mode 15 [row 22 - 8 to 15] + pslldq m3, 2 + pinsrb m3, [r3 + 6], 1 + pinsrb m3, [r3 + 8], 0 + pmaddubsw m5, m3, [r5 + 25 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 876 * 16 + 8], m5 + + ; mode 15 [row 23 - 8 to 15] + pmaddubsw m5, m3, [r5 + 8 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 878 * 16 + 8], m5 + + ; mode 15 [row 24 - 8 to 15] + pslldq m3, 2 + pinsrb m3, [r3 + 8], 1 + pinsrb m3, [r3 + 9], 0 + pmaddubsw m5, m3, [r5 + 23 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 880 * 16 + 8], m5 + + ; mode 15 [row 25 - 8 to 15] + pmaddubsw m5, m3, [r5 + 6 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 882 * 16 + 8], m5 + + ; mode 15 [row 26 - 8 to 15] + pslldq m3, 2 + pinsrb m3, [r3 + 9], 1 + pinsrb m3, [r3 + 11], 0 + pmaddubsw m5, m3, [r5 + 21 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 884 * 16 + 8], m5 + + ; mode 15 [row 27 - 8 to 15] + pmaddubsw m5, m3, [r5 + 4 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 886 * 16 + 8], m5 + + ; mode 15 [row 28 - 8 to 15] + pslldq m3, 2 + pinsrb m3, [r3 + 11], 1 + pinsrb m3, [r3 + 13], 0 + pmaddubsw m5, m3, [r5 + 19 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 888 * 16 + 8], m5 + + ; mode 15 [row 29 - 8 to 15] + pmaddubsw m5, m3, [r5 + 2 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 890 * 16 + 8], m5 + + ; mode 15 [row 30 - 8 to 15] + pslldq m3, 2 + pinsrb m3, [r3 + 13], 1 + pinsrb m3, [r3 + 15], 0 + pmaddubsw m5, m3, [r5 + 17 * 16] + pmulhrsw m5, [pw_1024] + packuswb m5, m5 + movh [r0 + 892 * 16 + 8], m5 + + ; mode 15 [row 31, 8 to 15] + pshufb m5, m3, [tab_S2] + movh [r0 + 894 * 16 + 8], m5 + + ; mode 14 [row 27] + pinsrb m2, [r3 + 5], 0 + pslldq m7, 2 + pinsrb m7, [r3 + 25], 1 + pinsrb m7, [r3 + 27], 0 + pmaddubsw m3, m7, [r5 + 20 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrb m2, [r3 + 5], 1 + pinsrb m2, [r3 + 7], 0 + pmaddubsw m5, m2, [r5 + 20 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 822 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 5], 0 + pmaddubsw m3, m1, [r5 + 20 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 13], 0 + pmaddubsw m5, m4, [r5 + 20 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 823 * 16], m3 + + ; mode 15 [row 20 - 16 to 31] + pmaddubsw m3, m1, [r5 + 27 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 27 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 873 * 16], m3 + + ; mode 15 [row 21 - 16 to 31] + pmaddubsw m3, m1, [r5 + 10 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 10 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 875 * 16], m3 + + ; mode 14 [row 28] + pmaddubsw m3, m7, [r5 + 7 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 7 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 824 * 16], m3 + pmaddubsw m3, m1, [r5 + 7 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 7 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 825 * 16], m3 + + ; mode 14 [row 29] + pslldq m7, 2 + pinsrb m7, [r3 + 27], 1 + pinsrb m7, [r3 + 30], 0 + pmaddubsw m3, m7, [r5 + 26 * 16] + pmulhrsw m3, [pw_1024] + pslldq m2, 2 + pinsrb m2, [r3 + 7], 1 + pinsrb m2, [r3 + 10], 0 + pmaddubsw m5, m2, [r5 + 26 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 826 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 4], 0 + pmaddubsw m3, m1, [r5 + 26 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 12], 0 + pmaddubsw m5, m4, [r5 + 26 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 827 * 16], m3 + + ; mode 14 [row 30] + pmaddubsw m3, m7, [r5 + 13 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m2, [r5 + 13 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 828 * 16], m3 + pmaddubsw m3, m1, [r5 + 13 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 13 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 829 * 16], m3 + + ; mode 15 [row 22] + pmaddubsw m3, m1, [r5 + 25 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 25 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 877 * 16], m3 + + ; mode 15 [row 23] + pmaddubsw m3, m1, [r5 + 8 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 8 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 879 * 16], m3 + + ; mode 14 [row 31] + pshufb m3, m7, [tab_S2] + movh [r0 + 830 * 16], m3 + pshufb m3, m2, [tab_S2] + movh [r0 + 830 * 16 + 8], m3 + pshufb m3, m1, [tab_S2] + movh [r0 + 831 * 16], m3 + pshufb m3, m4, [tab_S2] + movh [r0 + 831 * 16 + 8], m3 + + ; mode 13 [row 31] + pshufb m0, m6, [tab_S2] + movh [r0 + 766 * 16], m0 + movh m0, [r4] + movh [r0 + 766 * 16 + 8], m0 + movu m0, [r4 + 8] + movu [r0 + 767 * 16], m0 + + ; mode 15 [row 24] + pslldq m1, 2 + pinsrw m1, [r4 + 3], 0 + pmaddubsw m3, m1, [r5 + 23 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 11], 0 + pmaddubsw m5, m4, [r5 + 23 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 881 * 16], m3 + + ; mode 15 [row 25] + pmaddubsw m3, m1, [r5 + 6 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 6 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 883 * 16], m3 + + ; mode 15 [row 26] + pslldq m1, 2 + pinsrw m1, [r4 + 2], 0 + pmaddubsw m3, m1, [r5 + 21 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 10], 0 + pmaddubsw m5, m4, [r5 + 21 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 885 * 16], m3 + + ; mode 15 [row 27] + pmaddubsw m3, m1, [r5 + 4 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 4 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 887 * 16], m3 + + ; mode 15 [row 28] + pslldq m1, 2 + pinsrw m1, [r4 + 1], 0 + pmaddubsw m3, m1, [r5 + 19 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 9], 0 + pmaddubsw m5, m4, [r5 + 19 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 889 * 16], m3 + + ; mode 15 [row 29] + pmaddubsw m3, m1, [r5 + 2 * 16] + pmulhrsw m3, [pw_1024] + pmaddubsw m5, m4, [r5 + 2 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 891 * 16], m3 + + ; mode 15 [row 30] + pslldq m1, 2 + pinsrw m1, [r4 + 0], 0 + pmaddubsw m3, m1, [r5 + 17 * 16] + pmulhrsw m3, [pw_1024] + pslldq m4, 2 + pinsrw m4, [r4 + 8], 0 + pmaddubsw m5, m4, [r5 + 17 * 16] + pmulhrsw m5, [pw_1024] + packuswb m3, m5 + movu [r0 + 893 * 16], m3 + + ; mode 15 [row 31] + pshufb m5, m1, [tab_S2] + movh [r0 + 895 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 895 * 16 + 8], m5 + + ; mode 16 [row 0] + movu m6, [r5 + 11 * 16] + movu m7, [pw_1024] + movh m0, [r4 ] + movh m1, [r4 + 1 ] + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movh m2, [r4 + 8] + movh m3, [r4 + 9] + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 896 * 16], m1 + + movh m1, [r4 + 16] + movh m3, [r4 + 17] + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movh m4, [r4 + 24] + movh m5, [r4 + 25] + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 897 * 16], m3 + + ; mode16 [row 1] + movu m6, [r5 + 22 * 16] + pslldq m0, 2 + pinsrb m0, [r4], 1 + pinsrb m0, [r3 + 2], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 898 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 15], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 23], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 899 * 16], m3 + + ; mode16 [row 2] + movu m6, [r5 + 1 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 900 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 901 * 16], m3 + + ; mode16 [row 3] + movu m6, [r5 + 12 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 2], 1 + pinsrb m0, [r3 + 3], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 902 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 14], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 22], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 903 * 16], m3 + + ; mode16 [row 4] + movu m6, [r5 + 23 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 3], 1 + pinsrb m0, [r3 + 5], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 904 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 13], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 21], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 905 * 16], m3 + + ; mode16 [row 5] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 906 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 907 * 16], m3 + + ; mode16 [row 6] + movu m6, [r5 + 13 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 5], 1 + pinsrb m0, [r3 + 6], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 5], 1 + pinsrb m2, [r4 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 908 * 16], m3 + pslldq m1, 2 + pinsrw m1, [r4 + 12], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 20], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 909 * 16], m3 + + ; mode16 [row 7] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 6], 1 + pinsrb m0, [r3 + 8], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 910 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 11], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 19], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 911 * 16], m3 + + ; mode16 [row 8] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 912 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 913 * 16], m3 + + ; mode16 [row 9] + movu m6, [r5 + 14 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 8], 1 + pinsrb m0, [r3 + 9], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 914 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 10], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 18], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 915 * 16], m3 + + ; mode16 [row 10] + movu m6, [r5 + 25 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 9], 1 + pinsrb m0, [r3 + 11], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 916 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 9], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrb m4, [r4 + 18], 1 + pinsrb m4, [r4 + 17], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 917 * 16], m3 + + ; mode16 [row 11] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 918 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 919 * 16], m3 + + ; mode16 [row 12] + movu m6, [r5 + 15 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 11], 1 + pinsrb m0, [r3 + 12], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 0], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 920 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 8], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 16], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 921 * 16], m3 + + ; mode16 [row 13] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 12], 1 + pinsrb m0, [r3 + 14], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 0], 1 + pinsrb m2, [r3 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 922 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 7], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 15], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 923 * 16], m3 + + ; mode16 [row 14] + movu m6, [r5 + 5 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 924 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 925 * 16], m3 + + ; mode16 [row 15] + movu m6, [r5 + 16 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 14], 1 + pinsrb m0, [r3 + 15], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 2], 1 + pinsrb m2, [r3 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 926 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 6], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 14], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 927 * 16], m3 + + ; mode16 [row 16] + movu m6, [r5 + 27 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 15], 1 + pinsrb m0, [r3 + 17], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 3], 1 + pinsrb m2, [r3 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 928 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 5], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 13], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 929 * 16], m3 + + ; mode16 [row 17] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 930 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 931 * 16], m3 + + ; mode16 [row 18] + movu m6, [r5 + 17 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 17], 1 + pinsrb m0, [r3 + 18], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 5], 1 + pinsrb m2, [r3 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 932 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 4], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 12], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 933 * 16], m3 + + ; mode16 [row 19] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 18], 1 + pinsrb m0, [r3 + 20], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 6], 1 + pinsrb m2, [r3 + 8], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 934 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 3], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 11], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 935 * 16], m3 + + ; mode16 [row 20] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 936 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 937 * 16], m3 + + ; mode16 [row 21] + movu m6, [r5 + 18 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 20], 1 + pinsrb m0, [r3 + 21], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 8], 1 + pinsrb m2, [r3 + 9], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 938 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 2], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 10], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 939 * 16], m3 + + ; mode16 [row 22] + movu m6, [r5 + 29 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 21], 1 + pinsrb m0, [r3 + 23], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 9], 1 + pinsrb m2, [r3 + 11], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 940 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 1], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 9], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 941 * 16], m3 + + ; mode16 [row 23] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 942 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 943 * 16], m3 + + ; mode16 [row 24] + movu m6, [r5 + 19 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 23], 1 + pinsrb m0, [r3 + 24], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 11], 1 + pinsrb m2, [r3 + 12], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 944 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 0], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 8], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 945 * 16], m3 + + ; mode16 [row 25] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 24], 1 + pinsrb m0, [r3 + 26], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 12], 1 + pinsrb m2, [r3 + 14], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 946 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r4 + 0], 1 + pinsrb m1, [r3 + 2], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 7], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 947 * 16], m3 + + ; mode16 [row 26] + movu m6, [r5 + 9 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 948 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 949 * 16], m3 + + ; mode16 [row 27] + movu m6, [r5 + 20 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 26], 1 + pinsrb m0, [r3 + 27], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 14], 1 + pinsrb m2, [r3 + 15], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 950 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 2], 1 + pinsrb m1, [r3 + 3], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 6], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 951 * 16], m3 + + ; mode16 [row 28] + movu m6, [r5 + 31 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 27], 1 + pinsrb m0, [r3 + 29], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 15], 1 + pinsrb m2, [r3 + 17], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 952 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 3], 1 + pinsrb m1, [r3 + 5], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 5], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 953 * 16], m3 + + ; mode16 [row 29] + movu m6, [r5 + 10 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 954 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 955 * 16], m3 + + ; mode16 [row 30] + movu m6, [r5 + 21 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 29], 1 + pinsrb m0, [r3 + 30], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 17], 1 + pinsrb m2, [r3 + 18], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 956 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 5], 1 + pinsrb m1, [r3 + 6], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 4], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 957 * 16], m3 + + ; mode16 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 958 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 958 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 959 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 959 * 16 + 8], m5 + + ; mode 17 [row 0] + movu m6, [r5 + 6 * 16] + movu m7, [pw_1024] + movh m0, [r4 ] + movh m1, [r4 + 1 ] + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movh m2, [r4 + 8] + movh m3, [r4 + 9] + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 960 * 16], m1 + + movh m1, [r4 + 16] + movh m3, [r4 + 17] + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movh m4, [r4 + 24] + movh m5, [r4 + 25] + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 961 * 16], m3 + + ; mode17 [row 1] + movu m6, [r5 + 12 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 0], 1 + pinsrb m0, [r3 + 1], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 962 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 15], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 23], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 963 * 16], m3 + + ; mode17 [row 2] + movu m6, [r5 + 18 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 1], 1 + pinsrb m0, [r3 + 2], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 964 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 14], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 22], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 965 * 16], m3 + + ; mode17 [row 3] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 2], 1 + pinsrb m0, [r3 + 4], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 966 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 13], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 21], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 967 * 16], m3 + + ; mode17 [row 4] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 4], 1 + pinsrb m0, [r3 + 5], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 968 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 12], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 20], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 969 * 16], m3 + + ; mode17 [row 5] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 970 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 971 * 16], m3 + + ; mode17 [row 6] + movu m6, [r5 + 10 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 5], 1 + pinsrb m0, [r3 + 6], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 972 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 11], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 19], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 973 * 16], m3 + + ; mode17 [row 7] + movu m6, [r5 + 16 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 6], 1 + pinsrb m0, [r3 + 7], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 974 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 10], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 18], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 975 * 16], m3 + + ; mode17 [row 8] + movu m6, [r5 + 22 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 7], 1 + pinsrb m0, [r3 + 9], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 976 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 9], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 17], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 977 * 16], m3 + + ; mode17 [row 9] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 9], 1 + pinsrb m0, [r3 + 10], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrw m2, [r4 + 0], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 978 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 8], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 16], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 979 * 16], m3 + + ; mode17 [row 10] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 980 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 981 * 16], m3 + + ; mode17 [row 11] + movu m6, [r5 + 8 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 10], 1 + pinsrb m0, [r3 + 11], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 0], 1 + pinsrb m2, [r3 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 982 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 7], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 15], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 983 * 16], m3 + + ; mode17 [row 12] + movu m6, [r5 + 14 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 11], 1 + pinsrb m0, [r3 + 12], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 1], 1 + pinsrb m2, [r3 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 984 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 6], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 14], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 985 * 16], m3 + + ; mode17 [row 13] + movu m6, [r5 + 20 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 12], 1 + pinsrb m0, [r3 + 14], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 2], 1 + pinsrb m2, [r3 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 986 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 5], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 13], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 987 * 16], m3 + + ; mode17 [row 14] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 14], 1 + pinsrb m0, [r3 + 15], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 4], 1 + pinsrb m2, [r3 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 988 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 4], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 12], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 989 * 16], m3 + + ; mode17 [row 15] + pshufb m5, m0, [tab_S2] + movh [r0 + 990 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 990 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 991 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 991 * 16 + 8], m5 + + ; mode17 [row 16] + movu m6, [r5 + 6 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 15], 1 + pinsrb m0, [r3 + 16], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 5], 1 + pinsrb m2, [r3 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 992 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 3], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 11], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 993 * 16], m3 + + ; mode17 [row 17] + movu m6, [r5 + 12 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 16], 1 + pinsrb m0, [r3 + 17], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 6], 1 + pinsrb m2, [r3 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 994 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 2], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 10], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 995 * 16], m3 + + ; mode17 [row 18] + movu m6, [r5 + 18 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 17], 1 + pinsrb m0, [r3 + 18], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 7], 1 + pinsrb m2, [r3 + 9], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 996 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 1], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 9], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 997 * 16], m3 + + ; mode17 [row 19] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 18], 1 + pinsrb m0, [r3 + 20], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 9], 1 + pinsrb m2, [r3 + 10], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 998 * 16], m3 + + pslldq m1, 2 + pinsrw m1, [r4 + 0], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 8], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 999 * 16], m3 + + ; mode17 [row 20] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 20], 1 + pinsrb m0, [r3 + 21], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 10], 1 + pinsrb m2, [r3 + 11], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1000 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r4 + 0], 1 + pinsrb m1, [r3 + 1], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + ;pinsrb m4, [r4 + 8], 1 + ;pinsrb m4, [r4 + 7], 0 + pinsrw m4, [r4 + 7], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1001 * 16], m3 + + ; mode17 [row 21] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1002 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1003 * 16], m3 + + ; mode17 [row 22] + movu m6, [r5 + 10 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 21], 1 + pinsrb m0, [r3 + 22], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 11], 1 + pinsrb m2, [r3 + 12], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1004 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 1], 1 + pinsrb m1, [r3 + 2], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 6], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1005 * 16], m3 + + ; mode17 [row 23] + movu m6, [r5 + 16 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 22], 1 + pinsrb m0, [r3 + 23], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 12], 1 + pinsrb m2, [r3 + 14], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1006 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 2], 1 + pinsrb m1, [r3 + 4], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 5], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1007 * 16], m3 + + ; mode17 [row 24] + movu m6, [r5 + 22 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 23], 1 + pinsrb m0, [r3 + 25], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 14], 1 + pinsrb m2, [r3 + 15], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1008 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 4], 1 + pinsrb m1, [r3 + 5], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 4], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1009 * 16], m3 + + ; mode17 [row 25] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 25], 1 + pinsrb m0, [r3 + 26], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 15], 1 + pinsrb m2, [r3 + 16], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1010 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 5], 1 + pinsrb m1, [r3 + 6], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 3], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1011 * 16], m3 + + ; mode17 [row 26] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1012 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1013 * 16], m3 + + ; mode17 [row 27] + movu m6, [r5 + 8 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 26], 1 + pinsrb m0, [r3 + 27], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 16], 1 + pinsrb m2, [r3 + 17], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1014 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 6], 1 + pinsrb m1, [r3 + 7], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 2], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1015 * 16], m3 + + ; mode17 [row 28] + movu m6, [r5 + 14 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 27], 1 + pinsrb m0, [r3 + 28], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 17], 1 + pinsrb m2, [r3 + 18], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1016 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 7], 1 + pinsrb m1, [r3 + 9], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 1], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1017 * 16], m3 + + ; mode17 [row 29] + movu m6, [r5 + 20 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 28], 1 + pinsrb m0, [r3 + 30], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 18], 1 + pinsrb m2, [r3 + 20], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1018 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 9], 1 + pinsrb m1, [r3 + 10], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrw m4, [r4 + 0], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1019 * 16], m3 + + ; mode17 [row 30] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r3 + 30], 1 + pinsrb m0, [r3 + 31], 0 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 20], 1 + pinsrb m2, [r3 + 21], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1020 * 16], m3 + + pslldq m1, 2 + pinsrb m1, [r3 + 10], 1 + pinsrb m1, [r3 + 11], 0 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pslldq m4, 2 + pinsrb m4, [r4 + 0], 1 + pinsrb m4, [r3 + 1], 0 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1021 * 16], m3 + + ; mode17 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 1022 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1022 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1023 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 1023 * 16 + 8], m5 + + ;mode 18[row 0] + movu m0, [r3] + movu [r0 + 1024 * 16], m0 + movu m1, [r3 + 16] + movu [r0 + 1025 * 16], m1 + + ;mode 18[row 1] + pslldq m0, 1 + pinsrb m0, [r4 + 1], 0 + movu [r0 + 1026 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 15], 0 + movu [r0 + 1027 * 16], m1 + + ;mode 18[row 2] + pslldq m0, 1 + pinsrb m0, [r4 + 2], 0 + movu [r0 + 1028 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 14], 0 + movu [r0 + 1029 * 16], m1 + + ;mode 18[row 3] + pslldq m0, 1 + pinsrb m0, [r4 + 3], 0 + movu [r0 + 1030 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 13], 0 + movu [r0 + 1031 * 16], m1 + + ;mode 18[row 4] + pslldq m0, 1 + pinsrb m0, [r4 + 4], 0 + movu [r0 + 1032 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 12], 0 + movu [r0 + 1033 * 16], m1 + + ;mode 18[row 5] + pslldq m0, 1 + pinsrb m0, [r4 + 5], 0 + movu [r0 + 1034 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 11], 0 + movu [r0 + 1035 * 16], m1 + + ;mode 18[row 6] + pslldq m0, 1 + pinsrb m0, [r4 + 6], 0 + movu [r0 + 1036 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 10], 0 + movu [r0 + 1037 * 16], m1 + + ;mode 18[row 7] + pslldq m0, 1 + pinsrb m0, [r4 + 7], 0 + movu [r0 + 1038 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 9], 0 + movu [r0 + 1039 * 16], m1 + + ;mode 18[row 8] + pslldq m0, 1 + pinsrb m0, [r4 + 8], 0 + movu [r0 + 1040 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 8], 0 + movu [r0 + 1041 * 16], m1 + + ;mode 18[row 9] + pslldq m0, 1 + pinsrb m0, [r4 + 9], 0 + movu [r0 + 1042 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 7], 0 + movu [r0 + 1043 * 16], m1 + + ;mode 18[row 10] + pslldq m0, 1 + pinsrb m0, [r4 + 10], 0 + movu [r0 + 1044 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 6], 0 + movu [r0 + 1045 * 16], m1 + + ;mode 18[row 11] + pslldq m0, 1 + pinsrb m0, [r4 + 11], 0 + movu [r0 + 1046 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 5], 0 + movu [r0 + 1047 * 16], m1 + + ;mode 18[row 12] + pslldq m0, 1 + pinsrb m0, [r4 + 12], 0 + movu [r0 + 1048 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 4], 0 + movu [r0 + 1049 * 16], m1 + + ;mode 18[row 13] + pslldq m0, 1 + pinsrb m0, [r4 + 13], 0 + movu [r0 + 1050 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 3], 0 + movu [r0 + 1051 * 16], m1 + + ;mode 18[row 14] + pslldq m0, 1 + pinsrb m0, [r4 + 14], 0 + movu [r0 + 1052 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 2], 0 + movu [r0 + 1053 * 16], m1 + + ;mode 18[row 15] + pslldq m0, 1 + pinsrb m0, [r4 + 15], 0 + movu [r0 + 1054 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 1], 0 + movu [r0 + 1055 * 16], m1 + + ;mode 18[row 16] + pslldq m0, 1 + pinsrb m0, [r4 + 16], 0 + movu [r0 + 1056 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r3 + 0], 0 + movu [r0 + 1057 * 16], m1 + + ;mode 18[row 17] + pslldq m0, 1 + pinsrb m0, [r4 + 17], 0 + movu [r0 + 1058 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 1], 0 + movu [r0 + 1059 * 16], m1 + + ;mode 18[row 18] + pslldq m0, 1 + pinsrb m0, [r4 + 18], 0 + movu [r0 + 1060 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 2], 0 + movu [r0 + 1061 * 16], m1 + + ;mode 18[row 19] + pslldq m0, 1 + pinsrb m0, [r4 + 19], 0 + movu [r0 + 1062 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 3], 0 + movu [r0 + 1063 * 16], m1 + + ;mode 18[row 20] + pslldq m0, 1 + pinsrb m0, [r4 + 20], 0 + movu [r0 + 1064 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 4], 0 + movu [r0 + 1065 * 16], m1 + + ;mode 18[row 21] + pslldq m0, 1 + pinsrb m0, [r4 + 21], 0 + movu [r0 + 1066 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 5], 0 + movu [r0 + 1067 * 16], m1 + + ;mode 18[row 22] + pslldq m0, 1 + pinsrb m0, [r4 + 22], 0 + movu [r0 + 1068 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 6], 0 + movu [r0 + 1069 * 16], m1 + + ;mode 18[row 23] + pslldq m0, 1 + pinsrb m0, [r4 + 23], 0 + movu [r0 + 1070 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 7], 0 + movu [r0 + 1071 * 16], m1 + + ;mode 18[row 24] + pslldq m0, 1 + pinsrb m0, [r4 + 24], 0 + movu [r0 + 1072 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 8], 0 + movu [r0 + 1073 * 16], m1 + + ;mode 18[row 25] + pslldq m0, 1 + pinsrb m0, [r4 + 25], 0 + movu [r0 + 1074 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 9], 0 + movu [r0 + 1075 * 16], m1 + + ;mode 18[row 26] + pslldq m0, 1 + pinsrb m0, [r4 + 26], 0 + movu [r0 + 1076 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 10], 0 + movu [r0 + 1077 * 16], m1 + + ;mode 18[row 27] + pslldq m0, 1 + pinsrb m0, [r4 + 27], 0 + movu [r0 + 1078 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 11], 0 + movu [r0 + 1079 * 16], m1 + + ;mode 18[row 28] + pslldq m0, 1 + pinsrb m0, [r4 + 28], 0 + movu [r0 + 1080 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 12], 0 + movu [r0 + 1081 * 16], m1 + + ;mode 18[row 29] + pslldq m0, 1 + pinsrb m0, [r4 + 29], 0 + movu [r0 + 1082 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 13], 0 + movu [r0 + 1083 * 16], m1 + + ;mode 18[row 30] + pslldq m0, 1 + pinsrb m0, [r4 + 30], 0 + movu [r0 + 1084 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 14], 0 + movu [r0 + 1085 * 16], m1 + + ;mode 18[row 31] + pslldq m0, 1 + pinsrb m0, [r4 + 31], 0 + movu [r0 + 1086 * 16], m0 + pslldq m1, 1 + pinsrb m1, [r4 + 15], 0 + movu [r0 + 1087 * 16], m1 + + ; mode 19 [row 0] + movu m6, [r5 + 6 * 16] + movu m0, [r3 ] + movu m1, [r3 + 1 ] + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r3 + 8] + movu m3, [r3 + 9] + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 1088 * 16], m1 + + movu m1, [r3 + 16] + movu m3, [r3 + 17] + punpcklbw m1, m3 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + movu m3, [r3 + 24] + movu m5, [r3 + 25] + punpcklbw m3, m5 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1089 * 16], m4 + + ; mode 19 [row 1] + movu m6, [r5 + 12 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 0], 1 + pinsrb m0, [r4 + 1], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1090 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 15], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 23], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1091 * 16], m4 + + ; mode 19 [row 2] + movu m6, [r5 + 18 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 1], 1 + pinsrb m0, [r4 + 2], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1092 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 14], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 22], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1093 * 16], m4 + + ; mode 19 [row 3] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 2], 1 + pinsrb m0, [r4 + 4], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1094 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 13], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 21], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1095 * 16], m4 + + ; mode 19 [row 4] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 4], 1 + pinsrb m0, [r4 + 5], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1096 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 12], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 20], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1097 * 16], m4 + + ; mode 19 [row 5] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1098 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1099 * 16], m4 + + ; mode 19 [row 6] + movu m6, [r5 + 10 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 5], 1 + pinsrb m0, [r4 + 6], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1100 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 11], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 19], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1101 * 16], m4 + + ; mode 19 [row 7] + movu m6, [r5 + 16 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 6], 1 + pinsrb m0, [r4 + 7], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1102 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 10], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 18], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1103 * 16], m4 + + ; mode 19 [row 8] + movu m6, [r5 + 22 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 7], 1 + pinsrb m0, [r4 + 9], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1104 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 9], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 17], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1105 * 16], m4 + + ; mode 19 [row 9] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 9], 1 + pinsrb m0, [r4 + 10], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 0], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1106 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 8], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 16], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1107 * 16], m4 + + ; mode 19 [row 10] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1108 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1109 * 16], m4 + + ; mode 19 [row 11] + movu m6, [r5 + 8 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 10], 1 + pinsrb m0, [r4 + 11], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 0], 1 + pinsrb m2, [r4 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1110 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 7], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 15], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1111 * 16], m4 + + ; mode 19 [row 12] + movu m6, [r5 + 14 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 11], 1 + pinsrb m0, [r4 + 12], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 1], 1 + pinsrb m2, [r4 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1112 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 6], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 14], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1113 * 16], m4 + + ; mode 19 [row 13] + movu m6, [r5 + 20 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 12], 1 + pinsrb m0, [r4 + 14], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 2], 1 + pinsrb m2, [r4 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1114 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 5], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 13], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1115 * 16], m4 + + ; mode 19 [row 14] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 14], 1 + pinsrb m0, [r4 + 15], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 4], 1 + pinsrb m2, [r4 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1116 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 4], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 12], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1117 * 16], m4 + + ; mode19 [row 15] + pshufb m5, m0, [tab_S2] + movh [r0 + 1118 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1118 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1119 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1119 * 16 + 8], m5 + + ; mode 19 [row 16] + movu m6, [r5 + 6 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 15], 1 + pinsrb m0, [r4 + 16], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 5], 1 + pinsrb m2, [r4 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1120 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 3], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 11], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1121 * 16], m4 + + ; mode 19 [row 17] + movu m6, [r5 + 12 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 16], 1 + pinsrb m0, [r4 + 17], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 6], 1 + pinsrb m2, [r4 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1122 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 2], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 10], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1123 * 16], m4 + + ; mode 19 [row 18] + movu m6, [r5 + 18 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 17], 1 + pinsrb m0, [r4 + 18], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 7], 1 + pinsrb m2, [r4 + 9], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1124 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 1], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 9], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1125 * 16], m4 + + ; mode 19 [row 19] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 18], 1 + pinsrb m0, [r4 + 20], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 9], 1 + pinsrb m2, [r4 + 10], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1126 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 0], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 8], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1127 * 16], m4 + + ; mode 19 [row 20] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 20], 1 + pinsrb m0, [r4 + 21], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 10], 1 + pinsrb m2, [r4 + 11], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1128 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 0], 1 + pinsrb m1, [r4 + 1], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrb m3, [r3 + 8], 1 + pinsrb m3, [r3 + 7], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1129 * 16], m4 + + ; mode 19 [row 21] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1130 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1131 * 16], m4 + + ; mode 19 [row 22] + movu m6, [r5 + 10 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 21], 1 + pinsrb m0, [r4 + 22], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 11], 1 + pinsrb m2, [r4 + 12], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1132 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 1], 1 + pinsrb m1, [r4 + 2], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 6], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1133 * 16], m4 + + ; mode 19 [row 23] + movu m6, [r5 + 16 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 22], 1 + pinsrb m0, [r4 + 23], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 12], 1 + pinsrb m2, [r4 + 14], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1134 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 2], 1 + pinsrb m1, [r4 + 4], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 5], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1135 * 16], m4 + + ; mode 19 [row 24] + movu m6, [r5 + 22 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 23], 1 + pinsrb m0, [r4 + 25], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 14], 1 + pinsrb m2, [r4 + 15], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1136 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 4], 1 + pinsrb m1, [r4 + 5], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 4], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1137 * 16], m4 + + ; mode 19 [row 25] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 25], 1 + pinsrb m0, [r4 + 26], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 15], 1 + pinsrb m2, [r4 + 16], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1138 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 5], 1 + pinsrb m1, [r4 + 6], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 3], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1139 * 16], m4 + + ; mode 19 [row 26] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1140 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1141 * 16], m4 + + ; mode 19 [row 27] + movu m6, [r5 + 8 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 26], 1 + pinsrb m0, [r4 + 27], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 16], 1 + pinsrb m2, [r4 + 17], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1142 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 6], 1 + pinsrb m1, [r4 + 7], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 2], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1143 * 16], m4 + + ; mode 19 [row 28] + movu m6, [r5 + 14 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 27], 1 + pinsrb m0, [r4 + 28], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 17], 1 + pinsrb m2, [r4 + 18], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1144 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 7], 1 + pinsrb m1, [r4 + 9], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 1], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1145 * 16], m4 + + ; mode 19 [row 29] + movu m6, [r5 + 20 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 28], 1 + pinsrb m0, [r4 + 30], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 18], 1 + pinsrb m2, [r4 + 20], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1146 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 9], 1 + pinsrb m1, [r4 + 10], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 0], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1147 * 16], m4 + + ; mode 19 [row 30] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 30], 1 + pinsrb m0, [r4 + 31], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 20], 1 + pinsrb m2, [r4 + 21], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1148 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 10], 1 + pinsrb m1, [r4 + 11], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrb m3, [r4 + 0], 1 + pinsrb m3, [r4 + 1], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1149 * 16], m4 + + ; mode19 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 1150 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1150 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1151 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1151 * 16 + 8], m5 + + ; mode 20 [row 0] + movu m6, [r5 + 11 * 16] + movu m0, [r3 ] + movu m1, [r3 + 1 ] + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r3 + 8] + movu m3, [r3 + 9] + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 1152 * 16], m1 + + movu m1, [r3 + 16] + movu m3, [r3 + 17] + punpcklbw m1, m3 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + movu m3, [r3 + 24] + movu m5, [r3 + 25] + punpcklbw m3, m5 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1153 * 16], m4 + + ; mode 20 [row 1] + movu m6, [r5 + 22 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 0], 1 + pinsrb m0, [r4 + 2], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1154 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 15], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 23], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1155 * 16], m4 + + ; mode 20 [row 2] + movu m6, [r5 + 1 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1156 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1157 * 16], m4 + + ; mode 20 [row 3] + movu m6, [r5 + 12 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 2], 1 + pinsrb m0, [r4 + 3], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1158 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 14], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 22], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1159 * 16], m4 + + ; mode 20 [row 4] + movu m6, [r5 + 23 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 3], 1 + pinsrb m0, [r4 + 5], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1160 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 13], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 21], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1161 * 16], m4 + + ; mode 20 [row 5] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1162 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1163 * 16], m4 + + ; mode 20 [row 6] + movu m6, [r5 + 13 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 5], 1 + pinsrb m0, [r4 + 6], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1164 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 12], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 20], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1165 * 16], m4 + + ; mode 20 [row 7] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 6], 1 + pinsrb m0, [r4 + 8], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1166 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 11], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 19], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1167 * 16], m4 + + ; mode 20 [row 8] + movu m6, [r5 + 3 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1168 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1169 * 16], m4 + + ; mode 20 [row 9] + movu m6, [r5 + 14 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 8], 1 + pinsrb m0, [r4 + 9], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 3], 1 + pinsrb m2, [r3 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1170 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 10], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 18], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1171 * 16], m4 + + ; mode 20 [row 10] + movu m6, [r5 + 25 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 9], 1 + pinsrb m0, [r4 + 11], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1172 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 9], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 17], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1173 * 16], m4 + + ; mode 20 [row 11] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1174 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1175 * 16], m4 + + ; mode 20 [row 12] + movu m6, [r5 + 15 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 11], 1 + pinsrb m0, [r4 + 12], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r3 + 1], 1 + pinsrb m2, [r3 + 0], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1176 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 8], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 16], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1177 * 16], m4 + + ; mode 20 [row 13] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 12], 1 + pinsrb m0, [r4 + 14], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 0], 1 + pinsrb m2, [r4 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1178 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 7], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 15], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1179 * 16], m4 + + ; mode 20 [row 14] + movu m6, [r5 + 5 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1180 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1181 * 16], m4 + + ; mode 20 [row 15] + movu m6, [r5 + 16 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 14], 1 + pinsrb m0, [r4 + 15], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 2], 1 + pinsrb m2, [r4 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1182 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 6], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 14], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1183 * 16], m4 + + ; mode 20 [row 16] + movu m6, [r5 + 27 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 15], 1 + pinsrb m0, [r4 + 17], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 3], 1 + pinsrb m2, [r4 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1184 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 5], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 13], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1185 * 16], m4 + + ; mode 20 [row 17] + movu m6, [r5 + 6 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1186 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1187 * 16], m4 + + ; mode 20 [row 18] + movu m6, [r5 + 17 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 17], 1 + pinsrb m0, [r4 + 18], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 5], 1 + pinsrb m2, [r4 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1188 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 4], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 12], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1189 * 16], m4 + + ; mode 20 [row 19] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 18], 1 + pinsrb m0, [r4 + 20], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 6], 1 + pinsrb m2, [r4 + 8], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1190 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 3], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 11], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1191 * 16], m4 + + ; mode 20 [row 20] + movu m6, [r5 + 7 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1192 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1193 * 16], m4 + + ; mode 20 [row 21] + movu m6, [r5 + 18 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 20], 1 + pinsrb m0, [r4 + 21], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 8], 1 + pinsrb m2, [r4 + 9], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1194 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 2], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 10], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1195 * 16], m4 + + ; mode 20 [row 22] + movu m6, [r5 + 29 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 21], 1 + pinsrb m0, [r4 + 23], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 9], 1 + pinsrb m2, [r4 + 11], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1196 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 1], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 9], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1197 * 16], m4 + + ; mode 20 [row 23] + movu m6, [r5 + 8 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1198 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1199 * 16], m4 + + ; mode 20 [row 24] + movu m6, [r5 + 19 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 23], 1 + pinsrb m0, [r4 + 24], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 11], 1 + pinsrb m2, [r4 + 12], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1200 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 0], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 8], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1201 * 16], m4 + + ; mode 20 [row 25] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 24], 1 + pinsrb m0, [r4 + 26], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 12], 1 + pinsrb m2, [r4 + 14], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1202 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 0], 1 + pinsrb m1, [r4 + 2], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 7], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1203 * 16], m4 + + ; mode 20 [row 26] + movu m6, [r5 + 9 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1204 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1205 * 16], m4 + + ; mode 20 [row 27] + movu m6, [r5 + 20 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 26], 1 + pinsrb m0, [r4 + 27], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 14], 1 + pinsrb m2, [r4 + 15], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1206 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 2], 1 + pinsrb m1, [r4 + 3], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 6], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1207 * 16], m4 + + ; mode 20 [row 28] + movu m6, [r5 + 31 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 27], 1 + pinsrb m0, [r4 + 29], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 15], 1 + pinsrb m2, [r4 + 17], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1208 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 3], 1 + pinsrb m1, [r4 + 5], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 5], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1209 * 16], m4 + + ; mode 20 [row 29] + movu m6, [r5 + 10 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1210 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1211 * 16], m4 + + ; mode 20 [row 30] + movu m6, [r5 + 21 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 29], 1 + pinsrb m0, [r4 + 30], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 17], 1 + pinsrb m2, [r4 + 18], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1212 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r4 + 5], 1 + pinsrb m1, [r4 + 6], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 4], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1213 * 16], m4 + + ; mode20 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 1214 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1214 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1215 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1215 * 16 + 8], m5 + + ; mode 21 [row 0] + movu m6, [r5 + 15 * 16] + movu m0, [r3 ] + movu m1, [r3 + 1 ] + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r3 + 8] + movu m3, [r3 + 9] + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 1216 * 16], m1 + + movu m1, [r3 + 16] + movu m3, [r3 + 17] + punpcklbw m1, m3 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + movu m3, [r3 + 24] + movu m5, [r3 + 25] + punpcklbw m3, m5 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1217 * 16], m4 + + ; mode 21 [row 1] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 0], 1 + pinsrb m0, [r4 + 2], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1218 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 15], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 23], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1219 * 16], m4 + + ; mode 21 [row 2] + movu m6, [r5 + 13 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1220 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1221 * 16], m4 + + ; mode 21 [row 3] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 2], 1 + pinsrb m0, [r4 + 4], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1222 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 14], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 22], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1223 * 16], m4 + + ; mode 21 [row 4] + movu m6, [r5 + 11 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1224 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1225 * 16], m4 + + ; mode 21 [row 5] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 4], 1 + pinsrb m0, [r4 + 6], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1226 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 13], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 21], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1227 * 16], m4 + + ; mode 21 [row 6] + movu m6, [r5 + 9 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1228 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1229 * 16], m4 + + ; mode 21 [row 7] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 6], 1 + pinsrb m0, [r4 + 8], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1230 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 12], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 20], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1231 * 16], m4 + + ; mode 21 [row 8] + movu m6, [r5 + 7 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1232 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1233 * 16], m4 + + ; mode 21 [row 9] + movu m6, [r5 + 22 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 8], 1 + pinsrb m0, [r4 + 9], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1234 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 11], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 19], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1235 * 16], m4 + + ; mode 21 [row 10] + movu m6, [r5 + 5 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1236 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1237 * 16], m4 + + ; mode 21 [row 11] + movu m6, [r5 + 20 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 9], 1 + pinsrb m0, [r4 + 11], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1238 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 10], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 18], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1239 * 16], m4 + + ; mode 21 [row 12] + movu m6, [r5 + 3 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1240 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1241 * 16], m4 + + ; mode 21 [row 13] + movu m6, [r5 + 18 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 11], 1 + pinsrb m0, [r4 + 13], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1242 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 9], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 17], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1243 * 16], m4 + + ; mode 21 [row 14] + movu m6, [r5 + 1 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1244 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1245 * 16], m4 + + ; mode 21 [row 15] + movu m6, [r5 + 16 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 13], 1 + pinsrb m0, [r4 + 15], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 0], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1246 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 8], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 16], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1247 * 16], m4 + + ; mode 21 [row 16] + movu m6, [r5 + 31 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 15], 1 + pinsrb m0, [r4 + 17], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 0], 1 + pinsrb m2, [r4 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1248 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 7], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 15], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1249 * 16], m4 + + ; mode 21 [row 17] + movu m6, [r5 + 14 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1250 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1251 * 16], m4 + + ; mode 21 [row 18] + movu m6, [r5 + 29 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 17], 1 + pinsrb m0, [r4 + 19], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 2], 1 + pinsrb m2, [r4 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1252 * 16], m4 + pslldq m1, 2 + pinsrb m1, [r3 + 7], 1 + pinsrb m1, [r3 + 6], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrb m3, [r3 + 15], 1 + pinsrb m3, [r3 + 14], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1253 * 16], m4 + + ; mode 21 [row 19] + movu m6, [r5 + 12 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1254 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1255 * 16], m4 + + ; mode 21 [row 20] + movu m6, [r5 + 27 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 19], 1 + pinsrb m0, [r4 + 21], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 4], 1 + pinsrb m2, [r4 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1256 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 5], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 13], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1257 * 16], m4 + + ; mode 21 [row 21] + movu m6, [r5 + 10 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1258 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1259 * 16], m4 + + ; mode 21 [row 22] + movu m6, [r5 + 25 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 21], 1 + pinsrb m0, [r4 + 23], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 6], 1 + pinsrb m2, [r4 + 8], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1260 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 4], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 12], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1261 * 16], m4 + + ; mode 21 [row 23] + movu m6, [r5 + 8 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1262 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1263 * 16], m4 + + ; mode 21 [row 24] + movu m6, [r5 + 23 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 23], 1 + pinsrb m0, [r4 + 24], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 8], 1 + pinsrb m2, [r4 + 9], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1264 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 3], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 11], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1265 * 16], m4 + + ; mode 21 [row 25] + movu m6, [r5 + 6 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1266 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1267 * 16], m4 + + ; mode 21 [row 26] + movu m6, [r5 + 21 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 24], 1 + pinsrb m0, [r4 + 26], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 9], 1 + pinsrb m2, [r4 + 11], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1268 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 2], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 10], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1269 * 16], m4 + + ; mode 21 [row 27] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1270 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1271 * 16], m4 + + ; mode 21 [row 28] + movu m6, [r5 + 19 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 26], 1 + pinsrb m0, [r4 + 28], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 11], 1 + pinsrb m2, [r4 + 13], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1272 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 1], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 9], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1273 * 16], m4 + + ; mode 21 [row 29] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1274 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1275 * 16], m4 + + ; mode 21 [row 30] + movu m6, [r5 + 17 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 28], 1 + pinsrb m0, [r4 + 30], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 13], 1 + pinsrb m2, [r4 + 15], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1276 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 0], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 8], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1277 * 16], m4 + + ; mode21 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 1278 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1278 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1279 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1279 * 16 + 8], m5 + + ; mode 22 [row 0] + movu m6, [r5 + 19 * 16] + movu m0, [r3 ] + movu m1, [r3 + 1 ] + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r3 + 8] + movu m3, [r3 + 9] + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 1280 * 16], m1 + + movu m1, [r3 + 16] + movu m3, [r3 + 17] + punpcklbw m1, m3 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + movu m3, [r3 + 24] + movu m5, [r3 + 25] + punpcklbw m3, m5 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1281 * 16], m4 + + ; mode 22 [row 1] + movu m6, [r5 + 6 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1282 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1283 * 16], m4 + + ; mode 22 [row 2] + movu m6, [r5 + 25 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 0], 1 + pinsrb m0, [r4 + 2], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1284 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 15], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 23], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1285 * 16], m4 + + ; mode 22 [row 3] + movu m6, [r5 + 12 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1286 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1287 * 16], m4 + + ; mode 22 [row 4] + movu m6, [r5 + 31 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 2], 1 + pinsrb m0, [r4 + 5], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1288 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 14], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 22], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1289 * 16], m4 + + ; mode 22 [row 5] + movu m6, [r5 + 18 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1290 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1291 * 16], m4 + + ; mode 22 [row 6] + movu m6, [r5 + 5 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1292 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1293 * 16], m4 + + ; mode 22 [row 7] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 5], 1 + pinsrb m0, [r4 + 7], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1294 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 13], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 21], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1295 * 16], m4 + + ; mode 22 [row 8] + movu m6, [r5 + 11 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1296 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1297 * 16], m4 + + ; mode 22 [row 9] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 7], 1 + pinsrb m0, [r4 + 10], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1298 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 12], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 20], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1299 * 16], m4 + + ; mode 22 [row 10] + movu m6, [r5 + 17 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1300 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1301 * 16], m4 + + ; mode 22 [row 11] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1302 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1303 * 16], m4 + + ; mode 22 [row 12] + movu m6, [r5 + 23 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 10], 1 + pinsrb m0, [r4 + 12], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1304 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 11], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 19], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1305 * 16], m4 + + ; mode 22 [row 13] + movu m6, [r5 + 10 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1306 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1307 * 16], m4 + + ; mode 22 [row 14] + movu m6, [r5 + 29 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 12], 1 + pinsrb m0, [r4 + 15], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1308 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 10], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 18], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1309 * 16], m4 + + ; mode 22 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1310 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1311 * 16], m4 + + ; mode 22 [row 16] + movu m6, [r5 + 3 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1312 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1313 * 16], m4 + + ; mode 22 [row 17] + movu m6, [r5 + 22 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 15], 1 + pinsrb m0, [r4 + 17], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1314 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 9], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 17], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1315 * 16], m4 + + ; mode 22 [row 18] + movu m6, [r5 + 9 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1316 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1317 * 16], m4 + + ; mode 22 [row 19] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 17], 1 + pinsrb m0, [r4 + 20], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 0], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1318 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 8], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 16], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1319 * 16], m4 + + ; mode 22 [row 20] + movu m6, [r5 + 15 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1320 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1321 * 16], m4 + + ; mode 22 [row 21] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1322 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1323 * 16], m4 + + ; mode 22 [row 22] + movu m6, [r5 + 21 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 20], 1 + pinsrb m0, [r4 + 22], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 0], 1 + pinsrb m2, [r4 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1324 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 7], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 15], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1325 * 16], m4 + + ; mode 22 [row 23] + movu m6, [r5 + 8 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1326 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1327 * 16], m4 + + ; mode 22 [row 24] + movu m6, [r5 + 27 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 22], 1 + pinsrb m0, [r4 + 25], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 2], 1 + pinsrb m2, [r4 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1328 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 6], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 14], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1329 * 16], m4 + + ; mode 22 [row 25] + movu m6, [r5 + 14 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1330 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1331 * 16], m4 + + ; mode 22 [row 26] + movu m6, [r5 + 1 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1332 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1333 * 16], m4 + + ; mode 22 [row 27] + movu m6, [r5 + 20 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 25], 1 + pinsrb m0, [r4 + 27], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 5], 1 + pinsrb m2, [r4 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1334 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 5], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 13], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1335 * 16], m4 + + ; mode 22 [row 28] + movu m6, [r5 + 7 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1336 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1337 * 16], m4 + + ; mode 22 [row 29] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 27], 1 + pinsrb m0, [r4 + 30], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrb m2, [r4 + 7], 1 + pinsrb m2, [r4 + 10], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1338 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 4], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 12], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1339 * 16], m4 + + ; mode 22 [row 30] + movu m6, [r5 + 13 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1340 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1341 * 16], m4 + + ; mode22 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 1342 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1342 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1343 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1343 * 16 + 8], m5 + + ; mode 23 [row 0] + movu m6, [r5 + 23 * 16] + movu m0, [r3 ] + movu m1, [r3 + 1 ] + punpcklbw m0, m1 + pmaddubsw m1, m0, m6 + pmulhrsw m1, m7 + movu m2, [r3 + 8] + movu m3, [r3 + 9] + punpcklbw m2, m3 + pmaddubsw m3, m2, m6 + pmulhrsw m3, m7 + packuswb m1, m3 + movu [r0 + 1344 * 16], m1 + + movu m1, [r3 + 16] + movu m3, [r3 + 17] + punpcklbw m1, m3 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + movu m3, [r3 + 24] + movu m5, [r3 + 25] + punpcklbw m3, m5 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1345 * 16], m4 + + ; mode 23 [row 1] + movu m6, [r5 + 14 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1346 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1347 * 16], m4 + + ; mode 23 [row 2] + movu m6, [r5 + 5 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1348 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1349 * 16], m4 + + ; mode 23 [row 3] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 0], 1 + pinsrb m0, [r4 + 4], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1350 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 15], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 23], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1351 * 16], m4 + + ; mode 23 [row 4] + movu m6, [r5 + 19 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1352 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1353 * 16], m4 + + ; mode 23 [row 5] + movu m6, [r5 + 10 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1354 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1355 * 16], m4 + + ; mode 23 [row 6] + movu m6, [r5 + 1 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1356 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1357 * 16], m4 + + ; mode 23 [row 7] + movu m6, [r5 + 24 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 4], 1 + pinsrb m0, [r4 + 7], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1358 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 14], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 22], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1359 * 16], m4 + + ; mode 23 [row 8] + movu m6, [r5 + 15 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1360 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1361 * 16], m4 + + ; mode 23 [row 9] + movu m6, [r5 + 6 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1362 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1363 * 16], m4 + + ; mode 23 [row 10] + movu m6, [r5 + 29 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 7], 1 + pinsrb m0, [r4 + 11], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1364 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 13], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 21], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1365 * 16], m4 + + ; mode 23 [row 11] + movu m6, [r5 + 20 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1366 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1367 * 16], m4 + + ; mode 23 [row 12] + movu m6, [r5 + 11 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1368 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1369 * 16], m4 + + ; mode 23 [row 13] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1370 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1371 * 16], m4 + + ; mode 23 [row 14] + movu m6, [r5 + 25 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 11], 1 + pinsrb m0, [r4 + 14], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1372 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 12], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 20], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1373 * 16], m4 + + ; mode 23 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1374 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1375 * 16], m4 + + ; mode 23 [row 16] + movu m6, [r5 + 7 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1376 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1377 * 16], m4 + + ; mode 23 [row 17] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 14], 1 + pinsrb m0, [r4 + 18], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 3], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1378 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 11], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 19], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1379 * 16], m4 + + ; mode 23 [row 18] + movu m6, [r5 + 21 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1380 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1381 * 16], m4 + + ; mode 23 [row 19] + movu m6, [r5 + 12 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1382 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1383 * 16], m4 + + ; mode 23 [row 20] + movu m6, [r5 + 3 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1384 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1385 * 16], m4 + + ; mode 23 [row 21] + movu m6, [r5 + 26 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 18], 1 + pinsrb m0, [r4 + 21], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 2], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1386 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 10], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 18], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1387 * 16], m4 + + ; mode 23 [row 22] + movu m6, [r5 + 17 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1388 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1389 * 16], m4 + + ; mode 23 [row 23] + movu m6, [r5 + 8 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1390 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1391 * 16], m4 + + ; mode 23 [row 24] + movu m6, [r5 + 31 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 21], 1 + pinsrb m0, [r4 + 25], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 1], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1392 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 9], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 17], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1393 * 16], m4 + + ; mode 23 [row 25] + movu m6, [r5 + 22 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1394 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1395 * 16], m4 + + ; mode 23 [row 26] + movu m6, [r5 + 13 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1396 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1397 * 16], m4 + + ; mode 23 [row 27] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1398 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1399 * 16], m4 + + ; mode 23 [row 28] + movu m6, [r5 + 27 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 25], 1 + pinsrb m0, [r4 + 28], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 0], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1400 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 8], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 16], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1401 * 16], m4 + + ; mode 23 [row 29] + movu m6, [r5 + 18 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1402 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1403 * 16], m4 + + ; mode 23 [row 30] + movu m6, [r5 + 9 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1404 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1405 * 16], m4 + + ; mode23 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 1406 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1406 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1407 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1407 * 16 + 8], m5 + + ; mode 24 [row 0] + movu m6, [r5 + 27 * 16] + movu m0, [r3 ] + movu m1, [r3 + 1 ] + punpcklbw m0, m1 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + movu m2, [r3 + 8] + movu m3, [r3 + 9] + punpcklbw m2, m3 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1408 * 16], m4 + + movu m1, [r3 + 16] + movu m3, [r3 + 17] + punpcklbw m1, m3 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + movu m3, [r3 + 24] + movu m5, [r3 + 25] + punpcklbw m3, m5 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1409 * 16], m4 + + ; mode 24 [row 1] + movu m6, [r5 + 22 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1410 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1411 * 16], m4 + + ; mode 24 [row 2] + movu m6, [r5 + 17 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1412 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1413 * 16], m4 + + ; mode 24 [row 3] + movu m6, [r5 + 12 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1414 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1415 * 16], m4 + + ; mode 24 [row 4] + movu m6, [r5 + 7 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1416 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1417 * 16], m4 + + ; mode 24 [row 5] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1418 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1419 * 16], m4 + + ; mode 24 [row 6] + movu m6, [r5 + 29 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 0], 1 + pinsrb m0, [r4 + 6], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1420 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 15], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 23], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1421 * 16], m4 + + ; mode 24 [row 7] + movu m6, [r5 + 24 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1422 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1423 * 16], m4 + + ; mode 24 [row 8] + movu m6, [r5 + 19 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1424 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1425 * 16], m4 + + ; mode 24 [row 9] + movu m6, [r5 + 14 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1426 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1427 * 16], m4 + + ; mode 24 [row 10] + movu m6, [r5 + 9 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1428 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1429 * 16], m4 + + ; mode 24 [row 11] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1430 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1431 * 16], m4 + + ; mode 24 [row 12] + movu m6, [r5 + 31 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 6], 1 + pinsrb m0, [r4 + 13], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 6], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1432 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 14], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 22], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1433 * 16], m4 + + ; mode 24 [row 13] + movu m6, [r5 + 26 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1434 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1435 * 16], m4 + + ; mode 24 [row 14] + movu m6, [r5 + 21 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1436 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1437 * 16], m4 + + ; mode 24 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1438 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1439 * 16], m4 + + ; mode 24 [row 16] + movu m6, [r5 + 11 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1440 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1441 * 16], m4 + + ; mode 24 [row 17] + movu m6, [r5 + 6 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1442 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1443 * 16], m4 + + ; mode 24 [row 18] + movu m6, [r5 + 1 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1444 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1445 * 16], m4 + + ; mode 24 [row 19] + movu m6, [r5 + 28 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 13], 1 + pinsrb m0, [r4 + 19], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 5], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1446 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 13], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 21], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1447 * 16], m4 + + ; mode 24 [row 20] + movu m6, [r5 + 23 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1448 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1449 * 16], m4 + + ; mode 24 [row 21] + movu m6, [r5 + 18 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1450 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1451 * 16], m4 + + ; mode 24 [row 22] + movu m6, [r5 + 13 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1452 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1453 * 16], m4 + + ; mode 24 [row 23] + movu m6, [r5 + 8 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1454 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1455 * 16], m4 + + ; mode 24 [row 24] + movu m6, [r5 + 3 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1456 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1457 * 16], m4 + + ; mode 24 [row 25] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 19], 1 + pinsrb m0, [r4 + 26], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 4], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1458 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 12], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 20], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1459 * 16], m4 + + ; mode 24 [row 26] + movu m6, [r5 + 25 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1460 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1461 * 16], m4 + + ; mode 24 [row 27] + movu m6, [r5 + 20 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1462 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1463 * 16], m4 + + ; mode 24 [row 28] + movu m6, [r5 + 15 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1464 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1465 * 16], m4 + + ; mode 24 [row 29] + movu m6, [r5 + 10 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1466 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1467 * 16], m4 + + ; mode 24 [row 30] + movu m6, [r5 + 5 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1468 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1469 * 16], m4 + + ; mode 24 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 1470 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1470 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1471 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1471 * 16 + 8], m5 + + ; mode 25 [row 0] + movu m6, [r5 + 30 * 16] + movu m0, [r3 ] + movu m1, [r3 + 1 ] + punpcklbw m0, m1 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + movu m2, [r3 + 8] + movu m3, [r3 + 9] + punpcklbw m2, m3 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1472 * 16], m4 + + movu m1, [r3 + 16] + movu m3, [r3 + 17] + punpcklbw m1, m3 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + movu m3, [r3 + 24] + movu m5, [r3 + 25] + punpcklbw m3, m5 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1473 * 16], m4 + + ; mode 25 [row 1] + movu m6, [r5 + 28 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1474 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1475 * 16], m4 + + ; mode 25 [row 2] + movu m6, [r5 + 26 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1476 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1477 * 16], m4 + + ; mode 25 [row 3] + movu m6, [r5 + 24 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1478 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1479 * 16], m4 + + ; mode 25 [row 4] + movu m6, [r5 + 22 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1480 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1481 * 16], m4 + + ; mode 25 [row 5] + movu m6, [r5 + 20 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1482 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1483 * 16], m4 + + ; mode 25 [row 6] + movu m6, [r5 + 18 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1484 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1485 * 16], m4 + + ; mode 25 [row 7] + movu m6, [r5 + 16 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1486 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1487 * 16], m4 + + ; mode 25 [row 8] + movu m6, [r5 + 14 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1488 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1489 * 16], m4 + + ; mode 25 [row 9] + movu m6, [r5 + 12 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1490 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1491 * 16], m4 + + ; mode 25 [row 10] + movu m6, [r5 + 10 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1492 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1493 * 16], m4 + + ; mode 25 [row 11] + movu m6, [r5 + 8 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1494 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1495 * 16], m4 + + ; mode 25 [row 12] + movu m6, [r5 + 6 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1496 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1497 * 16], m4 + + ; mode 25 [row 13] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1498 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1499 * 16], m4 + + ; mode 25 [row 14] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1500 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1501 * 16], m4 + + ; mode 25 [row 15] + pshufb m5, m0, [tab_S2] + movh [r0 + 1502 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1502 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1503 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1503 * 16 + 8], m5 + + ; mode 25 [row 16] + movu m6, [r5 + 30 * 16] + pslldq m0, 2 + pinsrb m0, [r4 + 0], 1 + pinsrb m0, [r4 + 16], 0 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pslldq m2, 2 + pinsrw m2, [r3 + 7], 0 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1504 * 16], m4 + pslldq m1, 2 + pinsrw m1, [r3 + 15], 0 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pslldq m3, 2 + pinsrw m3, [r3 + 23], 0 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1505 * 16], m4 + + ; mode 25 [row 17] + movu m6, [r5 + 28 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1506 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1507 * 16], m4 + + ; mode 25 [row 18] + movu m6, [r5 + 26 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1508 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1509 * 16], m4 + + ; mode 25 [row 19] + movu m6, [r5 + 24 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1510 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1511 * 16], m4 + + ; mode 25 [row 20] + movu m6, [r5 + 22 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1512 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1513 * 16], m4 + + ; mode 25 [row 21] + movu m6, [r5 + 20 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1514 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1515 * 16], m4 + + ; mode 25 [row 22] + movu m6, [r5 + 18 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1516 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1517 * 16], m4 + + ; mode 25 [row 23] + movu m6, [r5 + 16 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1518 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1519 * 16], m4 + + ; mode 25 [row 24] + movu m6, [r5 + 14 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1520 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1521 * 16], m4 + + ; mode 25 [row 25] + movu m6, [r5 + 12 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1522 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1523 * 16], m4 + + ; mode 25 [row 26] + movu m6, [r5 + 10 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1524 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1525 * 16], m4 + + ; mode 25 [row 27] + movu m6, [r5 + 8 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1526 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1527 * 16], m4 + + ; mode 25 [row 28] + movu m6, [r5 + 6 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1528 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1529 * 16], m4 + + ; mode 25 [row 29] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1530 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1531 * 16], m4 + + ; mode 25 [row 30] + movu m6, [r5 + 2 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1532 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1533 * 16], m4 + + ; mode 25 [row 31] + pshufb m5, m0, [tab_S2] + movh [r0 + 1534 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1534 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1535 * 16], m5 + pshufb m5, m3, [tab_S2] + movh [r0 + 1535 * 16 + 8], m5 + + ; mode 26 + movu m1, [r1 + 1] + movu m2, [r1 + 17] + movu [r0 + 1536 * 16], m1 + movu [r0 + 1537 * 16], m2 + movu [r0 + 1538 * 16], m1 + movu [r0 + 1539 * 16], m2 + movu [r0 + 1540 * 16], m1 + movu [r0 + 1541 * 16], m2 + movu [r0 + 1542 * 16], m1 + movu [r0 + 1543 * 16], m2 + movu [r0 + 1544 * 16], m1 + movu [r0 + 1545 * 16], m2 + movu [r0 + 1546 * 16], m1 + movu [r0 + 1547 * 16], m2 + movu [r0 + 1548 * 16], m1 + movu [r0 + 1549 * 16], m2 + movu [r0 + 1550 * 16], m1 + movu [r0 + 1551 * 16], m2 + + movu [r0 + 1552 * 16], m1 + movu [r0 + 1553 * 16], m2 + movu [r0 + 1554 * 16], m1 + movu [r0 + 1555 * 16], m2 + movu [r0 + 1556 * 16], m1 + movu [r0 + 1557 * 16], m2 + movu [r0 + 1558 * 16], m1 + movu [r0 + 1559 * 16], m2 + movu [r0 + 1560 * 16], m1 + movu [r0 + 1561 * 16], m2 + movu [r0 + 1562 * 16], m1 + movu [r0 + 1563 * 16], m2 + movu [r0 + 1564 * 16], m1 + movu [r0 + 1565 * 16], m2 + movu [r0 + 1566 * 16], m1 + movu [r0 + 1567 * 16], m2 + + movu [r0 + 1568 * 16], m1 + movu [r0 + 1569 * 16], m2 + movu [r0 + 1570 * 16], m1 + movu [r0 + 1571 * 16], m2 + movu [r0 + 1572 * 16], m1 + movu [r0 + 1573 * 16], m2 + movu [r0 + 1574 * 16], m1 + movu [r0 + 1575 * 16], m2 + movu [r0 + 1576 * 16], m1 + movu [r0 + 1577 * 16], m2 + movu [r0 + 1578 * 16], m1 + movu [r0 + 1579 * 16], m2 + movu [r0 + 1580 * 16], m1 + movu [r0 + 1581 * 16], m2 + movu [r0 + 1582 * 16], m1 + movu [r0 + 1583 * 16], m2 + + movu [r0 + 1584 * 16], m1 + movu [r0 + 1585 * 16], m2 + movu [r0 + 1586 * 16], m1 + movu [r0 + 1587 * 16], m2 + movu [r0 + 1588 * 16], m1 + movu [r0 + 1589 * 16], m2 + movu [r0 + 1590 * 16], m1 + movu [r0 + 1591 * 16], m2 + movu [r0 + 1592 * 16], m1 + movu [r0 + 1593 * 16], m2 + movu [r0 + 1594 * 16], m1 + movu [r0 + 1595 * 16], m2 + movu [r0 + 1596 * 16], m1 + movu [r0 + 1597 * 16], m2 + movu [r0 + 1598 * 16], m1 + movu [r0 + 1599 * 16], m2 + + ; mode 27 [row 0] + movu m6, [r5 + 2 * 16] + movu m0, [r3 + 1 ] + movu m1, [r3 + 2 ] + punpcklbw m0, m1 + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + movu m2, [r3 + 9] + movu m3, [r3 + 10] + punpcklbw m2, m3 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1600 * 16], m4 + + movu m1, [r3 + 17] + movu m3, [r3 + 18] + punpcklbw m1, m3 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + movu m3, [r3 + 25] + movu m5, [r3 + 26] + punpcklbw m3, m5 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1601 * 16], m4 + + ; mode 27 [row 1] + movu m6, [r5 + 4 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1602 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1603 * 16], m4 + + ; mode 27 [row 2] + movu m6, [r5 + 6 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1604 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1605 * 16], m4 + + ; mode 27 [row 3] + movu m6, [r5 + 8 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1606 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1607 * 16], m4 + + ; mode 27 [row 4] + movu m6, [r5 + 10 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1608 * 16], m4 + + ; mode 28 [row 1 -first half] + movu [r0 + 1666 * 16], m4 + + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1609 * 16], m4 + + ; mode 28 [row 1 - second half] + movu [r0 + 1667 * 16], m4 + + ; mode 27 [row 5] + movu m6, [r5 + 12 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1610 * 16], m4 + + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1611 * 16], m4 + + ; mode 27 [row 6] + movu m6, [r5 + 14 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1612 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1613 * 16], m4 + + ; mode 27 [row 7] + movu m6, [r5 + 16 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1614 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1615 * 16], m4 + + ; mode 27 [row 8] + movu m6, [r5 + 18 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1616 * 16], m4 + + ; mode 29 [row 1 - first half] + movu [r0 + 1730 * 16], m4 + + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1617 * 16], m4 + + ; mode 29 [row 1 - second half] + movu [r0 + 1731 * 16], m4 + + ; mode 27 [row 9] + movu m6, [r5 + 20 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1618 * 16], m4 + + ; mode 28 [row 3 -first half] + movu [r0 + 1670 * 16], m4 + + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1619 * 16], m4 + + ; mode 28 [row 3 -second half] + movu [r0 + 1671 * 16], m4 + + ; mode 27 [row 10] + movu m6, [r5 + 22 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1620 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1621 * 16], m4 + + ; mode 27 [row 11] + movu m6, [r5 + 24 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1622 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1623 * 16], m4 + + ; mode 27 [row 12] + movu m6, [r5 + 26 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1624 * 16], m4 + + ; mode 30 [row 1 - first half] + movu [r0 + 1794 * 16], m4 + + ; mode 33 [row 0 - first half] + movu [r0 + 1984 * 16], m4 + + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1625 * 16], m4 + + ; mode 30 [row 1 - second half] + movu [r0 + 1795 * 16], m4 + + ; mode 33 [row 0 - second half] + movu [r0 + 1985 * 16], m4 + + ; mode 27 [row 13] + movu m6, [r5 + 28 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1626 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1627 * 16], m4 + + ; mode 27 [row 14] + movu m6, [r5 + 30 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1628 * 16], m4 + + ; mode 28 [row 5 first half] + movu [r0 + 1674 * 16], m4 + + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1629 * 16], m4 + + ; mode 28 [row 5 second half] + movu [r0 + 1675 * 16], m4 + + ; mode 28 [row 0] + movu m6, [r5 + 5 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1664 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1665 * 16], m4 + + ; mode 28 [row 2] + movu m6, [r5 + 15 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1668 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1669 * 16], m4 + + ; mode 28 [row 4] + movu m6, [r5 + 25 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1672 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1673 * 16], m4 + + ; mode 30 [row 0] + movu m6, [r5 + 13 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1792 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1793 * 16], m4 + + ; mode 29 [row 0] + movu m6, [r5 + 9 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1728 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1729 * 16], m4 + + ; mode 29 [row 2] + movu m6, [r5 + 27 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1732 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1733 * 16], m4 + + ; mode 31 [row 0] + movu m6, [r5 + 17 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1856 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1857 * 16], m4 + + ; mode 32 [row 0] + movu m6, [r5 + 21 * 16] + pmaddubsw m4, m0, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1920 * 16], m4 + pmaddubsw m4, m1, m6 + pmulhrsw m4, m7 + pmaddubsw m5, m3, m6 + pmulhrsw m5, m7 + packuswb m4, m5 + movu [r0 + 1921 * 16], m4 + + ; mode 27 [row 15] + movu m0, [r3 + 2] + movd m1, [r3 + 3] + palignr m1, m0, 1 + punpcklbw m0, m1 + movu m2, [r3 + 10] + movd m3, [r3 + 11] + palignr m3, m2, 1 + punpcklbw m2, m3 + movu m1, [r3 + 18] + movd m3, [r3 + 19] + palignr m3, m1, 1 + punpcklbw m1, m3 + movu m4, [r3 + 26] + movd m5, [r3 + 27] + palignr m5, m4, 1 + punpcklbw m4, m5 + + pshufb m5, m0, [tab_S2] + movh [r0 + 1630 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1630 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1631 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 1631 * 16 + 8], m5 + + ; mode 27 [row 16] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1632 * 16], m3 + + ; mode 31 [row 1 - first half] + movu [r0 + 1858 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1633 * 16], m3 + + ; mode 31 [row 1 - second half] + movu [r0 + 1859 * 16], m3 + + ; mode 27 [row 17] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1634 * 16], m3 + + ; mode 29 [row 3 - first half] + movu [r0 + 1734 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1635 * 16], m3 + + ; mode 29 [row 3 - second half] + movu [r0 + 1735 * 16], m3 + + ; mode 27 [row 18] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1636 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1637 * 16], m3 + + ; mode 27 [row 19] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1638 * 16], m3 + + ; mode 28 [row 7 - first half] + movu [r0 + 1678 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1639 * 16], m3 + + ; mode 28 [row 7 - second half] + movu [r0 + 1679 * 16], m3 + + ; mode 27 [row 20] + movu m6, [r5 + 10 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1640 * 16], m3 + + ; mode 32 [row 1 - first half] + movu [r0 + 1922 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1641 * 16], m3 + + ; mode 32 [row 1 - second half] + movu [r0 + 1923 * 16], m3 + + ; mode 27 [row 21] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1642 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1643 * 16], m3 + + ; mode 27 [row 22] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1644 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1645 * 16], m3 + + ; mode 27 [row 23] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1646 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1647 * 16], m3 + + ; mode 27 [row 24] + movu m6, [r5 + 18 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1648 * 16], m3 + + ; mode 28 [row 9 - first half] + movu [r0 + 1682 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1649 * 16], m3 + + ; mode 28 [row 9 - second half] + movu [r0 + 1683 * 16], m3 + + ; mode 27 [row 25] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1650 * 16], m3 + + ; mode 30 [row 3 - first half] + movu [r0 + 1798 * 16], m3 + + ; mode 33 [row 1 - first half] + movu [r0 + 1986 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1651 * 16], m3 + + ; mode 30 [row 3 - second half] + movu [r0 + 1799 * 16], m3 + + ; mode 33 [row 1 - second half] + movu [r0 + 1987 * 16], m3 + + ; mode 27 [row 26] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1652 * 16], m3 + + ; mode 29 [row 5 - first half] + movu [r0 + 1738 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1653 * 16], m3 + + ; mode 29 [row 5 - second half] + movu [r0 + 1739 * 16], m3 + + ; mode 27 [row 27] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1654 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1655 * 16], m3 + + ; mode 27 [row 28] + movu m6, [r5 + 26 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1656 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1657 * 16], m3 + + ; mode 27 [row 29] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1658 * 16], m3 + + ; mode 28 [row 11 - first half] + movu [r0 + 1686 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1659 * 16], m3 + + ; mode 28 [row 11 - second half] + movu [r0 + 1687 * 16], m3 + + ; mode 27 [row 30] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1660 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1661 * 16], m3 + + ; mode 28 [row 6] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1676 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1677 * 16], m3 + + ; mode 28 [row 8] + movu m6, [r5 + 13 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1680 * 16], m3 + + ; mode 29 [row 4 - first half] + movu [r0 + 1736 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1681 * 16], m3 + + ; mode 29 [row 4 - second half] + movu [r0 + 1737 * 16], m3 + + ; mode 28 [row 10] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1684 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1685 * 16], m3 + + ; mode 29 [row 6] + movu m6, [r5 + 31 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1740 * 16], m3 + + ; mode 32 [row 2 - first half] + movu [r0 + 1924 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1741 * 16], m3 + + ; mode 32 [row 2 - second half] + movu [r0 + 1925 * 16], m3 + + ; mode 30 [row 2] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1796 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1797 * 16], m3 + + ; mode 31 [row 2] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1860 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1861 * 16], m3 + + ; mode 27 [row 15] + movu m0, [r3 + 3] + movd m1, [r3 + 4] + palignr m1, m0, 1 + punpcklbw m0, m1 + movu m2, [r3 + 11] + movd m3, [r3 + 12] + palignr m3, m2, 1 + punpcklbw m2, m3 + movu m1, [r3 + 19] + movd m3, [r3 + 20] + palignr m3, m1, 1 + punpcklbw m1, m3 + movu m4, [r3 + 27] + movd m5, [r3 + 28] + palignr m5, m4, 1 + punpcklbw m4, m5 + + pshufb m5, m0, [tab_S2] + movh [r0 + 1662 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1662 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1663 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 1663 * 16 + 8], m5 + + ; mode 28 [row 12] + movu m6, [r5 + 1 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1688 * 16], m3 + + ; mode 30 [row 4 - first half] + movu [r0 + 1800 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1689 * 16], m3 + + ; mode 30 [row 4 - second half] + movu [r0 + 1801 * 16], m3 + + ; mode 28 [row 13] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1690 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1691 * 16], m3 + + ; mode 28 [row 14] + movu m6, [r5 + 11 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1692 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1693 * 16], m3 + + ; mode 28 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1694 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1695 * 16], m3 + + ; mode 28 [row 16] + movu m6, [r5 + 21 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1696 * 16], m3 + + ; mode 31 [row 4 - first half] + movu [r0 + 1864 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1697 * 16], m3 + + ; mode 31 [row 4 - second half] + movu [r0 + 1865 * 16], m3 + + ; mode 28 [row 17] + movu m6, [r5 + 26 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1698 * 16], m3 + + ; mode 29 [row 9 - first half] + movu [r0 + 1746 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1699 * 16], m3 + + ; mode 29 [row 9 - second half] + movu [r0 + 1747 * 16], m3 + + ; mode 28 [row 18] + movu m6, [r5 + 31 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1700 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1701 * 16], m3 + + ; mode 29 [row 7] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1742 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1743 * 16], m3 + + ; mode 29 [row 8] + movu m6, [r5 + 17 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1744 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1745 * 16], m3 + + ; mode 30 [row 5] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1802 * 16], m3 + + ; mode 33 [row 2 - first half] + movu [r0 + 1988 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1803 * 16], m3 + + ; mode 33 [row 2 - second half] + movu [r0 + 1989 * 16], m3 + + ; mode 30 [row 6] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1804 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1805 * 16], m3 + + ; mode 31 [row 3] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1862 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1863 * 16], m3 + + ; mode 32 [row 3] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1926 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1927 * 16], m3 + + ; mode 28 [row 19] + movu m6, [r5 + 4 * 16] + movu m0, [r3 + 4] + movd m1, [r3 + 5] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 12] + movd m4, [r3 + 13] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1702 * 16], m3 + + movu m1, [r3 + 20] + movd m3, [r3 + 21] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 28] + movd m5, [r3 + 29] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1703 * 16], m3 + + ; mode 28 [row 20] + movu m6, [r5 + 9 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1704 * 16], m3 + + ; mode 32 [row 4 - first half] + movu [r0 + 1928 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1705 * 16], m3 + + ; mode 32 [row 4 - second half] + movu [r0 + 1929 * 16], m3 + + ; mode 28 [row 21] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1706 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1707 * 16], m3 + + ; mode 28 [row 22] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1708 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1709 * 16], m3 + + ; mode 28 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1710 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1711 * 16], m3 + + ; mode 28 [row 24] + movu m6, [r5 + 29 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1712 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1713 * 16], m3 + + ; mode 29 [row 10] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1748 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1749 * 16], m3 + + ; mode 29 [row 11] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1750 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1751 * 16], m3 + + ; mode 29 [row 12] + movu m6, [r5 + 21 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1752 * 16], m3 + + ; mode 30 [row 8 -first half] + movu [r0 + 1808 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1753 * 16], m3 + + ; mode 30 [row 8 -second half] + movu [r0 + 1809 * 16], m3 + + ; mode 29 [row 13] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1754 * 16], m3 + + ; mode 32 [row 5 - first half] + movu [r0 + 1930 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1755 * 16], m3 + + ; mode 32 [row 5 - second half] + movu [r0 + 1931 * 16], m3 + + ; mode 30 [row 7] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1806 * 16], m3 + + ; mode 33 [row 3 - first half] + movu [r0 + 1990 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1807 * 16], m3 + + ; mode 33 [row 3 - second half] + movu [r0 + 1991 * 16], m3 + + ; mode 31 [row 5] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1866 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1867 * 16], m3 + + ; mode 31 [row 6] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1868 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1869 * 16], m3 + + ; mode 28 [row 25] + movu m6, [r5 + 2 * 16] + movu m0, [r3 + 5] + movd m1, [r3 + 6] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 13] + movd m4, [r3 + 14] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1714 * 16], m3 + + movu m1, [r3 + 21] + movd m3, [r3 + 22] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 29] + movd m5, [r3 + 30] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1715 * 16], m3 + + ; mode 28 [row 26] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1716 * 16], m3 + + ; mode 29 [row 14 - first half] + movu [r0 + 1756 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1717 * 16], m3 + + ; mode 29 [row 14 - second half] + movu [r0 + 1757 * 16], m3 + + ; mode 28 [row 27] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1718 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1719 * 16], m3 + + ; mode 28 [row 28] + movu m6, [r5 + 17 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1720 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1721 * 16], m3 + + ; mode 28 [row 29] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1722 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1723 * 16], m3 + + ; mode 28 [row 30] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1724 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1725 * 16], m3 + + ; mode 29 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1758 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1759 * 16], m3 + + ; mode 29 [row 16] + movu m6, [r5 + 25 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1760 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1761 * 16], m3 + + ; mode 30 [row 9] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1810 * 16], m3 + + ; mode 33 [row 4 - first half] + movu [r0 + 1992 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1811 * 16], m3 + + ; mode 33 [row 4 - second half] + movu [r0 + 1993 * 16], m3 + + ; mode 30 [row 10] + movu m6, [r5 + 15 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1812 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1813 * 16], m3 + + ; mode 31 [row 7] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1870 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1871 * 16], m3 + + ; mode 31 [row 8] + movu m6, [r5 + 25 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1872 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1873 * 16], m3 + + ; mode 32 [row 6] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1932 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1933 * 16], m3 + + ; mode 30 [row 11] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1814 * 16], m3 + + ; mode 33 [row 5 - first half] + movu [r0 + 1994 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1815 * 16], m3 + + ; mode 33 [row 5 - second half] + movu [r0 + 1995 * 16], m3 + + ; mode 28 [row 31] + movu m0, [r3 + 6] + movd m1, [r3 + 7] + palignr m1, m0, 1 + punpcklbw m0, m1 + movu m2, [r3 + 14] + movd m3, [r3 + 15] + palignr m3, m2, 1 + punpcklbw m2, m3 + movu m1, [r3 + 22] + movd m3, [r3 + 23] + palignr m3, m1, 1 + punpcklbw m1, m3 + movu m4, [r3 + 30] + movd m5, [r3 + 31] + palignr m5, m4, 1 + punpcklbw m4, m5 + + pshufb m5, m0, [tab_S2] + movh [r0 + 1726 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1726 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1727 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 1727 * 16 + 8], m5 + + ; mode 29 [row 17] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1762 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1763 * 16], m3 + + ; mode 29 [row 18] + movu m6, [r5 + 11 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1764 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1765 * 16], m3 + + ; mode 29 [row 19] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1766 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1767 * 16], m3 + + ; mode 29 [row 20] + movu m6, [r5 + 29 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1768 * 16], m3 + + ; mode 32 [row 8 - first halif] + movu [r0 + 1936 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1769 * 16], m3 + + ; mode 32 [row 8 - second halif] + movu [r0 + 1937 * 16], m3 + + ; mode 30 [row 12] + movu m6, [r5 + 9 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1816 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1817 * 16], m3 + + ; mode 30 [row 13] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1818 * 16], m3 + + ; mode 33 [row 6 - first half] + movu [r0 + 1996 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1819 * 16], m3 + + ; mode 33 [row 6 - second half] + movu [r0 + 1997 * 16], m3 + + ; mode 31 [row 9] + movu m6, [r5 + 10 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1874 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1875 * 16], m3 + + ; mode 31 [row 10] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1876 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1877 * 16], m3 + + ; mode 32 [row 7] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1934 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1935 * 16], m3 + + ; mode 29 [row 21] + movu m6, [r5 + 6 * 16] + movu m0, [r3 + 7] + movd m1, [r3 + 8] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 15] + movd m4, [r3 + 16] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1770 * 16], m3 + + movu m1, [r3 + 23] + movd m3, [r3 + 24] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 31] + movd m5, [r3 + 32] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1771 * 16], m3 + + ; mode 29 [row 22] + movu m6, [r5 + 15 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1772 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1773 * 16], m3 + + ; mode 29 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1774 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1775 * 16], m3 + + ; mode 30 [row 14] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1820 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1821 * 16], m3 + + ; mode 30 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1822 * 16], m3 + + ; mode 33 [row 7 - first half] + movu [r0 + 1998 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1823 * 16], m3 + + ; mode 33 [row 7 - second half] + movu [r0 + 1999 * 16], m3 + + ; mode 30 [row 16] + movu m6, [r5 + 29 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1824 * 16], m3 + + ; mode 31 [row 12 - first half] + movu [r0 + 1880 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1825 * 16], m3 + + ; mode 31 [row 12 - second half] + movu [r0 + 1881 * 16], m3 + + ; mode 31 [row 11] + movu m6, [r5 + 12 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1878 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1879 * 16], m3 + + ; mode 32 [row 9] + movu m6, [r5 + 18 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1938 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1939 * 16], m3 + + ; mode 29 [row 24] + movu m6, [r5 + 1 * 16] + movu m0, [r3 + 8] + movd m1, [r3 + 9] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 16] + movd m4, [r3 + 17] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1776 * 16], m3 + + movu m1, [r3 + 24] + movd m3, [r3 + 25] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 32] + movd m5, [r3 + 33] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1777 * 16], m3 + + ; mode 29 [row 25] + movu m6, [r5 + 10 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1778 * 16], m3 + + ; mode 30 [row 17 - first half] + movu [r0 + 1826 * 16], m3 + + ; mode 33 [row 8 - first half] + movu [r0 + 2000 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1779 * 16], m3 + + ; mode 30 [row 17 - second half] + movu [r0 + 1827 * 16], m3 + + ; mode 33 [row 8 - second half] + movu [r0 + 2001 * 16], m3 + + ; mode 29 [row 26] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1780 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1781 * 16], m3 + + ; mode 29 [row 27] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1782 * 16], m3 + + ; mode 32 [row 11 - first half] + movu [r0 + 1942 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1783 * 16], m3 + + ; mode 32 [row 11 - second half] + movu [r0 + 1943 * 16], m3 + + ; mode 30 [row 18] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1828 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1829 * 16], m3 + + ; mode 31 [row 13] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1882 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1883 * 16], m3 + + ; mode 31 [row 14] + movu m6, [r5 + 31 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1884 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1885 * 16], m3 + + ; mode 32 [row 10] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1940 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1941 * 16], m3 + + ; mode 29 [row 28] + movu m6, [r5 + 5 * 16] + movu m0, [r3 + 9] + movd m1, [r3 + 10] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 17] + movd m4, [r3 + 18] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1784 * 16], m3 + + movu m1, [r3 + 25] + movd m3, [r3 + 26] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 33] + movd m5, [r3 + 34] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1785 * 16], m3 + + ; mode 29 [row 29] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1786 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1787 * 16], m3 + + ; mode 29 [row 30] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1788 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1789 * 16], m3 + + ; mode 30 [row 19] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1830 * 16], m3 + + ; mode 33 [row 9 - first half] + movu [r0 + 2002 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1831 * 16], m3 + + ; mode 33 [row 9 - second half] + movu [r0 + 2003 * 16], m3 + + ; mode 30 [row 20] + movu m6, [r5 + 17 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1832 * 16], m3 + + ; mode 32 [row 12 - first half] + movu [r0 + 1944 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1833 * 16], m3 + + ; mode 32 [row 12 - second half] + movu [r0 + 1945 * 16], m3 + + ; mode 30 [row 21] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1834 * 16], m3 + + ; mode 33 [row 10 - first half] + movu [r0 + 2004 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1835 * 16], m3 + + ; mode 33 [row 10 - second half] + movu [r0 + 2005 * 16], m3 + + ; mode 31 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1886 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1887 * 16], m3 + + ; mode 29 [row 31] + movu m0, [r3 + 10] + movd m1, [r3 + 11] + palignr m1, m0, 1 + punpcklbw m0, m1 + movu m2, [r3 + 18] + movd m3, [r3 + 19] + palignr m3, m2, 1 + punpcklbw m2, m3 + movu m1, [r3 + 26] + movd m3, [r3 + 27] + palignr m3, m1, 1 + punpcklbw m1, m3 + movu m4, [r3 + 34] + movd m5, [r3 + 35] + palignr m5, m4, 1 + punpcklbw m4, m5 + + pshufb m5, m0, [tab_S2] + movh [r0 + 1790 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1790 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1791 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 1791 * 16 + 8], m5 + + ; mode 30 [row 22] + movu m6, [r5 + 11 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1836 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1837 * 16], m3 + + ; mode 30 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1838 * 16], m3 + + ; mode 33 [row 11 - first half] + movu [r0 + 2006 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1839 * 16], m3 + + ; mode 33 [row 11 - second half] + movu [r0 + 2007 * 16], m3 + + ; mode 31 [row 16] + movu m6, [r5 + 1 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1888 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1889 * 16], m3 + + ; mode 31 [row 17] + movu m6, [r5 + 18 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1890 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1891 * 16], m3 + + ; mode 32 [row 13] + movu m6, [r5 + 6 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1946 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1947 * 16], m3 + + ; mode 32 [row 14] + movu m6, [r5 + 27 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1948 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1949 * 16], m3 + + ; mode 30 [row 24] + movu m6, [r5 + 5 * 16] + movu m0, [r3 + 11] + movd m1, [r3 + 12] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 19] + movd m4, [r3 + 20] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1840 * 16], m3 + + movu m1, [r3 + 27] + movd m3, [r3 + 28] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 35] + movd m5, [r3 + 36] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1841 * 16], m3 + + ; mode 30 [row 25] + movu m6, [r5 + 18 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1842 * 16], m3 + + ; mode 33 [row 12 - first half] + movu [r0 + 2008 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1843 * 16], m3 + + ; mode 33 [row 12 - second half] + movu [r0 + 2009 * 16], m3 + + ; mode 30 [row 26] + movu m6, [r5 + 31 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1844 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1845 * 16], m3 + + ; mode 31 [row 18] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1892 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1893 * 16], m3 + + ; mode 31 [row 19] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1894 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1895 * 16], m3 + + ; mode 32 [row 15] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1950 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1951 * 16], m3 + + ; mode 30 [row 27] + movu m6, [r5 + 12 * 16] + movu m0, [r3 + 12] + movd m1, [r3 + 13] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 20] + movd m4, [r3 + 21] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1846 * 16], m3 + + ; mode 33 [row 13 - first half] + movu [r0 + 2010 * 16], m3 + + movu m1, [r3 + 28] + movd m3, [r3 + 29] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 36] + movd m5, [r3 + 37] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1847 * 16], m3 + + ; mode 33 [row 13 - second half] + movu [r0 + 2011 * 16], m3 + + ; mode 30 [row 28] + movu m6, [r5 + 25 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1848 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1849 * 16], m3 + + ; mode 31 [row 20] + movu m6, [r5 + 5 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1896 * 16], m3 + + ; mode 32 [row 16 - first half] + movu [r0 + 1952 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1897 * 16], m3 + + ; mode 32 [row 16 - second half] + movu [r0 + 1953 * 16], m3 + + ; mode 31 [row 21] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1898 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1899 * 16], m3 + + ; mode 32 [row 17] + movu m6, [r5 + 26 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1954 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1955 * 16], m3 + + ; mode 30 [row 29] + movu m6, [r5 + 6 * 16] + movu m0, [r3 + 13] + movd m1, [r3 + 14] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 21] + movd m4, [r3 + 22] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1850 * 16], m3 + + ; mode 33 [row 14 - first half] + movu [r0 + 2012 * 16], m3 + + movu m1, [r3 + 29] + movd m3, [r3 + 30] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 37] + movd m5, [r3 + 38] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1851 * 16], m3 + + ; mode 33 [row 14 - second half] + movu [r0 + 2013 * 16], m3 + + ; mode 30 [row 30] + movu m6, [r5 + 19 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1852 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1853 * 16], m3 + + ; mode 31 [row 22] + movu m6, [r5 + 7 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1900 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1901 * 16], m3 + + ; mode 31 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1902 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1903 * 16], m3 + + ; mode 32 [row 18] + movu m6, [r5 + 15 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1956 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1957 * 16], m3 + + ; mode 30 [row 31] + movu m0, [r3 + 14] + movd m1, [r3 + 15] + palignr m1, m0, 1 + punpcklbw m0, m1 + movu m2, [r3 + 22] + movd m3, [r3 + 23] + palignr m3, m2, 1 + punpcklbw m2, m3 + movu m1, [r3 + 30] + movd m3, [r3 + 31] + palignr m3, m1, 1 + punpcklbw m1, m3 + movu m4, [r3 + 38] + movd m5, [r3 + 39] + palignr m5, m4, 1 + punpcklbw m4, m5 + + pshufb m5, m0, [tab_S2] + movh [r0 + 1854 * 16], m5 + + ; mode 33 [row 15 - first eight] + movh [r0 + 2014 * 16], m5 + + pshufb m5, m2, [tab_S2] + movh [r0 + 1854 * 16 + 8], m5 + + ; mode 33 [row 15 - second eight] + movh [r0 + 2014 * 16 + 8], m5 + + pshufb m5, m1, [tab_S2] + movh [r0 + 1855 * 16], m5 + + ; mode 33 [row 15 - third eight] + movh [r0 + 2015 * 16], m5 + + pshufb m5, m4, [tab_S2] + movh [r0 + 1855 * 16 + 8], m5 + + ; mode 33 [row 15 - fourth eight] + movh [r0 + 2015 * 16 + 8], m5 + + ; mode 31 [row 24] + movu m6, [r5 + 9 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1904 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1905 * 16], m3 + + ; mode 31 [row 25] + movu m6, [r5 + 26 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1906 * 16], m3 + + ; mode 33 [row 16 - first half] + movu [r0 + 2016 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1907 * 16], m3 + + ; mode 33 [row 16 - second half] + movu [r0 + 2017 * 16], m3 + + ; mode 32 [row 19] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1958 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1959 * 16], m3 + + ; mode 32 [row 20] + movu m6, [r5 + 25 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1960 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1961 * 16], m3 + + ; mode 31 [row 26] + movu m6, [r5 + 11 * 16] + movu m0, [r3 + 15] + movd m1, [r3 + 16] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 23] + movd m4, [r3 + 24] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1908 * 16], m3 + + movu m1, [r3 + 31] + movd m3, [r3 + 32] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 39] + movd m5, [r3 + 40] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1909 * 16], m3 + + ; mode 31 [row 27] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1910 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1911 * 16], m3 + + ; mode 32 [row 21] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1962 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1963 * 16], m3 + + ; mode 33 [row 17] + movu m6, [r5 + 20 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2018 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2019 * 16], m3 + + ; mode 31 [row 28] + movu m6, [r5 + 13 * 16] + movu m0, [r3 + 16] + movd m1, [r3 + 17] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 24] + movd m4, [r3 + 25] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1912 * 16], m3 + + movu m1, [r3 + 32] + movd m3, [r3 + 33] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 40] + movd m5, [r3 + 41] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1913 * 16], m3 + + ; mode 31 [row 29] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1914 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1915 * 16], m3 + + ; mode 32 [row 22] + movu m6, [r5 + 3 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1964 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1965 * 16], m3 + + ; mode 32 [row 23] + movu m6, [r5 + 24 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1966 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1967 * 16], m3 + + ; mode 33 [row 18] + movu m6, [r5 + 14 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2020 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2021 * 16], m3 + + ; mode 31 [row 30] + movu m6, [r5 + 15 * 16] + movu m0, [r3 + 17] + movd m1, [r3 + 18] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 25] + movd m4, [r3 + 26] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1916 * 16], m3 + + movu m1, [r3 + 33] + movd m3, [r3 + 34] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 41] + movd m5, [r3 + 42] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1917 * 16], m3 + + ; mode 32 [row 24] + movu m6, [r5 + 13 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1968 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1969 * 16], m3 + + ; mode 33 [row 19] + movu m6, [r5 + 8 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2022 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2023 * 16], m3 + + ; mode 31 [row 31] + movu m0, [r3 + 18] + movd m1, [r3 + 19] + palignr m1, m0, 1 + punpcklbw m0, m1 + movu m2, [r3 + 26] + movd m3, [r3 + 27] + palignr m3, m2, 1 + punpcklbw m2, m3 + movu m1, [r3 + 34] + movd m3, [r3 + 35] + palignr m3, m1, 1 + punpcklbw m1, m3 + movu m4, [r3 + 42] + movd m5, [r3 + 43] + palignr m5, m4, 1 + punpcklbw m4, m5 + + pshufb m5, m0, [tab_S2] + movh [r0 + 1918 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1918 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1919 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 1919 * 16 + 8], m5 + + ; mode 32 [row 25] + movu m6, [r5 + 2 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1970 * 16], m3 + + ; mode 33 [row 20 - first half] + movu [r0 + 2024 * 16], m3 + + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1971 * 16], m3 + + ; mode 33 [row 20 - second half] + movu [r0 + 2025 * 16], m3 + + ; mode 32 [row 26] + movu m6, [r5 + 23 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1972 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1973 * 16], m3 + + ; mode 33 [row 21] + movu m6, [r5 + 28 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2026 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2027 * 16], m3 + + ; mode 32 [row 27] + movu m6, [r5 + 12 * 16] + movu m0, [r3 + 19] + movd m1, [r3 + 20] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 27] + movd m4, [r3 + 28] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1974 * 16], m3 + + movu m1, [r3 + 35] + movd m3, [r3 + 36] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 43] + movd m5, [r3 + 44] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1975 * 16], m3 + + ; mode 33 [row 22] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2028 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2029 * 16], m3 + + ; mode 32 [row 28] + movu m6, [r5 + 1 * 16] + movu m0, [r3 + 20] + movd m1, [r3 + 21] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 28] + movd m4, [r3 + 29] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1976 * 16], m3 + + movu m1, [r3 + 36] + movd m3, [r3 + 37] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 44] + movd m5, [r3 + 45] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1977 * 16], m3 + + ; mode 32 [row 29] + movu m6, [r5 + 22 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1978 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1979 * 16], m3 + + ; mode 33 [row 23] + movu m6, [r5 + 16 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2030 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2031 * 16], m3 + + ; mode 32 [row 30] + movu m6, [r5 + 11 * 16] + movu m0, [r3 + 21] + movd m1, [r3 + 22] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 29] + movd m4, [r3 + 30] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1980 * 16], m3 + + movu m1, [r3 + 37] + movd m3, [r3 + 38] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 45] + movd m5, [r3 + 46] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 1981 * 16], m3 + + ; mode 33 [row 24] + movu m6, [r5 + 10 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2032 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2033 * 16], m3 + + ; mode 32 [row 31] + movu m0, [r3 + 22] + movd m1, [r3 + 23] + palignr m1, m0, 1 + punpcklbw m0, m1 + movu m2, [r3 + 30] + movd m3, [r3 + 31] + palignr m3, m2, 1 + punpcklbw m2, m3 + movu m1, [r3 + 38] + movd m3, [r3 + 39] + palignr m3, m1, 1 + punpcklbw m1, m3 + movu m4, [r3 + 46] + movd m5, [r3 + 47] + palignr m5, m4, 1 + punpcklbw m4, m5 + + pshufb m5, m0, [tab_S2] + movh [r0 + 1982 * 16], m5 + pshufb m5, m2, [tab_S2] + movh [r0 + 1982 * 16 + 8], m5 + pshufb m5, m1, [tab_S2] + movh [r0 + 1983 * 16], m5 + pshufb m5, m4, [tab_S2] + movh [r0 + 1983 * 16 + 8], m5 + + ; mode 33 [row 25] + movu m6, [r5 + 4 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2034 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2035 * 16], m3 + + ; mode 33 [row 26] + movu m6, [r5 + 30 * 16] + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2036 * 16], m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2037 * 16], m3 + + ; mode 33 [row 27] + movu m6, [r5 + 24 * 16] + movu m0, [r3 + 23] + movd m1, [r3 + 24] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 31] + movd m4, [r3 + 32] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2038 * 16], m3 + + movu m1, [r3 + 39] + movd m3, [r3 + 40] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 47] + movd m5, [r3 + 48] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2039 * 16], m3 + + ; mode 33 [row 28] + movu m6, [r5 + 18 * 16] + movu m0, [r3 + 24] + movd m1, [r3 + 25] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 32] + movd m4, [r3 + 33] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2040 * 16], m3 + + movu m1, [r3 + 40] + movd m3, [r3 + 41] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 48] + movd m5, [r3 + 49] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2041 * 16], m3 + + ; mode 33 [row 29] + movu m6, [r5 + 12 * 16] + movu m0, [r3 + 25] + movd m1, [r3 + 26] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 33] + movd m4, [r3 + 34] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2042 * 16], m3 + + movu m1, [r3 + 41] + movd m3, [r3 + 42] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 49] + movd m5, [r3 + 50] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2043 * 16], m3 + + ; mode 33 [row 30] + movu m6, [r5 + 6 * 16] + movu m0, [r3 + 26] + movd m1, [r3 + 27] + palignr m1, m0, 1 + punpcklbw m0, m1 + pmaddubsw m3, m0, m6 + pmulhrsw m3, m7 + movu m2, [r3 + 34] + movd m4, [r3 + 35] + palignr m4, m2, 1 + punpcklbw m2, m4 + pmaddubsw m5, m2, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2044 * 16], m3 + + movu m1, [r3 + 42] + movd m3, [r3 + 43] + palignr m3, m1, 1 + punpcklbw m1, m3 + pmaddubsw m3, m1, m6 + pmulhrsw m3, m7 + movu m4, [r3 + 50] + movd m5, [r3 + 51] + palignr m5, m4, 1 + punpcklbw m4, m5 + pmaddubsw m5, m4, m6 + pmulhrsw m5, m7 + packuswb m3, m5 + movu [r0 + 2045 * 16], m3 + + ; mode 33 [row 31] + movu m5, [r3 + 27] + movu [r0 + 2046 * 16], m5 + movu m5, [r3 + 43] + movu [r0 + 2047 * 16], m5 + + ;mode 34 [row 0] + movu m0, [r3 + 2] + movu [r0 + 2048 * 16], m0 + movu m1, [r3 + 18] + movu [r0 + 2049 * 16], m1 + + ;mode 34 [row 1] + movu m2, [r3 + 34] + palignr m3, m1, m0, 1 + movu [r0 + 2050 * 16], m3 + palignr m4, m2, m1, 1 + movu [r0 + 2051 * 16], m4 + + ;mode 34 [row 2] + palignr m3, m1, m0, 2 + movu [r0 + 2052 * 16], m3 + palignr m4, m2, m1, 2 + movu [r0 + 2053 * 16], m4 + + ;mode 34 [row 3] + palignr m3, m1, m0, 3 + movu [r0 + 2054 * 16], m3 + palignr m4, m2, m1, 3 + movu [r0 + 2055 * 16], m4 + + ;mode 34 [row 4] + palignr m3, m1, m0, 4 + movu [r0 + 2056 * 16], m3 + palignr m4, m2, m1, 4 + movu [r0 + 2057 * 16], m4 + + ;mode 34 [row 5] + palignr m3, m1, m0, 5 + movu [r0 + 2058 * 16], m3 + palignr m4, m2, m1, 5 + movu [r0 + 2059 * 16], m4 + + ;mode 34 [row 6] + palignr m3, m1, m0, 6 + movu [r0 + 2060 * 16], m3 + palignr m4, m2, m1, 6 + movu [r0 + 2061 * 16], m4 + + ;mode 34 [row 7] + palignr m3, m1, m0, 7 + movu [r0 + 2062 * 16], m3 + palignr m4, m2, m1, 7 + movu [r0 + 2063 * 16], m4 + + ;mode 34 [row 8] + palignr m3, m1, m0, 8 + movu [r0 + 2064 * 16], m3 + palignr m4, m2, m1, 8 + movu [r0 + 2065 * 16], m4 + + ;mode 34 [row 9] + palignr m3, m1, m0, 9 + movu [r0 + 2066 * 16], m3 + palignr m4, m2, m1, 9 + movu [r0 + 2067 * 16], m4 + + ;mode 34 [row 10] + palignr m3, m1, m0, 10 + movu [r0 + 2068 * 16], m3 + palignr m4, m2, m1, 10 + movu [r0 + 2069 * 16], m4 + + ;mode 34 [row 11] + palignr m3, m1, m0, 11 + movu [r0 + 2070 * 16], m3 + palignr m4, m2, m1, 11 + movu [r0 + 2071 * 16], m4 + + ;mode 34 [row 12] + palignr m3, m1, m0, 12 + movu [r0 + 2072 * 16], m3 + palignr m4, m2, m1, 12 + movu [r0 + 2073 * 16], m4 + + ;mode 34 [row 13] + palignr m3, m1, m0, 13 + movu [r0 + 2074 * 16], m3 + palignr m4, m2, m1, 13 + movu [r0 + 2075 * 16], m4 + + ;mode 34 [row 14] + palignr m3, m1, m0, 14 + movu [r0 + 2076 * 16], m3 + palignr m4, m2, m1, 14 + movu [r0 + 2077 * 16], m4 + + ;mode 34 [row 15] + palignr m3, m1, m0, 15 + movu [r0 + 2078 * 16], m3 + palignr m4, m2, m1, 15 + movu [r0 + 2079 * 16], m4 + + ;mode 34 [row 16] + palignr m3, m1, m0, 16 + movu [r0 + 2080 * 16], m3 + palignr m4, m2, m1, 16 + movu [r0 + 2081 * 16], m4 + + ;mode 34 [row 17] + movu m0, [r3 + 19] + movu [r0 + 2082 * 16], m0 + movu m1, [r3 + 35] + movu [r0 + 2083 * 16], m1 + + mov r2d, r6d + mov [r4], r2b + mov r2d, [rsp] + mov [r1 + 64], r2b + + ;mode 34 [row 18] + movu m2, [r3 + 51] + palignr m3, m1, m0, 1 + movu [r0 + 2084 * 16], m3 + palignr m4, m2, m1, 1 + movu [r0 + 2085 * 16], m4 + + ;mode 34 [row 19] + palignr m3, m1, m0, 2 + movu [r0 + 2086 * 16], m3 + palignr m4, m2, m1, 2 + movu [r0 + 2087 * 16], m4 + + ;mode 34 [row 20] + palignr m3, m1, m0, 3 + movu [r0 + 2088 * 16], m3 + palignr m4, m2, m1, 3 + movu [r0 + 2089 * 16], m4 + + ;mode 34 [row 21] + palignr m3, m1, m0, 4 + movu [r0 + 2090 * 16], m3 + palignr m4, m2, m1, 4 + movu [r0 + 2091 * 16], m4 + + ;mode 34 [row 22] + palignr m3, m1, m0, 5 + movu [r0 + 2092 * 16], m3 + palignr m4, m2, m1, 5 + movu [r0 + 2093 * 16], m4 + + ;mode 34 [row 23] + palignr m3, m1, m0, 6 + movu [r0 + 2094 * 16], m3 + palignr m4, m2, m1, 6 + movu [r0 + 2095 * 16], m4 + + ;mode 34 [row 24] + palignr m3, m1, m0, 7 + movu [r0 + 2096 * 16], m3 + palignr m4, m2, m1, 7 + movu [r0 + 2097 * 16], m4 + + ;mode 34 [row 25] + palignr m3, m1, m0, 8 + movu [r0 + 2098 * 16], m3 + palignr m4, m2, m1, 8 + movu [r0 + 2099 * 16], m4 + + ;mode 34 [row 26] + palignr m3, m1, m0, 9 + movu [r0 + 2100 * 16], m3 + palignr m4, m2, m1, 9 + movu [r0 + 2101 * 16], m4 + + ;mode 34 [row 27] + palignr m3, m1, m0, 10 + movu [r0 + 2102 * 16], m3 + palignr m4, m2, m1, 10 + movu [r0 + 2103 * 16], m4 + + ;mode 34 [row 28] + palignr m3, m1, m0, 11 + movu [r0 + 2104 * 16], m3 + palignr m4, m2, m1, 11 + movu [r0 + 2105 * 16], m4 + + ;mode 34 [row 29] + palignr m3, m1, m0, 12 + movu [r0 + 2106 * 16], m3 + palignr m4, m2, m1, 12 + movu [r0 + 2107 * 16], m4 + + ;mode 34 [row 30] + palignr m3, m1, m0, 13 + movu [r0 + 2108 * 16], m3 + palignr m4, m2, m1, 13 + movu [r0 + 2109 * 16], m4 + + ;mode 34 [row 31] + palignr m3, m1, m0, 14 + movu [r0 + 2110 * 16], m3 + palignr m4, m2, m1, 14 + movu [r0 + 2111 * 16], m4 + RET + + +;----------------------------------------------------------------------------- +; void all_angs_pred_4x4(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) +;----------------------------------------------------------------------------- +INIT_YMM avx2 +cglobal all_angs_pred_4x4, 4, 4, 6 + + mova m5, [pw_1024] + lea r2, [all_ang4] + lea r3, [all_ang4_shuff] + +; mode 2 + + vbroadcasti128 m0, [r1 + 9] + mova xm1, xm0 + psrldq xm1, 1 + pshufb xm1, [r3] + movu [r0], xm1 + +; mode 3 + + pshufb m1, m0, [r3 + 1 * mmsize] + pmaddubsw m1, [r2] + pmulhrsw m1, m5 + +; mode 4 + + pshufb m2, m0, [r3 + 2 * mmsize] + pmaddubsw m2, [r2 + 1 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (3 - 2) * 16], m1 + +; mode 5 + + pshufb m1, m0, [r3 + 2 * mmsize] + pmaddubsw m1, [r2 + 2 * mmsize] + pmulhrsw m1, m5 + +; mode 6 + + pshufb m2, m0, [r3 + 3 * mmsize] + pmaddubsw m2, [r2 + 3 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (5 - 2) * 16], m1 + + add r3, 4 * mmsize + add r2, 4 * mmsize + +; mode 7 + + pshufb m1, m0, [r3 + 0 * mmsize] + pmaddubsw m1, [r2 + 0 * mmsize] + pmulhrsw m1, m5 + +; mode 8 + + pshufb m2, m0, [r3 + 1 * mmsize] + pmaddubsw m2, [r2 + 1 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (7 - 2) * 16], m1 + +; mode 9 + + pshufb m1, m0, [r3 + 1 * mmsize] + pmaddubsw m1, [r2 + 2 * mmsize] + pmulhrsw m1, m5 + packuswb m1, m1 + vpermq m1, m1, 11011000b + movu [r0 + (9 - 2) * 16], xm1 + +; mode 10 + + pshufb xm1, xm0, [r3 + 2 * mmsize] + movu [r0 + (10 - 2) * 16], xm1 + + pxor xm1, xm1 + movd xm2, [r1 + 1] + pshufd xm3, xm2, 0 + punpcklbw xm3, xm1 + pinsrb xm2, [r1], 0 + pshufb xm4, xm2, xm1 + punpcklbw xm4, xm1 + psubw xm3, xm4 + psraw xm3, 1 + pshufb xm4, xm0, xm1 + punpcklbw xm4, xm1 + paddw xm3, xm4 + packuswb xm3, xm1 + + pextrb [r0 + 128], xm3, 0 + pextrb [r0 + 132], xm3, 1 + pextrb [r0 + 136], xm3, 2 + pextrb [r0 + 140], xm3, 3 + +; mode 11 + + vbroadcasti128 m0, [r1] + pshufb m1, m0, [r3 + 3 * mmsize] + pmaddubsw m1, [r2 + 3 * mmsize] + pmulhrsw m1, m5 + +; mode 12 + + add r2, 4 * mmsize + + pshufb m2, m0, [r3 + 3 * mmsize] + pmaddubsw m2, [r2 + 0 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (11 - 2) * 16], m1 + +; mode 13 + + add r3, 4 * mmsize + + pshufb m1, m0, [r3 + 0 * mmsize] + pmaddubsw m1, [r2 + 1 * mmsize] + pmulhrsw m1, m5 + +; mode 14 + + pshufb m2, m0, [r3 + 1 * mmsize] + pmaddubsw m2, [r2 + 2 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (13 - 2) * 16], m1 + +; mode 15 + + pshufb m1, m0, [r3 + 2 * mmsize] + pmaddubsw m1, [r2 + 3 * mmsize] + pmulhrsw m1, m5 + +; mode 16 + + add r2, 4 * mmsize + + pshufb m2, m0, [r3 + 3 * mmsize] + pmaddubsw m2, [r2 + 0 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (15 - 2) * 16], m1 + +; mode 17 + + add r3, 4 * mmsize + + pshufb m1, m0, [r3 + 0 * mmsize] + pmaddubsw m1, [r2 + 1 * mmsize] + pmulhrsw m1, m5 + packuswb m1, m1 + vpermq m1, m1, 11011000b + +; mode 18 + + pshufb m2, m0, [r3 + 1 * mmsize] + vinserti128 m1, m1, xm2, 1 + movu [r0 + (17 - 2) * 16], m1 + +; mode 19 + + pshufb m1, m0, [r3 + 2 * mmsize] + pmaddubsw m1, [r2 + 2 * mmsize] + pmulhrsw m1, m5 + +; mode 20 + + pshufb m2, m0, [r3 + 3 * mmsize] + pmaddubsw m2, [r2 + 3 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (19 - 2) * 16], m1 + +; mode 21 + + add r2, 4 * mmsize + add r3, 4 * mmsize + + pshufb m1, m0, [r3 + 0 * mmsize] + pmaddubsw m1, [r2 + 0 * mmsize] + pmulhrsw m1, m5 + +; mode 22 + + pshufb m2, m0, [r3 + 1 * mmsize] + pmaddubsw m2, [r2 + 1 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (21 - 2) * 16], m1 + +; mode 23 + + pshufb m1, m0, [r3 + 2 * mmsize] + pmaddubsw m1, [r2 + 2 * mmsize] + pmulhrsw m1, m5 + +; mode 24 + + pshufb m2, m0, [r3 + 3 * mmsize] + pmaddubsw m2, [r2 + 3 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (23 - 2) * 16], m1 + +; mode 25 + + add r2, 4 * mmsize + + pshufb m1, m0, [r3 + 3 * mmsize] + pmaddubsw m1, [r2 + 0 * mmsize] + pmulhrsw m1, m5 + packuswb m1, m1 + vpermq m1, m1, 11011000b + movu [r0 + (25 - 2) * 16], xm1 + +; mode 26 + + add r3, 4 * mmsize + + pshufb xm1, xm0, [r3 + 0 * mmsize] + movu [r0 + (26 - 2) * 16], xm1 + + pxor xm1, xm1 + movd xm2, [r1 + 9] + pshufd xm3, xm2, 0 + punpcklbw xm3, xm1 + pinsrb xm4, [r1 + 0], 0 + pshufb xm4, xm1 + punpcklbw xm4, xm1 + psubw xm3, xm4 + psraw xm3, 1 + psrldq xm2, xm0, 1 + pshufb xm2, xm1 + punpcklbw xm2, xm1 + paddw xm3, xm2 + packuswb xm3, xm1 + + pextrb [r0 + 384], xm3, 0 + pextrb [r0 + 388], xm3, 1 + pextrb [r0 + 392], xm3, 2 + pextrb [r0 + 396], xm3, 3 + +; mode 27 + + pshufb m1, m0, [r3 + 1 * mmsize] + pmaddubsw m1, [r2 + 1 * mmsize] + pmulhrsw m1, m5 + +; mode 28 + + pshufb m2, m0, [r3 + 1 * mmsize] + pmaddubsw m2, [r2 + 2 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (27 - 2) * 16], m1 + +; mode 29 + + pshufb m1, m0, [r3 + 2 * mmsize] + pmaddubsw m1, [r2 + 3 * mmsize] + pmulhrsw m1, m5 + +; mode 30 + + add r2, 4 * mmsize + + pshufb m2, m0, [r3 + 3 * mmsize] + pmaddubsw m2, [r2 + 0 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (29 - 2) * 16], m1 + +; mode 31 + + add r3, 4 * mmsize + + pshufb m1, m0, [r3 + 0 * mmsize] + pmaddubsw m1, [r2 + 1 * mmsize] + pmulhrsw m1, m5 + +; mode 32 + + pshufb m2, m0, [r3 + 0 * mmsize] + pmaddubsw m2, [r2 + 2 * mmsize] + pmulhrsw m2, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + movu [r0 + (31 - 2) * 16], m1 + +; mode 33 + + pshufb m1, m0, [r3 + 1 * mmsize] + pmaddubsw m1, [r2 + 3 * mmsize] + pmulhrsw m1, m5 + packuswb m1, m2 + vpermq m1, m1, 11011000b + +; mode 34 + + pshufb m0, [r3 + 2 * mmsize] + vinserti128 m1, m1, xm0, 1 + movu [r0 + (33 - 2) * 16], m1 + RET + +;----------------------------------------------------------------------------- +; void all_angs_pred_4x4(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) +;----------------------------------------------------------------------------- +INIT_XMM sse2 +cglobal all_angs_pred_4x4, 4, 4, 8 + +; mode 2 + + movh m6, [r1 + 9] + mova m2, m6 + psrldq m2, 1 + movd [r0], m2 ;byte[A, B, C, D] + psrldq m2, 1 + movd [r0 + 4], m2 ;byte[B, C, D, E] + psrldq m2, 1 + movd [r0 + 8], m2 ;byte[C, D, E, F] + psrldq m2, 1 + movd [r0 + 12], m2 ;byte[D, E, F, G] + +; mode 10/26 + + pxor m7, m7 + pshufd m5, m6, 0 + mova [r0 + 128], m5 ;mode 10 byte[9, A, B, C, 9, A, B, C, 9, A, B, C, 9, A, B, C] + + movd m4, [r1 + 1] + pshufd m4, m4, 0 + mova [r0 + 384], m4 ;mode 26 byte[1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4] + + movd m1, [r1] + punpcklbw m1, m7 + pshuflw m1, m1, 0x00 + punpcklqdq m1, m1 ;m1 = byte[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + + punpckldq m4, m5 + punpcklbw m4, m7 ;m4 = word[1, 2, 3, 4, 9, A, B, C] + pshuflw m2, m4, 0x00 + pshufhw m2, m2, 0x00 ;m2 = word[1, 1, 1, 1, 9, 9, 9, 9] + + psubw m4, m1 + psraw m4, 1 + + pshufd m2, m2, q1032 ;m2 = word[9, 9, 9, 9, 1, 1, 1, 1] + paddw m4, m2 + packuswb m4, m4 + +%if ARCH_X86_64 + movq r2, m4 + + mov [r0 + 128], r2b ;mode 10 + shr r2, 8 + mov [r0 + 132], r2b + shr r2, 8 + mov [r0 + 136], r2b + shr r2, 8 + mov [r0 + 140], r2b + shr r2, 8 + mov [r0 + 384], r2b ;mode 26 + shr r2d, 8 + mov [r0 + 388], r2b + shr r2d, 8 + mov [r0 + 392], r2b + shr r2d, 8 + mov [r0 + 396], r2b + +%else + movd r2d, m4 + + mov [r0 + 128], r2b ;mode 10 + shr r2d, 8 + mov [r0 + 132], r2b + shr r2d, 8 + mov [r0 + 136], r2b + shr r2d, 8 + mov [r0 + 140], r2b + + psrldq m4, 4 + movd r2d, m4 + + mov [r0 + 384], r2b ;mode 26 + shr r2d, 8 + mov [r0 + 388], r2b + shr r2d, 8 + mov [r0 + 392], r2b + shr r2d, 8 + mov [r0 + 396], r2b +%endif + +; mode 3 + + mova m2, [pw_16] + lea r3, [pw_ang_table + 7 * 16] + lea r2, [pw_ang_table + 23 * 16] + punpcklbw m6, m6 + psrldq m6, 1 + movh m1, m6 + psrldq m6, 2 + movh m0, m6 + psrldq m6, 2 + movh m3, m6 + psrldq m6, 2 + punpcklbw m1, m7 ;m1 = word[9, A, A, B, B, C, C, D] + punpcklbw m0, m7 ;m0 = word[A, B, B, C, C, D, D, E] + punpcklbw m3, m7 ;m3 = word[B, C, C, D, D, E, E, F] + punpcklbw m6, m7 ;m6 = word[C, D, D, E, E, F, F, G] + + mova m7, [r2 - 3 * 16] + + pmaddwd m5, m1, [r2 + 3 * 16] + pmaddwd m4, m0, m7 + + packssdw m5, m4 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m4, m3, [r3 + 7 * 16] + pmaddwd m6, [r3 + 1 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 16], m5 + movd [r0 + 68], m5 ;mode 6 row 1 + psrldq m5, 4 + movd [r0 + 76], m5 ;mode 6 row 3 + +; mode 4 + + pmaddwd m4, m0, [r2 + 8 * 16] + pmaddwd m6, m3, m7 + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m5, m1, [r2 - 2 * 16] + pmaddwd m6, m0, [r3 + 3 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + packuswb m5, m4 + mova [r0 + 32], m5 + +; mode 5 + + pmaddwd m5, m1, [r2 - 6 * 16] + pmaddwd m6, m0, [r3 - 5 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m4, m0, [r2 - 4 * 16] + pmaddwd m3, [r3 - 3 * 16] + + packssdw m4, m3 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 48], m5 + +; mode 6 + + pmaddwd m5, m1, [r3 + 6 * 16] + pmaddwd m6, m0, [r3 + 0 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + packuswb m5, m6 + movd [r0 + 64], m5 + psrldq m5, 4 + movd [r0 + 72], m5 + +; mode 7 + + pmaddwd m5, m1, [r3 + 2 * 16] + pmaddwd m6, m1, [r2 - 5 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + mova m3, [r2 + 4 * 16] + pmaddwd m4, m1, m3 + pmaddwd m0, [r3 - 3 * 16] + + packssdw m4, m0 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 80], m5 + +; mode 8 + + mova m0, [r3 - 2 * 16] + pmaddwd m5, m1, m0 + pmaddwd m6, m1, [r3 + 3 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m4, m1, [r3 + 8 * 16] + pmaddwd m7, m1 + + packssdw m4, m7 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 96], m5 + +; mode 9 + + pmaddwd m5, m1, [r3 - 5 * 16] + pmaddwd m6, m1, [r3 - 3 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m4, m1, [r3 - 1 * 16] + pmaddwd m6, m1, [r3 + 1 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 112], m5 + +; mode 11 + + movd m5, [r1] + punpcklwd m5, m1 + pand m5, [pb_0000000000000F0F] + pslldq m1, 4 + por m1, m5 ;m1 = word[0, 9, 9, A, A, B, B, C] + + pmaddwd m5, m1, [r2 + 7 * 16] + pmaddwd m6, m1, [r2 + 5 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m4, m1, [r2 + 3 * 16] + pmaddwd m6, m1, [r2 + 1 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 144], m5 + +; mode 12 + + pmaddwd m3, m1 + pmaddwd m6, m1, [r2 - 1 * 16] + + packssdw m3, m6 + paddw m3, m2 + psraw m3, 5 + + pmaddwd m4, m1, [r2 - 6 * 16] + pmaddwd m6, m1, [r3 + 5 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + packuswb m3, m4 + mova [r0 + 160], m3 + +; mode 13 + + mova m3, m1 + movd m7, [r1 + 4] + punpcklwd m7, m1 + pand m7, [pb_0000000000000F0F] + pslldq m3, 4 + por m3, m7 ;m3 = word[4, 0, 0, 9, 9, A, A, B] + + pmaddwd m5, m1, [r2 + 0 * 16] + pmaddwd m6, m1, [r3 + 7 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m4, m1, m0 + pmaddwd m6, m3, [r2 + 5 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 176], m5 + +; mode 14 + + pmaddwd m5, m1, [r2 - 4 * 16] + pmaddwd m6, m1, [r3 - 1 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + movd m6, [r1 + 2] + pand m3, [pw_FFFFFFFFFFFFFFF0] + pand m6, [pb_000000000000000F] + por m3, m6 ;m3 = word[2, 0, 0, 9, 9, A, A, B] + + pmaddwd m4, m3, [r2 + 2 * 16] + pmaddwd m6, m3, [r3 + 5 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 192], m5 + psrldq m5, 4 + movd [r0 + 240], m5 ;mode 17 row 0 + +; mode 15 + + pmaddwd m5, m1, [r3 + 8 * 16] + pmaddwd m6, m3, [r2 + 7 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m6, m3, [r3 + 6 * 16] + + mova m0, m3 + punpcklwd m7, m3 + pslldq m0, 4 + pand m7, [pb_0000000000000F0F] + por m0, m7 ;m0 = word[4, 2, 2, 0, 0, 9, 9, A] + + pmaddwd m4, m0, [r2 + 5 * 16] + + packssdw m6, m4 + paddw m6, m2 + psraw m6, 5 + + packuswb m5, m6 + mova [r0 + 208], m5 + +; mode 16 + + pmaddwd m5, m1, [r3 + 4 * 16] + pmaddwd m6, m3, [r2 - 1 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m3, [r3 - 6 * 16] + + movd m6, [r1 + 3] + pand m0, [pw_FFFFFFFFFFFFFFF0] + pand m6, [pb_000000000000000F] + por m0, m6 ;m0 = word[3, 2, 2, 0, 0, 9, 9, A] + + pmaddwd m0, [r3 + 5 * 16] + packssdw m3, m0 + paddw m3, m2 + psraw m3, 5 + + packuswb m5, m3 + mova [r0 + 224], m5 + +; mode 17 + + movd m4, [r1 + 1] + punpcklwd m4, m1 + pand m4, [pb_0000000000000F0F] + pslldq m1, 4 + por m1, m4 ;m1 = word[1, 0, 0, 9, 9, A, A, B] + + pmaddwd m6, m1, [r3 + 5 * 16] + + packssdw m6, m6 + paddw m6, m2 + psraw m6, 5 + + movd m5, [r1 + 2] + punpcklwd m5, m1 + pand m5, [pb_0000000000000F0F] + pslldq m1, 4 + por m1, m5 ;m1 = word[2, 1, 1, 0, 0, 9, 9, A] + + pmaddwd m4, m1, [r2 - 5 * 16] + + punpcklwd m7, m1 + pand m7, [pb_0000000000000F0F] + pslldq m1, 4 + por m1, m7 ;m1 = word[4, 2, 2, 1, 1, 0, 0, 9] + + pmaddwd m1, [r2 + 1 * 16] + packssdw m4, m1 + paddw m4, m2 + psraw m4, 5 + + packuswb m6, m4 + movd [r0 + 244], m6 + psrldq m6, 8 + movh [r0 + 248], m6 + +; mode 18 + + movh m1, [r1] + movd [r0 + 256], m1 ;byte[0, 1, 2, 3] + + movh m3, [r1 + 2] + punpcklqdq m3, m1 + psrldq m3, 7 + movd [r0 + 260], m3 ;byte[2, 1, 0, 9] + + movh m4, [r1 + 3] + punpcklqdq m4, m3 + psrldq m4, 7 + movd [r0 + 264], m4 ;byte[1, 0, 9, A] + + movh m0, [r1 + 4] + punpcklqdq m0, m4 + psrldq m0, 7 + movd [r0 + 268], m0 ;byte[0, 9, A, B] + +; mode 19 + + pxor m7, m7 + punpcklbw m4, m3 + punpcklbw m3, m1 + punpcklbw m1, m1 + punpcklbw m4, m7 ;m4 = word[A, 9, 9, 0, 0, 1, 1, 2] + punpcklbw m3, m7 ;m3 = word[9, 0, 0, 1, 1, 2, 2, 3] + psrldq m1, 1 + punpcklbw m1, m7 ;m1 = word[0, 1, 1, 2, 2, 3, 3, 4] + + pmaddwd m6, m1, [r3 - 1 * 16] + pmaddwd m7, m3, [r3 + 5 * 16] + + packssdw m6, m7 + paddw m6, m2 + psraw m6, 5 + + pmaddwd m5, m4, [r2 - 5 * 16] + + movd m7, [r1 + 12] + punpcklwd m7, m4 + pand m7, [pb_0000000000000F0F] + pslldq m4, 4 + por m4, m7 ;m4 = word[C, A, A, 9, 9, 0, 0, 1] + + pmaddwd m4, [r2 + 1 * 16] + packssdw m5, m4 + paddw m5, m2 + psraw m5, 5 + + packuswb m6, m5 + mova [r0 + 272], m6 + movd [r0 + 324], m6 ;mode 22 row 1 + +; mode 20 + + pmaddwd m5, m1, [r3 + 4 * 16] + + movd m4, [r1 + 10] + pand m3, [pw_FFFFFFFFFFFFFFF0] + pand m4, [pb_000000000000000F] + por m3, m4 ;m3 = word[A, 0, 0, 1, 1, 2, 2, 3] + + pmaddwd m6, m3, [r2 - 1 * 16] + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + pmaddwd m4, m3, [r3 - 6 * 16] + + punpcklwd m0, m3 + pand m0, [pb_0000000000000F0F] + mova m6, m3 + pslldq m6, 4 + por m0, m6 ;m0 = word[B, A, A, 0, 0, 1, 1, 2] + + pmaddwd m6, m0, [r3 + 5 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + packuswb m5, m4 + mova [r0 + 288], m5 + +; mode 21 + + pmaddwd m4, m1, [r3 + 8 * 16] + pmaddwd m6, m3, [r2 + 7 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m5, m3, [r3 + 6 * 16] + + pand m0, [pw_FFFFFFFFFFFFFFF0] + pand m7, [pb_000000000000000F] + por m0, m7 ;m0 = word[C, A, A, 0, 0, 1, 1, 2] + + pmaddwd m0, [r2 + 5 * 16] + packssdw m5, m0 + paddw m5, m2 + psraw m5, 5 + + packuswb m4, m5 + mova [r0 + 304], m4 + +; mode 22 + + pmaddwd m4, m1, [r2 - 4 * 16] + packssdw m4, m4 + paddw m4, m2 + psraw m4, 5 + + mova m0, [r3 + 5 * 16] + pmaddwd m5, m3, [r2 + 2 * 16] + pmaddwd m6, m3, m0 + + packssdw m5, m6 + paddw m5, m2 + psraw m5, 5 + + packuswb m4, m5 + movd [r0 + 320], m4 + psrldq m4, 8 + movh [r0 + 328], m4 + +; mode 23 + + pmaddwd m4, m1, [r2 + 0 * 16] + pmaddwd m5, m1, [r3 + 7 * 16] + + packssdw m4, m5 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m6, m1, [r3 - 2 * 16] + + pand m3, [pw_FFFFFFFFFFFFFFF0] + por m3, m7 ;m3 = word[C, 0, 0, 1, 1, 2, 2, 3] + + pmaddwd m3, [r2 + 5 * 16] + packssdw m6, m3 + paddw m6, m2 + psraw m6, 5 + + packuswb m4, m6 + mova [r0 + 336], m4 + +; mode 24 + + pmaddwd m4, m1, [r2 + 4 * 16] + pmaddwd m5, m1, [r2 - 1 * 16] + + packssdw m4, m5 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m6, m1, [r2 - 6 * 16] + pmaddwd m0, m1 + + packssdw m6, m0 + paddw m6, m2 + psraw m6, 5 + + packuswb m4, m6 + mova [r0 + 352], m4 + +; mode 25 + + pmaddwd m4, m1, [r2 + 7 * 16] + pmaddwd m5, m1, [r2 + 5 * 16] + + packssdw m4, m5 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m6, m1, [r2 + 3 * 16] + pmaddwd m1, [r2 + 1 * 16] + + packssdw m6, m1 + paddw m6, m2 + psraw m6, 5 + + packuswb m4, m6 + mova [r0 + 368], m4 + +; mode 27 + + movh m0, [r1 + 1] + pxor m7, m7 + punpcklbw m0, m0 + psrldq m0, 1 + movh m1, m0 + psrldq m0, 2 + movh m3, m0 + psrldq m0, 2 + punpcklbw m1, m7 ;m1 = word[1, 2, 2, 3, 3, 4, 4, 5] + punpcklbw m3, m7 ;m3 = word[2, 3, 3, 4, 4, 5, 5, 6] + punpcklbw m0, m7 ;m0 = word[3, 4, 4, 5, 5, 6, 6, 7] + + mova m7, [r3 - 3 * 16] + + pmaddwd m4, m1, [r3 - 5 * 16] + pmaddwd m5, m1, m7 + + packssdw m4, m5 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m6, m1, [r3 - 1 * 16] + pmaddwd m5, m1, [r3 + 1 * 16] + + packssdw m6, m5 + paddw m6, m2 + psraw m6, 5 + + packuswb m4, m6 + mova [r0 + 400], m4 + +; mode 28 + + pmaddwd m4, m1, [r3 - 2 * 16] + pmaddwd m5, m1, [r3 + 3 * 16] + + packssdw m4, m5 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m6, m1, [r3 + 8 * 16] + pmaddwd m5, m1, [r2 - 3 * 16] + + packssdw m6, m5 + paddw m6, m2 + psraw m6, 5 + + packuswb m4, m6 + mova [r0 + 416], m4 + +; mode 29 + + pmaddwd m4, m1, [r3 + 2 * 16] + pmaddwd m6, m1, [r2 - 5 * 16] + + packssdw m4, m6 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m6, m1, [r2 + 4 * 16] + pmaddwd m5, m3, m7 + + packssdw m6, m5 + paddw m6, m2 + psraw m6, 5 + + packuswb m4, m6 + mova [r0 + 432], m4 + +; mode 30 + + pmaddwd m4, m1, [r3 + 6 * 16] + pmaddwd m5, m1, [r2 + 3 * 16] + + packssdw m4, m5 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m6, m3, [r3 + 0 * 16] + pmaddwd m5, m3, [r2 - 3 * 16] + + packssdw m6, m5 + paddw m6, m2 + psraw m6, 5 + + packuswb m4, m6 + mova [r0 + 448], m4 + psrldq m4, 4 + movh [r0 + 496], m4 ;mode 33 row 0 + psrldq m4, 8 + movd [r0 + 500], m4 ;mode 33 row 1 + +; mode 31 + + pmaddwd m4, m1, [r2 - 6 * 16] + pmaddwd m5, m3, [r3 - 5 * 16] + + packssdw m4, m5 + paddw m4, m2 + psraw m4, 5 + + pmaddwd m6, m3, [r2 - 4 * 16] + pmaddwd m7, m0 + + packssdw m6, m7 + paddw m6, m2 + psraw m6, 5 + + packuswb m4, m6 + mova [r0 + 464], m4 + +; mode 32 + + pmaddwd m1, [r2 - 2 * 16] + pmaddwd m5, m3, [r3 + 3 * 16] + + packssdw m1, m5 + paddw m1, m2 + psraw m1, 5 + + pmaddwd m3, [r2 + 8 * 16] + pmaddwd m5, m0, [r2 - 3 * 16] + packssdw m3, m5 + paddw m3, m2 + psraw m3, 5 + + packuswb m1, m3 + mova [r0 + 480], m1 + +; mode 33 + + pmaddwd m0, [r3 + 7 * 16] + pxor m7, m7 + movh m4, [r1 + 4] + punpcklbw m4, m4 + psrldq m4, 1 + punpcklbw m4, m7 + + pmaddwd m4, [r3 + 1 * 16] + + packssdw m0, m4 + paddw m0, m2 + psraw m0, 5 + + packuswb m0, m0 + movh [r0 + 504], m0 + +; mode 34 + + movh m7, [r1 + 2] + movd [r0 + 512], m7 ;byte[2, 3, 4, 5] + + psrldq m7, 1 + movd [r0 + 516], m7 ;byte[3, 4, 5, 6] + + psrldq m7, 1 + movd [r0 + 520], m7 ;byte[4, 5, 6, 7] + + psrldq m7, 1 + movd [r0 + 524], m7 ;byte[5, 6, 7, 8] + +RET