Mercurial > hg > forks > libbpg
view x265/source/common/x86/intrapred8_allangs.asm @ 0:772086c29cc7
Initial import.
| author | Matti Hamalainen <ccr@tnsp.org> |
|---|---|
| date | Wed, 16 Nov 2016 11:16:33 +0200 |
| parents | |
| children |
line wrap: on
line source
;***************************************************************************** ;* Copyright (C) 2013 x265 project ;* ;* Authors: Min Chen <chenm003@163.com> <min.chen@multicorewareinc.com> ;* Praveen Tiwari <praveen@multicorewareinc.com> ;* ;* This program is free software; you can redistribute it and/or modify ;* it under the terms of the GNU General Public License as published by ;* the Free Software Foundation; either version 2 of the License, or ;* (at your option) any later version. ;* ;* This program is distributed in the hope that it will be useful, ;* but WITHOUT ANY WARRANTY; without even the implied warranty of ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;* GNU General Public License for more details. ;* ;* You should have received a copy of the GNU General Public License ;* along with this program; if not, write to the Free Software ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. ;* ;* This program is also available under a commercial proprietary license. ;* For more information, contact us at license @ x265.com. ;*****************************************************************************/ %include "x86inc.asm" %include "x86util.asm" SECTION_RODATA 32 all_ang4_shuff: db 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6 db 0, 1, 1, 2, 2, 3, 3, 4, 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6, 3, 4, 4, 5, 5, 6, 6, 7 db 0, 1, 1, 2, 2, 3, 3, 4, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6 db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5 db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 1, 2, 2, 3, 3, 4, 4, 5 db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4 db 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3 db 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12 db 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 4, 0, 0, 9, 9, 10, 10, 11 db 0, 9, 9, 10, 10, 11, 11, 12, 0, 9, 9, 10, 10, 11, 11, 12, 2, 0, 0, 9, 9, 10, 10, 11, 2, 0, 0, 9, 9, 10, 10, 11 db 0, 9, 9, 10, 10, 11, 11, 12, 2, 0, 0, 9, 9, 10, 10, 11, 2, 0, 0, 9, 9, 10, 10, 11, 4, 2, 2, 0, 0, 9, 9, 10 db 0, 9, 9, 10, 10, 11, 11, 12, 2, 0, 0, 9, 9, 10, 10, 11, 2, 0, 0, 9, 9, 10, 10, 11, 3, 2, 2, 0, 0, 9, 9, 10 db 0, 9, 9, 10, 10, 11, 11, 12, 1, 0, 0, 9, 9, 10, 10, 11, 2, 1, 1, 0, 0, 9, 9, 10, 4, 2, 2, 1, 1, 0, 0, 9 db 0, 1, 2, 3, 9, 0, 1, 2, 10, 9, 0, 1, 11, 10, 9, 0, 0, 1, 2, 3, 9, 0, 1, 2, 10, 9, 0, 1, 11, 10, 9, 0 db 0, 1, 1, 2, 2, 3, 3, 4, 9, 0, 0, 1, 1, 2, 2, 3, 10, 9, 9, 0, 0, 1, 1, 2, 12, 10, 10, 9, 9, 0, 0, 1 db 0, 1, 1, 2, 2, 3, 3, 4, 10, 0, 0, 1, 1, 2, 2, 3, 10, 0, 0, 1, 1, 2, 2, 3, 11, 10, 10, 0, 0, 1, 1, 2 db 0, 1, 1, 2, 2, 3, 3, 4, 10, 0, 0, 1, 1, 2, 2, 3, 10, 0, 0, 1, 1, 2, 2, 3, 12, 10, 10, 0, 0, 1, 1, 2 db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 10, 0, 0, 1, 1, 2, 2, 3, 10, 0, 0, 1, 1, 2, 2, 3 db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 12, 0, 0, 1, 1, 2, 2, 3 db 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4, 0, 1, 1, 2, 2, 3, 3, 4 db 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4 db 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5 db 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6 db 1, 2, 2, 3, 3, 4, 4, 5, 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6, 2, 3, 3, 4, 4, 5, 5, 6 db 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6, 2, 3, 3, 4, 4, 5, 5, 6, 3, 4, 4, 5, 5, 6, 6, 7 db 1, 2, 2, 3, 3, 4, 4, 5, 2, 3, 3, 4, 4, 5, 5, 6, 3, 4, 4, 5, 5, 6, 6, 7, 4, 5, 5, 6, 6, 7, 7, 8 db 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8 all_ang4: db 6, 26, 6, 26, 6, 26, 6, 26, 12, 20, 12, 20, 12, 20, 12, 20, 18, 14, 18, 14, 18, 14, 18, 14, 24, 8, 24, 8, 24, 8, 24, 8 db 11, 21, 11, 21, 11, 21, 11, 21, 22, 10, 22, 10, 22, 10, 22, 10, 1, 31, 1, 31, 1, 31, 1, 31, 12, 20, 12, 20, 12, 20, 12, 20 db 15, 17, 15, 17, 15, 17, 15, 17, 30, 2, 30, 2, 30, 2, 30, 2, 13, 19, 13, 19, 13, 19, 13, 19, 28, 4, 28, 4, 28, 4, 28, 4 db 19, 13, 19, 13, 19, 13, 19, 13, 6, 26, 6, 26, 6, 26, 6, 26, 25, 7, 25, 7, 25, 7, 25, 7, 12, 20, 12, 20, 12, 20, 12, 20 db 23, 9, 23, 9, 23, 9, 23, 9, 14, 18, 14, 18, 14, 18, 14, 18, 5, 27, 5, 27, 5, 27, 5, 27, 28, 4, 28, 4, 28, 4, 28, 4 db 27, 5, 27, 5, 27, 5, 27, 5, 22, 10, 22, 10, 22, 10, 22, 10, 17, 15, 17, 15, 17, 15, 17, 15, 12, 20, 12, 20, 12, 20, 12, 20 db 30, 2, 30, 2, 30, 2, 30, 2, 28, 4, 28, 4, 28, 4, 28, 4, 26, 6, 26, 6, 26, 6, 26, 6, 24, 8, 24, 8, 24, 8, 24, 8 db 2, 30, 2, 30, 2, 30, 2, 30, 4, 28, 4, 28, 4, 28, 4, 28, 6, 26, 6, 26, 6, 26, 6, 26, 8, 24, 8, 24, 8, 24, 8, 24 db 5, 27, 5, 27, 5, 27, 5, 27, 10, 22, 10, 22, 10, 22, 10, 22, 15, 17, 15, 17, 15, 17, 15, 17, 20, 12, 20, 12, 20, 12, 20, 12 db 9, 23, 9, 23, 9, 23, 9, 23, 18, 14, 18, 14, 18, 14, 18, 14, 27, 5, 27, 5, 27, 5, 27, 5, 4, 28, 4, 28, 4, 28, 4, 28 db 13, 19, 13, 19, 13, 19, 13, 19, 26, 6, 26, 6, 26, 6, 26, 6, 7, 25, 7, 25, 7, 25, 7, 25, 20, 12, 20, 12, 20, 12, 20, 12 db 17, 15, 17, 15, 17, 15, 17, 15, 2, 30, 2, 30, 2, 30, 2, 30, 19, 13, 19, 13, 19, 13, 19, 13, 4, 28, 4, 28, 4, 28, 4, 28 db 21, 11, 21, 11, 21, 11, 21, 11, 10, 22, 10, 22, 10, 22, 10, 22, 31, 1, 31, 1, 31, 1, 31, 1, 20, 12, 20, 12, 20, 12, 20, 12 db 26, 6, 26, 6, 26, 6, 26, 6, 20, 12, 20, 12, 20, 12, 20, 12, 14, 18, 14, 18, 14, 18, 14, 18, 8, 24, 8, 24, 8, 24, 8, 24 db 26, 6, 26, 6, 26, 6, 26, 6, 20, 12, 20, 12, 20, 12, 20, 12, 14, 18, 14, 18, 14, 18, 14, 18, 8, 24, 8, 24, 8, 24, 8, 24 db 21, 11, 21, 11, 21, 11, 21, 11, 10, 22, 10, 22, 10, 22, 10, 22, 31, 1, 31, 1, 31, 1, 31, 1, 20, 12, 20, 12, 20, 12, 20, 12 db 17, 15, 17, 15, 17, 15, 17, 15, 2, 30, 2, 30, 2, 30, 2, 30, 19, 13, 19, 13, 19, 13, 19, 13, 4, 28, 4, 28, 4, 28, 4, 28 db 13, 19, 13, 19, 13, 19, 13, 19, 26, 6, 26, 6, 26, 6, 26, 6, 7, 25, 7, 25, 7, 25, 7, 25, 20, 12, 20, 12, 20, 12, 20, 12 db 9, 23, 9, 23, 9, 23, 9, 23, 18, 14, 18, 14, 18, 14, 18, 14, 27, 5, 27, 5, 27, 5, 27, 5, 4, 28, 4, 28, 4, 28, 4, 28 db 5, 27, 5, 27, 5, 27, 5, 27, 10, 22, 10, 22, 10, 22, 10, 22, 15, 17, 15, 17, 15, 17, 15, 17, 20, 12, 20, 12, 20, 12, 20, 12 db 2, 30, 2, 30, 2, 30, 2, 30, 4, 28, 4, 28, 4, 28, 4, 28, 6, 26, 6, 26, 6, 26, 6, 26, 8, 24, 8, 24, 8, 24, 8, 24 db 30, 2, 30, 2, 30, 2, 30, 2, 28, 4, 28, 4, 28, 4, 28, 4, 26, 6, 26, 6, 26, 6, 26, 6, 24, 8, 24, 8, 24, 8, 24, 8 db 27, 5, 27, 5, 27, 5, 27, 5, 22, 10, 22, 10, 22, 10, 22, 10, 17, 15, 17, 15, 17, 15, 17, 15, 12, 20, 12, 20, 12, 20, 12, 20 db 23, 9, 23, 9, 23, 9, 23, 9, 14, 18, 14, 18, 14, 18, 14, 18, 5, 27, 5, 27, 5, 27, 5, 27, 28, 4, 28, 4, 28, 4, 28, 4 db 19, 13, 19, 13, 19, 13, 19, 13, 6, 26, 6, 26, 6, 26, 6, 26, 25, 7, 25, 7, 25, 7, 25, 7, 12, 20, 12, 20, 12, 20, 12, 20 db 15, 17, 15, 17, 15, 17, 15, 17, 30, 2, 30, 2, 30, 2, 30, 2, 13, 19, 13, 19, 13, 19, 13, 19, 28, 4, 28, 4, 28, 4, 28, 4 db 11, 21, 11, 21, 11, 21, 11, 21, 22, 10, 22, 10, 22, 10, 22, 10, 1, 31, 1, 31, 1, 31, 1, 31, 12, 20, 12, 20, 12, 20, 12, 20 db 6, 26, 6, 26, 6, 26, 6, 26, 12, 20, 12, 20, 12, 20, 12, 20, 18, 14, 18, 14, 18, 14, 18, 14, 24, 8, 24, 8, 24, 8, 24, 8 SECTION .text ; global constant cextern pw_1024 ; common constant with intrapred8.asm cextern ang_table cextern pw_ang_table cextern tab_S1 cextern tab_S2 cextern tab_Si cextern pw_16 cextern pb_000000000000000F cextern pb_0000000000000F0F cextern pw_FFFFFFFFFFFFFFF0 ;----------------------------------------------------------------------------- ; void all_angs_pred_4x4(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal all_angs_pred_4x4, 4, 4, 8 ; mode 2 movh m0, [r1 + 10] movd [r0], m0 palignr m1, m0, 1 movd [r0 + 4], m1 palignr m1, m0, 2 movd [r0 + 8], m1 palignr m1, m0, 3 movd [r0 + 12], m1 ; mode 3 mova m2, [pw_1024] pslldq m1, m0, 1 pinsrb m1, [r1 + 9], 0 punpcklbw m1, m0 lea r3, [ang_table] pmaddubsw m6, m1, [r3 + 26 * 16] pmulhrsw m6, m2 packuswb m6, m6 movd [r0 + 16], m6 palignr m0, m1, 2 mova m7, [r3 + 20 * 16] pmaddubsw m3, m0, m7 pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 20], m3 ; mode 6 [row 3] movd [r0 + 76], m3 palignr m3, m1, 4 pmaddubsw m4, m3, [r3 + 14 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 24], m4 palignr m4, m1, 6 pmaddubsw m4, [r3 + 8 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 28], m4 ; mode 4 pmaddubsw m5, m1, [r3 + 21 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 32], m5 pmaddubsw m5, m0, [r3 + 10 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 36], m5 pmaddubsw m5, m0, [r3 + 31 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 40], m5 pmaddubsw m4, m3, m7 pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 44], m4 ; mode 5 pmaddubsw m5, m1, [r3 + 17 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 48], m5 pmaddubsw m5, m0, [r3 + 2 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 52], m5 pmaddubsw m5, m0, [r3 + 19 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 56], m5 pmaddubsw m4, m3, [r3 + 4 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 60], m4 ; mode 6 pmaddubsw m5, m1, [r3 + 13 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 64], m5 movd [r0 + 68], m6 pmaddubsw m5, m0, [r3 + 7 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 72], m5 ; mode 7 pmaddubsw m5, m1, [r3 + 9 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 80], m5 pmaddubsw m5, m1, [r3 + 18 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 84], m5 pmaddubsw m5, m1, [r3 + 27 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 88], m5 pmaddubsw m5, m0, [r3 + 4 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 92], m5 ; mode 8 pmaddubsw m5, m1, [r3 + 5 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 96], m5 pmaddubsw m5, m1, [r3 + 10 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 100], m5 pmaddubsw m5, m1, [r3 + 15 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 104], m5 pmaddubsw m5, m1, [r3 + 20 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 108], m5 ; mode 9 pmaddubsw m5, m1, [r3 + 2 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 112], m5 pmaddubsw m5, m1, [r3 + 4 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 116], m5 pmaddubsw m5, m1, [r3 + 6 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 120], m5 pmaddubsw m5, m1, [r3 + 8 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 124], m5 ; mode 10 movd m3, [r1 + 9] pshufd m4, m3, 0 movu [r0 + 128], m4 pxor m5, m5 movd m7, [r1 + 1] pshufd m4, m7, 0 punpcklbw m4, m5 pinsrb m7, [r1], 0 pshufb m6, m7, m5 punpcklbw m6, m5 psubw m4, m6 psraw m4, 1 pshufb m6, m3, m5 punpcklbw m6, m5 paddw m4, m6 packuswb m4, m5 pextrb [r0 + 128], m4, 0 pextrb [r0 + 132], m4, 1 pextrb [r0 + 136], m4, 2 pextrb [r0 + 140], m4, 3 ; mode 11 pslldq m1, m1, 2 pinsrb m1, [r1], 0 pinsrb m1, [r1 + 9], 1 pmaddubsw m3, m1, [r3 + 30 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 144], m3 pmaddubsw m3, m1, [r3 + 28 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 148], m3 pmaddubsw m3, m1, [r3 + 26 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 152], m3 pmaddubsw m3, m1, [r3 + 24 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 156], m3 ; mode 12 pmaddubsw m3, m1, [r3 + 27 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 160], m3 pmaddubsw m3, m1, [r3 + 22 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 164], m3 pmaddubsw m3, m1, [r3 + 17 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 168], m3 pmaddubsw m3, m1, [r3 + 12 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 172], m3 ; mode 13 pmaddubsw m3, m1, [r3 + 23 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 176], m3 pmaddubsw m3, m1, [r3 + 14 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 180], m3 pmaddubsw m3, m1, [r3 + 5 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 184], m3 pslldq m5, m1, 2 pinsrb m5, [r1 + 0], 1 pinsrb m5, [r1 + 4], 0 pmaddubsw m4, m5, [r3 + 28 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 188], m4 ; mode 14 pmaddubsw m4, m1, [r3 + 19 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 192], m4 pmaddubsw m7, m1, [r3 + 6 * 16] pmulhrsw m7, m2 packuswb m7, m7 movd [r0 + 196], m7 pinsrb m5, [r1 + 2], 0 pmaddubsw m4, m5, [r3 + 25 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 200], m4 pmaddubsw m4, m5, [r3 + 12 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 204], m4 ; mode 15 pmaddubsw m4, m1, [r3 + 15 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 208], m4 pmaddubsw m4, m5, [r3 + 30 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 212], m4 pmaddubsw m4, m5, [r3 + 13 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 216], m4 pslldq m4, m5, 2 pinsrb m4, [r1 + 2], 1 pinsrb m4, [r1 + 4], 0 pmaddubsw m6, m4, [r3 + 28 * 16] pmulhrsw m6, m2 packuswb m6, m6 movd [r0 + 220], m6 ; mode 16 pmaddubsw m6, m1, [r3 + 11 * 16] pmulhrsw m6, m2 packuswb m6, m6 movd [r0 + 224], m6 pmaddubsw m6, m5, [r3 + 22 * 16] pmulhrsw m6, m2 packuswb m6, m6 movd [r0 + 228], m6 pmaddubsw m6, m5, [r3 + 1 * 16] pmulhrsw m6, m2 packuswb m6, m6 movd [r0 + 232], m6 pinsrb m4, [r1 + 3], 0 pmaddubsw m4, [r3 + 12 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 236], m4 ; mode 17 movd [r0 + 240], m7 pslldq m1, 2 pinsrb m1, [r1 + 1], 0 pinsrb m1, [r1 + 0], 1 pmaddubsw m3, m1, [r3 + 12 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 244], m3 pslldq m1, 2 pinsrb m1, [r1 + 1], 1 pinsrb m1, [r1 + 2], 0 pmaddubsw m3, m1, [r3 + 18 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 248], m3 pslldq m1, 2 pinsrb m1, [r1 + 2], 1 pinsrb m1, [r1 + 4], 0 pmaddubsw m1, [r3 + 24 * 16] pmulhrsw m1, m2 packuswb m1, m1 movd [r0 + 252], m1 ; mode 18 movh m1, [r1] movd [r0 + 256], m1 pslldq m3, m1, 1 pinsrb m3, [r1 + 9], 0 movd [r0 + 260], m3 pslldq m4, m3, 1 pinsrb m4, [r1 + 10], 0 movd [r0 + 264], m4 pslldq m4, 1 pinsrb m4, [r1 + 11], 0 movd [r0 + 268], m4 ; mode 19 palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m7, m1, [r3 + 6 * 16] pmulhrsw m7, m2 packuswb m7, m7 movd [r0 + 272], m7 pslldq m3, m1, 2 pinsrb m3, [r1], 1 pinsrb m3, [r1 + 9], 0 pmaddubsw m4, m3, [r3 + 12 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 276], m4 pslldq m4, m3, 2 pinsrb m4, [r1 + 9], 1 pinsrb m4, [r1 + 10], 0 pmaddubsw m5, m4, [r3 + 18 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 280], m5 pslldq m4, 2 pinsrb m4, [r1 + 10], 1 pinsrb m4, [r1 + 12], 0 pmaddubsw m4, [r3 + 24 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 284], m4 ; mode 20 pmaddubsw m4, m1, [r3 + 11 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 288], m4 pinsrb m3, [r1 + 10], 0 pmaddubsw m4, m3, [r3 + 22 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 292], m4 pmaddubsw m4, m3, [r3 + 1 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 296], m4 pslldq m6, m3, 2 pinsrb m6, [r1 + 10], 1 pinsrb m6, [r1 + 11], 0 pmaddubsw m5, m6, [r3 + 12 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 300], m5 ; mode 21 pmaddubsw m4, m1, [r3 + 15 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 304], m4 pmaddubsw m4, m3, [r3 + 30 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 308], m4 pmaddubsw m4, m3, [r3 + 13 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 312], m4 pinsrb m6, [r1 + 12], 0 pmaddubsw m6, [r3 + 28 * 16] pmulhrsw m6, m2 packuswb m6, m6 movd [r0 + 316], m6 ; mode 22 pmaddubsw m4, m1, [r3 + 19 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 320], m4 movd [r0 + 324], m7 pmaddubsw m4, m3, [r3 + 25 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 328], m4 pmaddubsw m4, m3, [r3 + 12 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 332], m4 ; mode 23 pmaddubsw m4, m1, [r3 + 23 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 336], m4 pmaddubsw m4, m1, [r3 + 14 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 340], m4 pmaddubsw m4, m1, [r3 + 5 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 344], m4 pinsrb m3, [r1 + 12], 0 pmaddubsw m3, [r3 + 28 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 348], m3 ; mode 24 pmaddubsw m3, m1, [r3 + 27 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 352], m3 pmaddubsw m3, m1, [r3 + 22 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 356], m3 pmaddubsw m3, m1, [r3 + 17 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 360], m3 pmaddubsw m3, m1, [r3 + 12 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 364], m3 ; mode 25 pmaddubsw m3, m1, [r3 + 30 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 368], m3 pmaddubsw m3, m1, [r3 + 28 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 372], m3 pmaddubsw m3, m1, [r3 + 26 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 376], m3 pmaddubsw m1, [r3 + 24 * 16] pmulhrsw m1, m2 packuswb m1, m1 movd [r0 + 380], m1 ; mode 26 movh m1, [r1 + 1] pshufd m3, m1, 0 movu [r0 + 384], m3 pxor m4, m4 movd m5, [r1 + 9] pshufd m5, m5, 0 punpcklbw m5, m4 pinsrb m6, [r1], 0 pshufb m6, m4 punpcklbw m6, m4 psubw m5, m6 psraw m5, 1 pshufb m6, m1, m4 punpcklbw m6, m4 paddw m5, m6 packuswb m5, m4 pextrb [r0 + 384], m5, 0 pextrb [r0 + 388], m5, 1 pextrb [r0 + 392], m5, 2 pextrb [r0 + 396], m5, 3 ; mode 27 palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, [r3 + 2 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 400], m3 pmaddubsw m3, m1, [r3 + 4 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 404], m3 pmaddubsw m3, m1, [r3 + 6 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 408], m3 pmaddubsw m3, m1, [r3 + 8 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 412], m3 ; mode 28 pmaddubsw m3, m1, [r3 + 5 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 416], m3 pmaddubsw m3, m1, [r3 + 10 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 420], m3 pmaddubsw m3, m1, [r3 + 15 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 424], m3 pmaddubsw m3, m1, [r3 + 20 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 428], m3 ; mode 29 pmaddubsw m3, m1, [r3 + 9 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 432], m3 pmaddubsw m3, m1, [r3 + 18 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 436], m3 pmaddubsw m3, m1, [r3 + 27 * 16] pmulhrsw m3, m2 packuswb m3, m3 movd [r0 + 440], m3 palignr m3, m1, 2 pmaddubsw m4, m3, [r3 + 4 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 444], m4 ; mode 30 pmaddubsw m4, m1, [r3 + 13 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 448], m4 pmaddubsw m7, m1, [r3 + 26 * 16] pmulhrsw m7, m2 packuswb m7, m7 movd [r0 + 452], m7 pmaddubsw m5, m3, [r3 + 7 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 456], m5 pmaddubsw m6, m3, [r3 + 20 * 16] pmulhrsw m6, m2 packuswb m6, m6 movd [r0 + 460], m6 ; mode 31 pmaddubsw m4, m1, [r3 + 17 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 464], m4 pmaddubsw m5, m3, [r3 + 2 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 468], m5 pmaddubsw m5, m3, [r3 + 19 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 472], m5 palignr m4, m3, 2 pmaddubsw m5, m4, [r3 + 4 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 476], m5 ; mode 32 pmaddubsw m5, m1, [r3 + 21 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 480], m5 pmaddubsw m5, m3, [r3 + 10 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 484], m5 pmaddubsw m5, m3, [r3 + 31 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 488], m5 pmaddubsw m5, m4, [r3 + 20 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 492], m5 ; mode 33 movd [r0 + 496], m7 movd [r0 + 500], m6 pmaddubsw m5, m4, [r3 + 14 * 16] pmulhrsw m5, m2 packuswb m5, m5 movd [r0 + 504], m5 psrldq m4, 2 pmaddubsw m4, [r3 + 8 * 16] pmulhrsw m4, m2 packuswb m4, m4 movd [r0 + 508], m4 ; mode 34 movh m7, [r1 + 2] movd [r0 + 512], m7 psrldq m7, 1 movd [r0 + 516], m7 psrldq m7, 1 movd [r0 + 520], m7 psrldq m7, 1 movd [r0 + 524], m7 RET ;------------------------------------------------------------------------------ ; void all_angs_pred_8x8(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) ;------------------------------------------------------------------------------ INIT_XMM sse4 cglobal all_angs_pred_8x8, 3,4,8 ; mode 2 movu m0, [r2 + 18] palignr m1, m0, 1 punpcklqdq m2, m0, m1 movu [r0], m2 palignr m1, m0, 2 palignr m2, m0, 3 punpcklqdq m1, m2 movu [r0 + 16], m1 palignr m1, m0, 4 palignr m2, m0, 5 punpcklqdq m1, m2 movu [r0 + 32], m1 palignr m1, m0, 6 palignr m2, m0, 7 punpcklqdq m1, m2 movu [r0 + 48], m1 ; mode 3 [row 0, 1] mova m7, [pw_1024] lea r3, [ang_table] movu m0, [r1 + 17] palignr m1, m0, 1 palignr m2, m0, 2 punpcklbw m3, m0, m1 pmaddubsw m4, m3, [r3 + 26 * 16] pmulhrsw m4, m7 punpcklbw m1, m2 pmaddubsw m5, m1, [r3 + 20 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 64], m4 ; mode 6 [row 1] movh [r0 + 264], m4 ; mode 6 [row 3] movhps [r0 + 280], m4 ; mode 4 [row 0, 1] pmaddubsw m4, m3, [r3 + 21 * 16] pmulhrsw m4, m7 pmaddubsw m5, m1, [r3 + 10 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 128], m4 ; mode 5 [row 0, 1] pmaddubsw m4, m3, [r3 + 17 * 16] pmulhrsw m4, m7 pmaddubsw m5, m1, [r3 + 2 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 192], m4 ; mode 6 [row 0] pmaddubsw m4, m3, [r3 + 13 * 16] pmulhrsw m4, m7 pxor m5, m5 packuswb m4, m5 movh [r0 + 256], m4 ; mode 7 [row 0, 1] pmaddubsw m4, m3, [r3 + 9 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 18 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 320], m4 ; mode 8 [row 0, 1] pmaddubsw m4, m3, [r3 + 5 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 10 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 384], m4 ; mode 8 [row 2, 3] pmaddubsw m4, m3, [r3 + 15 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 20 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 400], m4 ; mode 8 [row 4, 5] pmaddubsw m4, m3, [r3 + 25 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 30 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 416], m4 ; mode 8 [row 6, 7] pmaddubsw m4, m1, [r3 + 3 * 16] pmulhrsw m4, m7 pmaddubsw m5, m1, [r3 + 8 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 432], m4 ; mode 9 [row 0, 1] pmaddubsw m4, m3, [r3 + 2 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 4 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 448], m4 ; mode 9 [row 2, 3] pmaddubsw m4, m3, [r3 + 6 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 8 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 464], m4 ; mode 9 [row 4, 5] pmaddubsw m4, m3, [r3 + 10 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 12 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 480], m4 ; mode 9 [row 6, 7] pmaddubsw m4, m3, [r3 + 14 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 16 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 496], m4 ; mode 7 [row 2, 3] pmaddubsw m4, m3, [r3 + 27 * 16] pmulhrsw m4, m7 pmaddubsw m5, m1, [r3 + 4 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 336], m4 ; mode 7 [row 4, 5] pmaddubsw m4, m1, [r3 + 13 * 16] pmulhrsw m4, m7 pmaddubsw m5, m1, [r3 + 22 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 352], m4 ; mode 6 [row 2] pmaddubsw m4, m1, [r3 + 7 * 16] pmulhrsw m4, m7 pxor m5, m5 packuswb m4, m5 movh [r0 + 272], m4 ; mode 3 [row 2, 3] palignr m1, m0, 3 palignr m3, m0, 4 punpcklbw m2, m1 pmaddubsw m5, m2, [r3 + 14 * 16] pmulhrsw m5, m7 punpcklbw m1, m3 pmaddubsw m6, m1, [r3 + 8 * 16] pmulhrsw m6, m7 packuswb m5, m6 movu [r0 + 80], m5 ; mode 6 [row 7] movhps [r0 + 312], m5 ; mode 6 [row 5] movh [r0 + 296], m5 ; mode 4 [calculate and store row 4, 5] pmaddubsw m4, m1, [r3 + 9 * 16] pmulhrsw m4, m7 pmaddubsw m5, m1, [r3 + 30 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 160], m4 ; mode 5 [row 4, 5] pmaddubsw m4, m2, [r3 + 21 * 16] pmulhrsw m4, m7 pmaddubsw m5, m1, [r3 + 6 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 224], m4 ; mode 6 [row 4, 5] pmaddubsw m5, m2, [r3 + 1 * 16] pmulhrsw m5, m7 pxor m6, m6 packuswb m5, m6 movh [r0 + 288], m5 ; mode 6 [row 6, 7] pmaddubsw m5, m2, [r3 + 27 * 16] pmulhrsw m5, m7 pxor m6, m6 packuswb m5, m6 movh [r0 + 304], m5 ; mode 5 [calculate row 6] pmaddubsw m6, m1, [r3 + 23 * 16] pmulhrsw m6, m7 ; mode 3 [row 4, 5] palignr m1, m0, 5 punpcklbw m3, m1 pmaddubsw m4, m3, [r3 + 2 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 28 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 96], m4 ; mode 4 [calculate row 7] pmaddubsw m5, m3, [r3 + 19 * 16] pmulhrsw m5, m7 ; mode 5 [calculate row 6] pmaddubsw m4, m3, [r3 + 8 * 16] pmulhrsw m4, m7 packuswb m6, m4 movu [r0 + 240], m6 ; mode 3 [row 6, 7] palignr m2, m0, 6 palignr m3, m0, 7 punpcklbw m1, m2 pmaddubsw m4, m1, [r3 + 22 * 16] pmulhrsw m4, m7 punpcklbw m2, m3 pmaddubsw m2, [r3 + 16 * 16] pmulhrsw m2, m7 packuswb m4, m2 movu [r0 + 112], m4 ; mode 4 [calculate row 7] pmaddubsw m2, m1, [r3 + 8 * 16] pmulhrsw m2, m7 ; mode 4 [store row 6 and 7] packuswb m5, m2 movu [r0 + 176], m5 ; mode 4 [row 2, 3] palignr m1, m0, 1 palignr m2, m0, 2 palignr m3, m0, 3 punpcklbw m1, m2 pmaddubsw m4, m1, [r3 + 31 * 16] pmulhrsw m4, m7 punpcklbw m2, m3 pmaddubsw m5, m2, [r3 + 20 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 144], m4 ; mode 5 [row 2, 3] pmaddubsw m4, m1, [r3 + 19 * 16] pmulhrsw m4, m7 pmaddubsw m5, m2, [r3 + 4 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 208], m4 ; mode 7 [row 6, 7] pmaddubsw m4, m1, [r3 + 31 * 16] pmulhrsw m4, m7 pmaddubsw m5, m2, [r3 + 8 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 368], m4 ; mode 10 pshufb m1, m0, [tab_Si] movu [r0 + 512], m1 movu [r0 + 528], m1 movu [r0 + 544], m1 movu [r0 + 560], m1 pxor m0, m0 pshufb m1, m1, m0 punpcklbw m1, m0 movu m2, [r1] pshufb m3, m2, m0 punpcklbw m3, m0 psrldq m4, m2, 1 punpcklbw m4, m0 movu m2, [r1 + 9] punpcklbw m2, m0 psubw m4, m3 psubw m2, m3 psraw m4, 1 psraw m2, 1 paddw m4, m1 paddw m2, m1 packuswb m4, m2 pextrb [r0 + 512], m4, 0 pextrb [r0 + 520], m4, 1 pextrb [r0 + 528], m4, 2 pextrb [r0 + 536], m4, 3 pextrb [r0 + 544], m4, 4 pextrb [r0 + 552], m4, 5 pextrb [r0 + 560], m4, 6 pextrb [r0 + 568], m4, 7 ; mode 11 [row 0, 1] movu m0, [r1 + 16] pinsrb m0, [r1], 0 palignr m1, m0, 1 punpcklbw m2, m0, m1 pmaddubsw m3, m2, [r3 + 30 * 16] pmulhrsw m3, m7 pmaddubsw m4, m2, [r3 + 28 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 576], m3 ; mode 11 [row 2, 3] pmaddubsw m3, m2, [r3 + 26 * 16] pmulhrsw m3, m7 pmaddubsw m4, m2, [r3 + 24 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 592], m3 ; mode 11 [row 4, 5] pmaddubsw m3, m2, [r3 + 22 * 16] pmulhrsw m3, m7 pmaddubsw m4, m2, [r3 + 20 * 16] pmulhrsw m4, m7 packuswb m5, m3, m4 movu [r0 + 608], m5 ; mode 12 [row 0, 1] pmaddubsw m4, m2, [r3 + 27 * 16] pmulhrsw m4, m7 packuswb m4, m3 movu [r0 + 640], m4 ; mode 11 [row 6, 7] pmaddubsw m3, m2, [r3 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m4, m2, [r3 + 16 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 624], m3 ; mode 12 [row 2, 3] pmaddubsw m3, m2, [r3 + 17 * 16] pmulhrsw m3, m7 pmaddubsw m4, m2, [r3 + 12 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 656], m3 ; mode 12 [row 4, 5] pmaddubsw m3, m2, [r3 + 7 * 16] pmulhrsw m3, m7 pmaddubsw m4, m2, [r3 + 2 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 672], m3 ; mode 12 [row 6, 7] pslldq m3, m2, 2 pinsrb m3, [r1 + 0], 1 pinsrb m3, [r1 + 6], 0 pmaddubsw m4, m3, [r3 + 29 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 24 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 688], m4 ; mode 13 [row 0, 1] pmaddubsw m4, m2, [r3 + 23 * 16] pmulhrsw m4, m7 pmaddubsw m5, m2, [r3 + 14 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 704], m4 ; mode 13 [row 2, 3] pmaddubsw m4, m2, [r3 + 5 * 16] pmulhrsw m4, m7 pinsrb m3, [r1 + 4], 0 pmaddubsw m5, m3, [r3 + 28 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 720], m4 ; mode 13 [row 4, 5] pmaddubsw m4, m3, [r3 + 19 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 10 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 736], m4 ; mode 13 [row 6, 7] pmaddubsw m4, m3, [r3 + 1 * 16] pmulhrsw m4, m7 pslldq m5, m3, 2 pinsrb m5, [r1 + 4], 1 pinsrb m5, [r1 + 7], 0 pmaddubsw m5, [r3 + 24 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 752], m4 ; mode 14 [row 0, 1] pmaddubsw m4, m2, [r3 + 19 * 16] pmulhrsw m4, m7 pmaddubsw m5, m2, [r3 + 6 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 768], m4 ; mode 14 [row 2, 3] pinsrb m3, [r1 + 2], 0 pmaddubsw m4, m3, [r3 + 25 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 12 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 784], m4 ; mode 14 [row 4, 5] pslldq m1, m3, 2 pinsrb m1, [r1 + 2], 1 pinsrb m1, [r1 + 5], 0 pmaddubsw m4, m1, [r3 + 31 * 16] pmulhrsw m4, m7 pmaddubsw m5, m1, [r3 + 18 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 800], m4 ; mode 14 [row 6, 7] pmaddubsw m4, m1, [r3 + 5 * 16] pmulhrsw m4, m7 pslldq m1, 2 pinsrb m1, [r1 + 5], 1 pinsrb m1, [r1 + 7], 0 pmaddubsw m5, m1, [r3 + 24 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 816], m4 ; mode 15 [row 0, 1] pmaddubsw m4, m2, [r3 + 15 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 30 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 832], m4 ; mode 15 [row 2, 3] pmaddubsw m4, m3, [r3 + 13 * 16] pmulhrsw m4, m7 pslldq m1, m3, 2 pinsrb m1, [r1 + 2], 1 pinsrb m1, [r1 + 4], 0 pmaddubsw m5, m1, [r3 + 28 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 848], m4 ; mode 15 [row 4, 5] pmaddubsw m4, m1, [r3 + 11 * 16] pmulhrsw m4, m7 pslldq m1, 2 pinsrb m1, [r1 + 4], 1 pinsrb m1, [r1 + 6], 0 pmaddubsw m5, m1, [r3 + 26 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 864], m4 ; mode 15 [row 6, 7] pmaddubsw m4, m1, [r3 + 9 * 16] pmulhrsw m4, m7 pslldq m1, 2 pinsrb m1, [r1 + 6], 1 pinsrb m1, [r1 + 8], 0 pmaddubsw m1, [r3 + 24 * 16] pmulhrsw m1, m7 packuswb m4, m1 movu [r0 + 880], m4 ; mode 16 [row 0, 1] pmaddubsw m4, m2, [r3 + 11 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 22 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 896], m4 ; mode 16 [row 2, 3] pmaddubsw m4, m3, [r3 + 1 * 16] pmulhrsw m4, m7 pslldq m3, 2 pinsrb m3, [r1 + 2], 1 pinsrb m3, [r1 + 3], 0 pmaddubsw m5, m3, [r3 + 12 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 912], m4 ; mode 16 [row 4, 5] pslldq m3, 2 pinsrb m3, [r1 + 3], 1 pinsrb m3, [r1 + 5], 0 pmaddubsw m4, m3, [r3 + 23 * 16] pmulhrsw m4, m7 pmaddubsw m5, m3, [r3 + 2 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 928], m4 ; mode 16 [row 6, 7] pslldq m3, 2 pinsrb m3, [r1 + 5], 1 pinsrb m3, [r1 + 6], 0 pmaddubsw m4, m3, [r3 + 13 * 16] pmulhrsw m4, m7 pslldq m3, 2 pinsrb m3, [r1 + 6], 1 pinsrb m3, [r1 + 8], 0 pmaddubsw m3, [r3 + 24 * 16] pmulhrsw m3, m7 packuswb m4, m3 movu [r0 + 944], m4 ; mode 17 [row 0, 1] pmaddubsw m4, m2, [r3 + 6 * 16] pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r1 + 0], 1 pinsrb m2, [r1 + 1], 0 pmaddubsw m3, m2, [r3 + 12 * 16] pmulhrsw m3, m7 packuswb m4, m3 movu [r0 + 960], m4 ; mode 17 [row 2, 3] pslldq m2, 2 pinsrb m2, [r1 + 1], 1 pinsrb m2, [r1 + 2], 0 pmaddubsw m4, m2, [r3 + 18 * 16] pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r1 + 2], 1 pinsrb m2, [r1 + 4], 0 pmaddubsw m3, m2, [r3 + 24 * 16] pmulhrsw m3, m7 packuswb m4, m3 movu [r0 + 976], m4 ; mode 17 [row 4, 5] pslldq m2, 2 pinsrb m2, [r1 + 4], 1 pinsrb m2, [r1 + 5], 0 pmaddubsw m4, m2, [r3 + 30 * 16] pmulhrsw m4, m7 pmaddubsw m3, m2, [r3 + 4 * 16] pmulhrsw m3, m7 packuswb m4, m3 movu [r0 + 992], m4 ; mode 17 [row 6, 7] pslldq m2, 2 pinsrb m2, [r1 + 5], 1 pinsrb m2, [r1 + 6], 0 pmaddubsw m4, m2, [r3 + 10 * 16] pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r1 + 6], 1 pinsrb m2, [r1 + 7], 0 pmaddubsw m3, m2, [r3 + 16 * 16] pmulhrsw m3, m7 packuswb m4, m3 movu [r0 + 1008], m4 ; mode 18 [row 0, 1, 2, 3, 4, 5, 6, 7] movh m1, [r2] pslldq m2, m1, 1 pinsrb m2, [r2 + 1 + 16], 0 punpcklqdq m1, m2 movu [r0 + 1024], m1 pslldq m2, 1 pinsrb m2, [r2 + 2 + 16], 0 pslldq m0, m2, 1 pinsrb m0, [r2 + 3 + 16], 0 punpcklqdq m2, m0 movu [r0 + 1040], m2 pslldq m0, 1 pinsrb m0, [r2 + 4 + 16], 0 pslldq m2, m0, 1 pinsrb m2, [r2 + 5 + 16], 0 punpcklqdq m0, m2 movu [r0 + 1056], m0 pslldq m2, 1 pinsrb m2, [r2 + 6 + 16], 0 pslldq m0, m2, 1 pinsrb m0, [r2 + 7 + 16], 0 punpcklqdq m2, m0 movu [r0 + 1072], m2 ; mode 19 [row 0, 1] movu m0, [r1] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 6 * 16] pmulhrsw m1, m7 pslldq m2, m0, 2 pinsrb m2, [r1], 1 pinsrb m2, [r1 + 1 + 16], 0 pmaddubsw m3, m2, [r3 + 12 * 16] pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 1088], m1 ; mode 19 [row 2, 3] pslldq m2, 2 pinsrb m2, [r1 + 1 + 16], 1 pinsrb m2, [r1 + 2 + 16], 0 pmaddubsw m4, m2, [r3 + 18 * 16] pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r1 + 2 + 16], 1 pinsrb m2, [r1 + 4 + 16], 0 pmaddubsw m5, m2, [r3 + 24 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1104], m4 ; mode 19 [row 4, 5] pslldq m2, 2 pinsrb m2, [r1 + 4 + 16], 1 pinsrb m2, [r1 + 5 + 16], 0 pmaddubsw m4, m2, [r3 + 30 * 16] pmulhrsw m4, m7 pmaddubsw m5, m2, [r3 + 4 * 16] pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1120], m4 ; mode 19 [row 6, 7] pslldq m2, 2 pinsrb m2, [r1 + 5 + 16], 1 pinsrb m2, [r1 + 6 + 16], 0 pmaddubsw m4, m2, [r3 + 10 * 16] pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r1 + 6 + 16], 1 pinsrb m2, [r1 + 7 + 16], 0 pmaddubsw m2, [r3 + 16 * 16] pmulhrsw m2, m7 packuswb m4, m2 movu [r0 + 1136], m4 ; mode 20 [row 0, 1] pmaddubsw m3, m0, [r3 + 11 * 16] pmulhrsw m3, m7 pslldq m1, m0, 2 pinsrb m1, [r1 + 0], 1 pinsrb m1, [r1 + 2 + 16], 0 pmaddubsw m4, m1, [r3 + 22 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 1152], m3 ; mode 20 [row 2, 3] pmaddubsw m3, m1, [r3 + 1 * 16] pmulhrsw m3, m7 pslldq m2, m1, 2 pinsrb m2, [r1 + 2 + 16], 1 pinsrb m2, [r1 + 3 + 16], 0 pmaddubsw m4, m2, [r3 + 12 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 1168], m3 ; mode 20 [row 4, 5] pslldq m2, 2 pinsrb m2, [r1 + 3 + 16], 1 pinsrb m2, [r1 + 5 + 16], 0 pmaddubsw m3, m2, [r3 + 23 * 16] pmulhrsw m3, m7 pmaddubsw m4, m2, [r3 + 2 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 1184], m3 ; mode 20 [row 6, 7] pslldq m2, 2 pinsrb m2, [r1 + 5 + 16], 1 pinsrb m2, [r1 + 6 + 16], 0 pmaddubsw m3, m2, [r3 + 13 * 16] pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r1 + 6 + 16], 1 pinsrb m2, [r1 + 8 + 16], 0 pmaddubsw m4, m2, [r3 + 24 * 16] pmulhrsw m4, m7 packuswb m3, m4 movu [r0 + 1200], m3 ; mode 21 [row 0, 1] pmaddubsw m2, m0, [r3 + 15 * 16] pmulhrsw m2, m7 pmaddubsw m3, m1, [r3 + 30 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1216], m2 ; mode 21 [row 2, 3] pmaddubsw m2, m1, [r3 + 13 * 16] pmulhrsw m2, m7 pslldq m3, m1, 2 pinsrb m3, [r1 + 2 + 16], 1 pinsrb m3, [r1 + 4 + 16], 0 pmaddubsw m4, m3, [r3 + 28 * 16] pmulhrsw m4, m7 packuswb m2, m4 movu [r0 + 1232], m2 ; mode 21 [row 4, 5] pmaddubsw m2, m3, [r3 + 11 * 16] pmulhrsw m2, m7 pslldq m3, 2 pinsrb m3, [r1 + 4 + 16], 1 pinsrb m3, [r1 + 6 + 16], 0 pmaddubsw m4, m3, [r3 + 26 * 16] pmulhrsw m4, m7 packuswb m2, m4 movu [r0 + 1248], m2 ; mode 21 [row 6, 7] pmaddubsw m2, m3, [r3 + 9 * 16] pmulhrsw m2, m7 pslldq m3, 2 pinsrb m3, [r1 + 6 + 16], 1 pinsrb m3, [r1 + 8 + 16], 0 pmaddubsw m4, m3, [r3 + 24 * 16] pmulhrsw m4, m7 packuswb m2, m4 movu [r0 + 1264], m2 ; mode 22 [row 0, 1] pmaddubsw m2, m0, [r3 + 19 * 16] pmulhrsw m2, m7 pmaddubsw m4, m0, [r3 + 6 * 16] pmulhrsw m4, m7 packuswb m2, m4 movu [r0 + 1280], m2 ; mode 22 [row 2, 3] pmaddubsw m2, m1, [r3 + 25 * 16] pmulhrsw m2, m7 pmaddubsw m3, m1, [r3 + 12 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1296], m2 ; mode 22 [row 4, 5] pslldq m1, 2 pinsrb m1, [r1 + 5 + 16], 0 pinsrb m1, [r1 + 2 + 16], 1 pmaddubsw m2, m1, [r3 + 31 * 16] pmulhrsw m2, m7 pmaddubsw m3, m1, [r3 + 18 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1312], m2 ; mode 22 [row 6, 7] pmaddubsw m2, m1, [r3 + 5 * 16] pmulhrsw m2, m7 pslldq m1, 2 pinsrb m1, [r1 + 5 + 16], 1 pinsrb m1, [r1 + 7 + 16], 0 pmaddubsw m1, [r3 + 24 * 16] pmulhrsw m1, m7 packuswb m2, m1 movu [r0 + 1328], m2 ; mode 23 [row 0, 1] pmaddubsw m2, m0, [r3 + 23 * 16] pmulhrsw m2, m7 pmaddubsw m3, m0, [r3 + 14 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1344], m2 ; mode 23 [row 2, 3] pmaddubsw m2, m0, [r3 + 5 * 16] pmulhrsw m2, m7 pslldq m1, m0, 2 pinsrb m1, [r1], 1 pinsrb m1, [r1 + 4 + 16], 0 pmaddubsw m3, m1, [r3 + 28 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1360], m2 ; mode 23 [row 4, 5] pmaddubsw m2, m1, [r3 + 19 * 16] pmulhrsw m2, m7 pmaddubsw m3, m1, [r3 + 10 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1376], m2 ; mode 23 [row 6, 7] pmaddubsw m2, m1, [r3 + 1 * 16] pmulhrsw m2, m7 pslldq m3, m1, 2 pinsrb m3, [r1 + 4 + 16], 1 pinsrb m3, [r1 + 7 + 16], 0 pmaddubsw m3, [r3 + 24 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1392], m2 ; mode 24 [row 0, 1] pmaddubsw m2, m0, [r3 + 27 * 16] pmulhrsw m2, m7 pmaddubsw m5, m0, [r3 + 22 * 16] pmulhrsw m5, m7 packuswb m2, m5 movu [r0 + 1408], m2 ; mode 24 [row 2, 3] pmaddubsw m2, m0, [r3 + 17 * 16] pmulhrsw m2, m7 pmaddubsw m3, m0, [r3 + 12 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1424], m2 ; mode 24 [row 4, 5] pmaddubsw m2, m0, [r3 + 7 * 16] pmulhrsw m2, m7 pmaddubsw m3, m0, [r3 + 2 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1440], m2 ; mode 24 [row 6, 7] pinsrb m1, [r1 + 6 + 16], 0 pmaddubsw m2, m1, [r3 + 29 * 16] pmulhrsw m2, m7 pmaddubsw m1, [r3 + 24 * 16] pmulhrsw m1, m7 packuswb m2, m1 movu [r0 + 1456], m2 ; mode 25 [row 0, 1] pmaddubsw m2, m0, [r3 + 30 * 16] pmulhrsw m2, m7 pmaddubsw m1, m0, [r3 + 28 * 16] pmulhrsw m1, m7 packuswb m2, m1 movu [r0 + 1472], m2 ; mode 25 [row 2, 3] pmaddubsw m2, m0, [r3 + 26 * 16] pmulhrsw m2, m7 pmaddubsw m1, m0, [r3 + 24 * 16] pmulhrsw m1, m7 packuswb m2, m1 movu [r0 + 1488], m2 ; mode 25 [row 4, 5] pmaddubsw m1, m0, [r3 + 20 * 16] pmulhrsw m1, m7 packuswb m5, m1 movu [r0 + 1504], m5 ; mode 25 [row 6, 7] pmaddubsw m2, m0, [r3 + 18 * 16] pmulhrsw m2, m7 pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m7 packuswb m2, m1 movu [r0 + 1520], m2 ; mode 26 movu m0, [r1 + 1] pshufb m1, m0, [tab_Si] movu [r0 + 1536], m1 movu [r0 + 1552], m1 movu [r0 + 1568], m1 movu [r0 + 1584], m1 pxor m5, m5 pshufb m1, m1, m5 punpcklbw m1, m5 movu m2, [r1 + 16] pinsrb m2, [r1], 0 pshufb m3, m2, m5 punpcklbw m3, m5 psrldq m4, m2, 1 punpcklbw m4, m5 movu m2, [r1 + 9 + 16] punpcklbw m2, m5 psubw m4, m3 psubw m2, m3 psraw m4, 1 psraw m2, 1 paddw m4, m1 paddw m2, m1 packuswb m4, m2 pextrb [r0 + 1536], m4, 0 pextrb [r0 + 1544], m4, 1 pextrb [r0 + 1552], m4, 2 pextrb [r0 + 1560], m4, 3 pextrb [r0 + 1568], m4, 4 pextrb [r0 + 1576], m4, 5 pextrb [r0 + 1584], m4, 6 pextrb [r0 + 1592], m4, 7 ; mode 27 [row 0, 1] palignr m6, m0, 1 punpcklbw m4, m0, m6 pmaddubsw m1, m4, [r3 + 2 * 16] pmulhrsw m1, m7 pmaddubsw m2, m4, [r3 + 4 * 16] pmulhrsw m2, m7 packuswb m1, m2 movu [r0 + 1600], m1 ; mode 27 [row 2, 3] pmaddubsw m1, m4, [r3 + 6 * 16] pmulhrsw m1, m7 pmaddubsw m2, m4, [r3 + 8 * 16] pmulhrsw m2, m7 packuswb m1, m2 movu [r0 + 1616], m1 ; mode 27 [row 4, 5] pmaddubsw m3, m4, [r3 + 10 * 16] pmulhrsw m3, m7 pmaddubsw m2, m4, [r3 + 12 * 16] pmulhrsw m2, m7 packuswb m1, m3, m2 movu [r0 + 1632], m1 ; mode 27 [row 6, 7] pmaddubsw m1, m4, [r3 + 14 * 16] pmulhrsw m1, m7 pmaddubsw m2, m4, [r3 + 16 * 16] pmulhrsw m2, m7 packuswb m1, m2 movu [r0 + 1648], m1 ; mode 28 [row 0, 1] pmaddubsw m1, m4, [r3 + 5 * 16] pmulhrsw m1, m7 packuswb m1, m3 movu [r0 + 1664], m1 ; mode 28 [row 2, 3] pmaddubsw m1, m4, [r3 + 15 * 16] pmulhrsw m1, m7 pmaddubsw m2, m4, [r3 + 20 * 16] pmulhrsw m2, m7 packuswb m1, m2 movu [r0 + 1680], m1 ; mode 28 [row 4, 5] pmaddubsw m1, m4, [r3 + 25 * 16] pmulhrsw m1, m7 pmaddubsw m2, m4, [r3 + 30 * 16] pmulhrsw m2, m7 packuswb m1, m2 movu [r0 + 1696], m1 ; mode 28 [row 6, 7] palignr m1, m0, 2 punpcklbw m5, m6, m1 pmaddubsw m2, m5, [r3 + 3 * 16] pmulhrsw m2, m7 pmaddubsw m3, m5, [r3 + 8 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1712], m2 ; mode 29 [row 0, 1] pmaddubsw m2, m4, [r3 + 9 * 16] pmulhrsw m2, m7 pmaddubsw m3, m4, [r3 + 18 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1728], m2 ; mode 29 [row 2, 3] pmaddubsw m2, m4, [r3 + 27 * 16] pmulhrsw m2, m7 pmaddubsw m3, m5, [r3 + 4 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1744], m2 ; mode 29 [row 4, 5] pmaddubsw m2, m5, [r3 + 13 * 16] pmulhrsw m2, m7 pmaddubsw m3, m5, [r3 + 22 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1760], m2 ; mode 29 [row 6, 7] pmaddubsw m2, m5, [r3 + 31 * 16] pmulhrsw m2, m7 palignr m6, m0, 3 punpcklbw m1, m6 pmaddubsw m3, m1, [r3 + 8 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1776], m2 ; mode 32 [row 2] movh [r0 + 1936], m2 ; mode 30 [row 0, 1] pmaddubsw m2, m4, [r3 + 13 * 16] pmulhrsw m2, m7 pmaddubsw m3, m4, [r3 + 26 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1792], m2 ; mode 30 [row 2, 3] pmaddubsw m2, m5, [r3 + 7 * 16] pmulhrsw m2, m7 pmaddubsw m3, m5, [r3 + 20 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1808], m2 ; mode 33 [row 1] movhps [r0 + 1992], m2 ; mode 30 [row 4, 5] pmaddubsw m2, m1, [r3 + 1 * 16] pmulhrsw m2, m7 pmaddubsw m3, m1, [r3 + 14 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1824], m2 ; mode 33 [row 2] movhps [r0 + 2000], m2 ; mode 30 [row 6, 7] pmaddubsw m2, m1, [r3 + 27 * 16] pmulhrsw m2, m7 psrldq m0, 4 punpcklbw m6, m0 pmaddubsw m3, m6, [r3 + 8 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1840], m2 ; mode 33 [row 3] movhps [r0 + 2008], m2 ; mode 31 [row 0, 1] pmaddubsw m2, m4, [r3 + 17 * 16] pmulhrsw m2, m7 pmaddubsw m3, m5, [r3 + 2 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1856], m2 ; mode 31 [row 2, 3] pmaddubsw m2, m5, [r3 + 19 * 16] pmulhrsw m2, m7 pmaddubsw m3, m1, [r3 + 4 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1872], m2 ; mode 31 [row 4, 5] pmaddubsw m2, m1, [r3 + 21 * 16] pmulhrsw m2, m7 pmaddubsw m3, m6, [r3 + 6 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1888], m2 ; mode 31 [row 6, 7] pmaddubsw m2, m6, [r3 + 23 * 16] pmulhrsw m2, m7 movu m3, [r1 + 6] punpcklbw m0, m3 pmaddubsw m3, m0, [r3 + 8 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1904], m2 ; mode 32 [row 0, 1] pmaddubsw m2, m4, [r3 + 21 * 16] pmulhrsw m2, m7 pmaddubsw m3, m5, [r3 + 10 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1920], m2 ; mode 32 [row 3] pmaddubsw m2, m1, [r3 + 20 * 16] pmulhrsw m2, m7 pxor m3, m3 packuswb m2, m3 movh [r0 + 1944], m2 ; mode 32 [row 4, 5] pmaddubsw m2, m6, [r3 + 9 * 16] pmulhrsw m2, m7 pmaddubsw m3, m6, [r3 + 30 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1952], m2 ; mode 33 [row 4, 5] pmaddubsw m2, m0, [r3 + 2 * 16] pmulhrsw m2, m7 pmaddubsw m3, m0, [r3 + 28 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 2016], m2 ; mode 32 [row 6] pmaddubsw m2, m0, [r3 + 19 * 16] pmulhrsw m2, m7 ; mode 32 [row 7] movu m0, [r1 + 6] palignr m3, m0, 1 punpcklbw m0, m3 pmaddubsw m3, m0, [r3 + 8 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 1968], m2 ; mode 33 [row 6, 7] pmaddubsw m2, m0, [r3 + 22 * 16] pmulhrsw m2, m7 movu m0, [r1 + 7] palignr m3, m0, 1 punpcklbw m0, m3 pmaddubsw m3, m0, [r3 + 16 * 16] pmulhrsw m3, m7 packuswb m2, m3 movu [r0 + 2032], m2 ; mode 33 [row 0] pmaddubsw m2, m4, [r3 + 26 * 16] pmulhrsw m2, m7 pxor m3, m3 packuswb m2, m3 movh [r0 + 1984], m2 ; mode 34 [row 0, 1, 2, 3, 4, 5, 6, 7] movu m0, [r2 + 2] palignr m1, m0, 1 punpcklqdq m2, m0, m1 movu [r0 + 2048], m2 palignr m1, m0, 2 palignr m2, m0, 3 punpcklqdq m1, m2 movu [r0 + 2064], m1 palignr m1, m0, 4 palignr m2, m0, 5 punpcklqdq m1, m2 movu [r0 + 2080], m1 palignr m1, m0, 6 palignr m2, m0, 7 punpcklqdq m1, m2 movu [r0 + 2096], m1 RET ;-------------------------------------------------------------------------------- ; void all_angs_pred_16x16(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) ;-------------------------------------------------------------------------------- INIT_XMM sse4 cglobal all_angs_pred_16x16, 3,4,8 ; mode 2 movu m0, [r2 + 2 + 32] movu [r0 + 0 * 16], m0 movu m1, m0 movu m6, [r2 + 18 + 32] palignr m5, m6, m0, 1 movu [r0 + 1 * 16], m5 movu m4, m5 palignr m5, m6, m0, 2 movu [r0 + 2 * 16], m5 palignr m5, m6, m0, 3 movu [r0 + 3 * 16], m5 palignr m5, m6, m0, 4 movu [r0 + 4 * 16], m5 palignr m5, m6, m0, 5 movu [r0 + 5 * 16], m5 palignr m5, m6, m0, 6 movu [r0 + 6 * 16], m5 palignr m5, m6, m0, 7 movu [r0 + 7 * 16], m5 movu m7, m5 palignr m5, m6, m0, 8 movu [r0 + 8 * 16], m5 movu m2, m5 palignr m5, m6, m0, 9 movu [r0 + 9 * 16], m5 palignr m3, m6, m0, 10 movu [r0 + 10 * 16], m3 palignr m3, m6, m0, 11 movu [r0 + 11 * 16], m3 palignr m3, m6, m0, 12 movu [r0 + 12 * 16], m3 ; mode 3 [row 15] movu [r0 + (3-2)*16*16 + 15 * 16], m3 palignr m3, m6, m0, 13 movu [r0 + 13 * 16], m3 palignr m3, m6, m0, 14 movu [r0 + 14 * 16], m3 palignr m3, m6, m0, 15 movu [r0 + 15 * 16], m3 ; mode 3 [row 0] lea r3, [ang_table] movu m3, [pw_1024] movu m0, [r2 + 1 + 32] punpcklbw m0, m1 ; mode 17 [row 8 - second half] pmaddubsw m1, m0, [r3 + 22 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 248 * 16 + 8], m1 ; mode 17 [row 8 - second half] end pmaddubsw m1, m0, [r3 + 26 * 16] pmulhrsw m1, m3 punpcklbw m7, m2 pmaddubsw m2, m7, [r3 + 26 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 16 * 16], m1 ;mode 6 [row 1] movu [r0 + 65 * 16], m1 ; mode 4 [row 0] pmaddubsw m1, m0, [r3 + 21 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 21 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 32 * 16], m1 ; mode 5 [row 0] pmaddubsw m1, m0, [r3 + 17 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 17 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 48 * 16], m1 ; mode 6 [row 0] pmaddubsw m1, m0, [r3 + 13 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 13 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 64 * 16], m1 ; mode 7 [row 0] pmaddubsw m1, m0, [r3 + 9 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 9 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 80 * 16], m1 ; mode 7 [row 1] pmaddubsw m1, m0, [r3 + 18 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 18 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 81 * 16], m1 ; mode 7 [row 2] pmaddubsw m1, m0, [r3 + 27 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 27 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 82 * 16], m1 ; mode 8 [row 0] pmaddubsw m1, m0, [r3 + 5 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 5 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 96 * 16], m1 ; mode 8 [row 1] pmaddubsw m1, m0, [r3 + 10 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 10 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 97 * 16], m1 ; mode 8 [row 2] pmaddubsw m1, m0, [r3 + 15 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 15 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 98 * 16], m1 ; mode 8 [row 3] pmaddubsw m1, m0, [r3 + 20 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 20 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 99 * 16], m1 ; mode 8 [row 4] pmaddubsw m1, m0, [r3 + 25 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 25 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 100 * 16], m1 ; mode 8 [row 5] pmaddubsw m1, m0, [r3 + 30 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 30 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 101 * 16], m1 ; mode 15 [row 13 - second half] pmaddubsw m1, m0, [r3 + 18 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 221 * 16 + 8], m1 ; mode 15 [row 13 - second half] end ; mode 15 [row 14 - second half] pmaddubsw m1, m0, [r3 + 1 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 222 * 16 + 8], m1 ; mode 15 [row 14 - second half] end ; mode 16 [row 10 - second half] pmaddubsw m1, m0, [r3 + 25 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 234 * 16 + 8], m1 ; mode 16 [row 10 - second half] end ; mode 16 [row 11 - second half] pmaddubsw m1, m0, [r3 + 4 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 235 * 16 + 8], m1 ; mode 16 [row 11 - second half] end ; mode 3 [row 1] movu m6, [r3 + 20 * 16] movu m0, [r2 + 2 + 32] punpcklbw m0, m4 ; mode 17 [row 7 - second half] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 247 * 16 + 8], m1 ; mode 17 [row 7 - second half] end pmaddubsw m1, m0, m6 pmulhrsw m1, m3 movu m2, [r2 + 10 + 32] punpcklbw m2, m5 pmaddubsw m4, m2, m6 pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 17 * 16], m1 ;mode 6 [row 3] movu [r0 + 67 * 16], m1 ; mode 4 row [row 1] pmaddubsw m1, m0, [r3 + 10 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 10 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 33 * 16], m1 ; mode 4 row [row 2] pmaddubsw m1, m0, [r3 + 31 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 31 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 34 * 16], m1 ; mode 7 [row 6] movu [r0 + 86 * 16], m1 ; mode 5 row [row 1] pmaddubsw m1, m0, [r3 + 2 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 2 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 49 * 16], m1 ; mode 5 row [row 2] pmaddubsw m1, m0, [r3 + 19 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 19 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 50 * 16], m1 ; mode 6 [row 2] pmaddubsw m1, m0, [r3 + 7 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 7 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 66 * 16], m1 ; mode 7 [row 3] pmaddubsw m1, m0, [r3 + 4 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 83 * 16], m1 ; mode 7 [row 4] pmaddubsw m1, m0, [r3 + 13 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 13 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 84 * 16], m1 ; mode 8 [row 8] movu [r0 + 104 * 16], m1 ; mode 7 [row 5] pmaddubsw m1, m0, [r3 + 22 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 22 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 85 * 16], m1 ; mode 8 [row 6] pmaddubsw m1, m0, [r3 + 3 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 3 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 102 * 16], m1 ; mode 8 [row 7] pmaddubsw m1, m0, [r3 + 8 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 8 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 103 * 16], m1 ; mode 8 [row 9] pmaddubsw m1, m0, [r3 + 18 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 18 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 105 * 16], m1 ; mode 8 [row 10] pmaddubsw m1, m0, [r3 + 23 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 23 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 106 * 16], m1 ; mode 8 [row 11] pmaddubsw m1, m0, [r3 + 28 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 28 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 107 * 16], m1 ; mode 3 [row 2] movu m0, [r2 + 3 + 32] movd m1, [r2 + 19 + 32] palignr m1, m0, 1 punpcklbw m0, m1 ; mode 17 [row 6 - second half] pmaddubsw m1, m0, [r3 + 10 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 246 * 16 + 8], m1 ; mode 17 [row 6 - second half] end pmaddubsw m1, m0, [r3 + 14 * 16] pmulhrsw m1, m3 movu m2, [r2 + 11 + 32] movd m4, [r2 + 27 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 14 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 18 * 16], m1 ; mode 6 [row 5] movu [r0 + 69 * 16], m1 ; mode 4 row [row 3] pmaddubsw m1, m0, [r3 + 20 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 20 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 35 * 16], m1 ; mode 5 row [row 3] pmaddubsw m1, m0, [r3 + 4 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 51 * 16], m1 ; mode 5 row [row 4] pmaddubsw m1, m0, [r3 + 21 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 21 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 52 * 16], m1 ; mode 6 [row 4] pmaddubsw m1, m0, [r3 + 1 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 68 * 16], m1 ; mode 6 [row 6] pmaddubsw m1, m0, [r3 + 27 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 27 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 70 * 16], m1 ; mode 7 [row 7] pmaddubsw m1, m0, [r3 + 8 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 8 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 87 * 16], m1 ; mode 7 [row 8] pmaddubsw m1, m0, [r3 + 17 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 17 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 88 * 16], m1 ; mode 7 [row 9] pmaddubsw m1, m0, [r3 + 26 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 26 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 89 * 16], m1 ; mode 8 [row 12] pmaddubsw m1, m0, [r3 + 1 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 108 * 16], m1 ; mode 8 [row 13] pmaddubsw m1, m0, [r3 + 6 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 6 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 109 * 16], m1 ; mode 8 [row 14] pmaddubsw m1, m0, [r3 + 11 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 11 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 110 * 16], m1 ; mode 8 [row 15] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 111 * 16], m1 ; mode 3 [row 3] movu m0, [r2 + 4 + 32] movd m1, [r2 + 20 + 32] palignr m1, m0, 1 punpcklbw m0, m1 ; mode 17 [row 4 - second half] pmaddubsw m1, m0, [r3 + 30 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 244 * 16 + 8], m1 ; mode 17 [row 4 - second half] end ; mode 17 [row 5 - second half] pmaddubsw m1, m0, [r3 + 4 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 245 * 16 + 8], m1 ; mode 17 [row 5 - second half] end pmaddubsw m1, m0, [r3 + 8 * 16] pmulhrsw m1, m3 movu m2, [r2 + 12 + 32] movd m4, [r2 + 28 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 8 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 19 * 16], m1 ; mode 6 [row 7] movu [r0 + 71 * 16], m1 ; mode 4 row [row 4] pmaddubsw m1, m0, [r3 + 9 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 9 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 36 * 16], m1 ; mode 4 row [row 5] pmaddubsw m1, m0, [r3 + 30 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 30 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 37 * 16], m1 ; mode 7 row [row 13] movu [r0 + 93 * 16], m1 ; mode 5 row [row 5] pmaddubsw m1, m0, [r3 + 6 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 6 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 53 * 16], m1 ; mode 5 row [row 6] pmaddubsw m1, m0, [r3 + 23 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 23 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 54 * 16], m1 ; mode 6 [row 8] pmaddubsw m1, m0, [r3 + 21 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 21 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 72 * 16], m1 ; mode 7 [row 12] movu [r0 + 92 * 16], m1 ; mode 7 [row 10] pmaddubsw m1, m0, [r3 + 3 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 3 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 90 * 16], m1 ; mode 7 [row 11] pmaddubsw m1, m0, [r3 + 12 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 12 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 91 * 16], m1 ; mode 3 [row 4] movu m0, [r2 + 5 + 32] movd m1, [r2 + 20 + 32] palignr m1, m0, 1 punpcklbw m0, m1 ; mode 17 [row 3 - second half] pmaddubsw m1, m0, [r3 + 24 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 243 * 16 + 8], m1 ; mode 17 [row 3 - second half] end pmaddubsw m1, m0, [r3 + 2 * 16] pmulhrsw m1, m3 movu m2, [r2 + 13 + 32] movd m4, [r2 + 29 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 2 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 20 * 16], m1 ;mode 6 [row 9] movu [r0 + 73 * 16], m1 ; mode 4 row [row 6] movu m6, [r3 + 19 * 16] pmaddubsw m1, m0, m6 pmulhrsw m1, m3 pmaddubsw m4, m2, m6 pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 38 * 16], m1 ; mode 3 [row 5] pmaddubsw m1, m0, [r3 + 28 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 28 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 21 * 16], m1 ;mode 6 [row 11] movu [r0 + 75 * 16], m1 ; mode 5 row [row 7] pmaddubsw m1, m0, [r3 + 8 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 8 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 55 * 16], m1 ; mode 5 row [row 8] pmaddubsw m1, m0, [r3 + 25 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 25 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 56 * 16], m1 ; mode 6 [row 10] pmaddubsw m1, m0, [r3 + 15 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 15 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 74 * 16], m1 ; mode 7 [row 14] pmaddubsw m1, m0, [r3 + 7 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 7 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 94 * 16], m1 ; mode 7 [row 15] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 95 * 16], m1 ; mode 3 [row 6] movu m0, [r2 + 6 + 32] movd m1, [r2 + 22 + 32] palignr m1, m0, 1 punpcklbw m0, m1 ; mode 17 [row 2 - second half] pmaddubsw m1, m0, [r3 + 18 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 242 * 16 + 8], m1 ; mode 17 [row 2 - second half] end pmaddubsw m1, m0, [r3 + 22 * 16] pmulhrsw m1, m3 movu m2, [r2 + 14 + 32] movd m4, [r2 + 30 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 22 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 22 * 16], m1 ; mode 6 [row 13] movu [r0 + 77 * 16], m1 ; mode 4 row [row 7] pmaddubsw m1, m0, [r3 + 8 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 8 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 39 * 16], m1 ; mode 4 row [row 8] pmaddubsw m1, m0, [r3 + 29 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 40 * 16], m1 ; mode 5 row [row 9] pmaddubsw m1, m0, [r3 + 10 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 10 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 57 * 16], m1 ; mode 5 row [row 10] pmaddubsw m1, m0, [r3 + 27 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 27 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 58 * 16], m1 ; mode 6 [row 12] pmaddubsw m1, m0, [r3 + 9 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 9 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 76 * 16], m1 ; mode 3 [row 7] movu m0, [r2 + 7 + 32] movd m1, [r2 + 27 + 32] palignr m1, m0, 1 punpcklbw m0, m1 ; mode 17 [row 1 - second half] pmaddubsw m1, m0, [r3 + 12 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 241 * 16 + 8], m1 ; mode 17 [row 1 - second half] end pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 movu m2, [r2 + 15 + 32] movd m4, [r2 + 25 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 23 * 16], m1 ; mode 6 [row 15] movu [r0 + 79 * 16], m1 ; mode 4 row [row 9] pmaddubsw m1, m0, [r3 + 18 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 18 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 41 * 16], m1 ; mode 5 row [row 11] pmaddubsw m1, m0, [r3 + 12 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 12 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 59 * 16], m1 ; mode 5 row [row 12] pmaddubsw m1, m0, [r3 + 29 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 60 * 16], m1 ; mode 6 [row 14] pmaddubsw m1, m0, [r3 + 3 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 3 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 78 * 16], m1 ; mode 3 [row 8] movu m0, [r2 + 8 + 32] movd m1, [r2 + 24 + 32] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 10 * 16] pmulhrsw m1, m3 movu m2, [r2 + 16 + 32] psrldq m4, m2, 1 pinsrb m4, [r2 + 32], 15 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 10 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 24 * 16], m1 ; mode 4 row [row 10] pmaddubsw m1, m0, [r3 + 7 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 7 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 42 * 16], m1 ; mode 4 row [row 11] pmaddubsw m1, m0, [r3 + 28 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 28 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 43 * 16], m1 ; mode 5 row [row 13] pmaddubsw m1, m0, [r3 + 14 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 14 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 61 * 16], m1 ; mode 5 row [row 14] pmaddubsw m1, m0, [r3 + 31 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 31 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 62 * 16], m1 ; mode 3 [row 9] movu m0, [r2 + 9 + 32] movd m1, [r2 + 16 + 32] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 4 * 16] pmulhrsw m1, m3 movu m2, [r2 + 17 + 32] movd m4, [r2 + 33 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 25 * 16], m1 ; mode 4 row [row 12] pmaddubsw m1, m0, [r3 + 17 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 17 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 44 * 16], m1 ; mode 3 [row 10] pmaddubsw m1, m0, [r3 + 30 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 30 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 26 * 16], m1 ; mode 5 row [row 15] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 63 * 16], m1 ; mode 3 [row 11] movu m0, [r2 + 10 + 32] movd m1, [r2 + 26 + 32] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 24 * 16] pmulhrsw m1, m3 movu m2, [r2 + 18 + 32] movd m4, [r2 + 34 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 27 * 16], m1 ; mode 4 row [row 13] pmaddubsw m1, m0, [r3 + 6 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 6 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 45 * 16], m1 ; mode 4 row [row 14] pmaddubsw m1, m0, [r3 + 27 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 27 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 46 * 16], m1 ; mode 3 [row 12] movu m0, [r2 + 11 + 32] movd m1, [r2 + 27 + 32] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 18 * 16] pmulhrsw m1, m3 movu m2, [r2 + 19 + 32] movd m4, [r2 + 35 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 18 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 28 * 16], m1 ; mode 4 row [row 15] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m4, m2, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 47 * 16], m1 ; mode 3 [row 13] movu m0, [r2 + 12 + 32] movd m1, [r2 + 28 + 32] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 12 * 16] pmulhrsw m1, m3 movu m2, [r2 + 20 + 32] movd m4, [r2 + 36 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 12 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 29 * 16], m1 ; mode 3 [row 14] movu m0, [r2 + 13 + 32] movd m1, [r2 + 29 + 32] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 6 * 16] pmulhrsw m1, m3 movu m2, [r2 + 21 + 32] movd m4, [r2 + 37 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m2, [r3 + 6 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 30 * 16], m1 ; mode 9 movu m0, [r1 + 1 + 32] movd m1, [r1 + 17 + 32] palignr m1, m0, 1 ; mode 9 [row 15] movu [r0 + 127 * 16], m1 ; mode 9 [row 0] punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 2 * 16] pmulhrsw m1, m3 movu m7, [r1 + 9 + 32] movd m4, [r2 + 25 + 32] palignr m2, m7, 1 punpcklbw m7, m2 pmaddubsw m2, m7, [r3 + 2 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 112 * 16], m1 ; mode 9 [row 1] pmaddubsw m1, m0, [r3 + 4 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 4 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 113 * 16], m1 ; mode 9 [row 2] pmaddubsw m1, m0, [r3 + 6 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 6 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 114 * 16], m1 ; mode 9 [row 3] pmaddubsw m1, m0, [r3 + 8 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 8 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 115 * 16], m1 ; mode 9 [row 4] pmaddubsw m1, m0, [r3 + 10 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 10 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 116 * 16], m1 ; mode 9 [row 5] pmaddubsw m1, m0, [r3 + 12 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 12 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 117 * 16], m1 ; mode 9 [row 6] pmaddubsw m1, m0, [r3 + 14 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 14 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 118 * 16], m1 ; mode 9 [row 7] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 16 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 119 * 16], m1 ; mode 9 [row 8] pmaddubsw m1, m0, [r3 + 18 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 18 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 120 * 16], m1 ; mode 9 [row 9] pmaddubsw m1, m0, [r3 + 20 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 20 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 121 * 16], m1 ; mode 9 [row 10] pmaddubsw m1, m0, [r3 + 22 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 22 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 122 * 16], m1 ; mode 9 [row 11] pmaddubsw m1, m0, [r3 + 24 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 24 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 123 * 16], m1 ; mode 9 [row 12] pmaddubsw m1, m0, [r3 + 26 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 26 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 124 * 16], m1 ; mode 9 [row 13] pmaddubsw m1, m0, [r3 + 28 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 28 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 125 * 16], m1 ; mode 9 [row 14] pmaddubsw m1, m0, [r3 + 30 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 30 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 126 * 16], m1 ; mode 10 movu m1, [r1 + 1 + 32] movu [r0 + 128 * 16], m1 movu [r0 + 129 * 16], m1 movu [r0 + 130 * 16], m1 movu [r0 + 131 * 16], m1 movu [r0 + 132 * 16], m1 movu [r0 + 133 * 16], m1 movu [r0 + 134 * 16], m1 movu [r0 + 135 * 16], m1 movu [r0 + 136 * 16], m1 movu [r0 + 137 * 16], m1 movu [r0 + 138 * 16], m1 movu [r0 + 139 * 16], m1 movu [r0 + 140 * 16], m1 movu [r0 + 141 * 16], m1 movu [r0 + 142 * 16], m1 movu [r0 + 143 * 16], m1 pxor m0, m0 pshufb m1, m1, m0 punpcklbw m1, m0 pinsrb m2, [r1], 0 pshufb m2, m2, m0 punpcklbw m2, m0 movu m4, [r1 + 1] punpcklbw m5, m4, m0 punpckhbw m4, m0 psubw m5, m2 psubw m4, m2 psraw m5, 1 psraw m4, 1 paddw m5, m1 paddw m4, m1 packuswb m5, m4 pextrb [r0 + 128 * 16], m5, 0 pextrb [r0 + 129 * 16], m5, 1 pextrb [r0 + 130 * 16], m5, 2 pextrb [r0 + 131 * 16], m5, 3 pextrb [r0 + 132 * 16], m5, 4 pextrb [r0 + 133 * 16], m5, 5 pextrb [r0 + 134 * 16], m5, 6 pextrb [r0 + 135 * 16], m5, 7 pextrb [r0 + 136 * 16], m5, 8 pextrb [r0 + 137 * 16], m5, 9 pextrb [r0 + 138 * 16], m5, 10 pextrb [r0 + 139 * 16], m5, 11 pextrb [r0 + 140 * 16], m5, 12 pextrb [r0 + 141 * 16], m5, 13 pextrb [r0 + 142 * 16], m5, 14 pextrb [r0 + 143 * 16], m5, 15 ; mode 11 movu m0, [r1 + 32] pinsrb m0, [r1], 0 ; mode 11 [row 15] movu [r0 + 159 * 16], m0 ; mode 11 [row 0] movu m1, [r1 + 1 + 32] punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 30 * 16] pmulhrsw m1, m3 movu m7, [r1 + 8 + 32] movu m2, [r1 + 9 + 32] punpcklbw m7, m2 pmaddubsw m2, m7, [r3 + 30 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 144 * 16], m1 ; mode 11 [row 1] pmaddubsw m1, m0, [r3 + 28 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 28 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 145 * 16], m1 ; mode 11 [row 2] pmaddubsw m1, m0, [r3 + 26 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 26 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 146 * 16], m1 ; mode 11 [row 3] pmaddubsw m1, m0, [r3 + 24 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 24 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 147 * 16], m1 ; mode 11 [row 4] pmaddubsw m1, m0, [r3 + 22 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 22 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 148 * 16], m1 ; mode 11 [row 5] pmaddubsw m1, m0, [r3 + 20 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 20 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 149 * 16], m1 ; mode 11 [row 6] pmaddubsw m1, m0, [r3 + 18 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 18 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 150 * 16], m1 ; mode 11 [row 7] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 16 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 151 * 16], m1 ; mode 11 [row 8] pmaddubsw m1, m0, [r3 + 14 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 14 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 152 * 16], m1 ; mode 11 [row 9] pmaddubsw m1, m0, [r3 + 12 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 12 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 153 * 16], m1 ; mode 11 [row 10] pmaddubsw m1, m0, [r3 + 10 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 10 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 154 * 16], m1 ; mode 11 [row 11] pmaddubsw m1, m0, [r3 + 8 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 8 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 155 * 16], m1 ; mode 11 [row 12] pmaddubsw m1, m0, [r3 + 6 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 6 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 156 * 16], m1 ; mode 11 [row 13] pmaddubsw m1, m0, [r3 + 4 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 4 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 157 * 16], m1 ; mode 11 [row 14] pmaddubsw m1, m0, [r3 + 2 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 2 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 158 * 16], m1 ; mode 12 [row 0] movu m0, [r2 + 32] pinsrb m0, [r2], 0 movu m1, [r2 + 1 + 32] punpcklbw m0, m1 pmaddubsw m1, m0, [r3 + 27 * 16] pmulhrsw m1, m3 movu m7, [r2 + 8 + 32] movd m2, [r2 + 24 + 32] palignr m2, m7, 1 punpcklbw m7, m2 pmaddubsw m2, m7, [r3 + 27 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 160 * 16], m1 ; mode 12 [row 1] pmaddubsw m1, m0, [r3 + 22 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 22 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 161 * 16], m1 ; mode 12 [row 2] pmaddubsw m1, m0, [r3 + 17 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 17 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 162 * 16], m1 ; mode 12 [row 3] pmaddubsw m1, m0, [r3 + 12 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 12 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 163 * 16], m1 ; mode 12 [row 4] pmaddubsw m1, m0, [r3 + 7 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 7 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 164 * 16], m1 ; mode 12 [row 5] pmaddubsw m1, m0, [r3 + 2 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 2 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 165 * 16], m1 ; mode 13 [row 0] pmaddubsw m1, m0, [r3 + 23 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 23 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 176 * 16], m1 ; mode 13 [row 1] pmaddubsw m1, m0, [r3 + 14 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 14 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 177 * 16], m1 ; mode 13 [row 2] pmaddubsw m1, m0, [r3 + 5 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 5 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 178 * 16], m1 ; mode 14 [row 0] pmaddubsw m1, m0, [r3 + 19 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 19 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 192 * 16], m1 ; mode 14 [row 1] pmaddubsw m1, m0, [r3 + 6 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 6 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 193 * 16], m1 ; mode 17 [row 0] movu [r0 + 240 * 16], m1 ; mode 15 [row 0] pmaddubsw m1, m0, [r3 + 15 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 15 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 208 * 16], m1 ; mode 15 [row 15 - second half] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 223 * 16 + 8], m1 ; mode 15 [row 15 - second half] end ; mode 16 [row 0] pmaddubsw m1, m0, [r3 + 11 * 16] pmulhrsw m1, m3 pmaddubsw m2, m7, [r3 + 11 * 16] pmulhrsw m2, m3 packuswb m1, m2 movu [r0 + 224 * 16], m1 ; mode 17 [row 9 - second half] pmaddubsw m1, m0, [r3 + 28 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 249 * 16 + 8], m1 ; mode 17 [row 9 - second half] end ; mode 17 [row 10 - second half] pmaddubsw m1, m0, [r3 + 2 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 250 * 16 + 8], m1 ; mode 17 [row 10 - second half] end ; mode 17 [row 1 - first half] pslldq m6, m0, 2 pinsrb m6, [r2], 1 pinsrb m6, [r2 + 1], 0 pmaddubsw m1, m6, [r3 + 12 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 241 * 16], m1 ; mode 17 [row 11 - second half] pmaddubsw m1, m6, [r3 + 8 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 251 * 16 + 8], m1 ; mode 17 [row 11 - second half] end ; mode 17 [row 2 - first half] pslldq m6, 2 pinsrb m6, [r2 + 1], 1 pinsrb m6, [r2 + 2], 0 pmaddubsw m1, m6, [r3 + 18 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 242 * 16], m1 ; mode 17 [row 12 - second half] pmaddubsw m1, m6, [r3 + 14 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 252 * 16 + 8], m1 ; mode 17 [row 12 - second half] end ; mode 17 [row 3 - first half] pslldq m6, 2 pinsrb m6, [r2 + 2], 1 pinsrb m6, [r2 + 4], 0 pmaddubsw m1, m6, [r3 + 24 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 243 * 16], m1 ; mode 17 [row 13 - first half] pmaddubsw m1, m6, [r3 + 20 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 253 * 16 + 8], m1 ; mode 17 [row 4 - first half] pslldq m6, 2 pinsrb m6, [r2 + 4], 1 pinsrb m6, [r2 + 5], 0 pmaddubsw m1, m6, [r3 + 30 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 244 * 16], m1 ; mode 17 [row 5 - first half] pmaddubsw m1, m6, [r3 + 4 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 245 * 16], m1 ; mode 17 [row 14 - second half] pmaddubsw m1, m6, [r3 + 26 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 254 * 16 + 8], m1 ; mode 17 [row 14 - second half] end ; mode 17 [row 6 - first half] pslldq m6, 2 pinsrb m6, [r2 + 5], 1 pinsrb m6, [r2 + 6], 0 pmaddubsw m1, m6, [r3 + 10 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 246 * 16], m1 ; mode 17 [row 7 - first half] pslldq m6, 2 pinsrb m6, [r2 + 6], 1 pinsrb m6, [r2 + 7], 0 pmaddubsw m1, m6, [r3 + 16 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 247 * 16], m1 ; mode 17 [row 8 - first half] pslldq m6, 2 pinsrb m6, [r2 + 7], 1 pinsrb m6, [r2 + 9], 0 pmaddubsw m1, m6, [r3 + 22 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 248 * 16], m1 ; mode 17 [row 9 - first half] pslldq m6, 2 pinsrb m6, [r2 + 9], 1 pinsrb m6, [r2 + 10], 0 pmaddubsw m1, m6, [r3 + 28 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 249 * 16], m1 ; mode 17 [row 10 - first half] pmaddubsw m1, m6, [r3 + 2 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 250 * 16], m1 ; mode 17 [row 11 - first half] pslldq m6, 2 pinsrb m6, [r2 + 10], 1 pinsrb m6, [r2 + 11], 0 pmaddubsw m1, m6, [r3 + 8 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 251 * 16], m1 ; mode 17 [row 12 - first half] pslldq m6, 2 pinsrb m6, [r2 + 11], 1 pinsrb m6, [r2 + 12], 0 pmaddubsw m1, m6, [r3 + 14 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 252 * 16], m1 ; mode 17 [row 13 - first half] pslldq m6, 2 pinsrb m6, [r2 + 12], 1 pinsrb m6, [r2 + 14], 0 pmaddubsw m1, m6, [r3 + 20 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 253 * 16], m1 ; mode 17 [row 14 - first half] pslldq m6, 2 pinsrb m6, [r2 + 14], 1 pinsrb m6, [r2 + 15], 0 pmaddubsw m1, m6, [r3 + 26 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 254 * 16], m1 ; mode 16 [row 12 - second half] pmaddubsw m1, m0, [r3 + 15 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 236 * 16 + 8], m1 ; mode 16 [row 12 - second half] ; mode 12 [row 6] pslldq m2, m0, 2 pinsrb m2, [r2], 1 pinsrb m2, [r2 + 6], 0 pmaddubsw m1, m2, [r3 + 29 * 16] pmulhrsw m1, m3 movu m0, [r2 + 7 + 32] psrldq m4, m0, 1 punpcklbw m0, m4 pmaddubsw m4, m0, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 166 * 16], m1 ; mode 12 [row 7] pmaddubsw m1, m2, [r3 + 24 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 167 * 16], m1 ; mode 12 [row 8] pmaddubsw m1, m2, [r3 + 19 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 19 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 168 * 16], m1 ; mode 12 [row 9] pmaddubsw m1, m2, [r3 + 14 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 14 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 169 * 16], m1 ; mode 12 [row 10] pmaddubsw m1, m2, [r3 + 9 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 9 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 170 * 16], m1 ; mode 12 [row 11] pmaddubsw m1, m2, [r3 + 4 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 171 * 16], m1 ; mode 13 [row 3] pinsrb m7, m2, [r2 + 4], 0 pmaddubsw m1, m7, [r3 + 28 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 28 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 179 * 16], m1 ; mode 13 [row 4] pmaddubsw m1, m7, [r3 + 19 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 19 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 180 * 16], m1 ; mode 13 [row 5] pmaddubsw m1, m7, [r3 + 10 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 10 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 181 * 16], m1 ; mode 13 [row 6] pmaddubsw m1, m7, [r3 + 1 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 182 * 16], m1 ; mode 14 [row 2] pinsrb m5, m7, [r2 + 2], 0 pmaddubsw m1, m5, [r3 + 25 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 25 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 194 * 16], m1 ; mode 14 [row 3] pmaddubsw m1, m5, [r3 + 12 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 12 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 195 * 16], m1 ; mode 15 [row 1] pmaddubsw m1, m5, [r3 + 30 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 30 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 209 * 16], m1 ; mode 15 [row 2] pmaddubsw m1, m5, [r3 + 13 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 13 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 210 * 16], m1 ; mode 16 [row 1] pmaddubsw m1, m5, [r3 + 22 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 22 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 225 * 16], m1 ; mode 16 [row 2] pmaddubsw m1, m5, [r3 + 1 * 16] pmulhrsw m1, m3 pmaddubsw m4, m0, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m1, m4 movu [r0 + 226 * 16], m1 ; mode 16 [row 13 - second half] pmaddubsw m1, m5, [r3 + 26 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 237 * 16 + 8], m1 ; mode 16 [row 13 - second half] ; mode 16 [row 14 - second half] pmaddubsw m1, m5, [r3 + 5 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 238 * 16 + 8], m1 ; mode 16 [row 14 - second half] ; mode 16 [row 3] pslldq m6, m5, 2 pinsrb m6, [r2 + 2], 1 pinsrb m6, [r2 + 3], 0 pmaddubsw m1, m6, [r3 + 12 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 227 * 16], m1 ; mode 16 [row 15 - second half] pmaddubsw m1, m6, [r3 + 16 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 239 * 16 + 8], m1 ; mode 16 [row 15 - second half] end ; mode 16 [row 4- first half] pslldq m6, 2 pinsrb m6, [r2 + 3], 1 pinsrb m6, [r2 + 5], 0 pmaddubsw m1, m6, [r3 + 23 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 228 * 16], m1 ; mode 16 [row 5- first half] pmaddubsw m1, m6, [r3 + 2 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 229 * 16], m1 ; mode 16 [row 6- first half] pslldq m6, 2 pinsrb m6, [r2 + 5], 1 pinsrb m6, [r2 + 6], 0 pmaddubsw m1, m6, [r3 + 13 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 230 * 16], m1 ; mode 16 [row 7- first half] pslldq m6, 2 pinsrb m6, [r2 + 6], 1 pinsrb m6, [r2 + 8], 0 pmaddubsw m1, m6, [r3 + 24 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 231 * 16], m1 ; mode 16 [row 8- first half] pmaddubsw m1, m6, [r3 + 3 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 232 * 16], m1 ; mode 19 [row 0 - second half] end ; mode 16 [row 9- first half] pslldq m6, 2 pinsrb m6, [r2 + 8], 1 pinsrb m6, [r2 + 9], 0 pmaddubsw m1, m6, [r3 + 14 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 233 * 16], m1 ; mode 16 [row 10 - first half] pslldq m6, 2 pinsrb m6, [r2 + 9], 1 pinsrb m6, [r2 + 11], 0 pmaddubsw m1, m6, [r3 + 25 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 234 * 16], m1 ; mode 16 [row 11 - first half] pmaddubsw m1, m6, [r3 + 4 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 235 * 16], m1 ; mode 16 [row 12 - first half] pslldq m6, 2 pinsrb m6, [r2 + 11], 1 pinsrb m6, [r2 + 12], 0 pmaddubsw m1, m6, [r3 + 15 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 236 * 16], m1 ; mode 16 [row 13 - first half] pslldq m6, 2 pinsrb m6, [r2 + 12], 1 pinsrb m6, [r2 + 14], 0 pmaddubsw m1, m6, [r3 + 26 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 237 * 16], m1 ; mode 16 [row 14 - first half] pmaddubsw m1, m6, [r3 + 5 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 238 * 16], m1 ; mode 16 [row 15 - first half] pslldq m6, 2 pinsrb m6, [r2 + 14], 1 pinsrb m6, [r2 + 15], 0 pmaddubsw m1, m6, [r3 + 16 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 239 * 16], m1 ; mode 14 [row 4] pslldq m5, 2 pinsrb m5, [r2 + 2], 1 pinsrb m5, [r2 + 5], 0 movu m4, [r2 + 6 + 32] psrldq m0, m4, 1 punpcklbw m4, m0 ; mode 16 [row 3 - second half] pmaddubsw m1, m4, [r3 + 12 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 227 * 16 + 8], m1 ; mode 16 [row 3 - second half] end pmaddubsw m1, m5, [r3 + 31 * 16] pmulhrsw m1, m3 pmaddubsw m0, m4, [r3 + 31 * 16] pmulhrsw m0, m3 packuswb m1, m0 movu [r0 + 196 * 16], m1 ; mode 14 [row 5] pmaddubsw m1, m5, [r3 + 18 * 16] pmulhrsw m1, m3 pmaddubsw m0, m4, [r3 + 18 * 16] pmulhrsw m0, m3 packuswb m1, m0 movu [r0 + 197 * 16], m1 ; mode 14 [row 6] pmaddubsw m1, m5, [r3 + 5 * 16] pmulhrsw m1, m3 pmaddubsw m0, m4, [r3 + 5 * 16] pmulhrsw m0, m3 packuswb m1, m0 movu [r0 + 198 * 16], m1 ; mode 15 [row 3] movu m6, m5 pinsrb m6, [r2 + 4], 0 pmaddubsw m1, m6, [r3 + 28 * 16] pmulhrsw m1, m3 pmaddubsw m0, m4, [r3 + 28 * 16] pmulhrsw m0, m3 packuswb m1, m0 movu [r0 + 211 * 16], m1 ; mode 15 [row 4] pmaddubsw m1, m6, [r3 + 11 * 16] pmulhrsw m1, m3 pmaddubsw m0, m4, [r3 + 11 * 16] pmulhrsw m0, m3 packuswb m1, m0 movu [r0 + 212 * 16], m1 ; mode 15 [row 5 - first half] pslldq m6, 2 pinsrb m6, [r2 + 4], 1 pinsrb m6, [r2 + 6], 0 pmaddubsw m1, m6, [r3 + 26 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 213 * 16], m1 ; mode 15 [row 6 - first half] pmaddubsw m1, m6, [r3 + 9 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 214 * 16], m1 ; mode 15 [row 7 - first half] pslldq m6, 2 pinsrb m6, [r2 + 6], 1 pinsrb m6, [r2 + 8], 0 pmaddubsw m1, m6, [r3 + 24 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 215 * 16], m1 ; mode 15 [row 8 - first half] pmaddubsw m1, m6, [r3 + 7 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 216 * 16], m1 ; mode 15 [row 9 - first half] pslldq m6, 2 pinsrb m6, [r2 + 8], 1 pinsrb m6, [r2 + 9], 0 pmaddubsw m1, m6, [r3 + 22 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 217 * 16], m1 ; mode 15 [row 10 - first half] pmaddubsw m1, m6, [r3 + 5 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 218 * 16], m1 ; mode 15 [row 11 - first half] pslldq m6, 2 pinsrb m6, [r2 + 9], 1 pinsrb m6, [r2 + 11], 0 pmaddubsw m1, m6, [r3 + 20 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 219 * 16], m1 ; mode 15 [row 12 - first half] pmaddubsw m1, m6, [r3 + 3 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 220 * 16], m1 ; mode 15 [row 13 - first half] pslldq m6, 2 pinsrb m6, [r2 + 11], 1 pinsrb m6, [r2 + 13], 0 pmaddubsw m1, m6, [r3 + 18 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 221 * 16], m1 ; mode 15 [row 14 - first half] pmaddubsw m1, m6, [r3 + 1 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 222 * 16], m1 ; mode 15 [row 15 - first half] pslldq m6, 2 pinsrb m6, [r2 + 13], 1 pinsrb m6, [r2 + 15], 0 pmaddubsw m1, m6, [r3 + 16 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 223 * 16], m1 ; mode 14 [row 7] pslldq m5, 2 pinsrb m5, [r2 + 5], 1 pinsrb m5, [r2 + 7], 0 movu m0, [r2 + 5 + 32] psrldq m6, m0, 1 punpcklbw m0, m6 ; mode 15 [row 5 - second half] pmaddubsw m1, m0, [r3 + 26 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 213 * 16 + 8], m1 ; mode 15 [row 5 - second half] end ; mode 15 [row 6 - second half] pmaddubsw m1, m0, [r3 + 9 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 214 * 16 + 8], m1 ; mode 15 [row 6 - second half] end ; mode 16 [row 4 - second half] pmaddubsw m1, m0, [r3 + 23 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 228 * 16 + 8], m1 ; mode 16 [row 4 - second half] end ; mode 16 [row 5 - second half] pmaddubsw m1, m0, [r3 + 2 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 229 * 16 + 8], m1 ; mode 16 [row 5 - second half] end pmaddubsw m1, m5, [r3 + 24 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 24 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 199 * 16], m1 ; mode 14 [row 8] pmaddubsw m1, m5, [r3 + 11 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 11 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 200 * 16], m1 ; mode 14 [row 9] pslldq m5, 2 pinsrb m5, [r2 + 7], 1 pinsrb m5, [r2 + 10], 0 movu m0, [r2 + 4 + 32] psrldq m6, m0, 1 punpcklbw m0, m6 ; mode 15 [row 7 - second half] pmaddubsw m1, m0, [r3 + 24 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 215 * 16 + 8], m1 ; mode 15 [row 7 - second half] end ; mode 15 [row 8 - second half] pmaddubsw m1, m0, [r3 + 7 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 216 * 16 + 8], m1 ; mode 15 [row 8 - second half] end ; mode 16 [row 6 - second half] pmaddubsw m1, m0, [r3 + 13 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 230 * 16 + 8], m1 ; mode 16 [row 6 - second half] end ; mode 15 [row 6 - second half] end pmaddubsw m1, m5, [r3 + 30 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 30 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 201 * 16], m1 ; mode 14 [row 10] pmaddubsw m1, m5, [r3 + 17 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 17 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 202 * 16], m1 ; mode 14 [row 11] pmaddubsw m1, m5, [r3 + 4 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 4 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 203 * 16], m1 ; mode 14 [row 12] pslldq m5, 2 pinsrb m5, [r2 + 10], 1 pinsrb m5, [r2 + 12], 0 movu m0, [r2 + 3 + 32] psrldq m6, m0, 1 punpcklbw m0, m6 ; mode 15 [row 9 - second half] pmaddubsw m1, m0, [r3 + 22 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 217 * 16 + 8], m1 ; mode 15 [row 9 - second half] end ; mode 15 [row 10 - second half] pmaddubsw m1, m0, [r3 + 5 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 218 * 16 + 8], m1 ; mode 15 [row 10 - second half] end ; mode 16 [row 7 - second half] pmaddubsw m1, m0, [r3 + 24 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 231 * 16 + 8], m1 ; mode 16 [row 7 - second half] end ; mode 16 [row 8 - second half] pmaddubsw m1, m0, [r3 + 3 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 232 * 16 + 8], m1 ; mode 16 [row 8 - second half] end pmaddubsw m1, m5, [r3 + 23 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 23 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 204 * 16], m1 ; mode 14 [row 13] pmaddubsw m1, m5, [r3 + 10 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 10 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 205 * 16], m1 ; mode 14 [row 14] pslldq m5, 2 pinsrb m5, [r2 + 12], 1 pinsrb m5, [r2 + 15], 0 movu m0, [r2 + 2 + 32] psrldq m6, m0, 1 punpcklbw m0, m6 ; mode 15 [row 11 - second half] pmaddubsw m1, m0, [r3 + 20 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 219 * 16 + 8], m1 ; mode 15 [row 11 - second half] end ; mode 15 [row 12 - second half] pmaddubsw m1, m0, [r3 + 3 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 220 * 16 + 8], m1 ; mode 15 [row 12 - second half] end ; mode 16 [row 9 - second half] pmaddubsw m1, m0, [r3 + 14 * 16] pmulhrsw m1, m3 packuswb m1, m1 movh [r0 + 233 * 16 + 8], m1 ; mode 16 [row 9 - second half] end pmaddubsw m1, m5, [r3 + 29 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 29 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 206 * 16], m1 ; mode 14 [row 15] pmaddubsw m1, m5, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m6, m0, [r3 + 16 * 16] pmulhrsw m6, m3 packuswb m1, m6 movu [r0 + 207 * 16], m1 ; mode 12 [row 12] pslldq m0, m2, 2 pinsrb m0, [r2 + 6], 1 pinsrb m0, [r2 + 13], 0 pmaddubsw m1, m0, [r3 + 31 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 31 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 172 * 16], m1 ; mode 12 [row 13] pmaddubsw m1, m0, [r3 + 26 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 26 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 173 * 16], m1 ; mode 12 [row 14] pmaddubsw m1, m0, [r3 + 21 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 21 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 174 * 16], m1 ; mode 12 [row 15] pmaddubsw m1, m0, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 175 * 16], m1 ; mode 13 [row 7] pslldq m7, 2 pinsrb m7, [r2 + 4], 1 pinsrb m7, [r2 + 7], 0 pmaddubsw m1, m7, [r3 + 24 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 24 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 183 * 16], m1 ; mode 13 [row 8] pmaddubsw m1, m7, [r3 + 15 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 15 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 184 * 16], m1 ; mode 13 [row 9] pmaddubsw m1, m7, [r3 + 6 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 6 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 185 * 16], m1 ; mode 13 [row 10] pslldq m7, 2 pinsrb m7, [r2 + 7], 1 pinsrb m7, [r2 + 11], 0 pmaddubsw m1, m7, [r3 + 29 * 16] pmulhrsw m1, m3 movu m4, [r2 + 5 + 32] psrldq m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, [r3 + 29 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 186 * 16], m1 ; mode 13 [row 11] pmaddubsw m1, m7, [r3 + 20 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 20 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 187 * 16], m1 ; mode 13 [row 12] pmaddubsw m1, m7, [r3 + 11 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 11 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 188 * 16], m1 ; mode 13 [row 13] pmaddubsw m1, m7, [r3 + 2 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 2 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 189 * 16], m1 ; mode 13 [row 14] pslldq m7, 2 pinsrb m7, [r2 + 11], 1 pinsrb m7, [r2 + 14], 0 pmaddubsw m1, m7, [r3 + 25 * 16] pmulhrsw m1, m3 movu m4, [r2 + 4 + 32] psrldq m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, [r3 + 25 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 190 * 16], m1 ; mode 13 [row 15] pmaddubsw m1, m7, [r3 + 16 * 16] pmulhrsw m1, m3 pmaddubsw m5, m4, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m1, m5 movu [r0 + 191 * 16], m1 ; mode 17 [row 15] movu m0, [r2] pshufb m1, m0, [tab_S1] movu [r0 + 255 * 16], m1 movu m2, [r2 + 32] pinsrb m2, [r2], 0 movd [r0 + 255 * 16 + 12], m2 ; mode 18 [row 0] movu [r0 + 256 * 16], m0 ; mode 18 [row 1] pslldq m4, m0, 1 pinsrb m4, [r2 + 1 + 32], 0 movu [r0 + 257 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 2 + 32], 0 movu [r0 + 258 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 3 + 32], 0 movu [r0 + 259 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 4 + 32], 0 movu [r0 + 260 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 5 + 32], 0 movu [r0 + 261 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 6 + 32], 0 movu [r0 + 262 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 7 + 32], 0 movu [r0 + 263 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 8 + 32], 0 movu [r0 + 264 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 9 + 32], 0 movu [r0 + 265 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 10 + 32], 0 movu [r0 + 266 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 11 + 32], 0 movu [r0 + 267 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 12 + 32], 0 movu [r0 + 268 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 13 + 32], 0 movu [r0 + 269 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 14 + 32], 0 movu [r0 + 270 * 16], m4 pslldq m4, 1 pinsrb m4, [r2 + 15 + 32], 0 movu [r0 + 271 * 16], m4 ; mode 19 [row 0] psrldq m2, m0, 1 punpcklbw m0, m2 movu m5, [r2 + 8] psrldq m6, m5, 1 punpcklbw m5, m6 pmaddubsw m4, m0, [r3 + 6 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 6 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 272 * 16], m4 ; mode 20 [row 0] pmaddubsw m4, m0, [r3 + 11 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 11 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 288 * 16], m4 ; mode 21 [row 0] pmaddubsw m4, m0, [r3 + 15 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 15 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 304 * 16], m4 ; mode 22 [row 0] pmaddubsw m4, m0, [r3 + 19 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 19 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 320 * 16], m4 ; mode 22 [row 1] pmaddubsw m4, m0, [r3 + 6 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 6 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 321 * 16], m4 ; mode 23 [row 0] pmaddubsw m4, m0, [r3 + 23 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 23 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 336 * 16], m4 ; mode 23 [row 1] pmaddubsw m4, m0, [r3 + 14 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 14 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 337 * 16], m4 ; mode 23 [row 2] pmaddubsw m4, m0, [r3 + 5 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 5 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 338 * 16], m4 ; mode 24 [row 0] pmaddubsw m4, m0, [r3 + 27 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 27 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 352 * 16], m4 ; mode 24 [row 1] pmaddubsw m4, m0, [r3 + 22 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 22 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 353 * 16], m4 ; mode 24 [row 2] pmaddubsw m4, m0, [r3 + 17 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 17 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 354 * 16], m4 ; mode 24 [row 3] pmaddubsw m4, m0, [r3 + 12 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 12 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 355 * 16], m4 ; mode 24 [row 4] pmaddubsw m4, m0, [r3 + 7 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 7 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 356 * 16], m4 ; mode 24 [row 5] pmaddubsw m4, m0, [r3 + 2 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 2 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 357 * 16], m4 ; mode 24 [row 6 - first half] pslldq m7, m0, 2 pinsrb m7, [r2 + 0], 1 pinsrb m7, [r2 + 6 + 32], 0 pmaddubsw m4, m7, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 358 * 16], m4 ; mode 24 [row 7 - first half] pmaddubsw m4, m7, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 359 * 16], m4 ; mode 24 [row 8 - first half] pmaddubsw m4, m7, [r3 + 19 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 360 * 16], m4 ; mode 24 [row 9 - first half] pmaddubsw m4, m7, [r3 + 14 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 361 * 16], m4 ; mode 24 [row 10 - first half] pmaddubsw m4, m7, [r3 + 9 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 362 * 16], m4 ; mode 24 [row 11 - first half] pmaddubsw m4, m7, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 363 * 16], m4 ; mode 24 [row 12 - first half] pslldq m7, 2 pinsrb m7, [r2 + 6 + 32], 1 pinsrb m7, [r2 + 13 + 32], 0 pmaddubsw m4, m7, [r3 + 31 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 364 * 16], m4 ; mode 24 [row 13 - first half] pmaddubsw m4, m7, [r3 + 26 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 365 * 16], m4 ; mode 24 [row 14 - first half] pmaddubsw m4, m7, [r3 + 21 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 366 * 16], m4 ; mode 24 [row 15 - first half] pmaddubsw m4, m7, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 367 * 16], m4 ; mode 23 [row 3 - first half] pslldq m7, m0, 2 pinsrb m7, [r2 + 0], 1 pinsrb m7, [r2 + 4 + 32], 0 pmaddubsw m4, m7, [r3 + 28 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 339 * 16], m4 ; mode 23 [row 4 - first half] pmaddubsw m4, m7, [r3 + 19 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 340 * 16], m4 ; mode 23 [row 5 - first half] pmaddubsw m4, m7, [r3 + 10 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 341 * 16], m4 ; mode 23 [row 6 - first half] pmaddubsw m4, m7, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 342 * 16], m4 ; mode 23 [row 7 - first half] pslldq m7, 2 pinsrb m7, [r2 + 4 + 32], 1 pinsrb m7, [r2 + 7 + 32], 0 pmaddubsw m4, m7, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 343 * 16], m4 ; mode 23 [row 8 - first half] pmaddubsw m4, m7, [r3 + 15 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 344 * 16], m4 ; mode 23 [row 9 - first half] pmaddubsw m4, m7, [r3 + 6 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 345 * 16], m4 ; mode 23 [row 10 - first half] pslldq m7, 2 pinsrb m7, [r2 + 7 + 32], 1 pinsrb m7, [r2 + 11 + 32], 0 pmaddubsw m4, m7, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 346 * 16], m4 ; mode 23 [row 11 - first half] pmaddubsw m4, m7, [r3 + 20 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 347 * 16], m4 ; mode 23 [row 12 - first half] pmaddubsw m4, m7, [r3 + 11 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 348 * 16], m4 ; mode 23 [row 13 - first half] pmaddubsw m4, m7, [r3 + 2 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 349 * 16], m4 ; mode 23 [row 14 - first half] pslldq m7, 2 pinsrb m7, [r2 + 11 + 32], 1 pinsrb m7, [r2 + 14 + 32], 0 pmaddubsw m4, m7, [r3 + 25 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 350 * 16], m4 ; mode 23 [row 15 - first half] pmaddubsw m4, m7, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 351 * 16], m4 ; mode 21 [row 15 - first half] pmaddubsw m4, m0, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 319 * 16 + 8], m4 ; mode 21 [row 15 - second half] end ; mode 20 [row 1 - first half] pslldq m7, m0, 2 pinsrb m7, [r2 + 0], 1 pinsrb m7, [r2 + 2 + 32], 0 pmaddubsw m4, m7, [r3 + 22 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 289 * 16], m4 ; mode 20 [row 2 - first half] pmaddubsw m4, m7, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 290 * 16], m4 ; mode 21 [row 1 - first half] pmaddubsw m4, m7, [r3 + 30 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 305 * 16], m4 ; mode 21 [row 2 - first half] pmaddubsw m4, m7, [r3 + 13 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 306 * 16], m4 ; mode 22 [row 2 - first half] pmaddubsw m4, m7, [r3 + 25 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 322 * 16], m4 ; mode 22 [row 3 - first half] pmaddubsw m4, m7, [r3 + 12 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 323 * 16], m4 ; mode 22 [row 4 - first half] pslldq m1, m7, 2 pinsrb m1, [r2 + 2 + 32], 1 pinsrb m1, [r2 + 5 + 32], 0 pmaddubsw m4, m1, [r3 + 31 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 324 * 16], m4 ; mode 22 [row 5 - first half] pmaddubsw m4, m1, [r3 + 18 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 325 * 16], m4 ; mode 22 [row 6 - first half] pmaddubsw m4, m1, [r3 + 5 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 326 * 16], m4 ; mode 22 [row 7 - first half] pslldq m1, 2 pinsrb m1, [r2 + 5 + 32], 1 pinsrb m1, [r2 + 7 + 32], 0 pmaddubsw m4, m1, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 327 * 16], m4 ; mode 22 [row 8 - first half] pmaddubsw m4, m1, [r3 + 11 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 328 * 16], m4 ; mode 22 [row 9 - first half] pslldq m1, 2 pinsrb m1, [r2 + 7 + 32], 1 pinsrb m1, [r2 + 10 + 32], 0 pmaddubsw m4, m1, [r3 + 30 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 329 * 16], m4 ; mode 22 [row 10 - first half] pmaddubsw m4, m1, [r3 + 17 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 330 * 16], m4 ; mode 22 [row 11 - first half] pmaddubsw m4, m1, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 331 * 16], m4 ; mode 22 [row 12 - first half] pslldq m1, 2 pinsrb m1, [r2 + 10 + 32], 1 pinsrb m1, [r2 + 12 + 32], 0 pmaddubsw m4, m1, [r3 + 23 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 332 * 16], m4 ; mode 22 [row 13 - first half] pmaddubsw m4, m1, [r3 + 10 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 333 * 16], m4 ; mode 22 [row 14 - first half] pslldq m1, 2 pinsrb m1, [r2 + 12 + 32], 1 pinsrb m1, [r2 + 15 + 32], 0 pmaddubsw m4, m1, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 334 * 16], m4 ; mode 22 [row 15 - first half] pmaddubsw m4, m1, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 335 * 16], m4 ; mode 21 [row 3 - first half] pslldq m6, m7, 2 pinsrb m6, [r2 + 2 + 32], 1 pinsrb m6, [r2 + 4 + 32], 0 pmaddubsw m4, m6, [r3 + 28 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 307 * 16], m4 ; mode 21 [row 4 - first half] pmaddubsw m4, m6, [r3 + 11 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 308 * 16], m4 ; mode 21 [row 5 - first half] pslldq m6, 2 pinsrb m6, [r2 + 4 + 32], 1 pinsrb m6, [r2 + 6 + 32], 0 pmaddubsw m4, m6, [r3 + 26 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 309 * 16], m4 ; mode 21 [row 6 - first half] pmaddubsw m4, m6, [r3 + 9 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 310 * 16], m4 ; mode 21 [row 7 - first half] pslldq m6, 2 pinsrb m6, [r2 + 6 + 32], 1 pinsrb m6, [r2 + 8 + 32], 0 pmaddubsw m4, m6, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 311 * 16], m4 ; mode 21 [row 8 - first half] pmaddubsw m4, m6, [r3 + 7 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 312 * 16], m4 ; mode 21 [row 9 - first half] pslldq m6, 2 pinsrb m6, [r2 + 8 + 32], 1 pinsrb m6, [r2 + 9 + 32], 0 pmaddubsw m4, m6, [r3 + 22 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 313 * 16], m4 ; mode 21 [row 10 - first half] pmaddubsw m4, m6, [r3 + 5 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 314 * 16], m4 ; mode 21 [row 11 - first half] pslldq m6, 2 pinsrb m6, [r2 + 9 + 32], 1 pinsrb m6, [r2 + 11 + 32], 0 pmaddubsw m4, m6, [r3 + 20 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 315 * 16], m4 ; mode 21 [row 12 - first half] pmaddubsw m4, m6, [r3 + 3 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 316 * 16], m4 ; mode 21 [row 13 - first half] pslldq m6, 2 pinsrb m6, [r2 + 11 + 32], 1 pinsrb m6, [r2 + 13 + 32], 0 pmaddubsw m4, m6, [r3 + 18 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 317 * 16], m4 ; mode 21 [row 14 - first half] pmaddubsw m4, m6, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 318 * 16], m4 ; mode 21 [row 15 - first half] pslldq m6, 2 pinsrb m6, [r2 + 32 + 13], 1 pinsrb m6, [r2 + 32 + 15], 0 pmaddubsw m4, m6, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 319 * 16], m4 ; mode 20 [row 13 - second half] pmaddubsw m4, m7, [r3 + 26 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 301 * 16 + 8], m4 ; mode 20 [row 13 - second half] ; mode 20 [row 14 - second half] pmaddubsw m4, m7, [r3 + 5 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 302 * 16 + 8], m4 ; mode 20 [row 14 - second half] ; mode 20 [row 3 - first half] pslldq m7, 2 pinsrb m7, [r2 + 32 + 2], 1 pinsrb m7, [r2 + 32 + 3], 0 pmaddubsw m4, m7, [r3 + 12 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 291 * 16], m4 ; mode 20 [row 15 - second half] pmaddubsw m4, m7, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 303 * 16 + 8], m4 ; mode 20 [row 15 - second half] ; mode 20 [row 4 - first half] pslldq m7, 2 pinsrb m7, [r2 + 32 + 3], 1 pinsrb m7, [r2 + 32 + 5], 0 pmaddubsw m4, m7, [r3 + 23 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 292 * 16], m4 ; mode 20 [row 5 - first half] pmaddubsw m4, m7, [r3 + 2 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 293 * 16], m4 ; mode 20 [row 6 - first half] pslldq m7, 2 pinsrb m7, [r2 + 32 + 5], 1 pinsrb m7, [r2 + 32 + 6], 0 pmaddubsw m4, m7, [r3 + 13 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 294 * 16], m4 ; mode 20 [row 7 - first half] pslldq m7, 2 pinsrb m7, [r2 + 32 + 6], 1 pinsrb m7, [r2 + 32 + 8], 0 pmaddubsw m4, m7, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 295 * 16], m4 ; mode 20 [row 8 - first half] pmaddubsw m4, m7, [r3 + 3 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 296 * 16], m4 ; mode 20 [row 9 - first half] pslldq m7, 2 pinsrb m7, [r2 + 32 + 8], 1 pinsrb m7, [r2 + 32 + 9], 0 pmaddubsw m4, m7, [r3 + 14 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 297 * 16], m4 ; mode 20 [row 10 - first half] pslldq m7, 2 pinsrb m7, [r2 + 32 + 9], 1 pinsrb m7, [r2 + 32 + 11], 0 pmaddubsw m4, m7, [r3 + 25 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 298 * 16], m4 ; mode 20 [row 11 - first half] pmaddubsw m4, m7, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 299 * 16], m4 ; mode 20 [row 12 - first half] movu m1, [r3 + 15 * 16] pslldq m7, 2 pinsrb m7, [r2 + 32 + 11], 1 pinsrb m7, [r2 + 32 + 12], 0 pmaddubsw m4, m7, [r3 + 15 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 300 * 16], m4 ; mode 20 [row 13 - first half] pslldq m7, 2 pinsrb m7, [r2 + 32 + 12], 1 pinsrb m7, [r2 + 32 + 14], 0 pmaddubsw m4, m7, [r3 + 26 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 301 * 16], m4 ; mode 20 [row 14 - first half] pmaddubsw m4, m7, [r3 + 5 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 302 * 16], m4 ; mode 20 [row 15 - first half] pslldq m7, 2 pinsrb m7, [r2 + 32 + 14], 1 pinsrb m7, [r2 + 32 + 15], 0 pmaddubsw m4, m7, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 303 * 16], m4 ; mode 19 [row 1] pslldq m0, 2 pinsrb m0, [r2], 1 pinsrb m0, [r2 + 32 + 1], 0 pslldq m5, 2 pinsrb m5, [r2 + 8], 1 pinsrb m5, [r2 + 7], 0 ; mode 20 [row 1 - second half] pmaddubsw m4, m5, [r3 + 22 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 289 * 16 + 8], m4 ; mode 20 [row 1 - second half] end ; mode 20 [row 2 - second half] pmaddubsw m4, m5, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 290 * 16 + 8], m4 ; mode 20 [row 2 - second half] end ; mode 21 [row 2 - second half] pmaddubsw m4, m5, [r3 + 30 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 305 * 16 + 8], m4 ; mode 21 [row 2 - second half] end ; mode 21 [row 3 - second half] pmaddubsw m4, m5, [r3 + 13 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 306 * 16 + 8], m4 ; mode 21 [row 3 - second half] end ; mode 21 [row 4 - second half] pmaddubsw m4, m5, [r3 + 11 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 307 * 16 + 8], m4 ; mode 21 [row 4 - second half] end ; mode 22 [row 2 - second half] pmaddubsw m4, m5, [r3 + 25 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 322 * 16 + 8], m4 ; mode 22 [row 2 - second half] end ; mode 22 [row 3 - second half] pmaddubsw m4, m5, [r3 + 12 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 323 * 16 + 8], m4 ; mode 22 [row 3 - second half] end ; mode 23 [row 3 - second half] pmaddubsw m4, m5, [r3 + 28 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 339 * 16 + 8], m4 ; mode 23 [row 3 - second half] end ; mode 23 [row 4 - second half] pmaddubsw m4, m5, [r3 + 19 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 340 * 16 + 8], m4 ; mode 23 [row 4 - second half] end ; mode 23 [row 5 - second half] pmaddubsw m4, m5, [r3 + 10 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 341 * 16 + 8], m4 ; mode 23 [row 5 - second half] end ; mode 23 [row 6 - second half] pmaddubsw m4, m5, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 342 * 16 + 8], m4 ; mode 23 [row 6 - second half] end ; mode 24 [row 6 - second half] pmaddubsw m4, m5, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 358 * 16 + 8], m4 ; mode 24 [row 6 - second half] end ; mode 24 [row 7 - second half] pmaddubsw m4, m5, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 359 * 16 + 8], m4 ; mode 24 [row 7 - second half] end ; mode 24 [row 8 - second half] pmaddubsw m4, m5, [r3 + 19 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 360 * 16 + 8], m4 ; mode 24 [row 8 - second half] end ; mode 24 [row 9 - second half] pmaddubsw m4, m5, [r3 + 14 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 361 * 16 + 8], m4 ; mode 24 [row 9 - second half] end ; mode 24 [row 10 - second half] pmaddubsw m4, m5, [r3 + 9 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 362 * 16 + 8], m4 ; mode 24 [row 10 - second half] end ; mode 24 [row 11 - second half] pmaddubsw m4, m5, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 363 * 16 + 8], m4 ; mode 24 [row 11 - second half] end pmaddubsw m4, m0, [r3 + 12 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 12 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 273 * 16], m4 ; mode 19 [row 2] pslldq m0, 2 pinsrb m0, [r2 + 32 + 1], 1 pinsrb m0, [r2 + 32 + 2], 0 pslldq m5, 2 pinsrb m5, [r2 + 7], 1 pinsrb m5, [r2 + 6], 0 ; mode 20 [row 3 - second half] pmaddubsw m4, m5, [r3 + 12 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 291 * 16 + 8], m4 ; mode 20 [row 3 - second half] end ; mode 21 [row 3 - second half] pmaddubsw m4, m5, [r3 + 28 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 307 * 16 + 8], m4 ; mode 21 [row 3 - second half] end ; mode 21 [row 4 - second half] pmaddubsw m4, m5, [r3 + 11 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 308 * 16 + 8], m4 ; mode 21 [row 4 - second half] end ; mode 22 [row 4 - second half] pmaddubsw m4, m5, [r3 + 31 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 324 * 16 + 8], m4 ; mode 22 [row 4 - second half] end ; mode 22 [row 5 - second half] pmaddubsw m4, m5, [r3 + 18 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 325 * 16 + 8], m4 ; mode 22 [row 5 - second half] end ; mode 22 [row 6 - second half] pmaddubsw m4, m5, [r3 + 5 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 326 * 16 + 8], m4 ; mode 22 [row 6 - second half] end ; mode 23 [row 7 - second half] pmaddubsw m4, m5, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 343 * 16 + 8], m4 ; mode 23 [row 7 - second half] end ; mode 23 [row 8 - second half] pmaddubsw m4, m5, [r3 + 15 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 344 * 16 + 8], m4 ; mode 23 [row 8 - second half] end ; mode 23 [row 9 - second half] pmaddubsw m4, m5, [r3 + 6 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 345 * 16 + 8], m4 ; mode 23 [row 9 - second half] end ; mode 24 [row 12 - second half] pmaddubsw m4, m5, [r3 + 31 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 364 * 16 + 8], m4 ; mode 24 [row 12 - second half] end ; mode 24 [row 13 - second half] pmaddubsw m4, m5, [r3 + 26 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 365 * 16 + 8], m4 ; mode 24 [row 13 - second half] end ; mode 24 [row 14 - second half] pmaddubsw m4, m5, [r3 + 21 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 366 * 16 + 8], m4 ; mode 24 [row 14 - second half] end ; mode 24 [row 15 - second half] pmaddubsw m4, m5, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 367 * 16 + 8], m4 ; mode 24 [row 15 - second half] end pmaddubsw m4, m0, [r3 + 18 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 18 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 274 * 16], m4 ; mode 19 [row 3] pslldq m0, 2 pinsrb m0, [r2 + 32 + 2], 1 pinsrb m0, [r2 + 32 + 4], 0 pslldq m5, 2 pinsrb m5, [r2 + 6], 1 pinsrb m5, [r2 + 5], 0 ; mode 20 [row 4 - second half] pmaddubsw m4, m5, [r3 + 23 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 292 * 16 + 8], m4 ; mode 20 [row 4 - second half] end ; mode 20 [row 5 - second half] pmaddubsw m4, m5, [r3 + 2 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 293 * 16 + 8], m4 ; mode 20 [row 5 - second half] end ; mode 21 [row 5 - second half] pmaddubsw m4, m5, [r3 + 26 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 309 * 16 + 8], m4 ; mode 21 [row 5 - second half] end ; mode 21 [row 6 - second half] pmaddubsw m4, m5, [r3 + 9 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 310 * 16 + 8], m4 ; mode 21 [row 6 - second half] end ; mode 22 [row 7 - second half] pmaddubsw m4, m5, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 327 * 16 + 8], m4 ; mode 22 [row 7 - second half] end ; mode 22 [row 8 - second half] pmaddubsw m4, m5, [r3 + 11 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 328 * 16 + 8], m4 ; mode 22 [row 7 - second half] end ; mode 23 [row 10 - second half] pmaddubsw m4, m5, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 346 * 16 + 8], m4 ; mode 23 [row 10 - second half] end ; mode 23 [row 11 - second half] pmaddubsw m4, m5, [r3 + 20 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 347 * 16 + 8], m4 ; mode 23 [row 11 - second half] end ; mode 23 [row 12 - second half] pmaddubsw m4, m5, [r3 + 11 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 348 * 16 + 8], m4 ; mode 23 [row 12 - second half] end ; mode 23 [row 13 - second half] pmaddubsw m4, m5, [r3 + 2 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 349 * 16 + 8], m4 ; mode 23 [row 13 - second half] end pmaddubsw m4, m0, [r3 + 24 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 24 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 275 * 16], m4 ; mode 19 [row 4] pslldq m0, 2 pinsrb m0, [r2 + 32 + 4], 1 pinsrb m0, [r2 + 32 + 5], 0 pslldq m5, 2 pinsrb m5, [r2 + 5], 1 pinsrb m5, [r2 + 4], 0 ; mode 20 [row 6 - second half] pmaddubsw m4, m5, [r3 + 13 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 294 * 16 + 8], m4 ; mode 20 [row 6 - second half] end ; mode 21 [row 7 - second half] pmaddubsw m4, m5, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 311 * 16 + 8], m4 ; mode 21 [row 7 - second half] end ; mode 21 [row 8 - second half] pmaddubsw m4, m5, [r3 + 7 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 312 * 16 + 8], m4 ; mode 21 [row 8 - second half] end ; mode 22 [row 9 - second half] pmaddubsw m4, m5, [r3 + 30 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 329 * 16 + 8], m4 ; mode 22 [row 9 - second half] end ; mode 22 [row 10 - second half] pmaddubsw m4, m5, [r3 + 17 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 330 * 16 + 8], m4 ; mode 22 [row 10 - second half] end ; mode 22 [row 11 - second half] pmaddubsw m4, m5, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 331 * 16 + 8], m4 ; mode 22 [row 11 - second half] end ; mode 23 [row 14 - second half] pmaddubsw m4, m5, [r3 + 25 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 350 * 16 + 8], m4 ; mode 23 [row 14 - second half] end ; mode 23 [row 15 - second half] pmaddubsw m4, m5, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 351 * 16 + 8], m4 ; mode 23 [row 15 - second half] end pmaddubsw m4, m0, [r3 + 30 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 30 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 276 * 16], m4 ; mode 19 [row 5] pmaddubsw m4, m0, [r3 + 4 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 4 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 277 * 16], m4 ; mode 19 [row 6] pslldq m0, 2 pinsrb m0, [r2 + 32 + 5], 1 pinsrb m0, [r2 + 32 + 6], 0 pslldq m5, 2 pinsrb m5, [r2 + 4], 1 pinsrb m5, [r2 + 3], 0 ; mode 20 [row 7 - second half] pmaddubsw m4, m5, [r3 + 24 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 295 * 16 + 8], m4 ; mode 20 [row 7 - second half] end ; mode 20 [row 8 - second half] pmaddubsw m4, m5, [r3 + 3 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 296 * 16 + 8], m4 ; mode 20 [row 8 - second half] end ; mode 21 [row 9 - second half] pmaddubsw m4, m5, [r3 + 22 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 313 * 16 + 8], m4 ; mode 21 [row 9 - second half] end ; mode 21 [row 10 - second half] pmaddubsw m4, m5, [r3 + 5 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 314 * 16 + 8], m4 ; mode 21 [row 10 - second half] end ; mode 22 [row 12 - second half] pmaddubsw m4, m5, [r3 + 23 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 332 * 16 + 8], m4 ; mode 22 [row 12 - second half] end ; mode 22 [row 12 - second half] pmaddubsw m4, m5, [r3 + 10 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 333 * 16 + 8], m4 ; mode 22 [row 12 - second half] end pmaddubsw m4, m0, [r3 + 10 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 10 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 278 * 16], m4 ; mode 19 [row 7] pslldq m0, 2 pinsrb m0, [r2 + 32 + 6], 1 pinsrb m0, [r2 + 32 + 7], 0 pslldq m5, 2 pinsrb m5, [r2 + 3], 1 pinsrb m5, [r2 + 2], 0 ; mode 20 [row 9 - second half] pmaddubsw m4, m5, [r3 + 14 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 297 * 16 + 8], m4 ; mode 20 [row 9 - second half] ; mode 21 [row 11 - second half] pmaddubsw m4, m5, [r3 + 20 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 315 * 16 + 8], m4 ; mode 21 [row 11 - second half] end ; mode 21 [row 12 - second half] pmaddubsw m4, m5, [r3 + 3 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 316 * 16 + 8], m4 ; mode 21 [row 12 - second half] end ; mode 22 [row 14 - second half] pmaddubsw m4, m5, [r3 + 29 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 334 * 16 + 8], m4 ; mode 22 [row 14 - second half] end ; mode 22 [row 15 - second half] pmaddubsw m4, m5, [r3 + 16 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 335 * 16 + 8], m4 ; mode 22 [row 15 - second half] end pmaddubsw m4, m0, [r3 + 16 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 16 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 279 * 16], m4 ; mode 19 [row 8] pslldq m0, 2 pinsrb m0, [r2 + 32 + 7], 1 pinsrb m0, [r2 + 32 + 9], 0 pslldq m5, 2 pinsrb m5, [r2 + 2], 1 pinsrb m5, [r2 + 1], 0 ; mode 20 [row 10 - second half] pmaddubsw m4, m5, [r3 + 25 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 298 * 16 + 8], m4 ; mode 20 [row 10 - second half] end ; mode 20 [row 11 - second half] pmaddubsw m4, m5, [r3 + 4 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 299 * 16 + 8], m4 ; mode 20 [row 11 - second half] end ; mode 21 [row 13 - second half] pmaddubsw m4, m5, [r3 + 18 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 317 * 16 + 8], m4 ; mode 21 [row 13 - second half] end ; mode 21 [row 14 - second half] pmaddubsw m4, m5, [r3 + 1 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 318 * 16 + 8], m4 ; mode 21 [row 14 - second half] end pmaddubsw m4, m0, [r3 + 22 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 22 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 280 * 16], m4 ; mode 19 [row 9] pslldq m0, 2 pinsrb m0, [r2 + 32 + 9], 1 pinsrb m0, [r2 + 32 + 10], 0 pslldq m5, 2 pinsrb m5, [r2 + 1], 1 pinsrb m5, [r2 + 0], 0 ; mode 20 [row 12 - second half] pmaddubsw m4, m5, [r3 + 15 * 16] pmulhrsw m4, m3 packuswb m4, m4 movh [r0 + 300 * 16 + 8], m4 ; mode 20 [row 12 - second half] end pmaddubsw m4, m0, [r3 + 28 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 28 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 281 * 16], m4 ; mode 19 [row 10] pmaddubsw m4, m0, [r3 + 2 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 2 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 282 * 16], m4 ; mode 19 [row 11] pslldq m0, 2 pinsrb m0, [r2 + 32 + 10], 1 pinsrb m0, [r2 + 32 + 11], 0 pmaddubsw m4, m0, [r3 + 8 * 16] pmulhrsw m4, m3 pslldq m5, 2 pinsrb m5, [r2], 1 pinsrb m5, [r2 + 32 + 1], 0 pmaddubsw m6, m5, [r3 + 8 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 283 * 16], m4 ; mode 19 [row 12] pslldq m0, 2 pinsrb m0, [r2 + 32 + 11], 1 pinsrb m0, [r2 + 32 + 12], 0 pslldq m5, 2 pinsrb m5, [r2 + 32 + 1], 1 pinsrb m5, [r2 + 32 + 2], 0 pmaddubsw m4, m0, [r3 + 14 * 16] pmulhrsw m4, m3 pmaddubsw m6, m5, [r3 + 14 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 284 * 16], m4 ; mode 19 [row 13] pslldq m0, 2 pinsrb m0, [r2 + 32 + 12], 1 pinsrb m0, [r2 + 32 + 14], 0 pmaddubsw m4, m0, [r3 + 20 * 16] pmulhrsw m4, m3 pslldq m5, 2 pinsrb m5, [r2 + 32 + 2], 1 pinsrb m5, [r2 + 32 + 4], 0 pmaddubsw m6, m5, [r3 + 20 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 285 * 16], m4 ; mode 19 [row 14] pslldq m0, 2 pinsrb m0, [r2 + 32 + 14], 1 pinsrb m0, [r2 + 32 + 15], 0 pmaddubsw m4, m0, [r3 + 26 * 16] pmulhrsw m4, m3 pslldq m5, 2 pinsrb m5, [r2 + 32 + 4], 1 pinsrb m5, [r2 + 32 + 5], 0 pmaddubsw m6, m5, [r3 + 26 * 16] pmulhrsw m6, m3 packuswb m4, m6 movu [r0 + 286 * 16], m4 ; mode 19 [row 15] movu m0, [r2 + 32] pshufb m0, [tab_S1] movu [r0 + 287 * 16], m0 movd m1, [r2] movd [r0 + 287 * 16 + 12], m1 ; mode 25 movu m1, [r1] ; mode 26 [all rows] psrldq m6, m1, 1 pinsrb m6, [r1 + 16], 15 movu m7, m6 movu [r0 + 384 * 16], m6 movu [r0 + 385 * 16], m6 movu [r0 + 386 * 16], m6 movu [r0 + 387 * 16], m6 movu [r0 + 388 * 16], m6 movu [r0 + 389 * 16], m6 movu [r0 + 390 * 16], m6 movu [r0 + 391 * 16], m6 movu [r0 + 392 * 16], m6 movu [r0 + 393 * 16], m6 movu [r0 + 394 * 16], m6 movu [r0 + 395 * 16], m6 movu [r0 + 396 * 16], m6 movu [r0 + 397 * 16], m6 movu [r0 + 398 * 16], m6 movu [r0 + 399 * 16], m6 pxor m0, m0 pshufb m6, m6, m0 punpcklbw m6, m0 pinsrb m2, [r1], 0 pshufb m2, m2, m0 punpcklbw m2, m0 movu m4, [r1 + 1 + 32] punpcklbw m5, m4, m0 punpckhbw m4, m0 psubw m5, m2 psubw m4, m2 psraw m5, 1 psraw m4, 1 paddw m5, m6 paddw m4, m6 packuswb m5, m4 pextrb [r0 + 384 * 16], m5, 0 pextrb [r0 + 385 * 16], m5, 1 pextrb [r0 + 386 * 16], m5, 2 pextrb [r0 + 387 * 16], m5, 3 pextrb [r0 + 388 * 16], m5, 4 pextrb [r0 + 389 * 16], m5, 5 pextrb [r0 + 390 * 16], m5, 6 pextrb [r0 + 391 * 16], m5, 7 pextrb [r0 + 392 * 16], m5, 8 pextrb [r0 + 393 * 16], m5, 9 pextrb [r0 + 394 * 16], m5, 10 pextrb [r0 + 395 * 16], m5, 11 pextrb [r0 + 396 * 16], m5, 12 pextrb [r0 + 397 * 16], m5, 13 pextrb [r0 + 398 * 16], m5, 14 pextrb [r0 + 399 * 16], m5, 15 ; mode 25 [row 15] movu [r0 + 383 * 16], m1 ; mode 25 [row 0] psrldq m2, m1, 1 punpcklbw m1, m2 movu m2, [r1 + 8] psrldq m4, m2, 1 punpcklbw m2, m4 pmaddubsw m4, m1, [r3 + 30 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 30 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 368 * 16], m4 ; mode 25 [row 1] pmaddubsw m4, m1, [r3 + 28 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 28 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 369 * 16], m4 ; mode 25 [row 2] pmaddubsw m4, m1, [r3 + 26 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 26 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 370 * 16], m4 ; mode 25 [row 3] pmaddubsw m4, m1, [r3 + 24 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 24 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 371 * 16], m4 ; mode 25 [row 4] pmaddubsw m4, m1, [r3 + 22 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 22 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 372 * 16], m4 ; mode 25 [row 5] pmaddubsw m4, m1, [r3 + 20 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 20 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 373 * 16], m4 ; mode 25 [row 6] pmaddubsw m4, m1, [r3 + 18 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 18 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 374 * 16], m4 ; mode 25 [row 7] pmaddubsw m4, m1, [r3 + 16 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 375 * 16], m4 ; mode 25 [row 8] pmaddubsw m4, m1, [r3 + 14 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 14 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 376 * 16], m4 ; mode 25 [row 9] pmaddubsw m4, m1, [r3 + 12 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 12 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 377 * 16], m4 ; mode 25 [row 10] pmaddubsw m4, m1, [r3 + 10 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 10 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 378 * 16], m4 ; mode 25 [row 11] pmaddubsw m4, m1, [r3 + 8 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 8 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 379 * 16], m4 ; mode 25 [row 12] pmaddubsw m4, m1, [r3 + 6 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 6 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 380 * 16], m4 ; mode 25 [row 13] pmaddubsw m4, m1, [r3 + 4 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 4 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 381 * 16], m4 ; mode 25 [row 14] pmaddubsw m4, m1, [r3 + 2 * 16] pmulhrsw m4, m3 pmaddubsw m5, m2, [r3 + 2 * 16] pmulhrsw m5, m3 packuswb m4, m5 movu [r0 + 382 * 16], m4 ; mode 27 [row 15] psrldq m6, m7, 1 punpcklbw m7, m6 pinsrb m6, [r1 + 17], 15 movu [r0 + 415 * 16], m6 ; mode 27 [row 0] movu m4, [r1 + 9] psrldq m5, m4, 1 punpcklbw m4, m5 pmaddubsw m6, m7, [r3 + 2 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 2 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 400 * 16], m6 ; mode 27 [row 1] pmaddubsw m6, m7, [r3 + 4 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 4 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 401 * 16], m6 ; mode 27 [row 2] pmaddubsw m6, m7, [r3 + 6 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 6 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 402 * 16], m6 ; mode 27 [row 3] pmaddubsw m6, m7, [r3 + 8 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 8 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 403 * 16], m6 ; mode 27 [row 4] pmaddubsw m6, m7, [r3 + 10 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 10 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 404 * 16], m6 ; mode 27 [row 5] pmaddubsw m6, m7, [r3 + 12 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 12 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 405 * 16], m6 ; mode 27 [row 6] pmaddubsw m6, m7, [r3 + 14 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 14 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 406 * 16], m6 ; mode 27 [row 7] pmaddubsw m6, m7, [r3 + 16 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 407 * 16], m6 ; mode 27 [row 8] pmaddubsw m6, m7, [r3 + 18 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 18 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 408 * 16], m6 ; mode 27 [row 9] pmaddubsw m6, m7, [r3 + 20 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 20 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 409 * 16], m6 ; mode 27 [row 10] pmaddubsw m6, m7, [r3 + 22 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 22 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 410 * 16], m6 ; mode 27 [row 11] pmaddubsw m6, m7, [r3 + 24 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 24 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 411 * 16], m6 ; mode 27 [row 12] pmaddubsw m6, m7, [r3 + 26 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 26 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 412 * 16], m6 ; mode 27 [row 13] pmaddubsw m6, m7, [r3 + 28 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 28 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 413 * 16], m6 ; mode 27 [row 14] pmaddubsw m6, m7, [r3 + 30 * 16] pmulhrsw m6, m3 pmaddubsw m5, m4, [r3 + 30 * 16] pmulhrsw m5, m3 packuswb m6, m5 movu [r0 + 414 * 16], m6 ; mode 28 [row 0] movu m1, [r2 + 1] psrldq m2, m1, 1 punpcklbw m1, m2 movu m4, [r2 + 9] psrldq m5, m4, 1 punpcklbw m4, m5 pmaddubsw m2, m1, [r3 + 5 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 5 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 416 * 16], m2 ; mode 28 [row 0] pmaddubsw m2, m1, [r3 + 5 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 5 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 416 * 16], m2 ; mode 28 [row 1] pmaddubsw m2, m1, [r3 + 10 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 10 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 417 * 16], m2 ; mode 28 [row 2] pmaddubsw m2, m1, [r3 + 15 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 15 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 418 * 16], m2 ; mode 28 [row 3] pmaddubsw m2, m1, [r3 + 20 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 20 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 419 * 16], m2 ; mode 28 [row 4] pmaddubsw m2, m1, [r3 + 25 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 25 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 420 * 16], m2 ; mode 28 [row 5] pmaddubsw m2, m1, [r3 + 30 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 30 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 421 * 16], m2 ; mode 29 [row 0] pmaddubsw m2, m1, [r3 + 9 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 9 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 432 * 16], m2 ; mode 29 [row 1] pmaddubsw m2, m1, [r3 + 18 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 18 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 433 * 16], m2 ; mode 29 [row 2] pmaddubsw m2, m1, [r3 + 27 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 27 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 434 * 16], m2 ; mode 30 [row 0] pmaddubsw m2, m1, [r3 + 13 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 13 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 448 * 16], m2 ; mode 30 [row 1] pmaddubsw m2, m1, [r3 + 26 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 26 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 449 * 16], m2 ; mode 33 [row 0] movu [r0 + 496 * 16], m2 ; mode 31 [row 0] pmaddubsw m2, m1, [r3 + 17 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 17 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 464 * 16], m2 ; mode 32 [row 0] pmaddubsw m2, m1, [r3 + 21 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 21 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 480 * 16], m2 ; mode 28 [row 6] movd m7, [r2 + 9] palignr m7, m1, 2 pmaddubsw m2, m7, [r3 + 3 * 16] pmulhrsw m2, m3 movd m6, [r2 + 17] palignr m6, m4, 2 pmaddubsw m5, m6, [r3 + 3 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 422 * 16], m2 ; mode 28 [row 7] pmaddubsw m2, m7, [r3 + 8 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 8 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 423 * 16], m2 ; mode 28 [row 8] pmaddubsw m2, m7, [r3 + 13 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 13 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 424 * 16], m2 ; mode 28 [row 9] pmaddubsw m2, m7, [r3 + 18 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 18 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 425 * 16], m2 ; mode 28 [row 10] pmaddubsw m2, m7, [r3 + 23 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 23 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 426 * 16], m2 ; mode 29 [row 3] pmaddubsw m2, m7, [r3 + 4 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 4 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 435 * 16], m2 ; mode 29 [row 4] pmaddubsw m2, m7, [r3 + 13 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 13 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 436 * 16], m2 ; mode 29 [row 5] pmaddubsw m2, m7, [r3 + 22 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 22 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 437 * 16], m2 ; mode 29 [row 6] pmaddubsw m2, m7, [r3 + 31 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 31 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 438 * 16], m2 ; mode 32 [row 2] movu [r0 + 482 * 16], m2 ; mode 30 [row 2] pmaddubsw m2, m7, [r3 + 7 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 7 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 450 * 16], m2 ; mode 30 [row 3] pmaddubsw m2, m7, [r3 + 20 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 20 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 451 * 16], m2 ; mode 33 [row 1] movu [r0 + 497 * 16], m2 ; mode 31 [row 1] pmaddubsw m2, m7, [r3 + 2 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 2 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 465 * 16], m2 ; mode 31 [row 2] pmaddubsw m2, m7, [r3 + 19 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 19 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 466 * 16], m2 ; mode 32 [row 1] pmaddubsw m2, m7, [r3 + 10 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 10 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 481 * 16], m2 ; mode 28 [row 11] pmaddubsw m2, m7, [r3 + 28 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 28 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 427 * 16], m2 ; mode 28 [row 12] movd m1, [r2 + 10] palignr m1, m7, 2 pmaddubsw m2, m1, [r3 + 1 * 16] pmulhrsw m2, m3 movd m4, [r2 + 18] palignr m4, m6, 2 pmaddubsw m5, m4, [r3 + 1 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 428 * 16], m2 ; mode 30 [row 4] movu [r0 + 452 * 16], m2 ; mode 28 [row 13] pmaddubsw m2, m1, [r3 + 6 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 6 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 429 * 16], m2 ; mode 28 [row 14] pmaddubsw m2, m1, [r3 + 11 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 11 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 430 * 16], m2 ; mode 28 [row 15] pmaddubsw m2, m1, [r3 + 16 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 431 * 16], m2 ; mode 29 [row 7] pmaddubsw m2, m1, [r3 + 8 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 8 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 439 * 16], m2 ; mode 29 [row 8] pmaddubsw m2, m1, [r3 + 17 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 17 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 440 * 16], m2 ; mode 29 [row 9] pmaddubsw m2, m1, [r3 + 26 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 26 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 441 * 16], m2 ; mode 30 [row 5] pmaddubsw m2, m1, [r3 + 14 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 14 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 453 * 16], m2 ; mode 33 [row 2] movu [r0 + 498 * 16], m2 ; mode 30 [row 6] pmaddubsw m2, m1, [r3 + 27 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 27 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 454 * 16], m2 ; mode 31 [row 3] pmaddubsw m2, m1, [r3 + 4 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 4 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 467 * 16], m2 ; mode 31 [row 4] pmaddubsw m2, m1, [r3 + 21 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 21 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 468 * 16], m2 ; mode 32 [row 3] pmaddubsw m2, m1, [r3 + 20 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 20 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 483 * 16], m2 ; mode 29 [row 10] movd m7, [r2 + 11] palignr m7, m1, 2 pmaddubsw m2, m7, [r3 + 3 * 16] pmulhrsw m2, m3 movd m6, [r2 + 19] palignr m6, m4, 2 pmaddubsw m5, m6, [r3 + 3 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 442 * 16], m2 ; mode 29 [row 11] pmaddubsw m2, m7, [r3 + 12 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 12 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 443 * 16], m2 ; mode 29 [row 12] pmaddubsw m2, m7, [r3 + 21 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 21 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 444 * 16], m2 ; mode 30 [row 8] movu [r0 + 456 * 16], m2 ; mode 29 [row 13] pmaddubsw m2, m7, [r3 + 30 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 30 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 445 * 16], m2 ; mode 32 [row 5] movu [r0 + 485 * 16], m2 ; mode 30 [row 7] pmaddubsw m2, m7, [r3 + 8 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 8 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 455 * 16], m2 ; mode 33 [row 3] movu [r0 + 499 * 16], m2 ; mode 31 [row 5] pmaddubsw m2, m7, [r3 + 6 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 6 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 469 * 16], m2 ; mode 31 [row 6] pmaddubsw m2, m7, [r3 + 23 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 23 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 470 * 16], m2 ; mode 32 [row 4] pmaddubsw m2, m7, [r3 + 9 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 9 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 484 * 16], m2 movu m1, m7 movu m4, m6 ; mode 29 [row 14] movu m1, [r2 + 12] palignr m1, m7, 2 pmaddubsw m2, m1, [r3 + 7 * 16] pmulhrsw m2, m3 movd m4, [r2 + 20] palignr m4, m6, 2 pmaddubsw m5, m4, [r3 + 7 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 446 * 16], m2 ; mode 29 [row 15] pmaddubsw m2, m1, [r3 + 16 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 447 * 16], m2 ; mode 30 [row 9] pmaddubsw m2, m1, [r3 + 2 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 2 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 457 * 16], m2 ; mode 33 [row 4] movu [r0 + 500 * 16], m2 ; mode 30 [row 10] pmaddubsw m2, m1, [r3 + 15 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 15 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 458 * 16], m2 ; mode 30 [row 11] pmaddubsw m2, m1, [r3 + 28 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 28 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 459 * 16], m2 ; mode 33 [row 5] movu [r0 + 501 * 16], m2 ; mode 31 [row 7] pmaddubsw m2, m1, [r3 + 8 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 8 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 471 * 16], m2 ; mode 31 [row 8] pmaddubsw m2, m1, [r3 + 25 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 25 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 472 * 16], m2 ; mode 32 [row 6] pmaddubsw m2, m1, [r3 + 19 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 19 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 486 * 16], m2 ; mode 30 [row 12] movd m7, [r2 + 13] palignr m7, m1, 2 pmaddubsw m2, m7, [r3 + 9 * 16] pmulhrsw m2, m3 movd m6, [r2 + 21] palignr m6, m4, 2 pmaddubsw m5, m6, [r3 + 9 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 460 * 16], m2 ; mode 30 [row 13] pmaddubsw m2, m7, [r3 + 22 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 22 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 461 * 16], m2 ; mode 33 [row 6] movu [r0 + 502 * 16], m2 ; mode 31 [row 9] pmaddubsw m2, m7, [r3 + 10 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 10 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 473 * 16], m2 ; mode 31 [row 10] pmaddubsw m2, m7, [r3 + 27 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 27 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 474 * 16], m2 ; mode 32 [row 7] pmaddubsw m2, m7, [r3 + 8 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 8 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 487 * 16], m2 ; mode 32 [row 8] pmaddubsw m2, m7, [r3 + 29 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 29 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 488 * 16], m2 movu m1, m7 movu m4, m6 ; mode 30 [row 14] movd m1, [r2 + 14] palignr m1, m7, 2 pmaddubsw m2, m1, [r3 + 3 * 16] pmulhrsw m2, m3 movd m4, [r2 + 22] palignr m4, m6, 2 pmaddubsw m5, m4, [r3 + 3 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 462 * 16], m2 ; mode 30 [row 15] pmaddubsw m2, m1, [r3 + 16 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 463 * 16], m2 ; mode 33 [row 7] movu [r0 + 503 * 16], m2 ; mode 31 [row 11] pmaddubsw m2, m1, [r3 + 12 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 12 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 475 * 16], m2 ; mode 31 [row 12] pmaddubsw m2, m1, [r3 + 29 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 29 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 476 * 16], m2 ; mode 32 [row 9] pmaddubsw m2, m1, [r3 + 18 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 18 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 489 * 16], m2 ; mode 31 [row 13] movd m7, [r2 + 15] palignr m7, m1, 2 pmaddubsw m2, m7, [r3 + 14 * 16] pmulhrsw m2, m3 movd m6, [r2 + 23] palignr m6, m4, 2 pmaddubsw m5, m6, [r3 + 14 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 477 * 16], m2 ; mode 31 [row 14] pmaddubsw m2, m7, [r3 + 31 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 31 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 478 * 16], m2 ; mode 32 [row 10] pmaddubsw m2, m7, [r3 + 7 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 7 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 490 * 16], m2 ; mode 32 [row 11] pmaddubsw m2, m7, [r3 + 28 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 28 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 491 * 16], m2 ; mode 33 [row 8] pmaddubsw m2, m7, [r3 + 10 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 10 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 504 * 16], m2 ; mode 31 [row 15] movd m1, [r2 + 16] palignr m1, m7, 2 pmaddubsw m2, m1, [r3 + 16 * 16] pmulhrsw m2, m3 movd m4, [r2 + 24] palignr m4, m6, 2 pmaddubsw m5, m4, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 479 * 16], m2 ; mode 32 [row 12] pmaddubsw m2, m1, [r3 + 17 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 17 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 492 * 16], m2 ; mode 33 [row 9] pmaddubsw m2, m1, [r3 + 4 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 4 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 505 * 16], m2 ; mode 33 [row 10] pmaddubsw m2, m1, [r3 + 30 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 30 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 506 * 16], m2 ; mode 33 [row 10] pmaddubsw m2, m1, [r3 + 4 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 4 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 505 * 16], m2 ; mode 32 [row 13] movd m7, [r2 + 17] palignr m7, m1, 2 pmaddubsw m2, m7, [r3 + 6 * 16] pmulhrsw m2, m3 movd m6, [r2 + 25] palignr m6, m4, 2 pmaddubsw m5, m6, [r3 + 6 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 493 * 16], m2 ; mode 32 [row 14] pmaddubsw m2, m7, [r3 + 27 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 27 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 494 * 16], m2 ; mode 33 [row 11] pmaddubsw m2, m7, [r3 + 24 * 16] pmulhrsw m2, m3 pmaddubsw m5, m6, [r3 + 24 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 507 * 16], m2 ; mode 32 [row 15] movd m1, [r2 + 18] palignr m1, m7, 2 pmaddubsw m2, m1, [r3 + 16 * 16] pmulhrsw m2, m3 psrldq m4, 2 pinsrb m4, [r2 + 26], 14 pinsrb m4, [r2 + 27], 15 movd m4, [r2 + 26] palignr m4, m6, 2 pmaddubsw m5, m4, [r3 + 16 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 495 * 16], m2 ; mode 33 [row 12] pmaddubsw m2, m1, [r3 + 18 * 16] pmulhrsw m2, m3 pmaddubsw m5, m4, [r3 + 18 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 508 * 16], m2 ; mode 33 [row 13] movd m7, [r2 + 19] palignr m7, m1, 2 pmaddubsw m2, m7, [r3 + 12 * 16] pmulhrsw m2, m3 movd m6, [r2 + 27] palignr m6, m4, 2 pmaddubsw m5, m6, [r3 + 12 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 509 * 16], m2 ; mode 33 [row 14] movd m1, [r2 + 20] palignr m1, m7, 2 pmaddubsw m2, m1, [r3 + 6 * 16] pmulhrsw m2, m3 movd m4, [r2 + 28] palignr m4, m6, 2 pmaddubsw m5, m4, [r3 + 6 * 16] pmulhrsw m5, m3 packuswb m2, m5 movu [r0 + 510 * 16], m2 ; mode 34 [row 0] movu m1, [r2 + 2] movu [r0 + 512 * 16], m1 movu m2, [r2 + 18] palignr m3, m2, m1, 1 movu [r0 + 513 * 16], m3 palignr m3, m2, m1, 2 movu [r0 + 514 * 16], m3 palignr m3, m2, m1, 3 movu [r0 + 515 * 16], m3 palignr m3, m2, m1, 4 movu [r0 + 516 * 16], m3 palignr m3, m2, m1, 5 movu [r0 + 517 * 16], m3 palignr m3, m2, m1, 6 movu [r0 + 518 * 16], m3 palignr m3, m2, m1, 7 movu [r0 + 519 * 16], m3 palignr m3, m2, m1, 8 movu [r0 + 520 * 16], m3 palignr m3, m2, m1, 9 movu [r0 + 521 * 16], m3 palignr m3, m2, m1, 10 movu [r0 + 522 * 16], m3 palignr m3, m2, m1, 11 movu [r0 + 523 * 16], m3 palignr m3, m2, m1, 12 movu [r0 + 524 * 16], m3 ; mode 33 [row 15] movu [r0 + 511 * 16], m3 ; mode 34 palignr m3, m2, m1, 13 movu [r0 + 525 * 16], m3 palignr m3, m2, m1, 14 movu [r0 + 526 * 16], m3 palignr m3, m2, m1, 15 movu [r0 + 527 * 16], m3 RET ;-------------------------------------------------------------------------------- ; void all_angs_pred_32x32(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) ;-------------------------------------------------------------------------------- INIT_XMM sse4 cglobal all_angs_pred_32x32, 3,7,8, 0-4 mov r6d, [r1 + 64] mov r3d, [r1] mov [rsp], r6d mov [r1 + 64], r3b mov r3d, [r2] mov r6d, [r2 + 64] mov [r2 + 64], r3b lea r3, [r2] lea r4, [r2 + 64] lea r2, [r1 + 64] ;mode 2[row 0] movu m0, [r4 + 2] movu [r0 + 0 * 16], m0 movu m1, [r4 + 18] movu [r0 + 1 * 16], m1 ;mode 9 [row 15] movu [r0 + 478 * 16], m0 movu [r0 + 479 * 16], m1 ;mode 2[row 1] movu m2, [r4 + 34] palignr m3, m1, m0, 1 movu [r0 + 2 * 16], m3 palignr m4, m2, m1, 1 movu [r0 + 3 * 16], m4 ; mode 9 [row 31] movu [r0 + 510 * 16], m3 movu [r0 + 511 * 16], m4 ;mode 2[row 17] movu [r0 + 34 * 16], m4 movu m5, [r4 + 35] movu [r0 + 35 * 16], m5 ;mode 2[row 2] palignr m3, m1, m0, 2 movu [r0 + 4 * 16], m3 palignr m4, m2, m1, 2 movu [r0 + 5 * 16], m4 ;mode 2[row 18] movu [r0 + 36 * 16], m4 movu m6, [r4 + 51] palignr m7, m6, m5, 1 movu [r0 + 37 * 16], m7 ;mode 2[row 3] palignr m3, m1, m0, 3 movu [r0 + 6 * 16], m3 palignr m4, m2, m1, 3 movu [r0 + 7 * 16], m4 ;mode 2[row 19] movu [r0 + 38 * 16], m4 palignr m7, m6, m5, 2 movu [r0 + 39 * 16], m7 ;mode 2[row 4] palignr m3, m1, m0, 4 movu [r0 + 8 * 16], m3 palignr m4, m2, m1, 4 movu [r0 + 9 * 16], m4 ; mode 8 [row 31] movu [r0 + 446 * 16], m3 movu [r0 + 447 * 16], m4 ;mode 2[row 20] movu [r0 + 40 * 16], m4 palignr m7, m6, m5, 3 movu [r0 + 41 * 16], m7 ; mode 4 [row 31] movu [r0 + 190 * 16], m4 movu [r0 + 191 * 16], m7 ;mode 2[row 5] palignr m3, m1, m0, 5 movu [r0 + 10 * 16], m3 palignr m4, m2, m1, 5 movu [r0 + 11 * 16], m4 ;mode 2[row 21] movu [r0 + 42 * 16], m4 palignr m7, m6, m5, 4 movu [r0 + 43 * 16], m7 ;mode 2[row 6] palignr m3, m1, m0, 6 movu [r0 + 12 * 16], m3 palignr m4, m2, m1, 6 movu [r0 + 13 * 16], m4 ;mode 2[row 22] movu [r0 + 44 * 16], m4 palignr m7, m6, m5, 5 movu [r0 + 45 * 16], m7 ;mode 2[row 7] palignr m3, m1, m0, 7 movu [r0 + 14 * 16], m3 palignr m4, m2, m1, 7 movu [r0 + 15 * 16], m4 ;mode 2[row 23] movu [r0 + 46 * 16], m4 palignr m7, m6, m5, 6 movu [r0 + 47 * 16], m7 ;mode 2[row 8] palignr m3, m1, m0, 8 movu [r0 + 16 * 16], m3 palignr m4, m2, m1, 8 movu [r0 + 17 * 16], m4 ;mode 7[row 31] movu [r0 + 382 * 16], m3 movu [r0 + 383 * 16], m4 ;mode 2[row 24] movu [r0 + 48 * 16], m4 palignr m7, m6, m5, 7 movu [r0 + 49 * 16], m7 ;mode 2[row 9] palignr m3, m1, m0, 9 movu [r0 + 18 * 16], m3 palignr m4, m2, m1, 9 movu [r0 + 19 * 16], m4 ;mode 2[row 25] movu [r0 + 50 * 16], m4 palignr m7, m6, m5, 8 movu [r0 + 51 * 16], m7 ; mode 3 [row 31] movu [r0 + 126 * 16], m4 movu [r0 + 127 * 16], m7 ;mode 2[row 10] palignr m3, m1, m0, 10 movu [r0 + 20 * 16], m3 palignr m4, m2, m1, 10 movu [r0 + 21 * 16], m4 ;mode 2[row 26] movu [r0 + 52 * 16], m4 palignr m7, m6, m5, 9 movu [r0 + 53 * 16], m7 ;mode 2[row 11] palignr m3, m1, m0, 11 movu [r0 + 22 * 16], m3 palignr m4, m2, m1, 11 movu [r0 + 23 * 16], m4 ;mode 2[row 27] movu [r0 + 54 * 16], m4 palignr m7, m6, m5, 10 movu [r0 + 55 * 16], m7 ;mode 2[row 12] palignr m3, m1, m0, 12 movu [r0 + 24 * 16], m3 palignr m4, m2, m1, 12 movu [r0 + 25 * 16], m4 ; mode 6 [row 31] movu [r0 + 318 * 16], m3 movu [r0 + 319 * 16], m4 ; mode 3 [row 15] movu [r0 + 94 * 16], m3 movu [r0 + 95 * 16], m4 ;mode 2[row 28] movu [r0 + 56 * 16], m4 palignr m7, m6, m5, 11 movu [r0 + 57 * 16], m7 ;mode 2[row 13] palignr m3, m1, m0, 13 movu [r0 + 26 * 16], m3 palignr m4, m2, m1, 13 movu [r0 + 27 * 16], m4 ;mode 2[row 29] movu [r0 + 58 * 16], m4 palignr m7, m6, m5, 12 movu [r0 + 59 * 16], m7 ;mode 2[row 14] palignr m3, m1, m0, 14 movu [r0 + 28 * 16], m3 palignr m4, m2, m1, 14 movu [r0 + 29 * 16], m4 ;mode 2[row 30] movu [r0 + 60 * 16], m4 palignr m7, m6, m5, 13 movu [r0 + 61 * 16], m7 ;mode 2[row 15] palignr m3, m1, m0, 15 movu [r0 + 30 * 16], m3 palignr m4, m2, m1, 15 movu [r0 + 31 * 16], m4 ;mode 2[row 31] movu [r0 + 62 * 16], m4 palignr m7, m6, m5, 14 movu [r0 + 63 * 16], m7 ;mode 2[row 16] movu [r0 + 32 * 16], m1 movu [r0 + 33 * 16], m2 ; mode 5[row 31] movu [r0 + 254 * 16], m1 movu [r0 + 255 * 16], m2 ; mode 3 [row 0] lea r5, [ang_table] movu m6, [r5 + 26 * 16] movu m7, [pw_1024 ] movu m1, [r4 + 1 ] punpcklbw m1, m0 pmaddubsw m0, m1, m6 pmulhrsw m0, m7 movu m2, [r4 + 9] movd m3, [r4 + 10] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m0, m3 movu [r0 + 64 * 16], m0 ; mode 6 [row 1 - first half] movu [r0 + 258 * 16], m0 ; mode 9 [row 12 - first half] movu [r0 + 472 * 16], m0 movu m0, [r4 + 17] movd m3, [r4 + 18] palignr m3, m0, 1 punpcklbw m0, m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m4, [r4 + 25] movd m5, [r4 + 26] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 65 * 16], m3 ; mode 6 [row 1 - second half] movu [r0 + 259 * 16], m3 ; mode 9 [row 12 - second half] movu [r0 + 473 * 16], m3 ; mode 4 [row 0] movu m6, [r5 + 21 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 128 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 129 * 16], m3 ; mode 5 [row 0] movu m6, [r5 + 17 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 192 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 193 * 16], m3 ; mode 6 [row 0] movu m6, [r5 + 13 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 256 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 257 * 16], m3 ; mode 7 [row 0] movu m6, [r5 + 9 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 320 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 321 * 16], m3 ; mode 7 [row 1] movu m6, [r5 + 18 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 322 * 16], m3 ; mode 9 [row 8 - first half] movu [r0 + 464 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 323 * 16], m3 ; mode 9 [row 8 - second half] movu [r0 + 465 * 16], m3 ; mode 7 [row 2] movu m6, [r5 + 27 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 324 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 325 * 16], m3 ; mode 8 [row 0] movu m6, [r5 + 5 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 384 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 385 * 16], m3 ; mode 8 [row 1] movu m6, [r5 + 10 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 386 * 16], m3 ; mode 9 [row 4 - first half] movu [r0 + 456 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 387 * 16], m3 ; mode 9 [row 4 - second half] movu [r0 + 457 * 16], m3 ; mode 8 [row 2] movu m6, [r5 + 15 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 388 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 389 * 16], m3 ; mode 8 [row 3] movu m6, [r5 + 20 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 390 * 16], m3 ; mode 9 [row 9 - first half] movu [r0 + 466 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 391 * 16], m3 ; mode 9 [row 9 - second half] movu [r0 + 467 * 16], m3 ; mode 8 [row 4] movu m6, [r5 + 25 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 392 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 393 * 16], m3 ; mode 8 [row 5] movu m6, [r5 + 30 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 394 * 16], m3 ; mode 9 [row 14 - first half] movu [r0 + 476 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 395 * 16], m3 ; mode 9 [row 14 - second half] movu [r0 + 477 * 16], m3 ; mode 9 [row 0] movu m6, [r5 + 2 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 448 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 449 * 16], m3 ; mode 9 [row 1] movu m6, [r5 + 4 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 450 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 451 * 16], m3 ; mode 9 [row 2] movu m6, [r5 + 6 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 452 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 453 * 16], m3 ; mode 9 [row 3] movu m6, [r5 + 8 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 454 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 455 * 16], m3 ; mode 9 [row 5] movu m6, [r5 + 12 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 458 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 459 * 16], m3 ; mode 9 [row 6] movu m6, [r5 + 14 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 460 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 461 * 16], m3 ; mode 9 [row 7] movu m6, [r5 + 16 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 462 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 463 * 16], m3 ; mode 9 [row 10] movu m6, [r5 + 22 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 468 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 469 * 16], m3 ; mode 9 [row 11] movu m6, [r5 + 24 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 470 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 471 * 16], m3 ; mode 9 [row 13] movu m6, [r5 + 28 * 16] pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 474 * 16], m3 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 475 * 16], m3 ; mode 3 [row 1] movu m6, [r5 + 20 * 16] movu m0, [r4 + 2] movd m1, [r4 + 3] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 10] movd m3, [r4 + 11] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 66 * 16], m1 ; mode 6 [row 3 - first half] movu [r0 + 262 * 16], m1 ; mode 9 [row 25 - first half] movu [r0 + 498 * 16], m1 movu m1, [r4 + 18] movd m3, [r4 + 19] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 26] movd m5, [r4 + 27] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 67 * 16], m3 ; mode 6 [row 3 - second half] movu [r0 + 263 * 16], m3 ; mode 9 [row 25 - second half] movu [r0 + 499 * 16], m3 ; mode 4 [row 1] movu m6, [r5 + 10 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 130 * 16], m3 ; mode 9 [row 20 - first half] movu [r0 + 488 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 131 * 16], m3 ; mode 9 [row 20 - second half] movu [r0 + 489 * 16], m3 ; mode 4 [row 2] movu m6, [r5 + 31 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 132 * 16], m3 ; mode 7 [row 6 - first half] movu [r0 + 332 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 133 * 16], m3 ; mode 7 [row 6 - second half] movu [r0 + 333 * 16], m3 ; mode 5 [row 1] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 194 * 16], m3 ; mode 5 [row 1 - first half] movu [r0 + 480 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 195 * 16], m3 ; mode 5 [row 1 - second half] movu [r0 + 481 * 16], m3 ; mode 5 [row 2] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 196 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 197 * 16], m3 ; mode 6 [row 2] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 260 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 261 * 16], m3 ; mode 7 [row 3] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 326 * 16], m3 ; mode 9 [row 17 - first half] movu [r0 + 482 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 327 * 16], m3 ; mode 9 [row 17 - second half] movu [r0 + 483 * 16], m3 ; mode 7 [row 4] movu m6, [r5 + 13 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 328 * 16], m3 ; mode 8 [row 8 - first half] movu [r0 + 400 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 329 * 16], m3 ; mode 8 [row 8 - second half] movu [r0 + 401 * 16], m3 ; mode 7 [row 5] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 330 * 16], m3 ; mode 9 [row 26 - first half] movu [r0 + 500 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 331 * 16], m3 ; mode 9 [row 26 - second half] movu [r0 + 501 * 16], m3 ; mode 8 [row 6] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 396 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 397 * 16], m3 ; mode 9 [row 18] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 484 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 485 * 16], m3 ; mode 9 [row 21] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 490 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 491 * 16], m3 ; mode 9 [row 22] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 492 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 493 * 16], m3 ; mode 9 [row 23] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 494 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 495 * 16], m3 ; mode 9 [row 27] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 502 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 503 * 16], m3 ; mode 9 [row 28] movu m6, [r5 + 26 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 504 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 505 * 16], m3 ; mode 9 [row 30] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 508 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 509 * 16], m3 ; mode 8 [row 7] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 398 * 16], m3 ; mode 9 [row 19 - first half] movu [r0 + 486 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 399 * 16], m3 ; mode 9 [row 19 - second half] movu [r0 + 487 * 16], m3 ; mode 8 [row 9] movu m6, [r5 + 18 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 402 * 16], m3 ; mode 9 [row 24 - first half] movu [r0 + 496 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 403 * 16], m3 ; mode 9 [row 24 - second half] movu [r0 + 497 * 16], m3 ; mode 8 [row 10] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 404 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 405 * 16], m3 ; mode 8 [row 11] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 406 * 16], m3 ; mode 9 [row 29 - first half] movu [r0 + 506 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 407 * 16], m3 ; mode 9 [row 29 - second half] movu [r0 + 507 * 16], m3 ; mode 3 [row 2] movu m6, [r5 + 14 * 16] movu m0, [r4 + 3] movd m1, [r4 + 4] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 11] movd m3, [r4 + 12] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 68 * 16], m1 ; mode 3 [row 2 - first half] movu [r0 + 266 * 16], m1 movu m1, [r4 + 19] movd m3, [r4 + 20] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 27] movd m5, [r4 + 28] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 69 * 16], m3 ; mode 3 [row 2 - second half] movu [r0 + 267 * 16], m3 ; mode 4 [row 3] movu m6, [r5 + 20 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 134 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 135 * 16], m3 ; mode 5 [row 3] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 198 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 199 * 16], m3 ; mode 5 [row 4] movu m6, [r5 + 21 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 200 * 16], m3 ; mode 8 [row 16 - first half] movu [r0 + 416 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 201 * 16], m3 ; mode 8 [row 16 - second half] movu [r0 + 417 * 16], m3 ; mode 6 [row 4] movu m6, [r5 + 1 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 264 * 16], m3 ; mode 6 [row 4 - first half] movu [r0 + 408 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 265 * 16], m3 ; mode 6 [row 4 - second half] movu [r0 + 409 * 16], m3 ; mode 6 [row 6] movu m6, [r5 + 27 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 268 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 269 * 16], m3 ; mode 7 [row 7] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 334 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 335 * 16], m3 ; mode 7 [row 8] movu m6, [r5 + 17 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 336 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 337 * 16], m3 ; mode 7 [row 9] movu m6, [r5 + 26 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 338 * 16], m3 ; mode 8 [row 17 - first half] movu [r0 + 418 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 339 * 16], m3 ; mode 8 [row 17 - second half] movu [r0 + 419 * 16], m3 ; mode 8 [row 13] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 410 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 411 * 16], m3 ; mode 8 [row 14] movu m6, [r5 + 11 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 412 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 413 * 16], m3 ; mode 8 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 414 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 415 * 16], m3 ; mode 8 [row 18] movu m6, [r5 + 31 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 420 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 421 * 16], m3 ; mode 3 [row 3] movu m6, [r5 + 8 * 16] movu m0, [r4 + 4] movd m1, [r4 + 5] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 12] movd m3, [r4 + 13] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 70 * 16], m1 ; mode 6 [row 7 - first half] movu [r0 + 270 * 16], m1 movu m1, [r4 + 20] movd m3, [r4 + 21] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 28] movd m5, [r4 + 29] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 71 * 16], m3 ; mode 6 [row 7 - second half] movu [r0 + 271 * 16], m3 ; mode 4 [row 4] movu m6, [r5 + 9 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 136 * 16], m3 ; mode 4 [row 4 - first half] movu [r0 + 424 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 137 * 16], m3 ; mode 4 [row 4 - second half] movu [r0 + 425 * 16], m3 ; mode 4 [row 5] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 138 * 16], m3 ; mode 7 [row 13 - first half] movu [r0 + 346 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 139 * 16], m3 ; mode 7 [row 13 - second half] movu [r0 + 347 * 16], m3 ; mode 5 [row 5] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 202 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 203 * 16], m3 ; mode 5 [row 6] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 204 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 205 * 16], m3 ; mode 6 [row 8] movu m6, [r5 + 21 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 272 * 16], m3 ; mode 7 [row 12 - first half] movu [r0 + 344 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 273 * 16], m3 ; mode 7 [row 12 - second half] movu [r0 + 345 * 16], m3 ; mode 7 [row 10] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 340 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 341 * 16], m3 ; mode 7 [row 11] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 342 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 343 * 16], m3 ; mode 8 [row 19] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 422 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 423 * 16], m3 ; mode 8 [row 21] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 426 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 427 * 16], m3 ; mode 8 [row 22] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 428 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 429 * 16], m3 ; mode 8 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 430 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 431 * 16], m3 ; mode 8 [row 24] movu m6, [r5 + 29 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 432 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 433 * 16], m3 ; mode 3 [row 4] movu m6, [r5 + 2 * 16] movu m0, [r4 + 5] movd m1, [r4 + 6] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 13] movd m3, [r4 + 14] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 72 * 16], m1 ; mode 3 [row 4 - first half] movu [r0 + 274 * 16], m1 ; mode 8 [row 25 - first half] movu [r0 + 434 * 16], m1 movu m1, [r4 + 21] movd m3, [r4 + 22] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 29] movd m5, [r4 + 30] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 73 * 16], m3 ; mode 3 [row 4 - second half] movu [r0 + 275 * 16], m3 ; mode 8 [row 25 - second half] movu [r0 + 435 * 16], m3 ; mode 3 [row 5] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 74 * 16], m3 ; mode 3 [row 5 - first half] movu [r0 + 278 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 75 * 16], m3 ; mode 3 [row 5 - second half] movu [r0 + 279 * 16], m3 ; mode 4 [row 6] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 140 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 141 * 16], m3 ; mode 5 [row 7] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 206 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 207 * 16], m3 ; mode 5 [row 8] movu m6, [r5 + 25 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 208 * 16], m3 ; mode 7 [row 16 - first half] movu [r0 + 352 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 209 * 16], m3 ; mode 7 [row 16 - second half] movu [r0 + 353 * 16], m3 ; mode 6 [row 10] movu m6, [r5 + 15 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 276 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 277 * 16], m3 ; mode 7 [row 14] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 348 * 16], m3 ; mode 8 [row 26 - first half] movu [r0 + 436 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 349 * 16], m3 ; mode 8 [row 26 - second half] movu [r0 + 437 * 16], m3 ; mode 7 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 350 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 351 * 16], m3 ; mode 8 [row 27] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 438 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 439 * 16], m3 ; mode 8 [row 28] movu m6, [r5 + 17 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 440 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 441 * 16], m3 ; mode 8 [row 29] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 442 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 443 * 16], m3 ; mode 8 [row 30] movu m6, [r5 + 27 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 444 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 445 * 16], m3 ; mode 3 [row 6] movu m6, [r5 + 22 * 16] movu m0, [r4 + 6] movd m1, [r4 + 7] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 14] movd m3, [r4 + 15] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 76 * 16], m1 ; mode 6 [row 13 - first half] movu [r0 + 282 * 16], m1 movu m1, [r4 + 22] movd m3, [r4 + 23] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 30] movd m5, [r4 + 31] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 77 * 16], m3 ; mode 6 [row 13 - second half] movu [r0 + 283 * 16], m3 ; mode 4 [row 7] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 142 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 143 * 16], m3 ; mode 4 [row 8] movu m6, [r5 + 29 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 144 * 16], m3 ; mode 4 [row 8 - first half] movu [r0 + 360 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 145 * 16], m3 ; mode 4 [row 8 - second half] movu [r0 + 361 * 16], m3 ; mode 5 [row 9] movu m6, [r5 + 10 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 210 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 211 * 16], m3 ; mode 5 [row 10] movu m6, [r5 + 27 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 212 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 213 * 16], m3 ; mode 7 [row 17] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 354 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 355 * 16], m3 ; mode 7 [row 18] movu m6, [r5 + 11 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 356 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 357 * 16], m3 ; mode 7 [row 19] movu m6, [r5 + 20 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 358 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 359 * 16], m3 ; mode 6 [row 12] movu m6, [r5 + 9 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 280 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 281 * 16], m3 ; mode 3 [row 7] movu m6, [r5 + 16 * 16] movu m0, [r4 + 7] movd m1, [r4 + 8] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 15] movd m3, [r4 + 16] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 78 * 16], m1 ; mode 6 [row 15 - first half] movu [r0 + 286 * 16], m1 movu m1, [r4 + 23] movd m3, [r4 + 24] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 31] movd m5, [r4 + 32] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 79 * 16], m3 ; mode 6 [row 15 - second half] movu [r0 + 287 * 16], m3 ; mode 4 [row 9] movu m6, [r5 + 18 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 146 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 147 * 16], m3 ; mode 5 [row 11] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 214 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 215 * 16], m3 ; mode 5 [row 12] movu m6, [r5 + 29 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 216 * 16], m3 ; mode 6 [row 16 - first half] movu [r0 + 288 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 217 * 16], m3 ; mode 6 [row 16 - second half] movu [r0 + 289 * 16], m3 ; mode 6 [row 14] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 284 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 285 * 16], m3 ; mode 7 [row 21] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 362 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 363 * 16], m3 ; mode 7 [row 22] movu m6, [r5 + 15 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 364 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 365 * 16], m3 ; mode 7 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 366 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 367 * 16], m3 ; mode 3 [row 8] movu m6, [r5 + 10 * 16] movu m0, [r4 + 8] movd m1, [r4 + 9] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 16] movd m3, [r4 + 17] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 80 * 16], m1 ; mode 7 [row 25 - first half] movu [r0 + 290 * 16], m1 ; mode 6 [row 17 - first half] movu [r0 + 370 * 16], m1 movu m1, [r4 + 24] movd m3, [r4 + 25] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 32] movd m5, [r4 + 33] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 81 * 16], m3 ; mode 7 [row 25 - second half] movu [r0 + 291 * 16], m3 ; mode 6 [row 17 - second half] movu [r0 + 371 * 16], m3 ; mode 4 [row 10] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 148 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 149 * 16], m3 ; mode 4 [row 11] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 150 * 16], m3 ; mode 7 [row 27 - first half] movu [r0 + 374 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 151 * 16], m3 ; mode 7 [row 27 - second half] movu [r0 + 375 * 16], m3 ; mode 5 [row 13] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 218 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 219 * 16], m3 ; mode 5 [row 14] movu m6, [r5 + 31 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 220 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 221 * 16], m3 ; mode 6 [row 18] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 292 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 293 * 16], m3 ; mode 7 [row 24] movu m6, [r5 + 1 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 368 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 369 * 16], m3 ; mode 7 [row 26] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 372 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 373 * 16], m3 ; mode 3 [row 9] movu m6, [r5 + 4 * 16] movu m0, [r4 + 9] movd m1, [r4 + 10] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 17] movd m3, [r4 + 18] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 82 * 16], m1 ; mode 6 [row 19 - first half] movu [r0 + 294 * 16], m1 movu m1, [r4 + 25] movd m3, [r4 + 26] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 33] movd m5, [r4 + 34] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 83 * 16], m3 ; mode 6 [row 19 - second half] movu [r0 + 295 * 16], m3 ; mode 4 [row 12] movu m6, [r5 + 17 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 152 * 16], m3 ; mode 4 [row 12 - first half] movu [r0 + 296 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 153 * 16], m3 ; mode 4 [row 12 - second half] movu [r0 + 297 * 16], m3 ; mode 3 [row 10] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 84 * 16], m3 ; mode 6 [row 21 - first half] movu [r0 + 298 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 85 * 16], m3 ; mode 6 [row 21 - second half] movu [r0 + 299 * 16], m3 ; mode 5 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 222 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 223 * 16], m3 ; mode 7 [row 28] movu m6, [r5 + 5 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 376 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 377 * 16], m3 ; mode 7 [row 29] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 378 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 379 * 16], m3 ; mode 7 [row 30] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 380 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 381 * 16], m3 ; mode 3 [row 11] movu m6, [r5 + 24 * 16] movu m0, [r4 + 10] movd m1, [r4 + 11] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 18] movd m3, [r4 + 19] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 86 * 16], m1 ; mode 6 [row 23 - first half] movu [r0 + 302 * 16], m1 movu m1, [r4 + 26] movd m3, [r4 + 27] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 34] movd m5, [r4 + 35] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 87 * 16], m3 ; mode 6 [row 23 - second half] movu [r0 + 303 * 16], m3 ; mode 4 [row 13] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 154 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 155 * 16], m3 ; mode 4 [row 14] movu m6, [r5 + 27 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 156 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 157 * 16], m3 ; mode 5 [row 16] movu m6, [r5 + 1 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 224 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 225 * 16], m3 ; mode 5 [row 17] movu m6, [r5 + 18 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 226 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 227 * 16], m3 ; mode 6 [row 22] movu m6, [r5 + 11 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 300 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 301 * 16], m3 ; mode 3 [row 12] movu m6, [r5 + 18 * 16] movu m0, [r4 + 11] movd m1, [r4 + 12] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 19] movd m3, [r4 + 20] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 88 * 16], m1 ; mode 6 [row 25 - first half] movu [r0 + 306 * 16], m1 movu m1, [r4 + 27] movd m3, [r4 + 28] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 35] movd m5, [r4 + 36] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 89 * 16], m3 ; mode 6 [row 25 - second half] movu [r0 + 307 * 16], m3 ; mode 4 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 158 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 159 * 16], m3 ; mode 5 [row 18] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 228 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 229 * 16], m3 ; mode 5 [row 19] movu m6, [r5 + 20 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 230 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 231 * 16], m3 ; mode 6 [row 24] movu m6, [r5 + 5 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 304 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 305 * 16], m3 ; mode 6 [row 26] movu m6, [r5 + 31 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 308 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 309 * 16], m3 ; mode 3 [row 13] movu m6, [r5 + 12 * 16] movu m0, [r4 + 12] movd m1, [r4 + 13] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 20] movd m3, [r4 + 21] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 90 * 16], m1 movu m1, [r4 + 28] movd m3, [r4 + 29] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 36] movd m5, [r4 + 37] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 91 * 16], m3 ; mode 4 [row 16] movu m6, [r5 + 5 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 160 * 16], m3 ; mode 5 [row 20 - first half] movu [r0 + 232 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 161 * 16], m3 ; mode 5 [row 20 - second half] movu [r0 + 233 * 16], m3 ; mode 4 [row 17] movu m6, [r5 + 26 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 162 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 163 * 16], m3 ; mode 5 [row 21] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 234 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 235 * 16], m3 ; mode 6 [row 27] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 310 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 311 * 16], m3 ; mode 6 [row 28] movu m6, [r5 + 25 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 312 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 313 * 16], m3 ; mode 3 [row 14] movu m6, [r5 + 6 * 16] movu m0, [r4 + 13] movd m1, [r4 + 14] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 21] movd m3, [r4 + 22] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 92 * 16], m1 ; mode 6 [row 29 - first half] movu [r0 + 314 * 16], m1 movu m1, [r4 + 29] movd m3, [r4 + 30] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 37] movd m5, [r4 + 38] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 93 * 16], m3 ; mode 6 [row 29 - second half] movu [r0 + 315 * 16], m3 ; mode 4 [row 18] movu m6, [r5 + 15 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 164 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 165 * 16], m3 ; mode 5 [row 22] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 236 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 237 * 16], m3 ; mode 5 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 238 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 239 * 16], m3 ; mode 6 [row 30] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 316 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 317 * 16], m3 ; mode 3 [row 16] movu m6, [r5 + 26 * 16] movu m0, [r4 + 14] movd m1, [r4 + 15] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 22] movd m3, [r4 + 23] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 96 * 16], m1 ; mode 5 [row 25 - first half] movu [r0 + 242 * 16], m1 movu m1, [r4 + 30] movd m3, [r4 + 31] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 38] movd m5, [r4 + 39] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 97 * 16], m3 ; mode 5 [row 25 - second half] movu [r0 + 243 * 16], m3 ; mode 4 [row 19] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 166 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 167 * 16], m3 ; mode 4 [row 20] movu m6, [r5 + 25 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 168 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 169 * 16], m3 ; mode 5 [row 24] movu m6, [r5 + 9 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 240 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 241 * 16], m3 ; mode 3 [row 17] movu m6, [r5 + 20 * 16] movu m0, [r4 + 15] movd m1, [r4 + 16] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 23] movd m3, [r4 + 24] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 98 * 16], m1 movu m1, [r4 + 31] movd m3, [r4 + 32] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 39] movd m5, [r4 + 40] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 99 * 16], m3 ; mode 4 [row 21] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 170 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 171 * 16], m3 ; mode 5 [row 26] movu m6, [r5 + 11 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 244 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 245 * 16], m3 ; mode 5 [row 27] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 246 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 247 * 16], m3 ; mode 3 [row 18] movu m6, [r5 + 14 * 16] movu m0, [r4 + 16] movd m1, [r4 + 17] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 24] movd m3, [r4 + 25] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 100 * 16], m1 movu m1, [r4 + 32] movd m3, [r4 + 33] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 40] movd m5, [r4 + 41] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 101 * 16], m3 ; mode 4 [row 22] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 172 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 173 * 16], m3 ; mode 4 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 174 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 175 * 16], m3 ; mode 5 [row 28] movu m6, [r5 + 13 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 248 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 249 * 16], m3 ; mode 5 [row 29] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 250 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 251 * 16], m3 ; mode 3 [row 19] movu m6, [r5 + 8 * 16] movu m0, [r4 + 17] movd m1, [r4 + 18] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 25] movd m3, [r4 + 26] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 102 * 16], m1 movu m1, [r4 + 33] movd m3, [r4 + 34] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 41] movd m5, [r4 + 42] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 103 * 16], m3 ; mode 4 [row 24] movu m6, [r5 + 13 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 176 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 177 * 16], m3 ; mode 5 [row 30] movu m6, [r5 + 15 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 252 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 253 * 16], m3 ; mode 3 [row 20] movu m6, [r5 + 2 * 16] movu m0, [r4 + 18] movd m1, [r4 + 19] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 26] movd m3, [r4 + 27] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 104 * 16], m1 movu m1, [r4 + 34] movd m3, [r4 + 35] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 42] movd m5, [r4 + 43] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 105 * 16], m3 ; mode 4 [row 25] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 178 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 179 * 16], m3 ; mode 4 [row 26] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 180 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 181 * 16], m3 ; mode 3 [row 21] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 106 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 107 * 16], m3 ; mode 3 [row 22] movu m6, [r5 + 22 * 16] movu m0, [r4 + 19] movd m1, [r4 + 20] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 27] movd m3, [r4 + 28] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 108 * 16], m1 movu m1, [r4 + 35] movd m3, [r4 + 36] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 43] movd m5, [r4 + 44] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 109 * 16], m3 ; mode 4 [row 27] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 182 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 183 * 16], m3 ; mode 3 [row 23] movu m6, [r5 + 16 * 16] movu m0, [r4 + 20] movd m1, [r4 + 21] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 28] movd m3, [r4 + 29] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 110 * 16], m1 movu m1, [r4 + 36] movd m3, [r4 + 37] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 44] movd m5, [r4 + 45] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 111 * 16], m3 ; mode 4 [row 28] movu m6, [r5 + 1 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 184 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 185 * 16], m3 ; mode 4 [row 29] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 186 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 187 * 16], m3 ; mode 3 [row 24] movu m6, [r5 + 10 * 16] movu m0, [r4 + 21] movd m1, [r4 + 22] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 29] movd m3, [r4 + 30] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 112 * 16], m1 movu m1, [r4 + 37] movd m3, [r4 + 38] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 45] movd m5, [r4 + 46] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 113 * 16], m3 ; mode 4 [row 30] movu m6, [r5 + 11 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 188 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 189 * 16], m3 ; mode 3 [row 25] movu m6, [r5 + 4 * 16] movu m0, [r4 + 22] movd m1, [r4 + 23] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 30] movd m3, [r4 + 31] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 114 * 16], m1 movu m1, [r4 + 38] movd m3, [r4 + 39] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 46] movd m5, [r4 + 47] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 115 * 16], m3 ; mode 3 [row 26] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 116 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 117 * 16], m3 ; mode 3 [row 27] movu m6, [r5 + 24 * 16] movu m0, [r4 + 23] movd m1, [r4 + 24] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 31] movd m3, [r4 + 32] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 118 * 16], m1 movu m1, [r4 + 39] movd m3, [r4 + 40] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 47] movd m5, [r4 + 48] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 119 * 16], m3 ; mode 3 [row 28] movu m6, [r5 + 18 * 16] movu m0, [r4 + 24] movd m1, [r4 + 25] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 32] movd m3, [r4 + 33] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 120 * 16], m1 movu m1, [r4 + 40] movd m3, [r4 + 41] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 48] movd m5, [r4 + 49] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 121 * 16], m3 ; mode 3 [row 29] movu m6, [r5 + 12 * 16] movu m0, [r4 + 25] movd m1, [r4 + 26] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 33] movd m3, [r4 + 34] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 122 * 16], m1 movu m1, [r4 + 41] movd m3, [r4 + 42] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 49] movd m5, [r4 + 50] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 123 * 16], m3 ; mode 3 [row 30] movu m6, [r5 + 6 * 16] movu m0, [r4 + 26] movd m1, [r4 + 27] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r4 + 34] movd m3, [r4 + 35] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 124 * 16], m1 movu m1, [r4 + 42] movd m3, [r4 + 43] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r4 + 50] movd m5, [r4 + 51] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 125 * 16], m3 ; mode 10 movu m1, [r2 + 1] movu m2, [r2 + 17] movu [r0 + 512 * 16], m1 movu [r0 + 513 * 16], m2 movu [r0 + 514 * 16], m1 movu [r0 + 515 * 16], m2 movu [r0 + 516 * 16], m1 movu [r0 + 517 * 16], m2 movu [r0 + 518 * 16], m1 movu [r0 + 519 * 16], m2 movu [r0 + 520 * 16], m1 movu [r0 + 521 * 16], m2 movu [r0 + 522 * 16], m1 movu [r0 + 523 * 16], m2 movu [r0 + 524 * 16], m1 movu [r0 + 525 * 16], m2 movu [r0 + 526 * 16], m1 movu [r0 + 527 * 16], m2 movu [r0 + 528 * 16], m1 movu [r0 + 529 * 16], m2 movu [r0 + 530 * 16], m1 movu [r0 + 531 * 16], m2 movu [r0 + 532 * 16], m1 movu [r0 + 533 * 16], m2 movu [r0 + 534 * 16], m1 movu [r0 + 535 * 16], m2 movu [r0 + 536 * 16], m1 movu [r0 + 537 * 16], m2 movu [r0 + 538 * 16], m1 movu [r0 + 539 * 16], m2 movu [r0 + 540 * 16], m1 movu [r0 + 541 * 16], m2 movu [r0 + 542 * 16], m1 movu [r0 + 543 * 16], m2 movu [r0 + 544 * 16], m1 movu [r0 + 545 * 16], m2 movu [r0 + 546 * 16], m1 movu [r0 + 547 * 16], m2 movu [r0 + 548 * 16], m1 movu [r0 + 549 * 16], m2 movu [r0 + 550 * 16], m1 movu [r0 + 551 * 16], m2 movu [r0 + 552 * 16], m1 movu [r0 + 553 * 16], m2 movu [r0 + 554 * 16], m1 movu [r0 + 555 * 16], m2 movu [r0 + 556 * 16], m1 movu [r0 + 557 * 16], m2 movu [r0 + 558 * 16], m1 movu [r0 + 559 * 16], m2 movu [r0 + 560 * 16], m1 movu [r0 + 561 * 16], m2 movu [r0 + 562 * 16], m1 movu [r0 + 563 * 16], m2 movu [r0 + 564 * 16], m1 movu [r0 + 565 * 16], m2 movu [r0 + 566 * 16], m1 movu [r0 + 567 * 16], m2 movu [r0 + 568 * 16], m1 movu [r0 + 569 * 16], m2 movu [r0 + 570 * 16], m1 movu [r0 + 571 * 16], m2 movu [r0 + 572 * 16], m1 movu [r0 + 573 * 16], m2 movu [r0 + 574 * 16], m1 movu [r0 + 575 * 16], m2 ; mode 11 [row 0] movu m0, [r4] ; mode 11 [row 15 - first half] movu [r0 + 606 * 16], m0 movu [r0 + 606 * 16], m0 ; mode 12 [row 31] pslldq m6, m0, 4 pinsrb m6, [r3 + 26], 0 pinsrb m6, [r3 + 19], 1 pinsrb m6, [r3 + 13], 2 pinsrb m6, [r3 + 6], 3 movu [r0 + 702 * 16], m6 movu m6, [r4 + 12] movu [r0 + 703 * 16], m6 ; mode 11 [row 31] pslldq m6, m0, 1 pinsrb m6, [r3 + 16], 0 movu [r0 + 638 * 16], m6 movu m6, [r4 + 15] movu [r0 + 639 * 16], m6 movd m1, [r4 + 1] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m1, m0, [r5 + 30 * 16] pmulhrsw m1, m7 movu m2, [r4 + 8] movd m3, [r4 + 9] palignr m3, m2, 1 punpcklbw m2, m3 pmaddubsw m3, m2, [r5 + 30 * 16] pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 576 * 16], m1 movu m1, [r4 + 16] ; mode 11 [row 15 - second half] movu [r0 + 607 * 16], m1 movd m3, [r4 + 17] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, [r5 + 30 * 16] pmulhrsw m3, m7 movu m4, [r4 + 24] movd m5, [r4 + 25] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, [r5 + 30 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 577 * 16], m3 ; mode 11 [row 1] pmaddubsw m3, m0, [r5 + 28 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 578 * 16], m3 pmaddubsw m3, m1, [r5 + 28 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 579 * 16], m3 ; mode 11 [row 2] pmaddubsw m3, m0, [r5 + 26 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 26 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 580 * 16], m3 pmaddubsw m3, m1, [r5 + 26 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 26 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 581 * 16], m3 ; mode 11 [row 3] pmaddubsw m3, m0, [r5 + 24 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 582 * 16], m3 pmaddubsw m3, m1, [r5 + 24 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 583 * 16], m3 ; mode 11 [row 4] pmaddubsw m3, m0, [r5 + 22 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 22 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 584 * 16], m3 ; mode 12 [row 1 - first half] movu [r0 + 642 * 16], m3 pmaddubsw m3, m1, [r5 + 22 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 22 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 585 * 16], m3 ; mode 12 [row 1 - second half] movu [r0 + 643 * 16], m3 ; mode 11 [row 5] pmaddubsw m3, m0, [r5 + 20 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 20 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 586 * 16], m3 pmaddubsw m3, m1, [r5 + 20 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 20 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 587 * 16], m3 ; mode 11 [row 6] pmaddubsw m3, m0, [r5 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 18 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 588 * 16], m3 pmaddubsw m3, m1, [r5 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 18 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 589 * 16], m3 ; mode 11 [row 7] pmaddubsw m3, m0, [r5 + 16 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 16 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 590 * 16], m3 pmaddubsw m3, m1, [r5 + 16 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 16 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 591 * 16], m3 ; mode 11 [row 8] pmaddubsw m3, m0, [r5 + 14 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 14 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 592 * 16], m3 ; mode 13 [row 1 - first half] movu [r0 + 706 * 16], m3 pmaddubsw m3, m1, [r5 + 14 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 14 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 593 * 16], m3 ; mode 13 [row 1 - second half] movu [r0 + 707 * 16], m3 ; mode 11 [row 9] pmaddubsw m3, m0, [r5 + 12 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 12 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 594 * 16], m3 ; mode 12 [row 3 - first half] movu [r0 + 646 * 16], m3 pmaddubsw m3, m1, [r5 + 12 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 12 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 595 * 16], m3 ; mode 12 [row 3 - second half] movu [r0 + 647 * 16], m3 ; mode 11 [row 10] pmaddubsw m3, m0, [r5 + 10 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 10 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 596 * 16], m3 pmaddubsw m3, m1, [r5 + 10 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 10 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 597 * 16], m3 ; mode 11 [row 11] pmaddubsw m3, m0, [r5 + 8 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 8 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 598 * 16], m3 pmaddubsw m3, m1, [r5 + 8 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 8 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 599 * 16], m3 ; mode 11 [row 12] pmaddubsw m3, m0, [r5 + 6 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 6 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 600 * 16], m3 ; mode 14 [row 1 - first half] movu [r0 + 770 * 16], m3 pmaddubsw m3, m1, [r5 + 6 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 6 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 601 * 16], m3 ; mode 14 [row 1 - second half] movu [r0 + 771 * 16], m3 ; mode 11 [row 13] pmaddubsw m3, m0, [r5 + 4 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 4 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 602 * 16], m3 pmaddubsw m3, m1, [r5 + 4 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 4 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 603 * 16], m3 ; mode 11 [row 14] pmaddubsw m3, m0, [r5 + 2 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 2 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 604 * 16], m3 ; mode 13 [row 5 - first half] movu [r0 + 650 * 16], m3 pmaddubsw m3, m1, [r5 + 2 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 2 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 605 * 16], m3 ; mode 13 [row 5 - second half] movu [r0 + 651 * 16], m3 ; mode 12 [row 0] pmaddubsw m3, m0, [r5 + 27 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 27 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 640 * 16], m3 pmaddubsw m3, m1, [r5 + 27 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 27 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 641 * 16], m3 ; mode 12 [row 2] pmaddubsw m3, m0, [r5 + 17 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 17 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 644 * 16], m3 pmaddubsw m3, m1, [r5 + 17 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 17 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 645 * 16], m3 ; mode 12 [row 4] pmaddubsw m3, m0, [r5 + 7 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 7 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 648 * 16], m3 pmaddubsw m3, m1, [r5 + 7 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 7 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 649 * 16], m3 ; mode 13 [row 0] pmaddubsw m3, m0, [r5 + 23 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 23 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 704 * 16], m3 pmaddubsw m3, m1, [r5 + 23 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 23 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 705 * 16], m3 ; mode 13 [row 2] pmaddubsw m3, m0, [r5 + 5 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 5 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 708 * 16], m3 pmaddubsw m3, m1, [r5 + 5 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 5 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 709 * 16], m3 ; mode 14 [row 0] pmaddubsw m3, m0, [r5 + 19 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 19 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 768 * 16], m3 pmaddubsw m3, m1, [r5 + 19 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 19 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 769 * 16], m3 ; mode 15 [row 0] pmaddubsw m3, m0, [r5 + 15 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 15 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 832 * 16], m3 pmaddubsw m3, m1, [r5 + 15 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 15 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 833 * 16], m3 ; mode 11 [row 16] pslldq m0, 2 pinsrb m0, [r4 + 0], 1 pinsrb m0, [r3 + 16], 0 pmaddubsw m3, m0, [r5 + 30 * 16] pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r4 + 8], 1 pinsrb m2, [r4 + 7], 0 pmaddubsw m5, m2, [r5 + 30 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 608 * 16], m3 pslldq m1, 2 pinsrb m1, [r4 + 16], 1 pinsrb m1, [r4 + 15], 0 pmaddubsw m3, m1, [r5 + 30 * 16] pmulhrsw m3, m7 pslldq m4, 2 pinsrb m4, [r4 + 24], 1 pinsrb m4, [r4 + 23], 0 pmaddubsw m5, m4, [r5 + 30 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 609 * 16], m3 ; mode 11 [row 17] pmaddubsw m3, m0, [r5 + 28 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 610 * 16], m3 pmaddubsw m3, m1, [r5 + 28 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 611 * 16], m3 ; mode 11 [row 18] pmaddubsw m3, m0, [r5 + 26 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 26 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 612 * 16], m3 pmaddubsw m3, m1, [r5 + 26 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 26 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 613 * 16], m3 ; mode 11 [row 19] pmaddubsw m3, m0, [r5 + 24 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 614 * 16], m3 pmaddubsw m3, m1, [r5 + 24 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 615 * 16], m3 ; mode 11 [row 20] pmaddubsw m3, m0, [r5 + 22 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 22 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 616 * 16], m3 pmaddubsw m3, m1, [r5 + 22 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 22 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 617 * 16], m3 ; mode 11 [row 21] pmaddubsw m3, m0, [r5 + 20 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 20 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 618 * 16], m3 pmaddubsw m3, m1, [r5 + 20 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 20 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 619 * 16], m3 ; mode 11 [row 22] pmaddubsw m3, m0, [r5 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 18 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 620 * 16], m3 pmaddubsw m3, m1, [r5 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 18 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 621 * 16], m3 ; mode 11 [row 23] pmaddubsw m3, m0, [r5 + 16 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 16 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 622 * 16], m3 pmaddubsw m3, m1, [r5 + 16 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 16 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 623 * 16], m3 ; mode 11 [row 24] pmaddubsw m3, m0, [r5 + 14 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 14 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 624 * 16], m3 pmaddubsw m3, m1, [r5 + 14 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 14 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 625 * 16], m3 ; mode 11 [row 25] pmaddubsw m3, m0, [r5 + 12 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 12 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 626 * 16], m3 pmaddubsw m3, m1, [r5 + 12 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 12 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 627 * 16], m3 ; mode 11 [row 26] pmaddubsw m3, m0, [r5 + 10 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 10 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 628 * 16], m3 pmaddubsw m3, m1, [r5 + 10 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 10 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 629 * 16], m3 ; mode 11 [row 27] pmaddubsw m3, m0, [r5 + 8 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 8 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 630 * 16], m3 pmaddubsw m3, m1, [r5 + 8 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 8 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 631 * 16], m3 ; mode 11 [row 28] pmaddubsw m3, m0, [r5 + 6 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 6 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 632 * 16], m3 pmaddubsw m3, m1, [r5 + 6 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 6 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 633 * 16], m3 ; mode 11 [row 29] pmaddubsw m3, m0, [r5 + 4 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 4 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 634 * 16], m3 pmaddubsw m3, m1, [r5 + 4 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 4 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 635 * 16], m3 ; mode 11 [row 30] pmaddubsw m3, m0, [r5 + 2 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 2 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 636 * 16], m3 pmaddubsw m3, m1, [r5 + 2 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 2 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 637 * 16], m3 ; mode 12 [row 6] pinsrb m0, [r3 + 6], 0 pmaddubsw m3, m0, [r5 + 29 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 29 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 652 * 16], m3 pmaddubsw m3, m1, [r5 + 29 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 29 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 653 * 16], m3 ; mode 12 [row 7] pmaddubsw m3, m0, [r5 + 24 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 654 * 16], m3 pmaddubsw m3, m1, [r5 + 24 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 655 * 16], m3 ; mode 12 [row 8] pmaddubsw m3, m0, [r5 + 19 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 19 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 656 * 16], m3 pmaddubsw m3, m1, [r5 + 19 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 19 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 657 * 16], m3 ; mode 12 [row 9] pmaddubsw m3, m0, [r5 + 14 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 14 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 658 * 16], m3 pmaddubsw m3, m1, [r5 + 14 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 14 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 659 * 16], m3 ; mode 12 [row 10] pmaddubsw m3, m0, [r5 + 9 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 9 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 660 * 16], m3 pmaddubsw m3, m1, [r5 + 9 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 9 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 661 * 16], m3 ; mode 12 [row 11] pmaddubsw m3, m0, [r5 + 4 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 4 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 662 * 16], m3 pmaddubsw m3, m1, [r5 + 4 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 4 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 663 * 16], m3 ; mode 13 [row 3] movu m6, m0 pinsrb m6, [r3 + 4], 0 pmaddubsw m3, m6, [r5 + 28 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 710 * 16], m3 pmaddubsw m3, m1, [r5 + 28 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 711 * 16], m3 ; mode 13 [row 4] pmaddubsw m3, m6, [r5 + 19 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 19 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 712 * 16], m3 pmaddubsw m3, m1, [r5 + 19 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 19 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 713 * 16], m3 ; mode 13 [row 5] pmaddubsw m3, m6, [r5 + 10 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 10 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 714 * 16], m3 pmaddubsw m3, m1, [r5 + 10 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 10 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 715 * 16], m3 ; mode 13 [row 6] pmaddubsw m3, m6, [r5 + 1 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 1 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 716 * 16], m3 pmaddubsw m3, m1, [r5 + 1 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 1 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 717 * 16], m3 ; mode 14 [row 2] movu m6, m0 pinsrb m6, [r4 + 0], 1 pinsrb m6, [r3 + 2], 0 pmaddubsw m3, m6, [r5 + 25 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 25 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 772 * 16], m3 pmaddubsw m3, m1, [r5 + 25 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 25 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 773 * 16], m3 ; mode 14 [row 3] pmaddubsw m3, m6, [r5 + 12 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 12 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 774 * 16], m3 pmaddubsw m3, m1, [r5 + 12 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 12 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 775 * 16], m3 ; mode 15 [row 1] pmaddubsw m3, m6, [r5 + 30 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 30 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 834 * 16], m3 pmaddubsw m3, m1, [r5 + 30 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 30 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 835 * 16], m3 ; mode 15 [row 2] pmaddubsw m3, m6, [r5 + 13 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 13 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 836 * 16], m3 pmaddubsw m3, m1, [r5 + 13 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 13 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 837 * 16], m3 ; mode 15 [row 3] pslldq m6, 2 pinsrb m6, [r3 + 2], 1 pinsrb m6, [r3 + 4], 0 pmaddubsw m3, m6, [r5 + 28 * 16] pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r4 + 7], 1 pinsrb m2, [r4 + 6], 0 pmaddubsw m5, m2, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 838 * 16], m3 pslldq m1, 2 pinsrb m1, [r4 + 15], 1 pinsrb m1, [r4 + 14], 0 pmaddubsw m3, m1, [r5 + 28 * 16] pmulhrsw m3, m7 pslldq m4, 2 pinsrb m4, [r4 + 23], 1 pinsrb m4, [r4 + 22], 0 pmaddubsw m5, m4, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 839 * 16], m3 ; mode 15 [row 4] pmaddubsw m3, m6, [r5 + 11 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 11 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 840 * 16], m3 pmaddubsw m3, m1, [r5 + 11 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 11 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 841 * 16], m3 ; mode 15 [row 5, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 4], 1 pinsrb m6, [r3 + 6], 0 pmaddubsw m3, m6, [r5 + 26 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 842 * 16], m3 ; mode 15 [row 6, 0-7] pmaddubsw m3, m6, [r5 + 9 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 844 * 16], m3 ; mode 15 [row 7, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 6], 1 pinsrb m6, [r3 + 8], 0 pmaddubsw m3, m6, [r5 + 24 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 846 * 16], m3 ; mode 15 [row 8, 0-7] pmaddubsw m3, m6, [r5 + 7 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 848 * 16], m3 ; mode 15 [row 9, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 8], 1 pinsrb m6, [r3 + 9], 0 pmaddubsw m3, m6, [r5 + 22 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 850 * 16], m3 ; mode 15 [row 10, 0-7] pmaddubsw m3, m6, [r5 + 5 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 852 * 16], m3 ; mode 15 [row 11, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 9], 1 pinsrb m6, [r3 + 11], 0 pmaddubsw m3, m6, [r5 + 20 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 854 * 16], m3 ; mode 15 [row 12, 0-7] pmaddubsw m3, m6, [r5 + 3 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 856 * 16], m3 ; mode 15 [row 13, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 11], 1 pinsrb m6, [r3 + 13], 0 pmaddubsw m3, m6, [r5 + 18 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 858 * 16], m3 ; mode 15 [row 14, 0-7] pmaddubsw m3, m6, [r5 + 1 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 860 * 16], m3 ; mode 15 [row 15, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 13], 1 pinsrb m6, [r3 + 15], 0 pmaddubsw m3, m6, [r5 + 16 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 862 * 16], m3 ; mode 15 [row 16, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 15], 1 pinsrb m6, [r3 + 17], 0 pmaddubsw m3, m6, [r5 + 31 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 864 * 16], m3 ; mode 15 [row 17, 0-7] pmaddubsw m3, m6, [r5 + 14 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 866 * 16], m3 ; mode 15 [row 18, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 17], 1 pinsrb m6, [r3 + 19], 0 pmaddubsw m3, m6, [r5 + 29 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 868 * 16], m3 ; mode 15 [row 19, 0-7] pmaddubsw m3, m6, [r5 + 12 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 870 * 16], m3 ; mode 15 [row 20, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 19], 1 pinsrb m6, [r3 + 21], 0 pmaddubsw m3, m6, [r5 + 27 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 872 * 16], m3 ; mode 15 [row 21, 0-7] pmaddubsw m3, m6, [r5 + 10 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 874 * 16], m3 ; mode 15 [row 22, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 21], 1 pinsrb m6, [r3 + 23], 0 pmaddubsw m3, m6, [r5 + 25 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 876 * 16], m3 ; mode 15 [row 23, 0-7] pmaddubsw m3, m6, [r5 + 8 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 878 * 16], m3 ; mode 15 [row 24, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 23], 1 pinsrb m6, [r3 + 24], 0 pmaddubsw m3, m6, [r5 + 23 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 880 * 16], m3 ; mode 15 [row 25, 0-7] pmaddubsw m3, m6, [r5 + 6 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 882 * 16], m3 ; mode 15 [row 26, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 24], 1 pinsrb m6, [r3 + 26], 0 pmaddubsw m3, m6, [r5 + 21 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 884 * 16], m3 ; mode 15 [row 27, 0-7] pmaddubsw m3, m6, [r5 + 4 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 886 * 16], m3 ; mode 15 [row 28, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 26], 1 pinsrb m6, [r3 + 28], 0 pmaddubsw m3, m6, [r5 + 19 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 888 * 16], m3 ; mode 15 [row 29, 0-7] pmaddubsw m3, m6, [r5 + 2 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 890 * 16], m3 ; mode 15 [row 30, 0-7] pslldq m6, 2 pinsrb m6, [r3 + 28], 1 pinsrb m6, [r3 + 30], 0 pmaddubsw m3, m6, [r5 + 17 * 16] pmulhrsw m3, m7 packuswb m3, m3 movh [r0 + 892 * 16], m3 ; mode 15 [row 31, 0-7] pshufb m3, m6, [tab_S2] movh [r0 + 894 * 16], m3 ; mode 12 [row 12] pslldq m0, 2 pinsrb m0, [r3 + 6], 1 pinsrb m0, [r3 + 13], 0 pmaddubsw m3, m0, [r5 + 31 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 31 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 664 * 16], m3 pmaddubsw m3, m1, [r5 + 31 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 31 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 665 * 16], m3 ; mode 12 [row 13] pmaddubsw m3, m0, [r5 + 26 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 26 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 666 * 16], m3 pmaddubsw m3, m1, [r5 + 26 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 26 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 667 * 16], m3 ; mode 12 [row 14] pmaddubsw m3, m0, [r5 + 21 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 21 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 668 * 16], m3 pmaddubsw m3, m1, [r5 + 21 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 21 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 669 * 16], m3 ; mode 12 [row 15] pmaddubsw m3, m0, [r5 + 16 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 16 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 670 * 16], m3 pmaddubsw m3, m1, [r5 + 16 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 16 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 671 * 16], m3 ; mode 12 [row 16] pmaddubsw m3, m0, [r5 + 11 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 11 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 672 * 16], m3 pmaddubsw m3, m1, [r5 + 11 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 11 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 673 * 16], m3 ; mode 12 [row 17] pmaddubsw m3, m0, [r5 + 6 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 6 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 674 * 16], m3 pmaddubsw m3, m1, [r5 + 6 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 6 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 675 * 16], m3 ; mode 12 [row 18] pmaddubsw m3, m0, [r5 + 1 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 1 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 676 * 16], m3 pmaddubsw m3, m1, [r5 + 1 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 1 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 677 * 16], m3 ; mode 13 [row 7] movu m6, m0 pinsrb m6, [r3 + 4], 2 pinsrb m6, [r3 + 4], 1 pinsrb m6, [r3 + 7], 0 pmaddubsw m3, m6, [r5 + 24 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 718 * 16], m3 pmaddubsw m3, m1, [r5 + 24 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 719 * 16], m3 ; mode 13 [row 8] pmaddubsw m3, m6, [r5 + 15 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 15 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 720 * 16], m3 pmaddubsw m3, m1, [r5 + 15 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 15 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 721 * 16], m3 ; mode 13 [row 9] pmaddubsw m3, m6, [r5 + 6 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 6 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 722 * 16], m3 pmaddubsw m3, m1, [r5 + 6 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 6 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 723 * 16], m3 ; mode 14 [row 4] pinsrb m6, [r3 + 2], 2 pinsrb m6, [r3 + 2], 1 pinsrb m6, [r3 + 5], 0 pmaddubsw m3, m6, [r5 + 31 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 31 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 776 * 16], m3 pmaddubsw m3, m1, [r5 + 31 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 31 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 777 * 16], m3 ; mode 14 [row 5] pmaddubsw m3, m6, [r5 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 18 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 778 * 16], m3 pmaddubsw m3, m1, [r5 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 18 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 779 * 16], m3 ; mode 14 [row 6] pmaddubsw m3, m6, [r5 + 5 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 5 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 780 * 16], m3 pmaddubsw m3, m1, [r5 + 5 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 5 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 781 * 16], m3 ; mode 14 [row 7] pslldq m6, 2 pinsrb m6, [r3 + 5], 1 pinsrb m6, [r3 + 7], 0 pmaddubsw m3, m6, [r5 + 24 * 16] pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 5], 0 pmaddubsw m5, m2, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 782 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 13], 0 pmaddubsw m3, m1, [r5 + 24 * 16] pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 21], 0 pmaddubsw m5, m4, [r5 + 24 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 783 * 16], m3 ; mode 14 [row 8] pmaddubsw m3, m6, [r5 + 11 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 11 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 784 * 16], m3 pmaddubsw m3, m1, [r5 + 11 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 11 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 785 * 16], m3 ; mode 15 [row 5, 8-31] pmaddubsw m5, m2, [r5 + 26 * 16] pmulhrsw m5, m7 packuswb m5, m5 movh [r0 + 842 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 26 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 26 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 843 * 16], m3 ; mode 15 [row 6, 8-31] pmaddubsw m5, m2, [r5 + 9 * 16] pmulhrsw m5, m7 packuswb m5, m5 movh [r0 + 844 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 9 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 9 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 845 * 16], m3 ; mode 12 [row 19] pslldq m0, 2 pinsrb m0, [r3 + 13], 1 pinsrb m0, [r3 + 19], 0 pmaddubsw m3, m0, [r5 + 28 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 678 * 16], m3 pmaddubsw m3, m1, [r5 + 28 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 28 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 679 * 16], m3 ; mode 12 [row 20] pmaddubsw m3, m0, [r5 + 23 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 23 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 680 * 16], m3 pmaddubsw m3, m1, [r5 + 23 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 23 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 681 * 16], m3 ; mode 12 [row 21] pmaddubsw m3, m0, [r5 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 18 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 682 * 16], m3 pmaddubsw m3, m1, [r5 + 18 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 18 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 683 * 16], m3 ; mode 12 [row 22] pmaddubsw m3, m0, [r5 + 13 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 13 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 684 * 16], m3 pmaddubsw m3, m1, [r5 + 13 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 13 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 685 * 16], m3 ; mode 12 [row 23] pmaddubsw m3, m0, [r5 + 8 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 8 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 686 * 16], m3 pmaddubsw m3, m1, [r5 + 8 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 8 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 687 * 16], m3 ; mode 12 [row 24] pmaddubsw m3, m0, [r5 + 3 * 16] pmulhrsw m3, m7 pmaddubsw m5, m2, [r5 + 3 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 688 * 16], m3 pmaddubsw m3, m1, [r5 + 3 * 16] pmulhrsw m3, m7 pmaddubsw m5, m4, [r5 + 3 * 16] pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 689 * 16], m3 ; mode 13 [row 10] movu m7, m6 movu m6, m0 pinsrb m6, [r3 + 4], 4 pinsrb m6, [r3 + 4], 3 pinsrb m6, [r3 + 7], 2 pinsrb m6, [r3 + 7], 1 pinsrb m6, [r3 + 11], 0 pmaddubsw m3, m6, [r5 + 29 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 29 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 724 * 16], m3 pmaddubsw m3, m1, [r5 + 29 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 29 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 725 * 16], m3 ; mode 13 [row 11] pmaddubsw m3, m6, [r5 + 20 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 20 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 726 * 16], m3 pmaddubsw m3, m1, [r5 + 20 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 20 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 727 * 16], m3 ; mode 13 [row 12] pmaddubsw m3, m6, [r5 + 11 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 11 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 728 * 16], m3 pmaddubsw m3, m1, [r5 + 11 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 11 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 729 * 16], m3 ; mode 13 [row 13] pmaddubsw m3, m6, [r5 + 2 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 2 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 730 * 16], m3 pmaddubsw m3, m1, [r5 + 2 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 2 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 731 * 16], m3 ; mode 14 [row 9] pslldq m7, 2 pinsrb m7, [r3 + 7], 1 pinsrb m7, [r3 + 10], 0 pmaddubsw m3, m7, [r5 + 30 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrw m2, [r4 + 4], 0 pmaddubsw m5, m2, [r5 + 30 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 786 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 12], 0 pmaddubsw m3, m1, [r5 + 30 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrb m4, [r4 + 21], 1 pinsrb m4, [r4 + 20], 0 pmaddubsw m5, m4, [r5 + 30 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 787 * 16], m3 ; mode 14 [row 10] pmaddubsw m3, m7, [r5 + 17 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 17 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 788 * 16], m3 pmaddubsw m3, m1, [r5 + 17 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 17 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 789 * 16], m3 ; mode 14 [row 11] pmaddubsw m3, m7, [r5 + 4 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 4 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 790 * 16], m3 pmaddubsw m3, m1, [r5 + 4 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 4 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 791 * 16], m3 movu m6, [pw_1024] ; mode 15 [row 7, 8-31] pmaddubsw m5, m2, [r5 + 24 * 16] pmulhrsw m5, m6 packuswb m5, m5 movh [r0 + 846 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 24 * 16] pmulhrsw m3, m6 pmaddubsw m5, m4, [r5 + 24 * 16] pmulhrsw m5, m6 packuswb m3, m5 movu [r0 + 847 * 16], m3 ; mode 15 [row 8, 8-31] pmaddubsw m5, m2, [r5 + 7 * 16] pmulhrsw m5, m6 packuswb m5, m5 movh [r0 + 848 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 7 * 16] pmulhrsw m3, m6 pmaddubsw m5, m4, [r5 + 7 * 16] pmulhrsw m5, m6 packuswb m3, m5 movu [r0 + 849 * 16], m3 ; mode 12 [row 25] pslldq m0, 2 pinsrb m0, [r3 + 19], 1 pinsrb m0, [r3 + 26], 0 pmaddubsw m3, m0, [r5 + 30 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 30 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 690 * 16], m3 pmaddubsw m3, m1, [r5 + 30 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 30 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 691 * 16], m3 ; mode 12 [row 26] pmaddubsw m3, m0, [r5 + 25 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 25 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 692 * 16], m3 pmaddubsw m3, m1, [r5 + 25 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 25 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 693 * 16], m3 ; mode 12 [row 27] pmaddubsw m3, m0, [r5 + 20 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 20 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 694 * 16], m3 pmaddubsw m3, m1, [r5 + 20 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 20 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 695 * 16], m3 ; mode 12 [row 28] pmaddubsw m3, m0, [r5 + 15 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 15 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 696 * 16], m3 pmaddubsw m3, m1, [r5 + 15 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 15 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 697 * 16], m3 ; mode 12 [row 29] pmaddubsw m3, m0, [r5 + 10 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 10 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 698 * 16], m3 pmaddubsw m3, m1, [r5 + 10 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 10 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 699 * 16], m3 ; mode 12 [row 30] pmaddubsw m3, m0, [r5 + 5 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 5 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 700 * 16], m3 pmaddubsw m3, m1, [r5 + 5 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 5 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 701 * 16], m3 ; mode 13 [row 14] movu m6, m0 pinsrb m6, [r3 + 4], 6 pinsrb m6, [r3 + 4], 5 pinsrb m6, [r3 + 7], 4 pinsrb m6, [r3 + 7], 3 pinsrb m6, [r3 + 11], 2 pinsrb m6, [r3 + 11], 1 pinsrb m6, [r3 + 14], 0 pmaddubsw m3, m6, [r5 + 25 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 25 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 732 * 16], m3 pmaddubsw m3, m1, [r5 + 25 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 25 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 733 * 16], m3 ; mode 13 [row 15] pmaddubsw m3, m6, [r5 + 16 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 16 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 734 * 16], m3 pmaddubsw m3, m1, [r5 + 16 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 16 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 735 * 16], m3 ; mode 13 [row 16] pmaddubsw m3, m6, [r5 + 7 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 7 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 736 * 16], m3 pmaddubsw m3, m1, [r5 + 7 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 7 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 737 * 16], m3 ; mode 13 [row 17] pslldq m6, 2 pinsrb m6, [r3 + 14], 1 pinsrb m6, [r3 + 18], 0 pmaddubsw m3, m6, [r5 + 30 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrw m2, [r4 + 3], 0 pmaddubsw m5, m2, [r5 + 30 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 738 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 11], 0 pmaddubsw m3, m1, [r5 + 30 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 19], 0 pmaddubsw m5, m4, [r5 + 30 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 739 * 16], m3 ; mode 13 [row 18] pmaddubsw m3, m6, [r5 + 21 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 21 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 740 * 16], m3 pmaddubsw m3, m1, [r5 + 21 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 21 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 741 * 16], m3 ; mode 13 [row 19] pmaddubsw m3, m6, [r5 + 12 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 12 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 742 * 16], m3 pmaddubsw m3, m1, [r5 + 12 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 12 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 743 * 16], m3 ; mode 13 [row 20] pmaddubsw m3, m6, [r5 + 3 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 3 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 744 * 16], m3 pmaddubsw m3, m1, [r5 + 3 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 3 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 745 * 16], m3 ; mode 14 [row 12] pslldq m7, 2 pinsrb m7, [r3 + 10], 1 pinsrb m7, [r3 + 12], 0 pmaddubsw m3, m7, [r5 + 23 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 23 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 792 * 16], m3 pmaddubsw m3, m1, [r5 + 23 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 23 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 793 * 16], m3 ; mode 14 [row 13] pmaddubsw m3, m7, [r5 + 10 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 10 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 794 * 16], m3 pmaddubsw m3, m1, [r5 + 10 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 10 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 795 * 16], m3 ; mode 15 [row 9] pmaddubsw m5, m2, [r5 + 22 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movu [r0 + 850 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 22 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 22 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 851 * 16], m3 ; mode 15 [row 10] pmaddubsw m5, m2, [r5 + 5 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movu [r0 + 852 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 5 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 5 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 853 * 16], m3 ; mode 13 [row 21] pslldq m6, 2 pinsrb m6, [r3 + 18], 1 pinsrb m6, [r3 + 21], 0 pmaddubsw m3, m6, [r5 + 26 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrw m2, [r4 + 2], 0 pmaddubsw m5, m2, [r5 + 26 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 746 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 10], 0 pmaddubsw m3, m1, [r5 + 26 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 18], 0 pmaddubsw m5, m4, [r5 + 26 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 747 * 16], m3 ; mode 13 [row 22] pmaddubsw m3, m6, [r5 + 17 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 17 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 748 * 16], m3 pmaddubsw m3, m1, [r5 + 17 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 17 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 749 * 16], m3 ; mode 13 [row 23] pmaddubsw m3, m6, [r5 + 8 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 8 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 750 * 16], m3 pmaddubsw m3, m1, [r5 + 8 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 8 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 751 * 16], m3 ; mode 14 [row 14] pslldq m7, 2 pinsrb m7, [r3 + 12], 1 pinsrb m7, [r3 + 15], 0 pmaddubsw m3, m7, [r5 + 29 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 29 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 796 * 16], m3 pmaddubsw m3, m1, [r5 + 29 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 29 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 797 * 16], m3 ; mode 14 [row 15] pmaddubsw m3, m7, [r5 + 16 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 16 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 798 * 16], m3 pmaddubsw m3, m1, [r5 + 16 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 16 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 799 * 16], m3 ; mode 14 [row 16] pmaddubsw m3, m7, [r5 + 3 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 3 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 800 * 16], m3 pmaddubsw m3, m1, [r5 + 3 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 3 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 801 * 16], m3 ; mode 15 [row 11] pmaddubsw m5, m2, [r5 + 20 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 854 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 20 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 20 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 855 * 16], m3 ; mode 15 [row 12] pmaddubsw m5, m2, [r5 + 3 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 856 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 3 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 3 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 857 * 16], m3 ; mode 13 [row 24] pslldq m6, 2 pinsrb m6, [r3 + 21], 1 pinsrb m6, [r3 + 25], 0 pmaddubsw m3, m6, [r5 + 31 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrw m2, [r4 + 1], 0 pmaddubsw m5, m2, [r5 + 31 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 752 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 9], 0 pmaddubsw m3, m1, [r5 + 31 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 17], 0 pmaddubsw m5, m4, [r5 + 31 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 753 * 16], m3 ; mode 13 [row 25] pmaddubsw m3, m6, [r5 + 22 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 22 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 754 * 16], m3 pmaddubsw m3, m1, [r5 + 22 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 22 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 755 * 16], m3 ; mode 13 [row 26] pmaddubsw m3, m6, [r5 + 13 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 13 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 756 * 16], m3 pmaddubsw m3, m1, [r5 + 13 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 13 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 757 * 16], m3 ; mode 13 [row 27] pmaddubsw m3, m6, [r5 + 4 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 4 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 758 * 16], m3 pmaddubsw m3, m1, [r5 + 4 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 4 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 759 * 16], m3 ; mode 14 [row 17] pslldq m7, 2 pinsrb m7, [r3 + 15], 1 pinsrb m7, [r3 + 17], 0 pmaddubsw m3, m7, [r5 + 22 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 22 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 802 * 16], m3 pmaddubsw m3, m1, [r5 + 22 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 22 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 803 * 16], m3 ; mode 14 [row 18] pmaddubsw m3, m7, [r5 + 9 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 9 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 804 * 16], m3 pmaddubsw m3, m1, [r5 + 9 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 9 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 805 * 16], m3 ; mode 15 [row 13] pmaddubsw m5, m2, [r5 + 18 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 858 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 18 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 18 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 859 * 16], m3 ; mode 15 [row 14] pmaddubsw m5, m2, [r5 + 1 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 860 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 1 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 1 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 861 * 16], m3 ; mode 13 [row 28] pslldq m6, 2 pinsrb m6, [r3 + 25], 1 pinsrb m6, [r3 + 28], 0 pmaddubsw m3, m6, [r5 + 27 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrw m2, [r4 + 0], 0 pmaddubsw m5, m2, [r5 + 27 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 760 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 8], 0 pmaddubsw m3, m1, [r5 + 27 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 16], 0 pmaddubsw m5, m4, [r5 + 27 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 761 * 16], m3 ; mode 13 [row 29] pmaddubsw m3, m6, [r5 + 18 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 18 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 762 * 16], m3 pmaddubsw m3, m1, [r5 + 18 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 18 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 763 * 16], m3 ; mode 13 [row 30] pmaddubsw m3, m6, [r5 + 9 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 9 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 764 * 16], m3 pmaddubsw m3, m1, [r5 + 9 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 9 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 765 * 16], m3 ; mode 14 [row 19] pslldq m7, 2 pinsrb m7, [r3 + 17], 1 pinsrb m7, [r3 + 20], 0 pmaddubsw m3, m7, [r5 + 28 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 28 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 806 * 16], m3 pmaddubsw m3, m1, [r5 + 28 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 28 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 807 * 16], m3 ; mode 14 [row 20] pmaddubsw m3, m7, [r5 + 15 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 15 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 808 * 16], m3 pmaddubsw m3, m1, [r5 + 15 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 15 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 809 * 16], m3 ; mode 14 [row 21] pmaddubsw m3, m7, [r5 + 2 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 2 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 810 * 16], m3 pmaddubsw m3, m1, [r5 + 2 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 2 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 811 * 16], m3 ; mode 15 [row 15] pmaddubsw m5, m2, [r5 + 16 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 862 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 16 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 16 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 863 * 16], m3 ; mode 14 [row 22] pslldq m7, 2 pinsrb m7, [r3 + 20], 1 pinsrb m7, [r3 + 22], 0 pmaddubsw m3, m7, [r5 + 21 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrb m2, [r4 + 0], 1 pinsrb m2, [r3 + 2], 0 pmaddubsw m5, m2, [r5 + 21 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 812 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 7], 0 pmaddubsw m3, m1, [r5 + 21 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 15], 0 pmaddubsw m5, m4, [r5 + 21 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 813 * 16], m3 ; mode 14 [row 23] pmaddubsw m3, m7, [r5 + 8 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 8 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 814 * 16], m3 pmaddubsw m3, m1, [r5 + 8 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 8 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 815 * 16], m3 ; mode 15 [row 16] pmaddubsw m5, m2, [r5 + 31 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 864 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 31 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 31 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 865 * 16], m3 ; mode 15 [row 17] pmaddubsw m5, m2, [r5 + 14 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 866 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 14 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 14 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 867 * 16], m3 ; mode 14 [row 24] pslldq m7, 2 pinsrb m7, [r3 + 22], 1 pinsrb m7, [r3 + 25], 0 pmaddubsw m3, m7, [r5 + 27 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrb m2, [r3 + 2], 1 pinsrb m2, [r3 + 5], 0 pmaddubsw m5, m2, [r5 + 27 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 816 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 6], 0 pmaddubsw m3, m1, [r5 + 27 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 14], 0 pmaddubsw m5, m4, [r5 + 27 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 817 * 16], m3 ; mode 14 [row 25] pmaddubsw m3, m7, [r5 + 14 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 14 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 818 * 16], m3 pmaddubsw m3, m1, [r5 + 14 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 14 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 819 * 16], m3 ; mode 14 [row 26] pmaddubsw m3, m7, [r5 + 1 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 1 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 820 * 16], m3 pmaddubsw m3, m1, [r5 + 1 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 1 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 821 * 16], m3 ; mode 15 [row 18] pinsrb m2, [r3 + 4], 0 pmaddubsw m5, m2, [r5 + 29 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 868 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 29 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 29 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 869 * 16], m3 ; mode 15 [row 19] pmaddubsw m5, m2, [r5 + 12 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 870 * 16 + 8], m5 pmaddubsw m3, m1, [r5 + 12 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 12 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 871 * 16], m3 ; mode 15 [row 20 - 8 to 15] pslldq m3, m2, 2 pinsrb m3, [r3 + 4], 1 pinsrb m3, [r3 + 6], 0 pmaddubsw m5, m3, [r5 + 27 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 872 * 16 + 8], m5 ; mode 15 [row 21 - 8 to 15] pmaddubsw m5, m3, [r5 + 10 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 874 * 16 + 8], m5 ; mode 15 [row 22 - 8 to 15] pslldq m3, 2 pinsrb m3, [r3 + 6], 1 pinsrb m3, [r3 + 8], 0 pmaddubsw m5, m3, [r5 + 25 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 876 * 16 + 8], m5 ; mode 15 [row 23 - 8 to 15] pmaddubsw m5, m3, [r5 + 8 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 878 * 16 + 8], m5 ; mode 15 [row 24 - 8 to 15] pslldq m3, 2 pinsrb m3, [r3 + 8], 1 pinsrb m3, [r3 + 9], 0 pmaddubsw m5, m3, [r5 + 23 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 880 * 16 + 8], m5 ; mode 15 [row 25 - 8 to 15] pmaddubsw m5, m3, [r5 + 6 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 882 * 16 + 8], m5 ; mode 15 [row 26 - 8 to 15] pslldq m3, 2 pinsrb m3, [r3 + 9], 1 pinsrb m3, [r3 + 11], 0 pmaddubsw m5, m3, [r5 + 21 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 884 * 16 + 8], m5 ; mode 15 [row 27 - 8 to 15] pmaddubsw m5, m3, [r5 + 4 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 886 * 16 + 8], m5 ; mode 15 [row 28 - 8 to 15] pslldq m3, 2 pinsrb m3, [r3 + 11], 1 pinsrb m3, [r3 + 13], 0 pmaddubsw m5, m3, [r5 + 19 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 888 * 16 + 8], m5 ; mode 15 [row 29 - 8 to 15] pmaddubsw m5, m3, [r5 + 2 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 890 * 16 + 8], m5 ; mode 15 [row 30 - 8 to 15] pslldq m3, 2 pinsrb m3, [r3 + 13], 1 pinsrb m3, [r3 + 15], 0 pmaddubsw m5, m3, [r5 + 17 * 16] pmulhrsw m5, [pw_1024] packuswb m5, m5 movh [r0 + 892 * 16 + 8], m5 ; mode 15 [row 31, 8 to 15] pshufb m5, m3, [tab_S2] movh [r0 + 894 * 16 + 8], m5 ; mode 14 [row 27] pinsrb m2, [r3 + 5], 0 pslldq m7, 2 pinsrb m7, [r3 + 25], 1 pinsrb m7, [r3 + 27], 0 pmaddubsw m3, m7, [r5 + 20 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrb m2, [r3 + 5], 1 pinsrb m2, [r3 + 7], 0 pmaddubsw m5, m2, [r5 + 20 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 822 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 5], 0 pmaddubsw m3, m1, [r5 + 20 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 13], 0 pmaddubsw m5, m4, [r5 + 20 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 823 * 16], m3 ; mode 15 [row 20 - 16 to 31] pmaddubsw m3, m1, [r5 + 27 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 27 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 873 * 16], m3 ; mode 15 [row 21 - 16 to 31] pmaddubsw m3, m1, [r5 + 10 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 10 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 875 * 16], m3 ; mode 14 [row 28] pmaddubsw m3, m7, [r5 + 7 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 7 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 824 * 16], m3 pmaddubsw m3, m1, [r5 + 7 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 7 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 825 * 16], m3 ; mode 14 [row 29] pslldq m7, 2 pinsrb m7, [r3 + 27], 1 pinsrb m7, [r3 + 30], 0 pmaddubsw m3, m7, [r5 + 26 * 16] pmulhrsw m3, [pw_1024] pslldq m2, 2 pinsrb m2, [r3 + 7], 1 pinsrb m2, [r3 + 10], 0 pmaddubsw m5, m2, [r5 + 26 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 826 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 4], 0 pmaddubsw m3, m1, [r5 + 26 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 12], 0 pmaddubsw m5, m4, [r5 + 26 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 827 * 16], m3 ; mode 14 [row 30] pmaddubsw m3, m7, [r5 + 13 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m2, [r5 + 13 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 828 * 16], m3 pmaddubsw m3, m1, [r5 + 13 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 13 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 829 * 16], m3 ; mode 15 [row 22] pmaddubsw m3, m1, [r5 + 25 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 25 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 877 * 16], m3 ; mode 15 [row 23] pmaddubsw m3, m1, [r5 + 8 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 8 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 879 * 16], m3 ; mode 14 [row 31] pshufb m3, m7, [tab_S2] movh [r0 + 830 * 16], m3 pshufb m3, m2, [tab_S2] movh [r0 + 830 * 16 + 8], m3 pshufb m3, m1, [tab_S2] movh [r0 + 831 * 16], m3 pshufb m3, m4, [tab_S2] movh [r0 + 831 * 16 + 8], m3 ; mode 13 [row 31] pshufb m0, m6, [tab_S2] movh [r0 + 766 * 16], m0 movh m0, [r4] movh [r0 + 766 * 16 + 8], m0 movu m0, [r4 + 8] movu [r0 + 767 * 16], m0 ; mode 15 [row 24] pslldq m1, 2 pinsrw m1, [r4 + 3], 0 pmaddubsw m3, m1, [r5 + 23 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 11], 0 pmaddubsw m5, m4, [r5 + 23 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 881 * 16], m3 ; mode 15 [row 25] pmaddubsw m3, m1, [r5 + 6 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 6 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 883 * 16], m3 ; mode 15 [row 26] pslldq m1, 2 pinsrw m1, [r4 + 2], 0 pmaddubsw m3, m1, [r5 + 21 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 10], 0 pmaddubsw m5, m4, [r5 + 21 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 885 * 16], m3 ; mode 15 [row 27] pmaddubsw m3, m1, [r5 + 4 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 4 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 887 * 16], m3 ; mode 15 [row 28] pslldq m1, 2 pinsrw m1, [r4 + 1], 0 pmaddubsw m3, m1, [r5 + 19 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 9], 0 pmaddubsw m5, m4, [r5 + 19 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 889 * 16], m3 ; mode 15 [row 29] pmaddubsw m3, m1, [r5 + 2 * 16] pmulhrsw m3, [pw_1024] pmaddubsw m5, m4, [r5 + 2 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 891 * 16], m3 ; mode 15 [row 30] pslldq m1, 2 pinsrw m1, [r4 + 0], 0 pmaddubsw m3, m1, [r5 + 17 * 16] pmulhrsw m3, [pw_1024] pslldq m4, 2 pinsrw m4, [r4 + 8], 0 pmaddubsw m5, m4, [r5 + 17 * 16] pmulhrsw m5, [pw_1024] packuswb m3, m5 movu [r0 + 893 * 16], m3 ; mode 15 [row 31] pshufb m5, m1, [tab_S2] movh [r0 + 895 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 895 * 16 + 8], m5 ; mode 16 [row 0] movu m6, [r5 + 11 * 16] movu m7, [pw_1024] movh m0, [r4 ] movh m1, [r4 + 1 ] punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movh m2, [r4 + 8] movh m3, [r4 + 9] punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 896 * 16], m1 movh m1, [r4 + 16] movh m3, [r4 + 17] punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movh m4, [r4 + 24] movh m5, [r4 + 25] punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 897 * 16], m3 ; mode16 [row 1] movu m6, [r5 + 22 * 16] pslldq m0, 2 pinsrb m0, [r4], 1 pinsrb m0, [r3 + 2], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 898 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 15], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 23], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 899 * 16], m3 ; mode16 [row 2] movu m6, [r5 + 1 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 900 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 901 * 16], m3 ; mode16 [row 3] movu m6, [r5 + 12 * 16] pslldq m0, 2 pinsrb m0, [r3 + 2], 1 pinsrb m0, [r3 + 3], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 902 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 14], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 22], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 903 * 16], m3 ; mode16 [row 4] movu m6, [r5 + 23 * 16] pslldq m0, 2 pinsrb m0, [r3 + 3], 1 pinsrb m0, [r3 + 5], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 904 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 13], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 21], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 905 * 16], m3 ; mode16 [row 5] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 906 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 907 * 16], m3 ; mode16 [row 6] movu m6, [r5 + 13 * 16] pslldq m0, 2 pinsrb m0, [r3 + 5], 1 pinsrb m0, [r3 + 6], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r4 + 5], 1 pinsrb m2, [r4 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 908 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 12], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 20], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 909 * 16], m3 ; mode16 [row 7] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r3 + 6], 1 pinsrb m0, [r3 + 8], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 910 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 11], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 19], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 911 * 16], m3 ; mode16 [row 8] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 912 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 913 * 16], m3 ; mode16 [row 9] movu m6, [r5 + 14 * 16] pslldq m0, 2 pinsrb m0, [r3 + 8], 1 pinsrb m0, [r3 + 9], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 914 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 10], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 18], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 915 * 16], m3 ; mode16 [row 10] movu m6, [r5 + 25 * 16] pslldq m0, 2 pinsrb m0, [r3 + 9], 1 pinsrb m0, [r3 + 11], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 916 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 9], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrb m4, [r4 + 18], 1 pinsrb m4, [r4 + 17], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 917 * 16], m3 ; mode16 [row 11] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 918 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 919 * 16], m3 ; mode16 [row 12] movu m6, [r5 + 15 * 16] pslldq m0, 2 pinsrb m0, [r3 + 11], 1 pinsrb m0, [r3 + 12], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 0], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 920 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 8], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 16], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 921 * 16], m3 ; mode16 [row 13] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r3 + 12], 1 pinsrb m0, [r3 + 14], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r4 + 0], 1 pinsrb m2, [r3 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 922 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 7], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 15], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 923 * 16], m3 ; mode16 [row 14] movu m6, [r5 + 5 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 924 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 925 * 16], m3 ; mode16 [row 15] movu m6, [r5 + 16 * 16] pslldq m0, 2 pinsrb m0, [r3 + 14], 1 pinsrb m0, [r3 + 15], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 2], 1 pinsrb m2, [r3 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 926 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 6], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 14], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 927 * 16], m3 ; mode16 [row 16] movu m6, [r5 + 27 * 16] pslldq m0, 2 pinsrb m0, [r3 + 15], 1 pinsrb m0, [r3 + 17], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 3], 1 pinsrb m2, [r3 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 928 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 5], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 13], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 929 * 16], m3 ; mode16 [row 17] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 930 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 931 * 16], m3 ; mode16 [row 18] movu m6, [r5 + 17 * 16] pslldq m0, 2 pinsrb m0, [r3 + 17], 1 pinsrb m0, [r3 + 18], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 5], 1 pinsrb m2, [r3 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 932 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 4], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 12], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 933 * 16], m3 ; mode16 [row 19] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r3 + 18], 1 pinsrb m0, [r3 + 20], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 6], 1 pinsrb m2, [r3 + 8], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 934 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 3], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 11], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 935 * 16], m3 ; mode16 [row 20] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 936 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 937 * 16], m3 ; mode16 [row 21] movu m6, [r5 + 18 * 16] pslldq m0, 2 pinsrb m0, [r3 + 20], 1 pinsrb m0, [r3 + 21], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 8], 1 pinsrb m2, [r3 + 9], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 938 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 2], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 10], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 939 * 16], m3 ; mode16 [row 22] movu m6, [r5 + 29 * 16] pslldq m0, 2 pinsrb m0, [r3 + 21], 1 pinsrb m0, [r3 + 23], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 9], 1 pinsrb m2, [r3 + 11], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 940 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 1], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 9], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 941 * 16], m3 ; mode16 [row 23] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 942 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 943 * 16], m3 ; mode16 [row 24] movu m6, [r5 + 19 * 16] pslldq m0, 2 pinsrb m0, [r3 + 23], 1 pinsrb m0, [r3 + 24], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 11], 1 pinsrb m2, [r3 + 12], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 944 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 0], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 8], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 945 * 16], m3 ; mode16 [row 25] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r3 + 24], 1 pinsrb m0, [r3 + 26], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 12], 1 pinsrb m2, [r3 + 14], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 946 * 16], m3 pslldq m1, 2 pinsrb m1, [r4 + 0], 1 pinsrb m1, [r3 + 2], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 7], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 947 * 16], m3 ; mode16 [row 26] movu m6, [r5 + 9 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 948 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 949 * 16], m3 ; mode16 [row 27] movu m6, [r5 + 20 * 16] pslldq m0, 2 pinsrb m0, [r3 + 26], 1 pinsrb m0, [r3 + 27], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 14], 1 pinsrb m2, [r3 + 15], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 950 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 2], 1 pinsrb m1, [r3 + 3], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 6], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 951 * 16], m3 ; mode16 [row 28] movu m6, [r5 + 31 * 16] pslldq m0, 2 pinsrb m0, [r3 + 27], 1 pinsrb m0, [r3 + 29], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 15], 1 pinsrb m2, [r3 + 17], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 952 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 3], 1 pinsrb m1, [r3 + 5], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 5], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 953 * 16], m3 ; mode16 [row 29] movu m6, [r5 + 10 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 954 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 955 * 16], m3 ; mode16 [row 30] movu m6, [r5 + 21 * 16] pslldq m0, 2 pinsrb m0, [r3 + 29], 1 pinsrb m0, [r3 + 30], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 17], 1 pinsrb m2, [r3 + 18], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 956 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 5], 1 pinsrb m1, [r3 + 6], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 4], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 957 * 16], m3 ; mode16 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 958 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 958 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 959 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 959 * 16 + 8], m5 ; mode 17 [row 0] movu m6, [r5 + 6 * 16] movu m7, [pw_1024] movh m0, [r4 ] movh m1, [r4 + 1 ] punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movh m2, [r4 + 8] movh m3, [r4 + 9] punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 960 * 16], m1 movh m1, [r4 + 16] movh m3, [r4 + 17] punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movh m4, [r4 + 24] movh m5, [r4 + 25] punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 961 * 16], m3 ; mode17 [row 1] movu m6, [r5 + 12 * 16] pslldq m0, 2 pinsrb m0, [r3 + 0], 1 pinsrb m0, [r3 + 1], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 962 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 15], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 23], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 963 * 16], m3 ; mode17 [row 2] movu m6, [r5 + 18 * 16] pslldq m0, 2 pinsrb m0, [r3 + 1], 1 pinsrb m0, [r3 + 2], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 964 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 14], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 22], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 965 * 16], m3 ; mode17 [row 3] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r3 + 2], 1 pinsrb m0, [r3 + 4], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 966 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 13], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 21], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 967 * 16], m3 ; mode17 [row 4] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r3 + 4], 1 pinsrb m0, [r3 + 5], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 968 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 12], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 20], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 969 * 16], m3 ; mode17 [row 5] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 970 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 971 * 16], m3 ; mode17 [row 6] movu m6, [r5 + 10 * 16] pslldq m0, 2 pinsrb m0, [r3 + 5], 1 pinsrb m0, [r3 + 6], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 972 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 11], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 19], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 973 * 16], m3 ; mode17 [row 7] movu m6, [r5 + 16 * 16] pslldq m0, 2 pinsrb m0, [r3 + 6], 1 pinsrb m0, [r3 + 7], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 974 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 10], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 18], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 975 * 16], m3 ; mode17 [row 8] movu m6, [r5 + 22 * 16] pslldq m0, 2 pinsrb m0, [r3 + 7], 1 pinsrb m0, [r3 + 9], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 976 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 9], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 17], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 977 * 16], m3 ; mode17 [row 9] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r3 + 9], 1 pinsrb m0, [r3 + 10], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrw m2, [r4 + 0], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 978 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 8], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 16], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 979 * 16], m3 ; mode17 [row 10] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 980 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 981 * 16], m3 ; mode17 [row 11] movu m6, [r5 + 8 * 16] pslldq m0, 2 pinsrb m0, [r3 + 10], 1 pinsrb m0, [r3 + 11], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r4 + 0], 1 pinsrb m2, [r3 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 982 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 7], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 15], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 983 * 16], m3 ; mode17 [row 12] movu m6, [r5 + 14 * 16] pslldq m0, 2 pinsrb m0, [r3 + 11], 1 pinsrb m0, [r3 + 12], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 1], 1 pinsrb m2, [r3 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 984 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 6], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 14], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 985 * 16], m3 ; mode17 [row 13] movu m6, [r5 + 20 * 16] pslldq m0, 2 pinsrb m0, [r3 + 12], 1 pinsrb m0, [r3 + 14], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 2], 1 pinsrb m2, [r3 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 986 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 5], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 13], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 987 * 16], m3 ; mode17 [row 14] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r3 + 14], 1 pinsrb m0, [r3 + 15], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 4], 1 pinsrb m2, [r3 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 988 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 4], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 12], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 989 * 16], m3 ; mode17 [row 15] pshufb m5, m0, [tab_S2] movh [r0 + 990 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 990 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 991 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 991 * 16 + 8], m5 ; mode17 [row 16] movu m6, [r5 + 6 * 16] pslldq m0, 2 pinsrb m0, [r3 + 15], 1 pinsrb m0, [r3 + 16], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 5], 1 pinsrb m2, [r3 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 992 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 3], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 11], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 993 * 16], m3 ; mode17 [row 17] movu m6, [r5 + 12 * 16] pslldq m0, 2 pinsrb m0, [r3 + 16], 1 pinsrb m0, [r3 + 17], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 6], 1 pinsrb m2, [r3 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 994 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 2], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 10], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 995 * 16], m3 ; mode17 [row 18] movu m6, [r5 + 18 * 16] pslldq m0, 2 pinsrb m0, [r3 + 17], 1 pinsrb m0, [r3 + 18], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 7], 1 pinsrb m2, [r3 + 9], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 996 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 1], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 9], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 997 * 16], m3 ; mode17 [row 19] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r3 + 18], 1 pinsrb m0, [r3 + 20], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 9], 1 pinsrb m2, [r3 + 10], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 998 * 16], m3 pslldq m1, 2 pinsrw m1, [r4 + 0], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 8], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 999 * 16], m3 ; mode17 [row 20] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r3 + 20], 1 pinsrb m0, [r3 + 21], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 10], 1 pinsrb m2, [r3 + 11], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1000 * 16], m3 pslldq m1, 2 pinsrb m1, [r4 + 0], 1 pinsrb m1, [r3 + 1], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 ;pinsrb m4, [r4 + 8], 1 ;pinsrb m4, [r4 + 7], 0 pinsrw m4, [r4 + 7], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1001 * 16], m3 ; mode17 [row 21] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1002 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1003 * 16], m3 ; mode17 [row 22] movu m6, [r5 + 10 * 16] pslldq m0, 2 pinsrb m0, [r3 + 21], 1 pinsrb m0, [r3 + 22], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 11], 1 pinsrb m2, [r3 + 12], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1004 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 1], 1 pinsrb m1, [r3 + 2], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 6], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1005 * 16], m3 ; mode17 [row 23] movu m6, [r5 + 16 * 16] pslldq m0, 2 pinsrb m0, [r3 + 22], 1 pinsrb m0, [r3 + 23], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 12], 1 pinsrb m2, [r3 + 14], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1006 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 2], 1 pinsrb m1, [r3 + 4], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 5], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1007 * 16], m3 ; mode17 [row 24] movu m6, [r5 + 22 * 16] pslldq m0, 2 pinsrb m0, [r3 + 23], 1 pinsrb m0, [r3 + 25], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 14], 1 pinsrb m2, [r3 + 15], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1008 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 4], 1 pinsrb m1, [r3 + 5], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 4], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1009 * 16], m3 ; mode17 [row 25] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r3 + 25], 1 pinsrb m0, [r3 + 26], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 15], 1 pinsrb m2, [r3 + 16], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1010 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 5], 1 pinsrb m1, [r3 + 6], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 3], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1011 * 16], m3 ; mode17 [row 26] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1012 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1013 * 16], m3 ; mode17 [row 27] movu m6, [r5 + 8 * 16] pslldq m0, 2 pinsrb m0, [r3 + 26], 1 pinsrb m0, [r3 + 27], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 16], 1 pinsrb m2, [r3 + 17], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1014 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 6], 1 pinsrb m1, [r3 + 7], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 2], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1015 * 16], m3 ; mode17 [row 28] movu m6, [r5 + 14 * 16] pslldq m0, 2 pinsrb m0, [r3 + 27], 1 pinsrb m0, [r3 + 28], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 17], 1 pinsrb m2, [r3 + 18], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1016 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 7], 1 pinsrb m1, [r3 + 9], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 1], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1017 * 16], m3 ; mode17 [row 29] movu m6, [r5 + 20 * 16] pslldq m0, 2 pinsrb m0, [r3 + 28], 1 pinsrb m0, [r3 + 30], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 18], 1 pinsrb m2, [r3 + 20], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1018 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 9], 1 pinsrb m1, [r3 + 10], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrw m4, [r4 + 0], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1019 * 16], m3 ; mode17 [row 30] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r3 + 30], 1 pinsrb m0, [r3 + 31], 0 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pslldq m2, 2 pinsrb m2, [r3 + 20], 1 pinsrb m2, [r3 + 21], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1020 * 16], m3 pslldq m1, 2 pinsrb m1, [r3 + 10], 1 pinsrb m1, [r3 + 11], 0 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pslldq m4, 2 pinsrb m4, [r4 + 0], 1 pinsrb m4, [r3 + 1], 0 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1021 * 16], m3 ; mode17 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 1022 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1022 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1023 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 1023 * 16 + 8], m5 ;mode 18[row 0] movu m0, [r3] movu [r0 + 1024 * 16], m0 movu m1, [r3 + 16] movu [r0 + 1025 * 16], m1 ;mode 18[row 1] pslldq m0, 1 pinsrb m0, [r4 + 1], 0 movu [r0 + 1026 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 15], 0 movu [r0 + 1027 * 16], m1 ;mode 18[row 2] pslldq m0, 1 pinsrb m0, [r4 + 2], 0 movu [r0 + 1028 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 14], 0 movu [r0 + 1029 * 16], m1 ;mode 18[row 3] pslldq m0, 1 pinsrb m0, [r4 + 3], 0 movu [r0 + 1030 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 13], 0 movu [r0 + 1031 * 16], m1 ;mode 18[row 4] pslldq m0, 1 pinsrb m0, [r4 + 4], 0 movu [r0 + 1032 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 12], 0 movu [r0 + 1033 * 16], m1 ;mode 18[row 5] pslldq m0, 1 pinsrb m0, [r4 + 5], 0 movu [r0 + 1034 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 11], 0 movu [r0 + 1035 * 16], m1 ;mode 18[row 6] pslldq m0, 1 pinsrb m0, [r4 + 6], 0 movu [r0 + 1036 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 10], 0 movu [r0 + 1037 * 16], m1 ;mode 18[row 7] pslldq m0, 1 pinsrb m0, [r4 + 7], 0 movu [r0 + 1038 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 9], 0 movu [r0 + 1039 * 16], m1 ;mode 18[row 8] pslldq m0, 1 pinsrb m0, [r4 + 8], 0 movu [r0 + 1040 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 8], 0 movu [r0 + 1041 * 16], m1 ;mode 18[row 9] pslldq m0, 1 pinsrb m0, [r4 + 9], 0 movu [r0 + 1042 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 7], 0 movu [r0 + 1043 * 16], m1 ;mode 18[row 10] pslldq m0, 1 pinsrb m0, [r4 + 10], 0 movu [r0 + 1044 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 6], 0 movu [r0 + 1045 * 16], m1 ;mode 18[row 11] pslldq m0, 1 pinsrb m0, [r4 + 11], 0 movu [r0 + 1046 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 5], 0 movu [r0 + 1047 * 16], m1 ;mode 18[row 12] pslldq m0, 1 pinsrb m0, [r4 + 12], 0 movu [r0 + 1048 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 4], 0 movu [r0 + 1049 * 16], m1 ;mode 18[row 13] pslldq m0, 1 pinsrb m0, [r4 + 13], 0 movu [r0 + 1050 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 3], 0 movu [r0 + 1051 * 16], m1 ;mode 18[row 14] pslldq m0, 1 pinsrb m0, [r4 + 14], 0 movu [r0 + 1052 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 2], 0 movu [r0 + 1053 * 16], m1 ;mode 18[row 15] pslldq m0, 1 pinsrb m0, [r4 + 15], 0 movu [r0 + 1054 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 1], 0 movu [r0 + 1055 * 16], m1 ;mode 18[row 16] pslldq m0, 1 pinsrb m0, [r4 + 16], 0 movu [r0 + 1056 * 16], m0 pslldq m1, 1 pinsrb m1, [r3 + 0], 0 movu [r0 + 1057 * 16], m1 ;mode 18[row 17] pslldq m0, 1 pinsrb m0, [r4 + 17], 0 movu [r0 + 1058 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 1], 0 movu [r0 + 1059 * 16], m1 ;mode 18[row 18] pslldq m0, 1 pinsrb m0, [r4 + 18], 0 movu [r0 + 1060 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 2], 0 movu [r0 + 1061 * 16], m1 ;mode 18[row 19] pslldq m0, 1 pinsrb m0, [r4 + 19], 0 movu [r0 + 1062 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 3], 0 movu [r0 + 1063 * 16], m1 ;mode 18[row 20] pslldq m0, 1 pinsrb m0, [r4 + 20], 0 movu [r0 + 1064 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 4], 0 movu [r0 + 1065 * 16], m1 ;mode 18[row 21] pslldq m0, 1 pinsrb m0, [r4 + 21], 0 movu [r0 + 1066 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 5], 0 movu [r0 + 1067 * 16], m1 ;mode 18[row 22] pslldq m0, 1 pinsrb m0, [r4 + 22], 0 movu [r0 + 1068 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 6], 0 movu [r0 + 1069 * 16], m1 ;mode 18[row 23] pslldq m0, 1 pinsrb m0, [r4 + 23], 0 movu [r0 + 1070 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 7], 0 movu [r0 + 1071 * 16], m1 ;mode 18[row 24] pslldq m0, 1 pinsrb m0, [r4 + 24], 0 movu [r0 + 1072 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 8], 0 movu [r0 + 1073 * 16], m1 ;mode 18[row 25] pslldq m0, 1 pinsrb m0, [r4 + 25], 0 movu [r0 + 1074 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 9], 0 movu [r0 + 1075 * 16], m1 ;mode 18[row 26] pslldq m0, 1 pinsrb m0, [r4 + 26], 0 movu [r0 + 1076 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 10], 0 movu [r0 + 1077 * 16], m1 ;mode 18[row 27] pslldq m0, 1 pinsrb m0, [r4 + 27], 0 movu [r0 + 1078 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 11], 0 movu [r0 + 1079 * 16], m1 ;mode 18[row 28] pslldq m0, 1 pinsrb m0, [r4 + 28], 0 movu [r0 + 1080 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 12], 0 movu [r0 + 1081 * 16], m1 ;mode 18[row 29] pslldq m0, 1 pinsrb m0, [r4 + 29], 0 movu [r0 + 1082 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 13], 0 movu [r0 + 1083 * 16], m1 ;mode 18[row 30] pslldq m0, 1 pinsrb m0, [r4 + 30], 0 movu [r0 + 1084 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 14], 0 movu [r0 + 1085 * 16], m1 ;mode 18[row 31] pslldq m0, 1 pinsrb m0, [r4 + 31], 0 movu [r0 + 1086 * 16], m0 pslldq m1, 1 pinsrb m1, [r4 + 15], 0 movu [r0 + 1087 * 16], m1 ; mode 19 [row 0] movu m6, [r5 + 6 * 16] movu m0, [r3 ] movu m1, [r3 + 1 ] punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r3 + 8] movu m3, [r3 + 9] punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 1088 * 16], m1 movu m1, [r3 + 16] movu m3, [r3 + 17] punpcklbw m1, m3 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 movu m3, [r3 + 24] movu m5, [r3 + 25] punpcklbw m3, m5 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1089 * 16], m4 ; mode 19 [row 1] movu m6, [r5 + 12 * 16] pslldq m0, 2 pinsrb m0, [r4 + 0], 1 pinsrb m0, [r4 + 1], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1090 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 15], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 23], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1091 * 16], m4 ; mode 19 [row 2] movu m6, [r5 + 18 * 16] pslldq m0, 2 pinsrb m0, [r4 + 1], 1 pinsrb m0, [r4 + 2], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1092 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 14], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 22], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1093 * 16], m4 ; mode 19 [row 3] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r4 + 2], 1 pinsrb m0, [r4 + 4], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1094 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 13], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 21], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1095 * 16], m4 ; mode 19 [row 4] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r4 + 4], 1 pinsrb m0, [r4 + 5], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1096 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 12], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 20], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1097 * 16], m4 ; mode 19 [row 5] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1098 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1099 * 16], m4 ; mode 19 [row 6] movu m6, [r5 + 10 * 16] pslldq m0, 2 pinsrb m0, [r4 + 5], 1 pinsrb m0, [r4 + 6], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1100 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 11], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 19], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1101 * 16], m4 ; mode 19 [row 7] movu m6, [r5 + 16 * 16] pslldq m0, 2 pinsrb m0, [r4 + 6], 1 pinsrb m0, [r4 + 7], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1102 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 10], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 18], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1103 * 16], m4 ; mode 19 [row 8] movu m6, [r5 + 22 * 16] pslldq m0, 2 pinsrb m0, [r4 + 7], 1 pinsrb m0, [r4 + 9], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1104 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 9], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 17], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1105 * 16], m4 ; mode 19 [row 9] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r4 + 9], 1 pinsrb m0, [r4 + 10], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 0], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1106 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 8], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 16], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1107 * 16], m4 ; mode 19 [row 10] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1108 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1109 * 16], m4 ; mode 19 [row 11] movu m6, [r5 + 8 * 16] pslldq m0, 2 pinsrb m0, [r4 + 10], 1 pinsrb m0, [r4 + 11], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r3 + 0], 1 pinsrb m2, [r4 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1110 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 7], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 15], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1111 * 16], m4 ; mode 19 [row 12] movu m6, [r5 + 14 * 16] pslldq m0, 2 pinsrb m0, [r4 + 11], 1 pinsrb m0, [r4 + 12], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 1], 1 pinsrb m2, [r4 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1112 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 6], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 14], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1113 * 16], m4 ; mode 19 [row 13] movu m6, [r5 + 20 * 16] pslldq m0, 2 pinsrb m0, [r4 + 12], 1 pinsrb m0, [r4 + 14], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 2], 1 pinsrb m2, [r4 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1114 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 5], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 13], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1115 * 16], m4 ; mode 19 [row 14] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r4 + 14], 1 pinsrb m0, [r4 + 15], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 4], 1 pinsrb m2, [r4 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1116 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 4], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 12], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1117 * 16], m4 ; mode19 [row 15] pshufb m5, m0, [tab_S2] movh [r0 + 1118 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1118 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1119 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1119 * 16 + 8], m5 ; mode 19 [row 16] movu m6, [r5 + 6 * 16] pslldq m0, 2 pinsrb m0, [r4 + 15], 1 pinsrb m0, [r4 + 16], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 5], 1 pinsrb m2, [r4 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1120 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 3], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 11], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1121 * 16], m4 ; mode 19 [row 17] movu m6, [r5 + 12 * 16] pslldq m0, 2 pinsrb m0, [r4 + 16], 1 pinsrb m0, [r4 + 17], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 6], 1 pinsrb m2, [r4 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1122 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 2], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 10], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1123 * 16], m4 ; mode 19 [row 18] movu m6, [r5 + 18 * 16] pslldq m0, 2 pinsrb m0, [r4 + 17], 1 pinsrb m0, [r4 + 18], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 7], 1 pinsrb m2, [r4 + 9], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1124 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 1], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 9], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1125 * 16], m4 ; mode 19 [row 19] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r4 + 18], 1 pinsrb m0, [r4 + 20], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 9], 1 pinsrb m2, [r4 + 10], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1126 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 0], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 8], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1127 * 16], m4 ; mode 19 [row 20] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r4 + 20], 1 pinsrb m0, [r4 + 21], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 10], 1 pinsrb m2, [r4 + 11], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1128 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 0], 1 pinsrb m1, [r4 + 1], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrb m3, [r3 + 8], 1 pinsrb m3, [r3 + 7], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1129 * 16], m4 ; mode 19 [row 21] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1130 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1131 * 16], m4 ; mode 19 [row 22] movu m6, [r5 + 10 * 16] pslldq m0, 2 pinsrb m0, [r4 + 21], 1 pinsrb m0, [r4 + 22], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 11], 1 pinsrb m2, [r4 + 12], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1132 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 1], 1 pinsrb m1, [r4 + 2], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 6], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1133 * 16], m4 ; mode 19 [row 23] movu m6, [r5 + 16 * 16] pslldq m0, 2 pinsrb m0, [r4 + 22], 1 pinsrb m0, [r4 + 23], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 12], 1 pinsrb m2, [r4 + 14], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1134 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 2], 1 pinsrb m1, [r4 + 4], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 5], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1135 * 16], m4 ; mode 19 [row 24] movu m6, [r5 + 22 * 16] pslldq m0, 2 pinsrb m0, [r4 + 23], 1 pinsrb m0, [r4 + 25], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 14], 1 pinsrb m2, [r4 + 15], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1136 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 4], 1 pinsrb m1, [r4 + 5], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 4], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1137 * 16], m4 ; mode 19 [row 25] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r4 + 25], 1 pinsrb m0, [r4 + 26], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 15], 1 pinsrb m2, [r4 + 16], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1138 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 5], 1 pinsrb m1, [r4 + 6], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 3], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1139 * 16], m4 ; mode 19 [row 26] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1140 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1141 * 16], m4 ; mode 19 [row 27] movu m6, [r5 + 8 * 16] pslldq m0, 2 pinsrb m0, [r4 + 26], 1 pinsrb m0, [r4 + 27], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 16], 1 pinsrb m2, [r4 + 17], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1142 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 6], 1 pinsrb m1, [r4 + 7], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 2], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1143 * 16], m4 ; mode 19 [row 28] movu m6, [r5 + 14 * 16] pslldq m0, 2 pinsrb m0, [r4 + 27], 1 pinsrb m0, [r4 + 28], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 17], 1 pinsrb m2, [r4 + 18], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1144 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 7], 1 pinsrb m1, [r4 + 9], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 1], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1145 * 16], m4 ; mode 19 [row 29] movu m6, [r5 + 20 * 16] pslldq m0, 2 pinsrb m0, [r4 + 28], 1 pinsrb m0, [r4 + 30], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 18], 1 pinsrb m2, [r4 + 20], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1146 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 9], 1 pinsrb m1, [r4 + 10], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 0], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1147 * 16], m4 ; mode 19 [row 30] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r4 + 30], 1 pinsrb m0, [r4 + 31], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 20], 1 pinsrb m2, [r4 + 21], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1148 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 10], 1 pinsrb m1, [r4 + 11], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrb m3, [r4 + 0], 1 pinsrb m3, [r4 + 1], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1149 * 16], m4 ; mode19 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 1150 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1150 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1151 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1151 * 16 + 8], m5 ; mode 20 [row 0] movu m6, [r5 + 11 * 16] movu m0, [r3 ] movu m1, [r3 + 1 ] punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r3 + 8] movu m3, [r3 + 9] punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 1152 * 16], m1 movu m1, [r3 + 16] movu m3, [r3 + 17] punpcklbw m1, m3 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 movu m3, [r3 + 24] movu m5, [r3 + 25] punpcklbw m3, m5 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1153 * 16], m4 ; mode 20 [row 1] movu m6, [r5 + 22 * 16] pslldq m0, 2 pinsrb m0, [r4 + 0], 1 pinsrb m0, [r4 + 2], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1154 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 15], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 23], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1155 * 16], m4 ; mode 20 [row 2] movu m6, [r5 + 1 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1156 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1157 * 16], m4 ; mode 20 [row 3] movu m6, [r5 + 12 * 16] pslldq m0, 2 pinsrb m0, [r4 + 2], 1 pinsrb m0, [r4 + 3], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1158 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 14], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 22], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1159 * 16], m4 ; mode 20 [row 4] movu m6, [r5 + 23 * 16] pslldq m0, 2 pinsrb m0, [r4 + 3], 1 pinsrb m0, [r4 + 5], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1160 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 13], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 21], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1161 * 16], m4 ; mode 20 [row 5] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1162 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1163 * 16], m4 ; mode 20 [row 6] movu m6, [r5 + 13 * 16] pslldq m0, 2 pinsrb m0, [r4 + 5], 1 pinsrb m0, [r4 + 6], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1164 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 12], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 20], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1165 * 16], m4 ; mode 20 [row 7] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r4 + 6], 1 pinsrb m0, [r4 + 8], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1166 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 11], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 19], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1167 * 16], m4 ; mode 20 [row 8] movu m6, [r5 + 3 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1168 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1169 * 16], m4 ; mode 20 [row 9] movu m6, [r5 + 14 * 16] pslldq m0, 2 pinsrb m0, [r4 + 8], 1 pinsrb m0, [r4 + 9], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r3 + 3], 1 pinsrb m2, [r3 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1170 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 10], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 18], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1171 * 16], m4 ; mode 20 [row 10] movu m6, [r5 + 25 * 16] pslldq m0, 2 pinsrb m0, [r4 + 9], 1 pinsrb m0, [r4 + 11], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1172 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 9], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 17], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1173 * 16], m4 ; mode 20 [row 11] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1174 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1175 * 16], m4 ; mode 20 [row 12] movu m6, [r5 + 15 * 16] pslldq m0, 2 pinsrb m0, [r4 + 11], 1 pinsrb m0, [r4 + 12], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r3 + 1], 1 pinsrb m2, [r3 + 0], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1176 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 8], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 16], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1177 * 16], m4 ; mode 20 [row 13] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r4 + 12], 1 pinsrb m0, [r4 + 14], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 0], 1 pinsrb m2, [r4 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1178 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 7], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 15], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1179 * 16], m4 ; mode 20 [row 14] movu m6, [r5 + 5 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1180 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1181 * 16], m4 ; mode 20 [row 15] movu m6, [r5 + 16 * 16] pslldq m0, 2 pinsrb m0, [r4 + 14], 1 pinsrb m0, [r4 + 15], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 2], 1 pinsrb m2, [r4 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1182 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 6], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 14], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1183 * 16], m4 ; mode 20 [row 16] movu m6, [r5 + 27 * 16] pslldq m0, 2 pinsrb m0, [r4 + 15], 1 pinsrb m0, [r4 + 17], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 3], 1 pinsrb m2, [r4 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1184 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 5], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 13], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1185 * 16], m4 ; mode 20 [row 17] movu m6, [r5 + 6 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1186 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1187 * 16], m4 ; mode 20 [row 18] movu m6, [r5 + 17 * 16] pslldq m0, 2 pinsrb m0, [r4 + 17], 1 pinsrb m0, [r4 + 18], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 5], 1 pinsrb m2, [r4 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1188 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 4], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 12], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1189 * 16], m4 ; mode 20 [row 19] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r4 + 18], 1 pinsrb m0, [r4 + 20], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 6], 1 pinsrb m2, [r4 + 8], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1190 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 3], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 11], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1191 * 16], m4 ; mode 20 [row 20] movu m6, [r5 + 7 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1192 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1193 * 16], m4 ; mode 20 [row 21] movu m6, [r5 + 18 * 16] pslldq m0, 2 pinsrb m0, [r4 + 20], 1 pinsrb m0, [r4 + 21], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 8], 1 pinsrb m2, [r4 + 9], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1194 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 2], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 10], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1195 * 16], m4 ; mode 20 [row 22] movu m6, [r5 + 29 * 16] pslldq m0, 2 pinsrb m0, [r4 + 21], 1 pinsrb m0, [r4 + 23], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 9], 1 pinsrb m2, [r4 + 11], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1196 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 1], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 9], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1197 * 16], m4 ; mode 20 [row 23] movu m6, [r5 + 8 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1198 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1199 * 16], m4 ; mode 20 [row 24] movu m6, [r5 + 19 * 16] pslldq m0, 2 pinsrb m0, [r4 + 23], 1 pinsrb m0, [r4 + 24], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 11], 1 pinsrb m2, [r4 + 12], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1200 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 0], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 8], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1201 * 16], m4 ; mode 20 [row 25] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r4 + 24], 1 pinsrb m0, [r4 + 26], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 12], 1 pinsrb m2, [r4 + 14], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1202 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 0], 1 pinsrb m1, [r4 + 2], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 7], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1203 * 16], m4 ; mode 20 [row 26] movu m6, [r5 + 9 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1204 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1205 * 16], m4 ; mode 20 [row 27] movu m6, [r5 + 20 * 16] pslldq m0, 2 pinsrb m0, [r4 + 26], 1 pinsrb m0, [r4 + 27], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 14], 1 pinsrb m2, [r4 + 15], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1206 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 2], 1 pinsrb m1, [r4 + 3], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 6], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1207 * 16], m4 ; mode 20 [row 28] movu m6, [r5 + 31 * 16] pslldq m0, 2 pinsrb m0, [r4 + 27], 1 pinsrb m0, [r4 + 29], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 15], 1 pinsrb m2, [r4 + 17], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1208 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 3], 1 pinsrb m1, [r4 + 5], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 5], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1209 * 16], m4 ; mode 20 [row 29] movu m6, [r5 + 10 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1210 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1211 * 16], m4 ; mode 20 [row 30] movu m6, [r5 + 21 * 16] pslldq m0, 2 pinsrb m0, [r4 + 29], 1 pinsrb m0, [r4 + 30], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 17], 1 pinsrb m2, [r4 + 18], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1212 * 16], m4 pslldq m1, 2 pinsrb m1, [r4 + 5], 1 pinsrb m1, [r4 + 6], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 4], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1213 * 16], m4 ; mode20 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 1214 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1214 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1215 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1215 * 16 + 8], m5 ; mode 21 [row 0] movu m6, [r5 + 15 * 16] movu m0, [r3 ] movu m1, [r3 + 1 ] punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r3 + 8] movu m3, [r3 + 9] punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 1216 * 16], m1 movu m1, [r3 + 16] movu m3, [r3 + 17] punpcklbw m1, m3 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 movu m3, [r3 + 24] movu m5, [r3 + 25] punpcklbw m3, m5 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1217 * 16], m4 ; mode 21 [row 1] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r4 + 0], 1 pinsrb m0, [r4 + 2], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1218 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 15], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 23], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1219 * 16], m4 ; mode 21 [row 2] movu m6, [r5 + 13 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1220 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1221 * 16], m4 ; mode 21 [row 3] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r4 + 2], 1 pinsrb m0, [r4 + 4], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1222 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 14], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 22], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1223 * 16], m4 ; mode 21 [row 4] movu m6, [r5 + 11 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1224 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1225 * 16], m4 ; mode 21 [row 5] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r4 + 4], 1 pinsrb m0, [r4 + 6], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1226 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 13], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 21], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1227 * 16], m4 ; mode 21 [row 6] movu m6, [r5 + 9 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1228 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1229 * 16], m4 ; mode 21 [row 7] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r4 + 6], 1 pinsrb m0, [r4 + 8], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1230 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 12], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 20], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1231 * 16], m4 ; mode 21 [row 8] movu m6, [r5 + 7 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1232 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1233 * 16], m4 ; mode 21 [row 9] movu m6, [r5 + 22 * 16] pslldq m0, 2 pinsrb m0, [r4 + 8], 1 pinsrb m0, [r4 + 9], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1234 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 11], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 19], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1235 * 16], m4 ; mode 21 [row 10] movu m6, [r5 + 5 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1236 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1237 * 16], m4 ; mode 21 [row 11] movu m6, [r5 + 20 * 16] pslldq m0, 2 pinsrb m0, [r4 + 9], 1 pinsrb m0, [r4 + 11], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1238 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 10], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 18], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1239 * 16], m4 ; mode 21 [row 12] movu m6, [r5 + 3 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1240 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1241 * 16], m4 ; mode 21 [row 13] movu m6, [r5 + 18 * 16] pslldq m0, 2 pinsrb m0, [r4 + 11], 1 pinsrb m0, [r4 + 13], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1242 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 9], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 17], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1243 * 16], m4 ; mode 21 [row 14] movu m6, [r5 + 1 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1244 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1245 * 16], m4 ; mode 21 [row 15] movu m6, [r5 + 16 * 16] pslldq m0, 2 pinsrb m0, [r4 + 13], 1 pinsrb m0, [r4 + 15], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 0], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1246 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 8], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 16], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1247 * 16], m4 ; mode 21 [row 16] movu m6, [r5 + 31 * 16] pslldq m0, 2 pinsrb m0, [r4 + 15], 1 pinsrb m0, [r4 + 17], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 0], 1 pinsrb m2, [r4 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1248 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 7], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 15], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1249 * 16], m4 ; mode 21 [row 17] movu m6, [r5 + 14 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1250 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1251 * 16], m4 ; mode 21 [row 18] movu m6, [r5 + 29 * 16] pslldq m0, 2 pinsrb m0, [r4 + 17], 1 pinsrb m0, [r4 + 19], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 2], 1 pinsrb m2, [r4 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1252 * 16], m4 pslldq m1, 2 pinsrb m1, [r3 + 7], 1 pinsrb m1, [r3 + 6], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrb m3, [r3 + 15], 1 pinsrb m3, [r3 + 14], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1253 * 16], m4 ; mode 21 [row 19] movu m6, [r5 + 12 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1254 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1255 * 16], m4 ; mode 21 [row 20] movu m6, [r5 + 27 * 16] pslldq m0, 2 pinsrb m0, [r4 + 19], 1 pinsrb m0, [r4 + 21], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 4], 1 pinsrb m2, [r4 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1256 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 5], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 13], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1257 * 16], m4 ; mode 21 [row 21] movu m6, [r5 + 10 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1258 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1259 * 16], m4 ; mode 21 [row 22] movu m6, [r5 + 25 * 16] pslldq m0, 2 pinsrb m0, [r4 + 21], 1 pinsrb m0, [r4 + 23], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 6], 1 pinsrb m2, [r4 + 8], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1260 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 4], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 12], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1261 * 16], m4 ; mode 21 [row 23] movu m6, [r5 + 8 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1262 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1263 * 16], m4 ; mode 21 [row 24] movu m6, [r5 + 23 * 16] pslldq m0, 2 pinsrb m0, [r4 + 23], 1 pinsrb m0, [r4 + 24], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 8], 1 pinsrb m2, [r4 + 9], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1264 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 3], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 11], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1265 * 16], m4 ; mode 21 [row 25] movu m6, [r5 + 6 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1266 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1267 * 16], m4 ; mode 21 [row 26] movu m6, [r5 + 21 * 16] pslldq m0, 2 pinsrb m0, [r4 + 24], 1 pinsrb m0, [r4 + 26], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 9], 1 pinsrb m2, [r4 + 11], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1268 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 2], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 10], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1269 * 16], m4 ; mode 21 [row 27] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1270 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1271 * 16], m4 ; mode 21 [row 28] movu m6, [r5 + 19 * 16] pslldq m0, 2 pinsrb m0, [r4 + 26], 1 pinsrb m0, [r4 + 28], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 11], 1 pinsrb m2, [r4 + 13], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1272 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 1], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 9], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1273 * 16], m4 ; mode 21 [row 29] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1274 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1275 * 16], m4 ; mode 21 [row 30] movu m6, [r5 + 17 * 16] pslldq m0, 2 pinsrb m0, [r4 + 28], 1 pinsrb m0, [r4 + 30], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 13], 1 pinsrb m2, [r4 + 15], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1276 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 0], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 8], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1277 * 16], m4 ; mode21 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 1278 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1278 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1279 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1279 * 16 + 8], m5 ; mode 22 [row 0] movu m6, [r5 + 19 * 16] movu m0, [r3 ] movu m1, [r3 + 1 ] punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r3 + 8] movu m3, [r3 + 9] punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 1280 * 16], m1 movu m1, [r3 + 16] movu m3, [r3 + 17] punpcklbw m1, m3 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 movu m3, [r3 + 24] movu m5, [r3 + 25] punpcklbw m3, m5 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1281 * 16], m4 ; mode 22 [row 1] movu m6, [r5 + 6 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1282 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1283 * 16], m4 ; mode 22 [row 2] movu m6, [r5 + 25 * 16] pslldq m0, 2 pinsrb m0, [r4 + 0], 1 pinsrb m0, [r4 + 2], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1284 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 15], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 23], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1285 * 16], m4 ; mode 22 [row 3] movu m6, [r5 + 12 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1286 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1287 * 16], m4 ; mode 22 [row 4] movu m6, [r5 + 31 * 16] pslldq m0, 2 pinsrb m0, [r4 + 2], 1 pinsrb m0, [r4 + 5], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1288 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 14], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 22], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1289 * 16], m4 ; mode 22 [row 5] movu m6, [r5 + 18 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1290 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1291 * 16], m4 ; mode 22 [row 6] movu m6, [r5 + 5 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1292 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1293 * 16], m4 ; mode 22 [row 7] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r4 + 5], 1 pinsrb m0, [r4 + 7], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1294 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 13], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 21], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1295 * 16], m4 ; mode 22 [row 8] movu m6, [r5 + 11 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1296 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1297 * 16], m4 ; mode 22 [row 9] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r4 + 7], 1 pinsrb m0, [r4 + 10], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1298 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 12], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 20], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1299 * 16], m4 ; mode 22 [row 10] movu m6, [r5 + 17 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1300 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1301 * 16], m4 ; mode 22 [row 11] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1302 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1303 * 16], m4 ; mode 22 [row 12] movu m6, [r5 + 23 * 16] pslldq m0, 2 pinsrb m0, [r4 + 10], 1 pinsrb m0, [r4 + 12], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1304 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 11], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 19], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1305 * 16], m4 ; mode 22 [row 13] movu m6, [r5 + 10 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1306 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1307 * 16], m4 ; mode 22 [row 14] movu m6, [r5 + 29 * 16] pslldq m0, 2 pinsrb m0, [r4 + 12], 1 pinsrb m0, [r4 + 15], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1308 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 10], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 18], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1309 * 16], m4 ; mode 22 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1310 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1311 * 16], m4 ; mode 22 [row 16] movu m6, [r5 + 3 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1312 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1313 * 16], m4 ; mode 22 [row 17] movu m6, [r5 + 22 * 16] pslldq m0, 2 pinsrb m0, [r4 + 15], 1 pinsrb m0, [r4 + 17], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1314 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 9], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 17], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1315 * 16], m4 ; mode 22 [row 18] movu m6, [r5 + 9 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1316 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1317 * 16], m4 ; mode 22 [row 19] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r4 + 17], 1 pinsrb m0, [r4 + 20], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 0], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1318 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 8], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 16], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1319 * 16], m4 ; mode 22 [row 20] movu m6, [r5 + 15 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1320 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1321 * 16], m4 ; mode 22 [row 21] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1322 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1323 * 16], m4 ; mode 22 [row 22] movu m6, [r5 + 21 * 16] pslldq m0, 2 pinsrb m0, [r4 + 20], 1 pinsrb m0, [r4 + 22], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 0], 1 pinsrb m2, [r4 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1324 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 7], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 15], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1325 * 16], m4 ; mode 22 [row 23] movu m6, [r5 + 8 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1326 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1327 * 16], m4 ; mode 22 [row 24] movu m6, [r5 + 27 * 16] pslldq m0, 2 pinsrb m0, [r4 + 22], 1 pinsrb m0, [r4 + 25], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 2], 1 pinsrb m2, [r4 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1328 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 6], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 14], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1329 * 16], m4 ; mode 22 [row 25] movu m6, [r5 + 14 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1330 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1331 * 16], m4 ; mode 22 [row 26] movu m6, [r5 + 1 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1332 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1333 * 16], m4 ; mode 22 [row 27] movu m6, [r5 + 20 * 16] pslldq m0, 2 pinsrb m0, [r4 + 25], 1 pinsrb m0, [r4 + 27], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 5], 1 pinsrb m2, [r4 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1334 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 5], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 13], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1335 * 16], m4 ; mode 22 [row 28] movu m6, [r5 + 7 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1336 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1337 * 16], m4 ; mode 22 [row 29] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r4 + 27], 1 pinsrb m0, [r4 + 30], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrb m2, [r4 + 7], 1 pinsrb m2, [r4 + 10], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1338 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 4], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 12], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1339 * 16], m4 ; mode 22 [row 30] movu m6, [r5 + 13 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1340 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1341 * 16], m4 ; mode22 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 1342 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1342 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1343 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1343 * 16 + 8], m5 ; mode 23 [row 0] movu m6, [r5 + 23 * 16] movu m0, [r3 ] movu m1, [r3 + 1 ] punpcklbw m0, m1 pmaddubsw m1, m0, m6 pmulhrsw m1, m7 movu m2, [r3 + 8] movu m3, [r3 + 9] punpcklbw m2, m3 pmaddubsw m3, m2, m6 pmulhrsw m3, m7 packuswb m1, m3 movu [r0 + 1344 * 16], m1 movu m1, [r3 + 16] movu m3, [r3 + 17] punpcklbw m1, m3 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 movu m3, [r3 + 24] movu m5, [r3 + 25] punpcklbw m3, m5 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1345 * 16], m4 ; mode 23 [row 1] movu m6, [r5 + 14 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1346 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1347 * 16], m4 ; mode 23 [row 2] movu m6, [r5 + 5 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1348 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1349 * 16], m4 ; mode 23 [row 3] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r4 + 0], 1 pinsrb m0, [r4 + 4], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1350 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 15], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 23], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1351 * 16], m4 ; mode 23 [row 4] movu m6, [r5 + 19 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1352 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1353 * 16], m4 ; mode 23 [row 5] movu m6, [r5 + 10 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1354 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1355 * 16], m4 ; mode 23 [row 6] movu m6, [r5 + 1 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1356 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1357 * 16], m4 ; mode 23 [row 7] movu m6, [r5 + 24 * 16] pslldq m0, 2 pinsrb m0, [r4 + 4], 1 pinsrb m0, [r4 + 7], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1358 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 14], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 22], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1359 * 16], m4 ; mode 23 [row 8] movu m6, [r5 + 15 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1360 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1361 * 16], m4 ; mode 23 [row 9] movu m6, [r5 + 6 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1362 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1363 * 16], m4 ; mode 23 [row 10] movu m6, [r5 + 29 * 16] pslldq m0, 2 pinsrb m0, [r4 + 7], 1 pinsrb m0, [r4 + 11], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1364 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 13], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 21], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1365 * 16], m4 ; mode 23 [row 11] movu m6, [r5 + 20 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1366 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1367 * 16], m4 ; mode 23 [row 12] movu m6, [r5 + 11 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1368 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1369 * 16], m4 ; mode 23 [row 13] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1370 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1371 * 16], m4 ; mode 23 [row 14] movu m6, [r5 + 25 * 16] pslldq m0, 2 pinsrb m0, [r4 + 11], 1 pinsrb m0, [r4 + 14], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1372 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 12], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 20], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1373 * 16], m4 ; mode 23 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1374 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1375 * 16], m4 ; mode 23 [row 16] movu m6, [r5 + 7 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1376 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1377 * 16], m4 ; mode 23 [row 17] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r4 + 14], 1 pinsrb m0, [r4 + 18], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 3], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1378 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 11], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 19], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1379 * 16], m4 ; mode 23 [row 18] movu m6, [r5 + 21 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1380 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1381 * 16], m4 ; mode 23 [row 19] movu m6, [r5 + 12 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1382 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1383 * 16], m4 ; mode 23 [row 20] movu m6, [r5 + 3 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1384 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1385 * 16], m4 ; mode 23 [row 21] movu m6, [r5 + 26 * 16] pslldq m0, 2 pinsrb m0, [r4 + 18], 1 pinsrb m0, [r4 + 21], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 2], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1386 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 10], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 18], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1387 * 16], m4 ; mode 23 [row 22] movu m6, [r5 + 17 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1388 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1389 * 16], m4 ; mode 23 [row 23] movu m6, [r5 + 8 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1390 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1391 * 16], m4 ; mode 23 [row 24] movu m6, [r5 + 31 * 16] pslldq m0, 2 pinsrb m0, [r4 + 21], 1 pinsrb m0, [r4 + 25], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 1], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1392 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 9], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 17], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1393 * 16], m4 ; mode 23 [row 25] movu m6, [r5 + 22 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1394 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1395 * 16], m4 ; mode 23 [row 26] movu m6, [r5 + 13 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1396 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1397 * 16], m4 ; mode 23 [row 27] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1398 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1399 * 16], m4 ; mode 23 [row 28] movu m6, [r5 + 27 * 16] pslldq m0, 2 pinsrb m0, [r4 + 25], 1 pinsrb m0, [r4 + 28], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 0], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1400 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 8], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 16], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1401 * 16], m4 ; mode 23 [row 29] movu m6, [r5 + 18 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1402 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1403 * 16], m4 ; mode 23 [row 30] movu m6, [r5 + 9 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1404 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1405 * 16], m4 ; mode23 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 1406 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1406 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1407 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1407 * 16 + 8], m5 ; mode 24 [row 0] movu m6, [r5 + 27 * 16] movu m0, [r3 ] movu m1, [r3 + 1 ] punpcklbw m0, m1 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 movu m2, [r3 + 8] movu m3, [r3 + 9] punpcklbw m2, m3 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1408 * 16], m4 movu m1, [r3 + 16] movu m3, [r3 + 17] punpcklbw m1, m3 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 movu m3, [r3 + 24] movu m5, [r3 + 25] punpcklbw m3, m5 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1409 * 16], m4 ; mode 24 [row 1] movu m6, [r5 + 22 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1410 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1411 * 16], m4 ; mode 24 [row 2] movu m6, [r5 + 17 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1412 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1413 * 16], m4 ; mode 24 [row 3] movu m6, [r5 + 12 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1414 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1415 * 16], m4 ; mode 24 [row 4] movu m6, [r5 + 7 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1416 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1417 * 16], m4 ; mode 24 [row 5] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1418 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1419 * 16], m4 ; mode 24 [row 6] movu m6, [r5 + 29 * 16] pslldq m0, 2 pinsrb m0, [r4 + 0], 1 pinsrb m0, [r4 + 6], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1420 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 15], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 23], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1421 * 16], m4 ; mode 24 [row 7] movu m6, [r5 + 24 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1422 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1423 * 16], m4 ; mode 24 [row 8] movu m6, [r5 + 19 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1424 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1425 * 16], m4 ; mode 24 [row 9] movu m6, [r5 + 14 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1426 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1427 * 16], m4 ; mode 24 [row 10] movu m6, [r5 + 9 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1428 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1429 * 16], m4 ; mode 24 [row 11] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1430 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1431 * 16], m4 ; mode 24 [row 12] movu m6, [r5 + 31 * 16] pslldq m0, 2 pinsrb m0, [r4 + 6], 1 pinsrb m0, [r4 + 13], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 6], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1432 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 14], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 22], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1433 * 16], m4 ; mode 24 [row 13] movu m6, [r5 + 26 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1434 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1435 * 16], m4 ; mode 24 [row 14] movu m6, [r5 + 21 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1436 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1437 * 16], m4 ; mode 24 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1438 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1439 * 16], m4 ; mode 24 [row 16] movu m6, [r5 + 11 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1440 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1441 * 16], m4 ; mode 24 [row 17] movu m6, [r5 + 6 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1442 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1443 * 16], m4 ; mode 24 [row 18] movu m6, [r5 + 1 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1444 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1445 * 16], m4 ; mode 24 [row 19] movu m6, [r5 + 28 * 16] pslldq m0, 2 pinsrb m0, [r4 + 13], 1 pinsrb m0, [r4 + 19], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 5], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1446 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 13], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 21], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1447 * 16], m4 ; mode 24 [row 20] movu m6, [r5 + 23 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1448 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1449 * 16], m4 ; mode 24 [row 21] movu m6, [r5 + 18 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1450 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1451 * 16], m4 ; mode 24 [row 22] movu m6, [r5 + 13 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1452 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1453 * 16], m4 ; mode 24 [row 23] movu m6, [r5 + 8 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1454 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1455 * 16], m4 ; mode 24 [row 24] movu m6, [r5 + 3 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1456 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1457 * 16], m4 ; mode 24 [row 25] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r4 + 19], 1 pinsrb m0, [r4 + 26], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 4], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1458 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 12], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 20], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1459 * 16], m4 ; mode 24 [row 26] movu m6, [r5 + 25 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1460 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1461 * 16], m4 ; mode 24 [row 27] movu m6, [r5 + 20 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1462 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1463 * 16], m4 ; mode 24 [row 28] movu m6, [r5 + 15 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1464 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1465 * 16], m4 ; mode 24 [row 29] movu m6, [r5 + 10 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1466 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1467 * 16], m4 ; mode 24 [row 30] movu m6, [r5 + 5 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1468 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1469 * 16], m4 ; mode 24 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 1470 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1470 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1471 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1471 * 16 + 8], m5 ; mode 25 [row 0] movu m6, [r5 + 30 * 16] movu m0, [r3 ] movu m1, [r3 + 1 ] punpcklbw m0, m1 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 movu m2, [r3 + 8] movu m3, [r3 + 9] punpcklbw m2, m3 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1472 * 16], m4 movu m1, [r3 + 16] movu m3, [r3 + 17] punpcklbw m1, m3 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 movu m3, [r3 + 24] movu m5, [r3 + 25] punpcklbw m3, m5 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1473 * 16], m4 ; mode 25 [row 1] movu m6, [r5 + 28 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1474 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1475 * 16], m4 ; mode 25 [row 2] movu m6, [r5 + 26 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1476 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1477 * 16], m4 ; mode 25 [row 3] movu m6, [r5 + 24 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1478 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1479 * 16], m4 ; mode 25 [row 4] movu m6, [r5 + 22 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1480 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1481 * 16], m4 ; mode 25 [row 5] movu m6, [r5 + 20 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1482 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1483 * 16], m4 ; mode 25 [row 6] movu m6, [r5 + 18 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1484 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1485 * 16], m4 ; mode 25 [row 7] movu m6, [r5 + 16 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1486 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1487 * 16], m4 ; mode 25 [row 8] movu m6, [r5 + 14 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1488 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1489 * 16], m4 ; mode 25 [row 9] movu m6, [r5 + 12 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1490 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1491 * 16], m4 ; mode 25 [row 10] movu m6, [r5 + 10 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1492 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1493 * 16], m4 ; mode 25 [row 11] movu m6, [r5 + 8 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1494 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1495 * 16], m4 ; mode 25 [row 12] movu m6, [r5 + 6 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1496 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1497 * 16], m4 ; mode 25 [row 13] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1498 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1499 * 16], m4 ; mode 25 [row 14] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1500 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1501 * 16], m4 ; mode 25 [row 15] pshufb m5, m0, [tab_S2] movh [r0 + 1502 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1502 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1503 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1503 * 16 + 8], m5 ; mode 25 [row 16] movu m6, [r5 + 30 * 16] pslldq m0, 2 pinsrb m0, [r4 + 0], 1 pinsrb m0, [r4 + 16], 0 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pslldq m2, 2 pinsrw m2, [r3 + 7], 0 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1504 * 16], m4 pslldq m1, 2 pinsrw m1, [r3 + 15], 0 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pslldq m3, 2 pinsrw m3, [r3 + 23], 0 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1505 * 16], m4 ; mode 25 [row 17] movu m6, [r5 + 28 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1506 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1507 * 16], m4 ; mode 25 [row 18] movu m6, [r5 + 26 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1508 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1509 * 16], m4 ; mode 25 [row 19] movu m6, [r5 + 24 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1510 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1511 * 16], m4 ; mode 25 [row 20] movu m6, [r5 + 22 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1512 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1513 * 16], m4 ; mode 25 [row 21] movu m6, [r5 + 20 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1514 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1515 * 16], m4 ; mode 25 [row 22] movu m6, [r5 + 18 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1516 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1517 * 16], m4 ; mode 25 [row 23] movu m6, [r5 + 16 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1518 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1519 * 16], m4 ; mode 25 [row 24] movu m6, [r5 + 14 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1520 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1521 * 16], m4 ; mode 25 [row 25] movu m6, [r5 + 12 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1522 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1523 * 16], m4 ; mode 25 [row 26] movu m6, [r5 + 10 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1524 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1525 * 16], m4 ; mode 25 [row 27] movu m6, [r5 + 8 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1526 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1527 * 16], m4 ; mode 25 [row 28] movu m6, [r5 + 6 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1528 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1529 * 16], m4 ; mode 25 [row 29] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1530 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1531 * 16], m4 ; mode 25 [row 30] movu m6, [r5 + 2 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1532 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1533 * 16], m4 ; mode 25 [row 31] pshufb m5, m0, [tab_S2] movh [r0 + 1534 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1534 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1535 * 16], m5 pshufb m5, m3, [tab_S2] movh [r0 + 1535 * 16 + 8], m5 ; mode 26 movu m1, [r1 + 1] movu m2, [r1 + 17] movu [r0 + 1536 * 16], m1 movu [r0 + 1537 * 16], m2 movu [r0 + 1538 * 16], m1 movu [r0 + 1539 * 16], m2 movu [r0 + 1540 * 16], m1 movu [r0 + 1541 * 16], m2 movu [r0 + 1542 * 16], m1 movu [r0 + 1543 * 16], m2 movu [r0 + 1544 * 16], m1 movu [r0 + 1545 * 16], m2 movu [r0 + 1546 * 16], m1 movu [r0 + 1547 * 16], m2 movu [r0 + 1548 * 16], m1 movu [r0 + 1549 * 16], m2 movu [r0 + 1550 * 16], m1 movu [r0 + 1551 * 16], m2 movu [r0 + 1552 * 16], m1 movu [r0 + 1553 * 16], m2 movu [r0 + 1554 * 16], m1 movu [r0 + 1555 * 16], m2 movu [r0 + 1556 * 16], m1 movu [r0 + 1557 * 16], m2 movu [r0 + 1558 * 16], m1 movu [r0 + 1559 * 16], m2 movu [r0 + 1560 * 16], m1 movu [r0 + 1561 * 16], m2 movu [r0 + 1562 * 16], m1 movu [r0 + 1563 * 16], m2 movu [r0 + 1564 * 16], m1 movu [r0 + 1565 * 16], m2 movu [r0 + 1566 * 16], m1 movu [r0 + 1567 * 16], m2 movu [r0 + 1568 * 16], m1 movu [r0 + 1569 * 16], m2 movu [r0 + 1570 * 16], m1 movu [r0 + 1571 * 16], m2 movu [r0 + 1572 * 16], m1 movu [r0 + 1573 * 16], m2 movu [r0 + 1574 * 16], m1 movu [r0 + 1575 * 16], m2 movu [r0 + 1576 * 16], m1 movu [r0 + 1577 * 16], m2 movu [r0 + 1578 * 16], m1 movu [r0 + 1579 * 16], m2 movu [r0 + 1580 * 16], m1 movu [r0 + 1581 * 16], m2 movu [r0 + 1582 * 16], m1 movu [r0 + 1583 * 16], m2 movu [r0 + 1584 * 16], m1 movu [r0 + 1585 * 16], m2 movu [r0 + 1586 * 16], m1 movu [r0 + 1587 * 16], m2 movu [r0 + 1588 * 16], m1 movu [r0 + 1589 * 16], m2 movu [r0 + 1590 * 16], m1 movu [r0 + 1591 * 16], m2 movu [r0 + 1592 * 16], m1 movu [r0 + 1593 * 16], m2 movu [r0 + 1594 * 16], m1 movu [r0 + 1595 * 16], m2 movu [r0 + 1596 * 16], m1 movu [r0 + 1597 * 16], m2 movu [r0 + 1598 * 16], m1 movu [r0 + 1599 * 16], m2 ; mode 27 [row 0] movu m6, [r5 + 2 * 16] movu m0, [r3 + 1 ] movu m1, [r3 + 2 ] punpcklbw m0, m1 pmaddubsw m4, m0, m6 pmulhrsw m4, m7 movu m2, [r3 + 9] movu m3, [r3 + 10] punpcklbw m2, m3 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1600 * 16], m4 movu m1, [r3 + 17] movu m3, [r3 + 18] punpcklbw m1, m3 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 movu m3, [r3 + 25] movu m5, [r3 + 26] punpcklbw m3, m5 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1601 * 16], m4 ; mode 27 [row 1] movu m6, [r5 + 4 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1602 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1603 * 16], m4 ; mode 27 [row 2] movu m6, [r5 + 6 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1604 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1605 * 16], m4 ; mode 27 [row 3] movu m6, [r5 + 8 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1606 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1607 * 16], m4 ; mode 27 [row 4] movu m6, [r5 + 10 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1608 * 16], m4 ; mode 28 [row 1 -first half] movu [r0 + 1666 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1609 * 16], m4 ; mode 28 [row 1 - second half] movu [r0 + 1667 * 16], m4 ; mode 27 [row 5] movu m6, [r5 + 12 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1610 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1611 * 16], m4 ; mode 27 [row 6] movu m6, [r5 + 14 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1612 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1613 * 16], m4 ; mode 27 [row 7] movu m6, [r5 + 16 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1614 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1615 * 16], m4 ; mode 27 [row 8] movu m6, [r5 + 18 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1616 * 16], m4 ; mode 29 [row 1 - first half] movu [r0 + 1730 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1617 * 16], m4 ; mode 29 [row 1 - second half] movu [r0 + 1731 * 16], m4 ; mode 27 [row 9] movu m6, [r5 + 20 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1618 * 16], m4 ; mode 28 [row 3 -first half] movu [r0 + 1670 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1619 * 16], m4 ; mode 28 [row 3 -second half] movu [r0 + 1671 * 16], m4 ; mode 27 [row 10] movu m6, [r5 + 22 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1620 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1621 * 16], m4 ; mode 27 [row 11] movu m6, [r5 + 24 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1622 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1623 * 16], m4 ; mode 27 [row 12] movu m6, [r5 + 26 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1624 * 16], m4 ; mode 30 [row 1 - first half] movu [r0 + 1794 * 16], m4 ; mode 33 [row 0 - first half] movu [r0 + 1984 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1625 * 16], m4 ; mode 30 [row 1 - second half] movu [r0 + 1795 * 16], m4 ; mode 33 [row 0 - second half] movu [r0 + 1985 * 16], m4 ; mode 27 [row 13] movu m6, [r5 + 28 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1626 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1627 * 16], m4 ; mode 27 [row 14] movu m6, [r5 + 30 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1628 * 16], m4 ; mode 28 [row 5 first half] movu [r0 + 1674 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1629 * 16], m4 ; mode 28 [row 5 second half] movu [r0 + 1675 * 16], m4 ; mode 28 [row 0] movu m6, [r5 + 5 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1664 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1665 * 16], m4 ; mode 28 [row 2] movu m6, [r5 + 15 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1668 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1669 * 16], m4 ; mode 28 [row 4] movu m6, [r5 + 25 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1672 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1673 * 16], m4 ; mode 30 [row 0] movu m6, [r5 + 13 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1792 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1793 * 16], m4 ; mode 29 [row 0] movu m6, [r5 + 9 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1728 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1729 * 16], m4 ; mode 29 [row 2] movu m6, [r5 + 27 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1732 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1733 * 16], m4 ; mode 31 [row 0] movu m6, [r5 + 17 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1856 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1857 * 16], m4 ; mode 32 [row 0] movu m6, [r5 + 21 * 16] pmaddubsw m4, m0, m6 pmulhrsw m4, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1920 * 16], m4 pmaddubsw m4, m1, m6 pmulhrsw m4, m7 pmaddubsw m5, m3, m6 pmulhrsw m5, m7 packuswb m4, m5 movu [r0 + 1921 * 16], m4 ; mode 27 [row 15] movu m0, [r3 + 2] movd m1, [r3 + 3] palignr m1, m0, 1 punpcklbw m0, m1 movu m2, [r3 + 10] movd m3, [r3 + 11] palignr m3, m2, 1 punpcklbw m2, m3 movu m1, [r3 + 18] movd m3, [r3 + 19] palignr m3, m1, 1 punpcklbw m1, m3 movu m4, [r3 + 26] movd m5, [r3 + 27] palignr m5, m4, 1 punpcklbw m4, m5 pshufb m5, m0, [tab_S2] movh [r0 + 1630 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1630 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1631 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 1631 * 16 + 8], m5 ; mode 27 [row 16] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1632 * 16], m3 ; mode 31 [row 1 - first half] movu [r0 + 1858 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1633 * 16], m3 ; mode 31 [row 1 - second half] movu [r0 + 1859 * 16], m3 ; mode 27 [row 17] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1634 * 16], m3 ; mode 29 [row 3 - first half] movu [r0 + 1734 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1635 * 16], m3 ; mode 29 [row 3 - second half] movu [r0 + 1735 * 16], m3 ; mode 27 [row 18] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1636 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1637 * 16], m3 ; mode 27 [row 19] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1638 * 16], m3 ; mode 28 [row 7 - first half] movu [r0 + 1678 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1639 * 16], m3 ; mode 28 [row 7 - second half] movu [r0 + 1679 * 16], m3 ; mode 27 [row 20] movu m6, [r5 + 10 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1640 * 16], m3 ; mode 32 [row 1 - first half] movu [r0 + 1922 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1641 * 16], m3 ; mode 32 [row 1 - second half] movu [r0 + 1923 * 16], m3 ; mode 27 [row 21] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1642 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1643 * 16], m3 ; mode 27 [row 22] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1644 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1645 * 16], m3 ; mode 27 [row 23] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1646 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1647 * 16], m3 ; mode 27 [row 24] movu m6, [r5 + 18 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1648 * 16], m3 ; mode 28 [row 9 - first half] movu [r0 + 1682 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1649 * 16], m3 ; mode 28 [row 9 - second half] movu [r0 + 1683 * 16], m3 ; mode 27 [row 25] movu m6, [r5 + 20 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1650 * 16], m3 ; mode 30 [row 3 - first half] movu [r0 + 1798 * 16], m3 ; mode 33 [row 1 - first half] movu [r0 + 1986 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1651 * 16], m3 ; mode 30 [row 3 - second half] movu [r0 + 1799 * 16], m3 ; mode 33 [row 1 - second half] movu [r0 + 1987 * 16], m3 ; mode 27 [row 26] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1652 * 16], m3 ; mode 29 [row 5 - first half] movu [r0 + 1738 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1653 * 16], m3 ; mode 29 [row 5 - second half] movu [r0 + 1739 * 16], m3 ; mode 27 [row 27] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1654 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1655 * 16], m3 ; mode 27 [row 28] movu m6, [r5 + 26 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1656 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1657 * 16], m3 ; mode 27 [row 29] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1658 * 16], m3 ; mode 28 [row 11 - first half] movu [r0 + 1686 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1659 * 16], m3 ; mode 28 [row 11 - second half] movu [r0 + 1687 * 16], m3 ; mode 27 [row 30] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1660 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1661 * 16], m3 ; mode 28 [row 6] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1676 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1677 * 16], m3 ; mode 28 [row 8] movu m6, [r5 + 13 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1680 * 16], m3 ; mode 29 [row 4 - first half] movu [r0 + 1736 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1681 * 16], m3 ; mode 29 [row 4 - second half] movu [r0 + 1737 * 16], m3 ; mode 28 [row 10] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1684 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1685 * 16], m3 ; mode 29 [row 6] movu m6, [r5 + 31 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1740 * 16], m3 ; mode 32 [row 2 - first half] movu [r0 + 1924 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1741 * 16], m3 ; mode 32 [row 2 - second half] movu [r0 + 1925 * 16], m3 ; mode 30 [row 2] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1796 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1797 * 16], m3 ; mode 31 [row 2] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1860 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1861 * 16], m3 ; mode 27 [row 15] movu m0, [r3 + 3] movd m1, [r3 + 4] palignr m1, m0, 1 punpcklbw m0, m1 movu m2, [r3 + 11] movd m3, [r3 + 12] palignr m3, m2, 1 punpcklbw m2, m3 movu m1, [r3 + 19] movd m3, [r3 + 20] palignr m3, m1, 1 punpcklbw m1, m3 movu m4, [r3 + 27] movd m5, [r3 + 28] palignr m5, m4, 1 punpcklbw m4, m5 pshufb m5, m0, [tab_S2] movh [r0 + 1662 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1662 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1663 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 1663 * 16 + 8], m5 ; mode 28 [row 12] movu m6, [r5 + 1 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1688 * 16], m3 ; mode 30 [row 4 - first half] movu [r0 + 1800 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1689 * 16], m3 ; mode 30 [row 4 - second half] movu [r0 + 1801 * 16], m3 ; mode 28 [row 13] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1690 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1691 * 16], m3 ; mode 28 [row 14] movu m6, [r5 + 11 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1692 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1693 * 16], m3 ; mode 28 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1694 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1695 * 16], m3 ; mode 28 [row 16] movu m6, [r5 + 21 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1696 * 16], m3 ; mode 31 [row 4 - first half] movu [r0 + 1864 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1697 * 16], m3 ; mode 31 [row 4 - second half] movu [r0 + 1865 * 16], m3 ; mode 28 [row 17] movu m6, [r5 + 26 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1698 * 16], m3 ; mode 29 [row 9 - first half] movu [r0 + 1746 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1699 * 16], m3 ; mode 29 [row 9 - second half] movu [r0 + 1747 * 16], m3 ; mode 28 [row 18] movu m6, [r5 + 31 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1700 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1701 * 16], m3 ; mode 29 [row 7] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1742 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1743 * 16], m3 ; mode 29 [row 8] movu m6, [r5 + 17 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1744 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1745 * 16], m3 ; mode 30 [row 5] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1802 * 16], m3 ; mode 33 [row 2 - first half] movu [r0 + 1988 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1803 * 16], m3 ; mode 33 [row 2 - second half] movu [r0 + 1989 * 16], m3 ; mode 30 [row 6] movu m6, [r5 + 27 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1804 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1805 * 16], m3 ; mode 31 [row 3] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1862 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1863 * 16], m3 ; mode 32 [row 3] movu m6, [r5 + 20 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1926 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1927 * 16], m3 ; mode 28 [row 19] movu m6, [r5 + 4 * 16] movu m0, [r3 + 4] movd m1, [r3 + 5] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 12] movd m4, [r3 + 13] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1702 * 16], m3 movu m1, [r3 + 20] movd m3, [r3 + 21] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 28] movd m5, [r3 + 29] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1703 * 16], m3 ; mode 28 [row 20] movu m6, [r5 + 9 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1704 * 16], m3 ; mode 32 [row 4 - first half] movu [r0 + 1928 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1705 * 16], m3 ; mode 32 [row 4 - second half] movu [r0 + 1929 * 16], m3 ; mode 28 [row 21] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1706 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1707 * 16], m3 ; mode 28 [row 22] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1708 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1709 * 16], m3 ; mode 28 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1710 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1711 * 16], m3 ; mode 28 [row 24] movu m6, [r5 + 29 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1712 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1713 * 16], m3 ; mode 29 [row 10] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1748 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1749 * 16], m3 ; mode 29 [row 11] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1750 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1751 * 16], m3 ; mode 29 [row 12] movu m6, [r5 + 21 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1752 * 16], m3 ; mode 30 [row 8 -first half] movu [r0 + 1808 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1753 * 16], m3 ; mode 30 [row 8 -second half] movu [r0 + 1809 * 16], m3 ; mode 29 [row 13] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1754 * 16], m3 ; mode 32 [row 5 - first half] movu [r0 + 1930 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1755 * 16], m3 ; mode 32 [row 5 - second half] movu [r0 + 1931 * 16], m3 ; mode 30 [row 7] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1806 * 16], m3 ; mode 33 [row 3 - first half] movu [r0 + 1990 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1807 * 16], m3 ; mode 33 [row 3 - second half] movu [r0 + 1991 * 16], m3 ; mode 31 [row 5] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1866 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1867 * 16], m3 ; mode 31 [row 6] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1868 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1869 * 16], m3 ; mode 28 [row 25] movu m6, [r5 + 2 * 16] movu m0, [r3 + 5] movd m1, [r3 + 6] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 13] movd m4, [r3 + 14] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1714 * 16], m3 movu m1, [r3 + 21] movd m3, [r3 + 22] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 29] movd m5, [r3 + 30] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1715 * 16], m3 ; mode 28 [row 26] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1716 * 16], m3 ; mode 29 [row 14 - first half] movu [r0 + 1756 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1717 * 16], m3 ; mode 29 [row 14 - second half] movu [r0 + 1757 * 16], m3 ; mode 28 [row 27] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1718 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1719 * 16], m3 ; mode 28 [row 28] movu m6, [r5 + 17 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1720 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1721 * 16], m3 ; mode 28 [row 29] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1722 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1723 * 16], m3 ; mode 28 [row 30] movu m6, [r5 + 27 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1724 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1725 * 16], m3 ; mode 29 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1758 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1759 * 16], m3 ; mode 29 [row 16] movu m6, [r5 + 25 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1760 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1761 * 16], m3 ; mode 30 [row 9] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1810 * 16], m3 ; mode 33 [row 4 - first half] movu [r0 + 1992 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1811 * 16], m3 ; mode 33 [row 4 - second half] movu [r0 + 1993 * 16], m3 ; mode 30 [row 10] movu m6, [r5 + 15 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1812 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1813 * 16], m3 ; mode 31 [row 7] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1870 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1871 * 16], m3 ; mode 31 [row 8] movu m6, [r5 + 25 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1872 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1873 * 16], m3 ; mode 32 [row 6] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1932 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1933 * 16], m3 ; mode 30 [row 11] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1814 * 16], m3 ; mode 33 [row 5 - first half] movu [r0 + 1994 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1815 * 16], m3 ; mode 33 [row 5 - second half] movu [r0 + 1995 * 16], m3 ; mode 28 [row 31] movu m0, [r3 + 6] movd m1, [r3 + 7] palignr m1, m0, 1 punpcklbw m0, m1 movu m2, [r3 + 14] movd m3, [r3 + 15] palignr m3, m2, 1 punpcklbw m2, m3 movu m1, [r3 + 22] movd m3, [r3 + 23] palignr m3, m1, 1 punpcklbw m1, m3 movu m4, [r3 + 30] movd m5, [r3 + 31] palignr m5, m4, 1 punpcklbw m4, m5 pshufb m5, m0, [tab_S2] movh [r0 + 1726 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1726 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1727 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 1727 * 16 + 8], m5 ; mode 29 [row 17] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1762 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1763 * 16], m3 ; mode 29 [row 18] movu m6, [r5 + 11 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1764 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1765 * 16], m3 ; mode 29 [row 19] movu m6, [r5 + 20 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1766 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1767 * 16], m3 ; mode 29 [row 20] movu m6, [r5 + 29 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1768 * 16], m3 ; mode 32 [row 8 - first halif] movu [r0 + 1936 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1769 * 16], m3 ; mode 32 [row 8 - second halif] movu [r0 + 1937 * 16], m3 ; mode 30 [row 12] movu m6, [r5 + 9 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1816 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1817 * 16], m3 ; mode 30 [row 13] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1818 * 16], m3 ; mode 33 [row 6 - first half] movu [r0 + 1996 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1819 * 16], m3 ; mode 33 [row 6 - second half] movu [r0 + 1997 * 16], m3 ; mode 31 [row 9] movu m6, [r5 + 10 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1874 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1875 * 16], m3 ; mode 31 [row 10] movu m6, [r5 + 27 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1876 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1877 * 16], m3 ; mode 32 [row 7] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1934 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1935 * 16], m3 ; mode 29 [row 21] movu m6, [r5 + 6 * 16] movu m0, [r3 + 7] movd m1, [r3 + 8] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 15] movd m4, [r3 + 16] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1770 * 16], m3 movu m1, [r3 + 23] movd m3, [r3 + 24] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 31] movd m5, [r3 + 32] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1771 * 16], m3 ; mode 29 [row 22] movu m6, [r5 + 15 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1772 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1773 * 16], m3 ; mode 29 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1774 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1775 * 16], m3 ; mode 30 [row 14] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1820 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1821 * 16], m3 ; mode 30 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1822 * 16], m3 ; mode 33 [row 7 - first half] movu [r0 + 1998 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1823 * 16], m3 ; mode 33 [row 7 - second half] movu [r0 + 1999 * 16], m3 ; mode 30 [row 16] movu m6, [r5 + 29 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1824 * 16], m3 ; mode 31 [row 12 - first half] movu [r0 + 1880 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1825 * 16], m3 ; mode 31 [row 12 - second half] movu [r0 + 1881 * 16], m3 ; mode 31 [row 11] movu m6, [r5 + 12 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1878 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1879 * 16], m3 ; mode 32 [row 9] movu m6, [r5 + 18 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1938 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1939 * 16], m3 ; mode 29 [row 24] movu m6, [r5 + 1 * 16] movu m0, [r3 + 8] movd m1, [r3 + 9] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 16] movd m4, [r3 + 17] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1776 * 16], m3 movu m1, [r3 + 24] movd m3, [r3 + 25] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 32] movd m5, [r3 + 33] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1777 * 16], m3 ; mode 29 [row 25] movu m6, [r5 + 10 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1778 * 16], m3 ; mode 30 [row 17 - first half] movu [r0 + 1826 * 16], m3 ; mode 33 [row 8 - first half] movu [r0 + 2000 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1779 * 16], m3 ; mode 30 [row 17 - second half] movu [r0 + 1827 * 16], m3 ; mode 33 [row 8 - second half] movu [r0 + 2001 * 16], m3 ; mode 29 [row 26] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1780 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1781 * 16], m3 ; mode 29 [row 27] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1782 * 16], m3 ; mode 32 [row 11 - first half] movu [r0 + 1942 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1783 * 16], m3 ; mode 32 [row 11 - second half] movu [r0 + 1943 * 16], m3 ; mode 30 [row 18] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1828 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1829 * 16], m3 ; mode 31 [row 13] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1882 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1883 * 16], m3 ; mode 31 [row 14] movu m6, [r5 + 31 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1884 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1885 * 16], m3 ; mode 32 [row 10] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1940 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1941 * 16], m3 ; mode 29 [row 28] movu m6, [r5 + 5 * 16] movu m0, [r3 + 9] movd m1, [r3 + 10] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 17] movd m4, [r3 + 18] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1784 * 16], m3 movu m1, [r3 + 25] movd m3, [r3 + 26] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 33] movd m5, [r3 + 34] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1785 * 16], m3 ; mode 29 [row 29] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1786 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1787 * 16], m3 ; mode 29 [row 30] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1788 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1789 * 16], m3 ; mode 30 [row 19] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1830 * 16], m3 ; mode 33 [row 9 - first half] movu [r0 + 2002 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1831 * 16], m3 ; mode 33 [row 9 - second half] movu [r0 + 2003 * 16], m3 ; mode 30 [row 20] movu m6, [r5 + 17 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1832 * 16], m3 ; mode 32 [row 12 - first half] movu [r0 + 1944 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1833 * 16], m3 ; mode 32 [row 12 - second half] movu [r0 + 1945 * 16], m3 ; mode 30 [row 21] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1834 * 16], m3 ; mode 33 [row 10 - first half] movu [r0 + 2004 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1835 * 16], m3 ; mode 33 [row 10 - second half] movu [r0 + 2005 * 16], m3 ; mode 31 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1886 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1887 * 16], m3 ; mode 29 [row 31] movu m0, [r3 + 10] movd m1, [r3 + 11] palignr m1, m0, 1 punpcklbw m0, m1 movu m2, [r3 + 18] movd m3, [r3 + 19] palignr m3, m2, 1 punpcklbw m2, m3 movu m1, [r3 + 26] movd m3, [r3 + 27] palignr m3, m1, 1 punpcklbw m1, m3 movu m4, [r3 + 34] movd m5, [r3 + 35] palignr m5, m4, 1 punpcklbw m4, m5 pshufb m5, m0, [tab_S2] movh [r0 + 1790 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1790 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1791 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 1791 * 16 + 8], m5 ; mode 30 [row 22] movu m6, [r5 + 11 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1836 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1837 * 16], m3 ; mode 30 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1838 * 16], m3 ; mode 33 [row 11 - first half] movu [r0 + 2006 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1839 * 16], m3 ; mode 33 [row 11 - second half] movu [r0 + 2007 * 16], m3 ; mode 31 [row 16] movu m6, [r5 + 1 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1888 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1889 * 16], m3 ; mode 31 [row 17] movu m6, [r5 + 18 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1890 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1891 * 16], m3 ; mode 32 [row 13] movu m6, [r5 + 6 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1946 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1947 * 16], m3 ; mode 32 [row 14] movu m6, [r5 + 27 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1948 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1949 * 16], m3 ; mode 30 [row 24] movu m6, [r5 + 5 * 16] movu m0, [r3 + 11] movd m1, [r3 + 12] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 19] movd m4, [r3 + 20] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1840 * 16], m3 movu m1, [r3 + 27] movd m3, [r3 + 28] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 35] movd m5, [r3 + 36] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1841 * 16], m3 ; mode 30 [row 25] movu m6, [r5 + 18 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1842 * 16], m3 ; mode 33 [row 12 - first half] movu [r0 + 2008 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1843 * 16], m3 ; mode 33 [row 12 - second half] movu [r0 + 2009 * 16], m3 ; mode 30 [row 26] movu m6, [r5 + 31 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1844 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1845 * 16], m3 ; mode 31 [row 18] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1892 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1893 * 16], m3 ; mode 31 [row 19] movu m6, [r5 + 20 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1894 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1895 * 16], m3 ; mode 32 [row 15] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1950 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1951 * 16], m3 ; mode 30 [row 27] movu m6, [r5 + 12 * 16] movu m0, [r3 + 12] movd m1, [r3 + 13] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 20] movd m4, [r3 + 21] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1846 * 16], m3 ; mode 33 [row 13 - first half] movu [r0 + 2010 * 16], m3 movu m1, [r3 + 28] movd m3, [r3 + 29] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 36] movd m5, [r3 + 37] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1847 * 16], m3 ; mode 33 [row 13 - second half] movu [r0 + 2011 * 16], m3 ; mode 30 [row 28] movu m6, [r5 + 25 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1848 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1849 * 16], m3 ; mode 31 [row 20] movu m6, [r5 + 5 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1896 * 16], m3 ; mode 32 [row 16 - first half] movu [r0 + 1952 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1897 * 16], m3 ; mode 32 [row 16 - second half] movu [r0 + 1953 * 16], m3 ; mode 31 [row 21] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1898 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1899 * 16], m3 ; mode 32 [row 17] movu m6, [r5 + 26 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1954 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1955 * 16], m3 ; mode 30 [row 29] movu m6, [r5 + 6 * 16] movu m0, [r3 + 13] movd m1, [r3 + 14] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 21] movd m4, [r3 + 22] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1850 * 16], m3 ; mode 33 [row 14 - first half] movu [r0 + 2012 * 16], m3 movu m1, [r3 + 29] movd m3, [r3 + 30] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 37] movd m5, [r3 + 38] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1851 * 16], m3 ; mode 33 [row 14 - second half] movu [r0 + 2013 * 16], m3 ; mode 30 [row 30] movu m6, [r5 + 19 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1852 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1853 * 16], m3 ; mode 31 [row 22] movu m6, [r5 + 7 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1900 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1901 * 16], m3 ; mode 31 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1902 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1903 * 16], m3 ; mode 32 [row 18] movu m6, [r5 + 15 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1956 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1957 * 16], m3 ; mode 30 [row 31] movu m0, [r3 + 14] movd m1, [r3 + 15] palignr m1, m0, 1 punpcklbw m0, m1 movu m2, [r3 + 22] movd m3, [r3 + 23] palignr m3, m2, 1 punpcklbw m2, m3 movu m1, [r3 + 30] movd m3, [r3 + 31] palignr m3, m1, 1 punpcklbw m1, m3 movu m4, [r3 + 38] movd m5, [r3 + 39] palignr m5, m4, 1 punpcklbw m4, m5 pshufb m5, m0, [tab_S2] movh [r0 + 1854 * 16], m5 ; mode 33 [row 15 - first eight] movh [r0 + 2014 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1854 * 16 + 8], m5 ; mode 33 [row 15 - second eight] movh [r0 + 2014 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1855 * 16], m5 ; mode 33 [row 15 - third eight] movh [r0 + 2015 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 1855 * 16 + 8], m5 ; mode 33 [row 15 - fourth eight] movh [r0 + 2015 * 16 + 8], m5 ; mode 31 [row 24] movu m6, [r5 + 9 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1904 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1905 * 16], m3 ; mode 31 [row 25] movu m6, [r5 + 26 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1906 * 16], m3 ; mode 33 [row 16 - first half] movu [r0 + 2016 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1907 * 16], m3 ; mode 33 [row 16 - second half] movu [r0 + 2017 * 16], m3 ; mode 32 [row 19] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1958 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1959 * 16], m3 ; mode 32 [row 20] movu m6, [r5 + 25 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1960 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1961 * 16], m3 ; mode 31 [row 26] movu m6, [r5 + 11 * 16] movu m0, [r3 + 15] movd m1, [r3 + 16] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 23] movd m4, [r3 + 24] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1908 * 16], m3 movu m1, [r3 + 31] movd m3, [r3 + 32] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 39] movd m5, [r3 + 40] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1909 * 16], m3 ; mode 31 [row 27] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1910 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1911 * 16], m3 ; mode 32 [row 21] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1962 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1963 * 16], m3 ; mode 33 [row 17] movu m6, [r5 + 20 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2018 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2019 * 16], m3 ; mode 31 [row 28] movu m6, [r5 + 13 * 16] movu m0, [r3 + 16] movd m1, [r3 + 17] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 24] movd m4, [r3 + 25] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1912 * 16], m3 movu m1, [r3 + 32] movd m3, [r3 + 33] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 40] movd m5, [r3 + 41] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1913 * 16], m3 ; mode 31 [row 29] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1914 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1915 * 16], m3 ; mode 32 [row 22] movu m6, [r5 + 3 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1964 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1965 * 16], m3 ; mode 32 [row 23] movu m6, [r5 + 24 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1966 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1967 * 16], m3 ; mode 33 [row 18] movu m6, [r5 + 14 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2020 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2021 * 16], m3 ; mode 31 [row 30] movu m6, [r5 + 15 * 16] movu m0, [r3 + 17] movd m1, [r3 + 18] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 25] movd m4, [r3 + 26] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1916 * 16], m3 movu m1, [r3 + 33] movd m3, [r3 + 34] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 41] movd m5, [r3 + 42] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1917 * 16], m3 ; mode 32 [row 24] movu m6, [r5 + 13 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1968 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1969 * 16], m3 ; mode 33 [row 19] movu m6, [r5 + 8 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2022 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2023 * 16], m3 ; mode 31 [row 31] movu m0, [r3 + 18] movd m1, [r3 + 19] palignr m1, m0, 1 punpcklbw m0, m1 movu m2, [r3 + 26] movd m3, [r3 + 27] palignr m3, m2, 1 punpcklbw m2, m3 movu m1, [r3 + 34] movd m3, [r3 + 35] palignr m3, m1, 1 punpcklbw m1, m3 movu m4, [r3 + 42] movd m5, [r3 + 43] palignr m5, m4, 1 punpcklbw m4, m5 pshufb m5, m0, [tab_S2] movh [r0 + 1918 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1918 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1919 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 1919 * 16 + 8], m5 ; mode 32 [row 25] movu m6, [r5 + 2 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1970 * 16], m3 ; mode 33 [row 20 - first half] movu [r0 + 2024 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1971 * 16], m3 ; mode 33 [row 20 - second half] movu [r0 + 2025 * 16], m3 ; mode 32 [row 26] movu m6, [r5 + 23 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1972 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1973 * 16], m3 ; mode 33 [row 21] movu m6, [r5 + 28 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2026 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2027 * 16], m3 ; mode 32 [row 27] movu m6, [r5 + 12 * 16] movu m0, [r3 + 19] movd m1, [r3 + 20] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 27] movd m4, [r3 + 28] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1974 * 16], m3 movu m1, [r3 + 35] movd m3, [r3 + 36] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 43] movd m5, [r3 + 44] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1975 * 16], m3 ; mode 33 [row 22] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2028 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2029 * 16], m3 ; mode 32 [row 28] movu m6, [r5 + 1 * 16] movu m0, [r3 + 20] movd m1, [r3 + 21] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 28] movd m4, [r3 + 29] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1976 * 16], m3 movu m1, [r3 + 36] movd m3, [r3 + 37] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 44] movd m5, [r3 + 45] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1977 * 16], m3 ; mode 32 [row 29] movu m6, [r5 + 22 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1978 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1979 * 16], m3 ; mode 33 [row 23] movu m6, [r5 + 16 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2030 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2031 * 16], m3 ; mode 32 [row 30] movu m6, [r5 + 11 * 16] movu m0, [r3 + 21] movd m1, [r3 + 22] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 29] movd m4, [r3 + 30] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1980 * 16], m3 movu m1, [r3 + 37] movd m3, [r3 + 38] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 45] movd m5, [r3 + 46] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 1981 * 16], m3 ; mode 33 [row 24] movu m6, [r5 + 10 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2032 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2033 * 16], m3 ; mode 32 [row 31] movu m0, [r3 + 22] movd m1, [r3 + 23] palignr m1, m0, 1 punpcklbw m0, m1 movu m2, [r3 + 30] movd m3, [r3 + 31] palignr m3, m2, 1 punpcklbw m2, m3 movu m1, [r3 + 38] movd m3, [r3 + 39] palignr m3, m1, 1 punpcklbw m1, m3 movu m4, [r3 + 46] movd m5, [r3 + 47] palignr m5, m4, 1 punpcklbw m4, m5 pshufb m5, m0, [tab_S2] movh [r0 + 1982 * 16], m5 pshufb m5, m2, [tab_S2] movh [r0 + 1982 * 16 + 8], m5 pshufb m5, m1, [tab_S2] movh [r0 + 1983 * 16], m5 pshufb m5, m4, [tab_S2] movh [r0 + 1983 * 16 + 8], m5 ; mode 33 [row 25] movu m6, [r5 + 4 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2034 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2035 * 16], m3 ; mode 33 [row 26] movu m6, [r5 + 30 * 16] pmaddubsw m3, m0, m6 pmulhrsw m3, m7 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2036 * 16], m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2037 * 16], m3 ; mode 33 [row 27] movu m6, [r5 + 24 * 16] movu m0, [r3 + 23] movd m1, [r3 + 24] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 31] movd m4, [r3 + 32] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2038 * 16], m3 movu m1, [r3 + 39] movd m3, [r3 + 40] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 47] movd m5, [r3 + 48] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2039 * 16], m3 ; mode 33 [row 28] movu m6, [r5 + 18 * 16] movu m0, [r3 + 24] movd m1, [r3 + 25] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 32] movd m4, [r3 + 33] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2040 * 16], m3 movu m1, [r3 + 40] movd m3, [r3 + 41] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 48] movd m5, [r3 + 49] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2041 * 16], m3 ; mode 33 [row 29] movu m6, [r5 + 12 * 16] movu m0, [r3 + 25] movd m1, [r3 + 26] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 33] movd m4, [r3 + 34] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2042 * 16], m3 movu m1, [r3 + 41] movd m3, [r3 + 42] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 49] movd m5, [r3 + 50] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2043 * 16], m3 ; mode 33 [row 30] movu m6, [r5 + 6 * 16] movu m0, [r3 + 26] movd m1, [r3 + 27] palignr m1, m0, 1 punpcklbw m0, m1 pmaddubsw m3, m0, m6 pmulhrsw m3, m7 movu m2, [r3 + 34] movd m4, [r3 + 35] palignr m4, m2, 1 punpcklbw m2, m4 pmaddubsw m5, m2, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2044 * 16], m3 movu m1, [r3 + 42] movd m3, [r3 + 43] palignr m3, m1, 1 punpcklbw m1, m3 pmaddubsw m3, m1, m6 pmulhrsw m3, m7 movu m4, [r3 + 50] movd m5, [r3 + 51] palignr m5, m4, 1 punpcklbw m4, m5 pmaddubsw m5, m4, m6 pmulhrsw m5, m7 packuswb m3, m5 movu [r0 + 2045 * 16], m3 ; mode 33 [row 31] movu m5, [r3 + 27] movu [r0 + 2046 * 16], m5 movu m5, [r3 + 43] movu [r0 + 2047 * 16], m5 ;mode 34 [row 0] movu m0, [r3 + 2] movu [r0 + 2048 * 16], m0 movu m1, [r3 + 18] movu [r0 + 2049 * 16], m1 ;mode 34 [row 1] movu m2, [r3 + 34] palignr m3, m1, m0, 1 movu [r0 + 2050 * 16], m3 palignr m4, m2, m1, 1 movu [r0 + 2051 * 16], m4 ;mode 34 [row 2] palignr m3, m1, m0, 2 movu [r0 + 2052 * 16], m3 palignr m4, m2, m1, 2 movu [r0 + 2053 * 16], m4 ;mode 34 [row 3] palignr m3, m1, m0, 3 movu [r0 + 2054 * 16], m3 palignr m4, m2, m1, 3 movu [r0 + 2055 * 16], m4 ;mode 34 [row 4] palignr m3, m1, m0, 4 movu [r0 + 2056 * 16], m3 palignr m4, m2, m1, 4 movu [r0 + 2057 * 16], m4 ;mode 34 [row 5] palignr m3, m1, m0, 5 movu [r0 + 2058 * 16], m3 palignr m4, m2, m1, 5 movu [r0 + 2059 * 16], m4 ;mode 34 [row 6] palignr m3, m1, m0, 6 movu [r0 + 2060 * 16], m3 palignr m4, m2, m1, 6 movu [r0 + 2061 * 16], m4 ;mode 34 [row 7] palignr m3, m1, m0, 7 movu [r0 + 2062 * 16], m3 palignr m4, m2, m1, 7 movu [r0 + 2063 * 16], m4 ;mode 34 [row 8] palignr m3, m1, m0, 8 movu [r0 + 2064 * 16], m3 palignr m4, m2, m1, 8 movu [r0 + 2065 * 16], m4 ;mode 34 [row 9] palignr m3, m1, m0, 9 movu [r0 + 2066 * 16], m3 palignr m4, m2, m1, 9 movu [r0 + 2067 * 16], m4 ;mode 34 [row 10] palignr m3, m1, m0, 10 movu [r0 + 2068 * 16], m3 palignr m4, m2, m1, 10 movu [r0 + 2069 * 16], m4 ;mode 34 [row 11] palignr m3, m1, m0, 11 movu [r0 + 2070 * 16], m3 palignr m4, m2, m1, 11 movu [r0 + 2071 * 16], m4 ;mode 34 [row 12] palignr m3, m1, m0, 12 movu [r0 + 2072 * 16], m3 palignr m4, m2, m1, 12 movu [r0 + 2073 * 16], m4 ;mode 34 [row 13] palignr m3, m1, m0, 13 movu [r0 + 2074 * 16], m3 palignr m4, m2, m1, 13 movu [r0 + 2075 * 16], m4 ;mode 34 [row 14] palignr m3, m1, m0, 14 movu [r0 + 2076 * 16], m3 palignr m4, m2, m1, 14 movu [r0 + 2077 * 16], m4 ;mode 34 [row 15] palignr m3, m1, m0, 15 movu [r0 + 2078 * 16], m3 palignr m4, m2, m1, 15 movu [r0 + 2079 * 16], m4 ;mode 34 [row 16] palignr m3, m1, m0, 16 movu [r0 + 2080 * 16], m3 palignr m4, m2, m1, 16 movu [r0 + 2081 * 16], m4 ;mode 34 [row 17] movu m0, [r3 + 19] movu [r0 + 2082 * 16], m0 movu m1, [r3 + 35] movu [r0 + 2083 * 16], m1 mov r2d, r6d mov [r4], r2b mov r2d, [rsp] mov [r1 + 64], r2b ;mode 34 [row 18] movu m2, [r3 + 51] palignr m3, m1, m0, 1 movu [r0 + 2084 * 16], m3 palignr m4, m2, m1, 1 movu [r0 + 2085 * 16], m4 ;mode 34 [row 19] palignr m3, m1, m0, 2 movu [r0 + 2086 * 16], m3 palignr m4, m2, m1, 2 movu [r0 + 2087 * 16], m4 ;mode 34 [row 20] palignr m3, m1, m0, 3 movu [r0 + 2088 * 16], m3 palignr m4, m2, m1, 3 movu [r0 + 2089 * 16], m4 ;mode 34 [row 21] palignr m3, m1, m0, 4 movu [r0 + 2090 * 16], m3 palignr m4, m2, m1, 4 movu [r0 + 2091 * 16], m4 ;mode 34 [row 22] palignr m3, m1, m0, 5 movu [r0 + 2092 * 16], m3 palignr m4, m2, m1, 5 movu [r0 + 2093 * 16], m4 ;mode 34 [row 23] palignr m3, m1, m0, 6 movu [r0 + 2094 * 16], m3 palignr m4, m2, m1, 6 movu [r0 + 2095 * 16], m4 ;mode 34 [row 24] palignr m3, m1, m0, 7 movu [r0 + 2096 * 16], m3 palignr m4, m2, m1, 7 movu [r0 + 2097 * 16], m4 ;mode 34 [row 25] palignr m3, m1, m0, 8 movu [r0 + 2098 * 16], m3 palignr m4, m2, m1, 8 movu [r0 + 2099 * 16], m4 ;mode 34 [row 26] palignr m3, m1, m0, 9 movu [r0 + 2100 * 16], m3 palignr m4, m2, m1, 9 movu [r0 + 2101 * 16], m4 ;mode 34 [row 27] palignr m3, m1, m0, 10 movu [r0 + 2102 * 16], m3 palignr m4, m2, m1, 10 movu [r0 + 2103 * 16], m4 ;mode 34 [row 28] palignr m3, m1, m0, 11 movu [r0 + 2104 * 16], m3 palignr m4, m2, m1, 11 movu [r0 + 2105 * 16], m4 ;mode 34 [row 29] palignr m3, m1, m0, 12 movu [r0 + 2106 * 16], m3 palignr m4, m2, m1, 12 movu [r0 + 2107 * 16], m4 ;mode 34 [row 30] palignr m3, m1, m0, 13 movu [r0 + 2108 * 16], m3 palignr m4, m2, m1, 13 movu [r0 + 2109 * 16], m4 ;mode 34 [row 31] palignr m3, m1, m0, 14 movu [r0 + 2110 * 16], m3 palignr m4, m2, m1, 14 movu [r0 + 2111 * 16], m4 RET ;----------------------------------------------------------------------------- ; void all_angs_pred_4x4(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) ;----------------------------------------------------------------------------- INIT_YMM avx2 cglobal all_angs_pred_4x4, 4, 4, 6 mova m5, [pw_1024] lea r2, [all_ang4] lea r3, [all_ang4_shuff] ; mode 2 vbroadcasti128 m0, [r1 + 9] mova xm1, xm0 psrldq xm1, 1 pshufb xm1, [r3] movu [r0], xm1 ; mode 3 pshufb m1, m0, [r3 + 1 * mmsize] pmaddubsw m1, [r2] pmulhrsw m1, m5 ; mode 4 pshufb m2, m0, [r3 + 2 * mmsize] pmaddubsw m2, [r2 + 1 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (3 - 2) * 16], m1 ; mode 5 pshufb m1, m0, [r3 + 2 * mmsize] pmaddubsw m1, [r2 + 2 * mmsize] pmulhrsw m1, m5 ; mode 6 pshufb m2, m0, [r3 + 3 * mmsize] pmaddubsw m2, [r2 + 3 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (5 - 2) * 16], m1 add r3, 4 * mmsize add r2, 4 * mmsize ; mode 7 pshufb m1, m0, [r3 + 0 * mmsize] pmaddubsw m1, [r2 + 0 * mmsize] pmulhrsw m1, m5 ; mode 8 pshufb m2, m0, [r3 + 1 * mmsize] pmaddubsw m2, [r2 + 1 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (7 - 2) * 16], m1 ; mode 9 pshufb m1, m0, [r3 + 1 * mmsize] pmaddubsw m1, [r2 + 2 * mmsize] pmulhrsw m1, m5 packuswb m1, m1 vpermq m1, m1, 11011000b movu [r0 + (9 - 2) * 16], xm1 ; mode 10 pshufb xm1, xm0, [r3 + 2 * mmsize] movu [r0 + (10 - 2) * 16], xm1 pxor xm1, xm1 movd xm2, [r1 + 1] pshufd xm3, xm2, 0 punpcklbw xm3, xm1 pinsrb xm2, [r1], 0 pshufb xm4, xm2, xm1 punpcklbw xm4, xm1 psubw xm3, xm4 psraw xm3, 1 pshufb xm4, xm0, xm1 punpcklbw xm4, xm1 paddw xm3, xm4 packuswb xm3, xm1 pextrb [r0 + 128], xm3, 0 pextrb [r0 + 132], xm3, 1 pextrb [r0 + 136], xm3, 2 pextrb [r0 + 140], xm3, 3 ; mode 11 vbroadcasti128 m0, [r1] pshufb m1, m0, [r3 + 3 * mmsize] pmaddubsw m1, [r2 + 3 * mmsize] pmulhrsw m1, m5 ; mode 12 add r2, 4 * mmsize pshufb m2, m0, [r3 + 3 * mmsize] pmaddubsw m2, [r2 + 0 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (11 - 2) * 16], m1 ; mode 13 add r3, 4 * mmsize pshufb m1, m0, [r3 + 0 * mmsize] pmaddubsw m1, [r2 + 1 * mmsize] pmulhrsw m1, m5 ; mode 14 pshufb m2, m0, [r3 + 1 * mmsize] pmaddubsw m2, [r2 + 2 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (13 - 2) * 16], m1 ; mode 15 pshufb m1, m0, [r3 + 2 * mmsize] pmaddubsw m1, [r2 + 3 * mmsize] pmulhrsw m1, m5 ; mode 16 add r2, 4 * mmsize pshufb m2, m0, [r3 + 3 * mmsize] pmaddubsw m2, [r2 + 0 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (15 - 2) * 16], m1 ; mode 17 add r3, 4 * mmsize pshufb m1, m0, [r3 + 0 * mmsize] pmaddubsw m1, [r2 + 1 * mmsize] pmulhrsw m1, m5 packuswb m1, m1 vpermq m1, m1, 11011000b ; mode 18 pshufb m2, m0, [r3 + 1 * mmsize] vinserti128 m1, m1, xm2, 1 movu [r0 + (17 - 2) * 16], m1 ; mode 19 pshufb m1, m0, [r3 + 2 * mmsize] pmaddubsw m1, [r2 + 2 * mmsize] pmulhrsw m1, m5 ; mode 20 pshufb m2, m0, [r3 + 3 * mmsize] pmaddubsw m2, [r2 + 3 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (19 - 2) * 16], m1 ; mode 21 add r2, 4 * mmsize add r3, 4 * mmsize pshufb m1, m0, [r3 + 0 * mmsize] pmaddubsw m1, [r2 + 0 * mmsize] pmulhrsw m1, m5 ; mode 22 pshufb m2, m0, [r3 + 1 * mmsize] pmaddubsw m2, [r2 + 1 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (21 - 2) * 16], m1 ; mode 23 pshufb m1, m0, [r3 + 2 * mmsize] pmaddubsw m1, [r2 + 2 * mmsize] pmulhrsw m1, m5 ; mode 24 pshufb m2, m0, [r3 + 3 * mmsize] pmaddubsw m2, [r2 + 3 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (23 - 2) * 16], m1 ; mode 25 add r2, 4 * mmsize pshufb m1, m0, [r3 + 3 * mmsize] pmaddubsw m1, [r2 + 0 * mmsize] pmulhrsw m1, m5 packuswb m1, m1 vpermq m1, m1, 11011000b movu [r0 + (25 - 2) * 16], xm1 ; mode 26 add r3, 4 * mmsize pshufb xm1, xm0, [r3 + 0 * mmsize] movu [r0 + (26 - 2) * 16], xm1 pxor xm1, xm1 movd xm2, [r1 + 9] pshufd xm3, xm2, 0 punpcklbw xm3, xm1 pinsrb xm4, [r1 + 0], 0 pshufb xm4, xm1 punpcklbw xm4, xm1 psubw xm3, xm4 psraw xm3, 1 psrldq xm2, xm0, 1 pshufb xm2, xm1 punpcklbw xm2, xm1 paddw xm3, xm2 packuswb xm3, xm1 pextrb [r0 + 384], xm3, 0 pextrb [r0 + 388], xm3, 1 pextrb [r0 + 392], xm3, 2 pextrb [r0 + 396], xm3, 3 ; mode 27 pshufb m1, m0, [r3 + 1 * mmsize] pmaddubsw m1, [r2 + 1 * mmsize] pmulhrsw m1, m5 ; mode 28 pshufb m2, m0, [r3 + 1 * mmsize] pmaddubsw m2, [r2 + 2 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (27 - 2) * 16], m1 ; mode 29 pshufb m1, m0, [r3 + 2 * mmsize] pmaddubsw m1, [r2 + 3 * mmsize] pmulhrsw m1, m5 ; mode 30 add r2, 4 * mmsize pshufb m2, m0, [r3 + 3 * mmsize] pmaddubsw m2, [r2 + 0 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (29 - 2) * 16], m1 ; mode 31 add r3, 4 * mmsize pshufb m1, m0, [r3 + 0 * mmsize] pmaddubsw m1, [r2 + 1 * mmsize] pmulhrsw m1, m5 ; mode 32 pshufb m2, m0, [r3 + 0 * mmsize] pmaddubsw m2, [r2 + 2 * mmsize] pmulhrsw m2, m5 packuswb m1, m2 vpermq m1, m1, 11011000b movu [r0 + (31 - 2) * 16], m1 ; mode 33 pshufb m1, m0, [r3 + 1 * mmsize] pmaddubsw m1, [r2 + 3 * mmsize] pmulhrsw m1, m5 packuswb m1, m2 vpermq m1, m1, 11011000b ; mode 34 pshufb m0, [r3 + 2 * mmsize] vinserti128 m1, m1, xm0, 1 movu [r0 + (33 - 2) * 16], m1 RET ;----------------------------------------------------------------------------- ; void all_angs_pred_4x4(pixel *dest, pixel *refPix, pixel *filtPix, int bLuma) ;----------------------------------------------------------------------------- INIT_XMM sse2 cglobal all_angs_pred_4x4, 4, 4, 8 ; mode 2 movh m6, [r1 + 9] mova m2, m6 psrldq m2, 1 movd [r0], m2 ;byte[A, B, C, D] psrldq m2, 1 movd [r0 + 4], m2 ;byte[B, C, D, E] psrldq m2, 1 movd [r0 + 8], m2 ;byte[C, D, E, F] psrldq m2, 1 movd [r0 + 12], m2 ;byte[D, E, F, G] ; mode 10/26 pxor m7, m7 pshufd m5, m6, 0 mova [r0 + 128], m5 ;mode 10 byte[9, A, B, C, 9, A, B, C, 9, A, B, C, 9, A, B, C] movd m4, [r1 + 1] pshufd m4, m4, 0 mova [r0 + 384], m4 ;mode 26 byte[1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4] movd m1, [r1] punpcklbw m1, m7 pshuflw m1, m1, 0x00 punpcklqdq m1, m1 ;m1 = byte[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] punpckldq m4, m5 punpcklbw m4, m7 ;m4 = word[1, 2, 3, 4, 9, A, B, C] pshuflw m2, m4, 0x00 pshufhw m2, m2, 0x00 ;m2 = word[1, 1, 1, 1, 9, 9, 9, 9] psubw m4, m1 psraw m4, 1 pshufd m2, m2, q1032 ;m2 = word[9, 9, 9, 9, 1, 1, 1, 1] paddw m4, m2 packuswb m4, m4 %if ARCH_X86_64 movq r2, m4 mov [r0 + 128], r2b ;mode 10 shr r2, 8 mov [r0 + 132], r2b shr r2, 8 mov [r0 + 136], r2b shr r2, 8 mov [r0 + 140], r2b shr r2, 8 mov [r0 + 384], r2b ;mode 26 shr r2d, 8 mov [r0 + 388], r2b shr r2d, 8 mov [r0 + 392], r2b shr r2d, 8 mov [r0 + 396], r2b %else movd r2d, m4 mov [r0 + 128], r2b ;mode 10 shr r2d, 8 mov [r0 + 132], r2b shr r2d, 8 mov [r0 + 136], r2b shr r2d, 8 mov [r0 + 140], r2b psrldq m4, 4 movd r2d, m4 mov [r0 + 384], r2b ;mode 26 shr r2d, 8 mov [r0 + 388], r2b shr r2d, 8 mov [r0 + 392], r2b shr r2d, 8 mov [r0 + 396], r2b %endif ; mode 3 mova m2, [pw_16] lea r3, [pw_ang_table + 7 * 16] lea r2, [pw_ang_table + 23 * 16] punpcklbw m6, m6 psrldq m6, 1 movh m1, m6 psrldq m6, 2 movh m0, m6 psrldq m6, 2 movh m3, m6 psrldq m6, 2 punpcklbw m1, m7 ;m1 = word[9, A, A, B, B, C, C, D] punpcklbw m0, m7 ;m0 = word[A, B, B, C, C, D, D, E] punpcklbw m3, m7 ;m3 = word[B, C, C, D, D, E, E, F] punpcklbw m6, m7 ;m6 = word[C, D, D, E, E, F, F, G] mova m7, [r2 - 3 * 16] pmaddwd m5, m1, [r2 + 3 * 16] pmaddwd m4, m0, m7 packssdw m5, m4 paddw m5, m2 psraw m5, 5 pmaddwd m4, m3, [r3 + 7 * 16] pmaddwd m6, [r3 + 1 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 16], m5 movd [r0 + 68], m5 ;mode 6 row 1 psrldq m5, 4 movd [r0 + 76], m5 ;mode 6 row 3 ; mode 4 pmaddwd m4, m0, [r2 + 8 * 16] pmaddwd m6, m3, m7 packssdw m4, m6 paddw m4, m2 psraw m4, 5 pmaddwd m5, m1, [r2 - 2 * 16] pmaddwd m6, m0, [r3 + 3 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 packuswb m5, m4 mova [r0 + 32], m5 ; mode 5 pmaddwd m5, m1, [r2 - 6 * 16] pmaddwd m6, m0, [r3 - 5 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 pmaddwd m4, m0, [r2 - 4 * 16] pmaddwd m3, [r3 - 3 * 16] packssdw m4, m3 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 48], m5 ; mode 6 pmaddwd m5, m1, [r3 + 6 * 16] pmaddwd m6, m0, [r3 + 0 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 packuswb m5, m6 movd [r0 + 64], m5 psrldq m5, 4 movd [r0 + 72], m5 ; mode 7 pmaddwd m5, m1, [r3 + 2 * 16] pmaddwd m6, m1, [r2 - 5 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 mova m3, [r2 + 4 * 16] pmaddwd m4, m1, m3 pmaddwd m0, [r3 - 3 * 16] packssdw m4, m0 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 80], m5 ; mode 8 mova m0, [r3 - 2 * 16] pmaddwd m5, m1, m0 pmaddwd m6, m1, [r3 + 3 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 pmaddwd m4, m1, [r3 + 8 * 16] pmaddwd m7, m1 packssdw m4, m7 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 96], m5 ; mode 9 pmaddwd m5, m1, [r3 - 5 * 16] pmaddwd m6, m1, [r3 - 3 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 pmaddwd m4, m1, [r3 - 1 * 16] pmaddwd m6, m1, [r3 + 1 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 112], m5 ; mode 11 movd m5, [r1] punpcklwd m5, m1 pand m5, [pb_0000000000000F0F] pslldq m1, 4 por m1, m5 ;m1 = word[0, 9, 9, A, A, B, B, C] pmaddwd m5, m1, [r2 + 7 * 16] pmaddwd m6, m1, [r2 + 5 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 pmaddwd m4, m1, [r2 + 3 * 16] pmaddwd m6, m1, [r2 + 1 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 144], m5 ; mode 12 pmaddwd m3, m1 pmaddwd m6, m1, [r2 - 1 * 16] packssdw m3, m6 paddw m3, m2 psraw m3, 5 pmaddwd m4, m1, [r2 - 6 * 16] pmaddwd m6, m1, [r3 + 5 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 packuswb m3, m4 mova [r0 + 160], m3 ; mode 13 mova m3, m1 movd m7, [r1 + 4] punpcklwd m7, m1 pand m7, [pb_0000000000000F0F] pslldq m3, 4 por m3, m7 ;m3 = word[4, 0, 0, 9, 9, A, A, B] pmaddwd m5, m1, [r2 + 0 * 16] pmaddwd m6, m1, [r3 + 7 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 pmaddwd m4, m1, m0 pmaddwd m6, m3, [r2 + 5 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 176], m5 ; mode 14 pmaddwd m5, m1, [r2 - 4 * 16] pmaddwd m6, m1, [r3 - 1 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 movd m6, [r1 + 2] pand m3, [pw_FFFFFFFFFFFFFFF0] pand m6, [pb_000000000000000F] por m3, m6 ;m3 = word[2, 0, 0, 9, 9, A, A, B] pmaddwd m4, m3, [r2 + 2 * 16] pmaddwd m6, m3, [r3 + 5 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 192], m5 psrldq m5, 4 movd [r0 + 240], m5 ;mode 17 row 0 ; mode 15 pmaddwd m5, m1, [r3 + 8 * 16] pmaddwd m6, m3, [r2 + 7 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 pmaddwd m6, m3, [r3 + 6 * 16] mova m0, m3 punpcklwd m7, m3 pslldq m0, 4 pand m7, [pb_0000000000000F0F] por m0, m7 ;m0 = word[4, 2, 2, 0, 0, 9, 9, A] pmaddwd m4, m0, [r2 + 5 * 16] packssdw m6, m4 paddw m6, m2 psraw m6, 5 packuswb m5, m6 mova [r0 + 208], m5 ; mode 16 pmaddwd m5, m1, [r3 + 4 * 16] pmaddwd m6, m3, [r2 - 1 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 pmaddwd m3, [r3 - 6 * 16] movd m6, [r1 + 3] pand m0, [pw_FFFFFFFFFFFFFFF0] pand m6, [pb_000000000000000F] por m0, m6 ;m0 = word[3, 2, 2, 0, 0, 9, 9, A] pmaddwd m0, [r3 + 5 * 16] packssdw m3, m0 paddw m3, m2 psraw m3, 5 packuswb m5, m3 mova [r0 + 224], m5 ; mode 17 movd m4, [r1 + 1] punpcklwd m4, m1 pand m4, [pb_0000000000000F0F] pslldq m1, 4 por m1, m4 ;m1 = word[1, 0, 0, 9, 9, A, A, B] pmaddwd m6, m1, [r3 + 5 * 16] packssdw m6, m6 paddw m6, m2 psraw m6, 5 movd m5, [r1 + 2] punpcklwd m5, m1 pand m5, [pb_0000000000000F0F] pslldq m1, 4 por m1, m5 ;m1 = word[2, 1, 1, 0, 0, 9, 9, A] pmaddwd m4, m1, [r2 - 5 * 16] punpcklwd m7, m1 pand m7, [pb_0000000000000F0F] pslldq m1, 4 por m1, m7 ;m1 = word[4, 2, 2, 1, 1, 0, 0, 9] pmaddwd m1, [r2 + 1 * 16] packssdw m4, m1 paddw m4, m2 psraw m4, 5 packuswb m6, m4 movd [r0 + 244], m6 psrldq m6, 8 movh [r0 + 248], m6 ; mode 18 movh m1, [r1] movd [r0 + 256], m1 ;byte[0, 1, 2, 3] movh m3, [r1 + 2] punpcklqdq m3, m1 psrldq m3, 7 movd [r0 + 260], m3 ;byte[2, 1, 0, 9] movh m4, [r1 + 3] punpcklqdq m4, m3 psrldq m4, 7 movd [r0 + 264], m4 ;byte[1, 0, 9, A] movh m0, [r1 + 4] punpcklqdq m0, m4 psrldq m0, 7 movd [r0 + 268], m0 ;byte[0, 9, A, B] ; mode 19 pxor m7, m7 punpcklbw m4, m3 punpcklbw m3, m1 punpcklbw m1, m1 punpcklbw m4, m7 ;m4 = word[A, 9, 9, 0, 0, 1, 1, 2] punpcklbw m3, m7 ;m3 = word[9, 0, 0, 1, 1, 2, 2, 3] psrldq m1, 1 punpcklbw m1, m7 ;m1 = word[0, 1, 1, 2, 2, 3, 3, 4] pmaddwd m6, m1, [r3 - 1 * 16] pmaddwd m7, m3, [r3 + 5 * 16] packssdw m6, m7 paddw m6, m2 psraw m6, 5 pmaddwd m5, m4, [r2 - 5 * 16] movd m7, [r1 + 12] punpcklwd m7, m4 pand m7, [pb_0000000000000F0F] pslldq m4, 4 por m4, m7 ;m4 = word[C, A, A, 9, 9, 0, 0, 1] pmaddwd m4, [r2 + 1 * 16] packssdw m5, m4 paddw m5, m2 psraw m5, 5 packuswb m6, m5 mova [r0 + 272], m6 movd [r0 + 324], m6 ;mode 22 row 1 ; mode 20 pmaddwd m5, m1, [r3 + 4 * 16] movd m4, [r1 + 10] pand m3, [pw_FFFFFFFFFFFFFFF0] pand m4, [pb_000000000000000F] por m3, m4 ;m3 = word[A, 0, 0, 1, 1, 2, 2, 3] pmaddwd m6, m3, [r2 - 1 * 16] packssdw m5, m6 paddw m5, m2 psraw m5, 5 pmaddwd m4, m3, [r3 - 6 * 16] punpcklwd m0, m3 pand m0, [pb_0000000000000F0F] mova m6, m3 pslldq m6, 4 por m0, m6 ;m0 = word[B, A, A, 0, 0, 1, 1, 2] pmaddwd m6, m0, [r3 + 5 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 packuswb m5, m4 mova [r0 + 288], m5 ; mode 21 pmaddwd m4, m1, [r3 + 8 * 16] pmaddwd m6, m3, [r2 + 7 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 pmaddwd m5, m3, [r3 + 6 * 16] pand m0, [pw_FFFFFFFFFFFFFFF0] pand m7, [pb_000000000000000F] por m0, m7 ;m0 = word[C, A, A, 0, 0, 1, 1, 2] pmaddwd m0, [r2 + 5 * 16] packssdw m5, m0 paddw m5, m2 psraw m5, 5 packuswb m4, m5 mova [r0 + 304], m4 ; mode 22 pmaddwd m4, m1, [r2 - 4 * 16] packssdw m4, m4 paddw m4, m2 psraw m4, 5 mova m0, [r3 + 5 * 16] pmaddwd m5, m3, [r2 + 2 * 16] pmaddwd m6, m3, m0 packssdw m5, m6 paddw m5, m2 psraw m5, 5 packuswb m4, m5 movd [r0 + 320], m4 psrldq m4, 8 movh [r0 + 328], m4 ; mode 23 pmaddwd m4, m1, [r2 + 0 * 16] pmaddwd m5, m1, [r3 + 7 * 16] packssdw m4, m5 paddw m4, m2 psraw m4, 5 pmaddwd m6, m1, [r3 - 2 * 16] pand m3, [pw_FFFFFFFFFFFFFFF0] por m3, m7 ;m3 = word[C, 0, 0, 1, 1, 2, 2, 3] pmaddwd m3, [r2 + 5 * 16] packssdw m6, m3 paddw m6, m2 psraw m6, 5 packuswb m4, m6 mova [r0 + 336], m4 ; mode 24 pmaddwd m4, m1, [r2 + 4 * 16] pmaddwd m5, m1, [r2 - 1 * 16] packssdw m4, m5 paddw m4, m2 psraw m4, 5 pmaddwd m6, m1, [r2 - 6 * 16] pmaddwd m0, m1 packssdw m6, m0 paddw m6, m2 psraw m6, 5 packuswb m4, m6 mova [r0 + 352], m4 ; mode 25 pmaddwd m4, m1, [r2 + 7 * 16] pmaddwd m5, m1, [r2 + 5 * 16] packssdw m4, m5 paddw m4, m2 psraw m4, 5 pmaddwd m6, m1, [r2 + 3 * 16] pmaddwd m1, [r2 + 1 * 16] packssdw m6, m1 paddw m6, m2 psraw m6, 5 packuswb m4, m6 mova [r0 + 368], m4 ; mode 27 movh m0, [r1 + 1] pxor m7, m7 punpcklbw m0, m0 psrldq m0, 1 movh m1, m0 psrldq m0, 2 movh m3, m0 psrldq m0, 2 punpcklbw m1, m7 ;m1 = word[1, 2, 2, 3, 3, 4, 4, 5] punpcklbw m3, m7 ;m3 = word[2, 3, 3, 4, 4, 5, 5, 6] punpcklbw m0, m7 ;m0 = word[3, 4, 4, 5, 5, 6, 6, 7] mova m7, [r3 - 3 * 16] pmaddwd m4, m1, [r3 - 5 * 16] pmaddwd m5, m1, m7 packssdw m4, m5 paddw m4, m2 psraw m4, 5 pmaddwd m6, m1, [r3 - 1 * 16] pmaddwd m5, m1, [r3 + 1 * 16] packssdw m6, m5 paddw m6, m2 psraw m6, 5 packuswb m4, m6 mova [r0 + 400], m4 ; mode 28 pmaddwd m4, m1, [r3 - 2 * 16] pmaddwd m5, m1, [r3 + 3 * 16] packssdw m4, m5 paddw m4, m2 psraw m4, 5 pmaddwd m6, m1, [r3 + 8 * 16] pmaddwd m5, m1, [r2 - 3 * 16] packssdw m6, m5 paddw m6, m2 psraw m6, 5 packuswb m4, m6 mova [r0 + 416], m4 ; mode 29 pmaddwd m4, m1, [r3 + 2 * 16] pmaddwd m6, m1, [r2 - 5 * 16] packssdw m4, m6 paddw m4, m2 psraw m4, 5 pmaddwd m6, m1, [r2 + 4 * 16] pmaddwd m5, m3, m7 packssdw m6, m5 paddw m6, m2 psraw m6, 5 packuswb m4, m6 mova [r0 + 432], m4 ; mode 30 pmaddwd m4, m1, [r3 + 6 * 16] pmaddwd m5, m1, [r2 + 3 * 16] packssdw m4, m5 paddw m4, m2 psraw m4, 5 pmaddwd m6, m3, [r3 + 0 * 16] pmaddwd m5, m3, [r2 - 3 * 16] packssdw m6, m5 paddw m6, m2 psraw m6, 5 packuswb m4, m6 mova [r0 + 448], m4 psrldq m4, 4 movh [r0 + 496], m4 ;mode 33 row 0 psrldq m4, 8 movd [r0 + 500], m4 ;mode 33 row 1 ; mode 31 pmaddwd m4, m1, [r2 - 6 * 16] pmaddwd m5, m3, [r3 - 5 * 16] packssdw m4, m5 paddw m4, m2 psraw m4, 5 pmaddwd m6, m3, [r2 - 4 * 16] pmaddwd m7, m0 packssdw m6, m7 paddw m6, m2 psraw m6, 5 packuswb m4, m6 mova [r0 + 464], m4 ; mode 32 pmaddwd m1, [r2 - 2 * 16] pmaddwd m5, m3, [r3 + 3 * 16] packssdw m1, m5 paddw m1, m2 psraw m1, 5 pmaddwd m3, [r2 + 8 * 16] pmaddwd m5, m0, [r2 - 3 * 16] packssdw m3, m5 paddw m3, m2 psraw m3, 5 packuswb m1, m3 mova [r0 + 480], m1 ; mode 33 pmaddwd m0, [r3 + 7 * 16] pxor m7, m7 movh m4, [r1 + 4] punpcklbw m4, m4 psrldq m4, 1 punpcklbw m4, m7 pmaddwd m4, [r3 + 1 * 16] packssdw m0, m4 paddw m0, m2 psraw m0, 5 packuswb m0, m0 movh [r0 + 504], m0 ; mode 34 movh m7, [r1 + 2] movd [r0 + 512], m7 ;byte[2, 3, 4, 5] psrldq m7, 1 movd [r0 + 516], m7 ;byte[3, 4, 5, 6] psrldq m7, 1 movd [r0 + 520], m7 ;byte[4, 5, 6, 7] psrldq m7, 1 movd [r0 + 524], m7 ;byte[5, 6, 7, 8] RET