Mercurial > hg > forks > libbpg
view x265/source/common/x86/ssd-a.asm @ 0:772086c29cc7
Initial import.
| author | Matti Hamalainen <ccr@tnsp.org> |
|---|---|
| date | Wed, 16 Nov 2016 11:16:33 +0200 |
| parents | |
| children |
line wrap: on
line source
;***************************************************************************** ;* ssd-a.asm: x86 ssd functions ;***************************************************************************** ;* Copyright (C) 2003-2013 x264 project ;* ;* Authors: Loren Merritt <lorenm@u.washington.edu> ;* Fiona Glaser <fiona@x264.com> ;* Laurent Aimar <fenrir@via.ecp.fr> ;* Alex Izvorski <aizvorksi@gmail.com> ;* ;* This program is free software; you can redistribute it and/or modify ;* it under the terms of the GNU General Public License as published by ;* the Free Software Foundation; either version 2 of the License, or ;* (at your option) any later version. ;* ;* This program is distributed in the hope that it will be useful, ;* but WITHOUT ANY WARRANTY; without even the implied warranty of ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;* GNU General Public License for more details. ;* ;* You should have received a copy of the GNU General Public License ;* along with this program; if not, write to the Free Software ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. ;* ;* This program is also available under a commercial proprietary license. ;* For more information, contact us at license @ x265.com. ;***************************************************************************** %include "x86inc.asm" %include "x86util.asm" SECTION_RODATA 32 SECTION .text cextern pw_00ff cextern hsub_mul ;============================================================================= ; SSD ;============================================================================= %if HIGH_BIT_DEPTH ;----------------------------------------------------------------------------- ; int pixel_ssd_WxH( uint16_t *, intptr_t, uint16_t *, intptr_t ) ;----------------------------------------------------------------------------- %macro SSD_ONE 2 cglobal pixel_ssd_ss_%1x%2, 4,7,8 FIX_STRIDES r1, r3 %if mmsize == %1*2 %define offset0_1 r1 %define offset0_2 r1*2 %define offset0_3 r5 %define offset1_1 r3 %define offset1_2 r3*2 %define offset1_3 r6 lea r5, [3*r1] lea r6, [3*r3] %elif mmsize == %1 %define offset0_1 mmsize %define offset0_2 r1 %define offset0_3 r1+mmsize %define offset1_1 mmsize %define offset1_2 r3 %define offset1_3 r3+mmsize %elif mmsize == %1/2 %define offset0_1 mmsize %define offset0_2 mmsize*2 %define offset0_3 mmsize*3 %define offset1_1 mmsize %define offset1_2 mmsize*2 %define offset1_3 mmsize*3 %endif %assign %%n %2/(2*mmsize/%1) %if %%n > 1 mov r4d, %%n %endif pxor m0, m0 .loop: movu m1, [r0] movu m2, [r0+offset0_1] movu m3, [r0+offset0_2] movu m4, [r0+offset0_3] movu m6, [r2] movu m7, [r2+offset1_1] psubw m1, m6 psubw m2, m7 movu m6, [r2+offset1_2] movu m7, [r2+offset1_3] psubw m3, m6 psubw m4, m7 %if %%n > 1 lea r0, [r0+r1*(%2/%%n)] lea r2, [r2+r3*(%2/%%n)] %endif pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 %if %%n > 1 dec r4d jg .loop %endif %if BIT_DEPTH == 12 && mmsize == 16 movu m5, m0 pxor m6, m6 punpckldq m0, m6 punpckhdq m5, m6 paddq m0, m5 movhlps m5, m0 paddq m0, m5 movq r6, xm0 %else HADDD m0, m5 movd eax,xm0 %endif %ifidn movu,movq ; detect MMX EMMS %endif RET %endmacro %macro SSD_TWO 2 cglobal pixel_ssd_ss_%1x%2, 4,7,8 FIX_STRIDES r1, r3 pxor m0, m0 mov r4d, %2/2 lea r5, [r1 * 2] lea r6, [r3 * 2] .loop: movu m1, [r0] movu m2, [r0 + 16] movu m3, [r0 + 32] movu m4, [r0 + 48] movu m6, [r2] movu m7, [r2 + 16] psubw m1, m6 psubw m2, m7 movu m6, [r2 + 32] movu m7, [r2 + 48] psubw m3, m6 psubw m4, m7 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 movu m1, [r0 + 64] movu m2, [r0 + 80] movu m6, [r2 + 64] movu m7, [r2 + 80] psubw m1, m6 psubw m2, m7 pmaddwd m1, m1 pmaddwd m2, m2 paddd m1, m2 paddd m0, m1 %if %1 == 64 movu m3, [r0 + 96] movu m4, [r0 + 112] movu m6, [r2 + 96] movu m7, [r2 + 112] psubw m3, m6 psubw m4, m7 pmaddwd m3, m3 pmaddwd m4, m4 paddd m3, m4 paddd m0, m3 %endif movu m1, [r0 + r1] movu m2, [r0 + r1 + 16] movu m3, [r0 + r1 + 32] movu m4, [r0 + r1 + 48] movu m6, [r2 + r3] movu m7, [r2 + r3 + 16] psubw m1, m6 psubw m2, m7 movu m6, [r2 + r3 + 32] movu m7, [r2 + r3 + 48] psubw m3, m6 psubw m4, m7 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 movu m1, [r0 + r1 + 64] movu m2, [r0 + r1 + 80] movu m6, [r2 + r3 + 64] movu m7, [r2 + r3 + 80] psubw m1, m6 psubw m2, m7 pmaddwd m1, m1 pmaddwd m2, m2 paddd m1, m2 paddd m0, m1 %if %1 == 64 movu m3, [r0 + r1 + 96] movu m4, [r0 + r1 + 112] movu m6, [r2 + r3 + 96] movu m7, [r2 + r3 + 112] psubw m3, m6 psubw m4, m7 pmaddwd m3, m3 pmaddwd m4, m4 paddd m3, m4 paddd m0, m3 %endif lea r0, [r0 + r5] lea r2, [r2 + r6] dec r4d jnz .loop HADDD m0, m5 movd eax, xm0 RET %endmacro %macro SSD_24 2 cglobal pixel_ssd_ss_%1x%2, 4,7,8 FIX_STRIDES r1, r3 pxor m0, m0 mov r4d, %2/2 lea r5, [r1 * 2] lea r6, [r3 * 2] .loop: movu m1, [r0] movu m2, [r0 + 16] movu m3, [r0 + 32] movu m5, [r2] movu m6, [r2 + 16] movu m7, [r2 + 32] psubw m1, m5 psubw m2, m6 psubw m3, m7 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 paddd m1, m2 paddd m0, m1 movu m1, [r0 + r1] movu m2, [r0 + r1 + 16] movu m4, [r0 + r1 + 32] movu m5, [r2 + r3] movu m6, [r2 + r3 + 16] movu m7, [r2 + r3 + 32] psubw m1, m5 psubw m2, m6 psubw m4, m7 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 lea r0, [r0 + r5] lea r2, [r2 + r6] dec r4d jnz .loop HADDD m0, m5 movd eax, xm0 RET %endmacro %macro SSD_12 2 cglobal pixel_ssd_ss_%1x%2, 4,7,8 FIX_STRIDES r1, r3 pxor m0, m0 mov r4d, %2/4 lea r5, [r1 * 2] lea r6, [r3 * 2] .loop: movu m1, [r0] movh m2, [r0 + 16] movu m3, [r0 + r1] punpcklqdq m2, [r0 + r1 + 16] movu m7, [r2] psubw m1, m7 movh m4, [r2 + 16] movu m7, [r2 + r3] psubw m3, m7 punpcklqdq m4, [r2 + r3 + 16] psubw m2, m4 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 paddd m1, m2 paddd m0, m1 movu m1, [r0 + r5] movh m2, [r0 + r5 + 16] lea r0, [r0 + r5] movu m6, [r0 + r1] punpcklqdq m2, [r0 + r1 + 16] movu m7, [r2 + r6] psubw m1, m7 movh m4, [r2 + r6 + 16] lea r2, [r2 + r6] movu m7, [r2 + r3] psubw m6, m7 punpcklqdq m4, [r2 + r3 + 16] psubw m2, m4 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m6, m6 paddd m1, m2 paddd m3, m6 paddd m0, m1 paddd m0, m3 lea r0, [r0 + r5] lea r2, [r2 + r6] dec r4d jnz .loop HADDD m0, m5 movd eax, xm0 RET %endmacro INIT_YMM avx2 cglobal pixel_ssd_16x16, 4,7,8 FIX_STRIDES r1, r3 lea r5, [3 * r1] lea r6, [3 * r3] mov r4d, 4 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r0 + r1] movu m3, [r0 + r1 * 2] movu m4, [r0 + r5] movu m6, [r2] movu m7, [r2 + r3] psubw m1, m6 psubw m2, m7 movu m6, [r2 + r3 * 2] movu m7, [r2 + r6] psubw m3, m6 psubw m4, m7 lea r0, [r0 + r1 * 4] lea r2, [r2 + r3 * 4] pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 dec r4d jg .loop HADDD m0, m5 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_ssd_32x32, 4,7,8 add r1, r1 add r3, r3 mov r4d, 16 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r0 + 32] movu m3, [r0 + r1] movu m4, [r0 + r1 + 32] movu m6, [r2] movu m7, [r2 + 32] psubw m1, m6 psubw m2, m7 movu m6, [r2 + r3] movu m7, [r2 + r3 + 32] psubw m3, m6 psubw m4, m7 lea r0, [r0 + r1 * 2] lea r2, [r2 + r3 * 2] pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 dec r4d jg .loop HADDD m0, m5 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_ssd_64x64, 4,7,8 FIX_STRIDES r1, r3 mov r4d, 64 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r0+32] movu m3, [r0+32*2] movu m4, [r0+32*3] movu m6, [r2] movu m7, [r2+32] psubw m1, m6 psubw m2, m7 movu m6, [r2+32*2] movu m7, [r2+32*3] psubw m3, m6 psubw m4, m7 lea r0, [r0+r1] lea r2, [r2+r3] pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 dec r4d jg .loop HADDD m0, m5 movd eax, xm0 RET INIT_MMX mmx2 SSD_ONE 4, 4 SSD_ONE 4, 8 SSD_ONE 4, 16 SSD_ONE 8, 4 SSD_ONE 8, 8 SSD_ONE 8, 16 SSD_ONE 16, 8 SSD_ONE 16, 16 INIT_XMM sse2 SSD_ONE 8, 4 SSD_ONE 8, 8 SSD_ONE 8, 16 SSD_ONE 8, 32 SSD_12 12, 16 SSD_ONE 16, 4 SSD_ONE 16, 8 SSD_ONE 16, 12 SSD_ONE 16, 16 SSD_ONE 16, 32 SSD_ONE 16, 64 SSD_24 24, 32 SSD_ONE 32, 8 SSD_ONE 32, 16 SSD_ONE 32, 24 SSD_ONE 32, 32 SSD_ONE 32, 64 SSD_TWO 48, 64 SSD_TWO 64, 16 SSD_TWO 64, 32 SSD_TWO 64, 48 SSD_TWO 64, 64 INIT_YMM avx2 SSD_ONE 16, 8 SSD_ONE 16, 16 SSD_ONE 32, 32 SSD_ONE 64, 64 SSD_ONE 16, 32 SSD_ONE 32, 64 %endif ; HIGH_BIT_DEPTH ;----------------------------------------------------------------------------- ; int pixel_ssd_WxH( uint16_t *, intptr_t, uint16_t *, intptr_t ) ;----------------------------------------------------------------------------- %if HIGH_BIT_DEPTH == 0 %macro SSD_SS 2 cglobal pixel_ssd_ss_%1x%2, 4,7,6 FIX_STRIDES r1, r3 %if mmsize == %1*4 || mmsize == %1*2 %define offset0_1 r1*2 %define offset0_2 r1*4 %define offset0_3 r5 %define offset1_1 r3*2 %define offset1_2 r3*4 %define offset1_3 r6 lea r5, [4*r1] lea r6, [4*r3] lea r5, [r5 + 2*r1] lea r6, [r6 + 2*r3] %elif mmsize == %1 %define offset0_1 16 %define offset0_2 r1*2 %define offset0_3 r1*2+16 %define offset1_1 16 %define offset1_2 r3*2 %define offset1_3 r3*2+16 %endif %if %1 == 4 %assign %%n %2/(mmsize/%1) %else %assign %%n %2/(2*mmsize/%1) %endif %if %%n > 1 mov r4d, %%n %endif pxor m0, m0 .loop: %if %1 == 4 movh m1, [r0] movh m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movh m1, [r0 + offset0_1] movh m2, [r2 + offset1_1] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movh m1, [r0 + offset0_2] movh m2, [r2 + offset1_2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movh m1, [r0 + offset0_3] movh m2, [r2 + offset1_3] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 %else movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + offset0_1] movu m2, [r2 + offset1_1] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + offset0_2] movu m2, [r2 + offset1_2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + offset0_3] movu m2, [r2 + offset1_3] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 %endif lea r0, [r0+r1*(%2/%%n)*2] lea r2, [r2+r3*(%2/%%n)*2] %if %%n > 1 dec r4d jg .loop %endif %if %1 == 4 %if notcpuflag(ssse3) pshufd m1, m0, 1 paddd m0, m1 %else phaddd m0, m0 %endif %else HADDD m0, m1 %endif movd eax, m0 RET %endmacro %macro SSD_SS_ONE 0 SSD_SS 4, 4 SSD_SS 4, 8 SSD_SS 4, 16 SSD_SS 8, 4 SSD_SS 8, 8 SSD_SS 8, 16 SSD_SS 8, 32 SSD_SS 16, 4 SSD_SS 16, 8 SSD_SS 16, 12 SSD_SS 16, 16 SSD_SS 16, 32 SSD_SS 16, 64 %endmacro %macro SSD_SS_12x16 0 cglobal pixel_ssd_ss_12x16, 4,7,6 FIX_STRIDES r1, r3 mov r4d, 8 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 pslldq m1, 8 psrldq m1, 8 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 pslldq m1, 8 psrldq m1, 8 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] dec r4d jnz .loop HADDD m0, m1 movd eax, m0 RET %endmacro %macro SSD_SS_32 1 cglobal pixel_ssd_ss_32x%1, 4,7,6 FIX_STRIDES r1, r3 mov r4d, %1/2 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 32] movu m2, [r2 + 32] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 48] movu m2, [r2 + 48] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 32] movu m2, [r2 + 32] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 48] movu m2, [r2 + 48] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] dec r4d jnz .loop HADDD m0, m1 movd eax, m0 RET %endmacro %macro SSD_SS_32xN 0 SSD_SS_32 8 SSD_SS_32 16 SSD_SS_32 24 SSD_SS_32 32 SSD_SS_32 64 %endmacro %macro SSD_SS_24 0 cglobal pixel_ssd_ss_24x32, 4,7,6 FIX_STRIDES r1, r3 mov r4d, 16 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 32] movu m2, [r2 + 32] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 32] movu m2, [r2 + 32] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] dec r4d jnz .loop HADDD m0, m1 movd eax, m0 RET %endmacro %macro SSD_SS_48 0 cglobal pixel_ssd_ss_48x64, 4,7,6 FIX_STRIDES r1, r3 mov r4d, 32 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 32] movu m2, [r2 + 32] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 48] movu m2, [r2 + 48] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 64] movu m2, [r2 + 64] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 80] movu m2, [r2 + 80] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 32] movu m2, [r2 + 32] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 48] movu m2, [r2 + 48] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 64] movu m2, [r2 + 64] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 80] movu m2, [r2 + 80] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] dec r4d jnz .loop HADDD m0, m1 movd eax, m0 RET %endmacro %macro SSD_SS_64 1 cglobal pixel_ssd_ss_64x%1, 4,7,6 FIX_STRIDES r1, r3 mov r4d, %1/2 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 32] movu m2, [r2 + 32] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 48] movu m2, [r2 + 48] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 64] movu m2, [r2 + 64] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 80] movu m2, [r2 + 80] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 96] movu m2, [r2 + 96] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 112] movu m2, [r2 + 112] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] movu m1, [r0] movu m2, [r2] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 16] movu m2, [r2 + 16] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 32] movu m2, [r2 + 32] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 48] movu m2, [r2 + 48] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 64] movu m2, [r2 + 64] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 80] movu m2, [r2 + 80] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 96] movu m2, [r2 + 96] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 movu m1, [r0 + 112] movu m2, [r2 + 112] psubw m1, m2 pmaddwd m1, m1 paddd m0, m1 lea r0, [r0 + 2*r1] lea r2, [r2 + 2*r3] dec r4d jnz .loop HADDD m0, m1 movd eax, m0 RET %endmacro %macro SSD_SS_64xN 0 SSD_SS_64 16 SSD_SS_64 32 SSD_SS_64 48 SSD_SS_64 64 %endmacro INIT_XMM sse2 SSD_SS_ONE SSD_SS_12x16 SSD_SS_24 SSD_SS_32xN SSD_SS_48 SSD_SS_64xN INIT_XMM sse4 SSD_SS_ONE SSD_SS_12x16 SSD_SS_24 SSD_SS_32xN SSD_SS_48 SSD_SS_64xN INIT_XMM avx SSD_SS_ONE SSD_SS_12x16 SSD_SS_24 SSD_SS_32xN SSD_SS_48 SSD_SS_64xN %endif ; !HIGH_BIT_DEPTH %if HIGH_BIT_DEPTH == 0 %macro SSD_LOAD_FULL 5 movu m1, [t0+%1] movu m2, [t2+%2] movu m3, [t0+%3] movu m4, [t2+%4] %if %5==1 add t0, t1 add t2, t3 %elif %5==2 lea t0, [t0+2*t1] lea t2, [t2+2*t3] %endif %endmacro %macro LOAD 5 movh m%1, %3 movh m%2, %4 %if %5 lea t0, [t0+2*t1] %endif %endmacro %macro JOIN 7 movh m%3, %5 movh m%4, %6 %if %7 lea t2, [t2+2*t3] %endif punpcklbw m%1, m7 punpcklbw m%3, m7 psubw m%1, m%3 punpcklbw m%2, m7 punpcklbw m%4, m7 psubw m%2, m%4 %endmacro %macro JOIN_SSE2 7 movh m%3, %5 movh m%4, %6 %if %7 lea t2, [t2+2*t3] %endif punpcklqdq m%1, m%2 punpcklqdq m%3, m%4 DEINTB %2, %1, %4, %3, 7 psubw m%2, m%4 psubw m%1, m%3 %endmacro %macro JOIN_SSSE3 7 movh m%3, %5 movh m%4, %6 %if %7 lea t2, [t2+2*t3] %endif punpcklbw m%1, m%3 punpcklbw m%2, m%4 %endmacro %macro LOAD_AVX2 5 mova xm%1, %3 vinserti128 m%1, m%1, %4, 1 %if %5 lea t0, [t0+2*t1] %endif %endmacro %macro JOIN_AVX2 7 mova xm%2, %5 vinserti128 m%2, m%2, %6, 1 %if %7 lea t2, [t2+2*t3] %endif SBUTTERFLY bw, %1, %2, %3 %endmacro %macro SSD_LOAD_HALF 5 LOAD 1, 2, [t0+%1], [t0+%3], 1 JOIN 1, 2, 3, 4, [t2+%2], [t2+%4], 1 LOAD 3, 4, [t0+%1], [t0+%3], %5 JOIN 3, 4, 5, 6, [t2+%2], [t2+%4], %5 %endmacro %macro SSD_CORE 7-8 %ifidn %8, FULL mova m%6, m%2 mova m%7, m%4 psubusb m%2, m%1 psubusb m%4, m%3 psubusb m%1, m%6 psubusb m%3, m%7 por m%1, m%2 por m%3, m%4 punpcklbw m%2, m%1, m%5 punpckhbw m%1, m%5 punpcklbw m%4, m%3, m%5 punpckhbw m%3, m%5 %endif pmaddwd m%1, m%1 pmaddwd m%2, m%2 pmaddwd m%3, m%3 pmaddwd m%4, m%4 %endmacro %macro SSD_CORE_SSE2 7-8 %ifidn %8, FULL DEINTB %6, %1, %7, %2, %5 psubw m%6, m%7 psubw m%1, m%2 SWAP %6, %2, %1 DEINTB %6, %3, %7, %4, %5 psubw m%6, m%7 psubw m%3, m%4 SWAP %6, %4, %3 %endif pmaddwd m%1, m%1 pmaddwd m%2, m%2 pmaddwd m%3, m%3 pmaddwd m%4, m%4 %endmacro %macro SSD_CORE_SSSE3 7-8 %ifidn %8, FULL punpckhbw m%6, m%1, m%2 punpckhbw m%7, m%3, m%4 punpcklbw m%1, m%2 punpcklbw m%3, m%4 SWAP %6, %2, %3 SWAP %7, %4 %endif pmaddubsw m%1, m%5 pmaddubsw m%2, m%5 pmaddubsw m%3, m%5 pmaddubsw m%4, m%5 pmaddwd m%1, m%1 pmaddwd m%2, m%2 pmaddwd m%3, m%3 pmaddwd m%4, m%4 %endmacro %macro SSD_ITER 6 SSD_LOAD_%1 %2,%3,%4,%5,%6 SSD_CORE 1, 2, 3, 4, 7, 5, 6, %1 paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 %endmacro ;----------------------------------------------------------------------------- ; int pixel_ssd_16x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- %macro SSD 2 %if %1 != %2 %assign function_align 8 %else %assign function_align 16 %endif cglobal pixel_ssd_%1x%2, 0,0,0 mov al, %1*%2/mmsize/2 %if %1 != %2 jmp mangle(private_prefix %+ _ %+ pixel_ssd_%1x%1 %+ SUFFIX %+ .startloop) %else .startloop: %if ARCH_X86_64 DECLARE_REG_TMP 0,1,2,3 PROLOGUE 0,0,8 %else PROLOGUE 0,5 DECLARE_REG_TMP 1,2,3,4 mov t0, r0m mov t1, r1m mov t2, r2m mov t3, r3m %endif %if cpuflag(ssse3) mova m7, [hsub_mul] %elifidn cpuname, sse2 mova m7, [pw_00ff] %elif %1 >= mmsize pxor m7, m7 %endif pxor m0, m0 ALIGN 16 .loop: %if %1 > mmsize SSD_ITER FULL, 0, 0, mmsize, mmsize, 1 %elif %1 == mmsize SSD_ITER FULL, 0, 0, t1, t3, 2 %else SSD_ITER HALF, 0, 0, t1, t3, 2 %endif dec al jg .loop %if mmsize==32 vextracti128 xm1, m0, 1 paddd xm0, xm1 HADDD xm0, xm1 movd eax, xm0 %else HADDD m0, m1 movd eax, m0 %endif %if (mmsize == 8) emms %endif RET %endif %endmacro %macro HEVC_SSD 0 SSD 32, 64 SSD 16, 64 SSD 32, 32 SSD 32, 16 SSD 16, 32 SSD 32, 8 SSD 8, 32 SSD 32, 24 SSD 24, 24 ; not used, but resolves x265_pixel_ssd_24x24_sse2.startloop symbol SSD 8, 4 SSD 8, 8 SSD 16, 16 SSD 16, 12 SSD 16, 8 SSD 8, 16 SSD 16, 4 %endmacro INIT_MMX mmx SSD 16, 16 SSD 16, 8 SSD 8, 8 SSD 8, 16 SSD 4, 4 SSD 8, 4 SSD 4, 8 SSD 4, 16 INIT_XMM sse2slow SSD 16, 16 SSD 8, 8 SSD 16, 8 SSD 8, 16 SSD 8, 4 INIT_XMM sse2 %define SSD_CORE SSD_CORE_SSE2 %define JOIN JOIN_SSE2 HEVC_SSD INIT_XMM ssse3 %define SSD_CORE SSD_CORE_SSSE3 %define JOIN JOIN_SSSE3 HEVC_SSD INIT_XMM avx HEVC_SSD INIT_MMX ssse3 SSD 4, 4 SSD 4, 8 SSD 4, 16 INIT_XMM xop SSD 16, 16 SSD 8, 8 SSD 16, 8 SSD 8, 16 SSD 8, 4 %define LOAD LOAD_AVX2 %define JOIN JOIN_AVX2 INIT_YMM avx2 SSD 16, 16 SSD 16, 8 SSD 32, 32 SSD 64, 64 %assign function_align 16 %endif ; !HIGH_BIT_DEPTH ;----------------------------------------------------------------------------- ; int pixel_ssd_12x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_12x16, 4, 5, 7, src1, stride1, src2, stride2 pxor m6, m6 mov r4d, 4 .loop: movu m0, [r0] movu m1, [r2] movu m2, [r0 + r1] movu m3, [r2 + r3] punpckhdq m4, m0, m2 punpckhdq m5, m1, m3 pmovzxbw m0, m0 pmovzxbw m1, m1 pmovzxbw m2, m2 pmovzxbw m3, m3 pmovzxbw m4, m4 pmovzxbw m5, m5 psubw m0, m1 psubw m2, m3 psubw m4, m5 pmaddwd m0, m0 pmaddwd m2, m2 pmaddwd m4, m4 paddd m0, m2 paddd m6, m4 paddd m6, m0 movu m0, [r0 + 2 * r1] movu m1, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] lea r2, [r2 + 2 * r3] movu m2, [r0 + r1] movu m3, [r2 + r3] punpckhdq m4, m0, m2 punpckhdq m5, m1, m3 pmovzxbw m0, m0 pmovzxbw m1, m1 pmovzxbw m2, m2 pmovzxbw m3, m3 pmovzxbw m4, m4 pmovzxbw m5, m5 psubw m0, m1 psubw m2, m3 psubw m4, m5 pmaddwd m0, m0 pmaddwd m2, m2 pmaddwd m4, m4 paddd m0, m2 paddd m6, m4 paddd m6, m0 dec r4d lea r0, [r0 + 2 * r1] lea r2, [r2 + 2 * r3] jnz .loop HADDD m6, m1 movd eax, m6 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_24x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_24x32, 4, 5, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r4d, 16 .loop: movu m1, [r0] pmovzxbw m0, m1 punpckhbw m1, m6 pmovzxbw m2, [r0 + 16] movu m4, [r2] pmovzxbw m3, m4 punpckhbw m4, m6 pmovzxbw m5, [r2 + 16] psubw m0, m3 psubw m1, m4 psubw m2, m5 pmaddwd m0, m0 pmaddwd m1, m1 pmaddwd m2, m2 paddd m0, m1 paddd m7, m2 paddd m7, m0 movu m1, [r0 + r1] pmovzxbw m0, m1 punpckhbw m1, m6 pmovzxbw m2, [r0 + r1 + 16] movu m4, [r2 + r3] pmovzxbw m3, m4 punpckhbw m4, m6 pmovzxbw m5, [r2 + r3 + 16] psubw m0, m3 psubw m1, m4 psubw m2, m5 pmaddwd m0, m0 pmaddwd m1, m1 pmaddwd m2, m2 paddd m0, m1 paddd m7, m2 paddd m7, m0 dec r4d lea r0, [r0 + 2 * r1] lea r2, [r2 + 2 * r3] jnz .loop HADDD m7, m1 movd eax, m7 RET %macro PIXEL_SSD_16x4 0 movu m1, [r0] pmovzxbw m0, m1 punpckhbw m1, m6 movu m3, [r2] pmovzxbw m2, m3 punpckhbw m3, m6 psubw m0, m2 psubw m1, m3 movu m5, [r0 + r1] pmovzxbw m4, m5 punpckhbw m5, m6 movu m3, [r2 + r3] pmovzxbw m2, m3 punpckhbw m3, m6 psubw m4, m2 psubw m5, m3 pmaddwd m0, m0 pmaddwd m1, m1 pmaddwd m4, m4 pmaddwd m5, m5 paddd m0, m1 paddd m4, m5 paddd m4, m0 paddd m7, m4 movu m1, [r0 + r6] pmovzxbw m0, m1 punpckhbw m1, m6 movu m3, [r2 + 2 * r3] pmovzxbw m2, m3 punpckhbw m3, m6 psubw m0, m2 psubw m1, m3 lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] movu m5, [r0 + r1] pmovzxbw m4, m5 punpckhbw m5, m6 movu m3, [r2 + r3] pmovzxbw m2, m3 punpckhbw m3, m6 psubw m4, m2 psubw m5, m3 pmaddwd m0, m0 pmaddwd m1, m1 pmaddwd m4, m4 pmaddwd m5, m5 paddd m0, m1 paddd m4, m5 paddd m4, m0 paddd m7, m4 %endmacro cglobal pixel_ssd_16x16_internal PIXEL_SSD_16x4 lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] PIXEL_SSD_16x4 lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] PIXEL_SSD_16x4 lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] PIXEL_SSD_16x4 ret ;----------------------------------------------------------------------------- ; int pixel_ssd_48x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_48x64, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r4, r0 mov r5, r2 lea r6, [r1 * 2] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 16] lea r2, [r5 + 16] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 32] lea r2, [r5 + 32] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_64x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_64x16, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r4, r0 mov r5, r2 lea r6, [r1 * 2] call pixel_ssd_16x16_internal lea r0, [r4 + 16] lea r2, [r5 + 16] call pixel_ssd_16x16_internal lea r0, [r4 + 32] lea r2, [r5 + 32] call pixel_ssd_16x16_internal lea r0, [r4 + 48] lea r2, [r5 + 48] call pixel_ssd_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_64x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_64x32, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r4, r0 mov r5, r2 lea r6, [r1 * 2] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 16] lea r2, [r5 + 16] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 32] lea r2, [r5 + 32] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 48] lea r2, [r5 + 48] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_64x48( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_64x48, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r4, r0 mov r5, r2 lea r6, [r1 * 2] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 16] lea r2, [r5 + 16] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 32] lea r2, [r5 + 32] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 48] lea r2, [r5 + 48] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_64x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_64x64, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r4, r0 mov r5, r2 lea r6, [r1 * 2] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 16] lea r2, [r5 + 16] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 32] lea r2, [r5 + 32] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r4 + 48] lea r2, [r5 + 48] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal lea r0, [r0 + r6] lea r2, [r2 + 2 * r3] call pixel_ssd_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp ( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_ssd_sp_4x4_internal movh m0, [r0] movh m1, [r0 + r1] punpcklqdq m0, m1 movd m2, [r2] movd m3, [r2 + r3] punpckldq m2, m3 pmovzxbw m2, m2 psubw m0, m2 movh m4, [r0 + 2 * r1] movh m5, [r0 + r4] punpcklqdq m4, m5 movd m6, [r2 + 2 * r3] lea r2, [r2 + 2 * r3] movd m1, [r2 + r3] punpckldq m6, m1 pmovzxbw m6, m6 psubw m4, m6 pmaddwd m0, m0 pmaddwd m4, m4 paddd m0, m4 paddd m7, m0 ret ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_4x4( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_4x4, 4, 5, 8, src1, stride1, src2, stride2 pxor m7, m7 add r1, r1 lea r4, [r1 * 3] call pixel_ssd_sp_4x4_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_4x8( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_4x8, 4, 5, 8, src1, stride1, src2, stride2 pxor m7, m7 add r1, r1 lea r4, [r1 * 3] call pixel_ssd_sp_4x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_4x4_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_4x16( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_4x16, 4, 5, 8, src1, stride1, src2, stride2 pxor m7, m7 add r1, r1 lea r4, [r1 * 3] call pixel_ssd_sp_4x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_4x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_4x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_4x4_internal HADDD m7, m1 movd eax, m7 RET cglobal pixel_ssd_sp_8x4_internal movu m0, [r0] movu m1, [r0 + r1] movh m2, [r2] movh m3, [r2 + r3] pmovzxbw m2, m2 pmovzxbw m3, m3 psubw m0, m2 psubw m1, m3 movu m4, [r0 + 2 * r1] movu m5, [r0 + r4] movh m2, [r2 + 2 * r3] movh m3, [r2 + r5] pmovzxbw m2, m2 pmovzxbw m3, m3 psubw m4, m2 psubw m5, m3 pmaddwd m0, m0 pmaddwd m1, m1 pmaddwd m4, m4 pmaddwd m5, m5 paddd m0, m1 paddd m4, m5 paddd m4, m0 paddd m7, m4 ret ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_8x4( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_8x4, 4, 6, 8, src1, stride1, src2, stride2 pxor m7, m7 add r1, r1 lea r4, [r1 * 3] lea r5, [r3 * 3] call pixel_ssd_sp_8x4_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_8x8( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_8x8, 4, 6, 8, src1, stride1, src2, stride2 pxor m7, m7 add r1, r1 lea r4, [r1 * 3] lea r5, [r3 * 3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_8x16( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_8x16, 4, 6, 8, src1, stride1, src2, stride2 pxor m7, m7 add r1, r1 lea r4, [r1 * 3] lea r5, [r3 * 3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_8x32( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_8x32, 4, 6, 8, src1, stride1, src2, stride2 pxor m7, m7 add r1, r1 lea r4, [r1 * 3] lea r5, [r3 * 3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_12x16( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_12x16, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 add r1, r1 lea r4, [r1 * 3] mov r5, r0 mov r6, r2 call pixel_ssd_sp_4x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_4x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_4x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_4x4_internal lea r0, [r5 + 8] lea r2, [r6 + 4] lea r5, [r3 * 3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal HADDD m7, m1 movd eax, m7 RET %macro PIXEL_SSD_SP_16x4 0 movu m0, [r0] movu m1, [r0 + 16] movu m3, [r2] pmovzxbw m2, m3 punpckhbw m3, m6 psubw m0, m2 psubw m1, m3 movu m4, [r0 + r1] movu m5, [r0 + r1 +16] movu m3, [r2 + r3] pmovzxbw m2, m3 punpckhbw m3, m6 psubw m4, m2 psubw m5, m3 pmaddwd m0, m0 pmaddwd m1, m1 pmaddwd m4, m4 pmaddwd m5, m5 paddd m0, m1 paddd m4, m5 paddd m4, m0 paddd m7, m4 movu m0, [r0 + 2 * r1] movu m1, [r0 + 2 * r1 + 16] movu m3, [r2 + 2 * r3] pmovzxbw m2, m3 punpckhbw m3, m6 psubw m0, m2 psubw m1, m3 lea r0, [r0 + 2 * r1] lea r2, [r2 + 2 * r3] movu m4, [r0 + r1] movu m5, [r0 + r1 + 16] movu m3, [r2 + r3] pmovzxbw m2, m3 punpckhbw m3, m6 psubw m4, m2 psubw m5, m3 pmaddwd m0, m0 pmaddwd m1, m1 pmaddwd m4, m4 pmaddwd m5, m5 paddd m0, m1 paddd m4, m5 paddd m4, m0 paddd m7, m4 %endmacro ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_16x4( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_16x4, 4, 6, 8, src1, stride1, src2, stride2 pxor m6, m6 pxor m7, m7 add r1, r1 PIXEL_SSD_SP_16x4 HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_16x8( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_16x8, 4, 4, 8, src1, stride1, src2, stride2 pxor m6, m6 pxor m7, m7 add r1, r1 PIXEL_SSD_SP_16x4 lea r0, [r0 + 2 * r1] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_16x12( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_16x12, 4, 6, 8, src1, stride1, src2, stride2 pxor m6, m6 pxor m7, m7 add r1, r1 lea r4, [r1 * 2] lea r5, [r3 * 2] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + r5] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + r5] PIXEL_SSD_SP_16x4 HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_16x16( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_16x16, 4, 6, 8, src1, stride1, src2, stride2 pxor m6, m6 pxor m7, m7 add r1, r1 lea r4, [r1 * 2] lea r5, [r3 * 2] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + r5] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + r5] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + r5] PIXEL_SSD_SP_16x4 HADDD m7, m1 movd eax, m7 RET cglobal pixel_ssd_sp_16x16_internal PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 ret ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_16x32( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_16x32, 4, 5, 8, src1, stride1, src2, stride2 pxor m6, m6 pxor m7, m7 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_16x64( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_16x64, 4, 6, 8, src1, stride1, src2, stride2 pxor m6, m6 pxor m7, m7 add r1, r1 lea r4, [r1 * 2] lea r5, [r3 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + r5] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + r5] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + r5] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_sp_24x32( int16_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_24x32, 4, 7, 8, src1, stride1, src2, stride2 pxor m6, m6 pxor m7, m7 add r1, r1 lea r4, [r1 * 2] mov r5, r0 mov r6, r2 call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] lea r4, [r1 * 3] lea r5, [r3 * 3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal lea r0, [r0 + 4 * r1] lea r2, [r2 + 4 * r3] call pixel_ssd_sp_8x4_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_32x8( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_32x8, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 lea r0, [r5 + 32] lea r2, [r6 + 16] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_32x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_32x16, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_32x24( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_32x24, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] PIXEL_SSD_SP_16x4 HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_32x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_32x32, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_32x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_32x64, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_48x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_48x64, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 64] lea r2, [r6 + 32] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_64x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_64x16, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 64] lea r2, [r6 + 32] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 96] lea r2, [r6 + 48] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_64x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_64x32, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 64] lea r2, [r6 + 32] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 96] lea r2, [r6 + 48] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_64x48( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_64x48, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 64] lea r2, [r6 + 32] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 96] lea r2, [r6 + 48] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_64x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- INIT_XMM sse4 cglobal pixel_ssd_sp_64x64, 4, 7, 8, src1, stride1, src2, stride2 pxor m7, m7 pxor m6, m6 mov r5, r0 mov r6, r2 add r1, r1 lea r4, [r1 * 2] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 32] lea r2, [r6 + 16] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 64] lea r2, [r6 + 32] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r5 + 96] lea r2, [r6 + 48] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal lea r0, [r0 + r4] lea r2, [r2 + 2 * r3] call pixel_ssd_sp_16x16_internal HADDD m7, m1 movd eax, m7 RET ;----------------------------------------------------------------------------- ; int pixel_ssd_s( int16_t *ref, intptr_t i_stride ) ;----------------------------------------------------------------------------- INIT_XMM sse2 cglobal pixel_ssd_s_4, 2,2,2 add r1, r1 movh m0, [r0] movhps m0, [r0 + r1] lea r0, [r0 + r1 * 2] movh m1, [r0] movhps m1, [r0 + r1] pmaddwd m0, m0 pmaddwd m1, m1 paddd m0, m1 ; calculate sum and return HADDD m0, m1 movd eax, m0 RET INIT_XMM sse2 cglobal pixel_ssd_s_8, 2,3,5 add r1, r1 lea r2, [r1 * 3] movu m0, [r0] movu m1, [r0 + r1] movu m2, [r0 + r1 * 2] movu m3, [r0 + r2] pmaddwd m0, m0 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 paddd m0, m1 paddd m2, m3 paddd m0, m2 lea r0, [r0 + r1 * 4] movu m4, [r0] movu m1, [r0 + r1] movu m2, [r0 + r1 * 2] movu m3, [r0 + r2] pmaddwd m4, m4 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 paddd m4, m1 paddd m2, m3 paddd m4, m2 paddd m0, m4 ; calculate sum and return HADDD m0, m1 movd eax, m0 RET INIT_XMM sse2 cglobal pixel_ssd_s_16, 2,3,5 add r1, r1 mov r2d, 4 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r0 + mmsize] movu m3, [r0 + r1] movu m4, [r0 + r1 + mmsize] lea r0, [r0 + r1 * 2] pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m1, m3 paddd m0, m1 movu m1, [r0] movu m2, [r0 + mmsize] movu m3, [r0 + r1] movu m4, [r0 + r1 + mmsize] lea r0, [r0 + r1 * 2] pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m1, m3 paddd m0, m1 dec r2d jnz .loop ; calculate sum and return HADDD m0, m1 movd eax, m0 RET INIT_XMM sse2 cglobal pixel_ssd_s_32, 2,3,5 add r1, r1 mov r2d, 16 pxor m0, m0 .loop: movu m1, [r0 + 0 * mmsize] movu m2, [r0 + 1 * mmsize] movu m3, [r0 + 2 * mmsize] movu m4, [r0 + 3 * mmsize] add r0, r1 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m1, m3 paddd m0, m1 movu m1, [r0 + 0 * mmsize] movu m2, [r0 + 1 * mmsize] movu m3, [r0 + 2 * mmsize] movu m4, [r0 + 3 * mmsize] add r0, r1 pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m1, m3 paddd m0, m1 dec r2d jnz .loop ; calculate sum and return HADDD m0, m1 movd eax, m0 RET INIT_YMM avx2 cglobal pixel_ssd_s_16, 2,4,5 add r1, r1 lea r3, [r1 * 3] mov r2d, 16/4 pxor m0, m0 .loop: movu m1, [r0] movu m2, [r0 + r1] movu m3, [r0 + 2 * r1] movu m4, [r0 + r3] lea r0, [r0 + r1 * 4] pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m1, m3 paddd m0, m1 dec r2d jnz .loop ; calculate sum and return HADDD m0, m1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_ssd_s_32, 2,4,5 add r1, r1 lea r3, [r1 * 3] mov r2d, 8 pxor m0, m0 .loop: movu m1, [r0 + 0 * mmsize] movu m2, [r0 + 1 * mmsize] movu m3, [r0 + r1 + 0 * mmsize] movu m4, [r0 + r1 + 1 * mmsize] pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m1, m3 paddd m0, m1 movu m1, [r0 + r1 * 2 + 0 * mmsize] movu m2, [r0 + r1 * 2 + 1 * mmsize] movu m3, [r0 + r3 + 0 * mmsize] movu m4, [r0 + r3 + 1 * mmsize] lea r0, [r0 + 4 * r1] pmaddwd m1, m1 pmaddwd m2, m2 pmaddwd m3, m3 pmaddwd m4, m4 paddd m1, m2 paddd m3, m4 paddd m1, m3 paddd m0, m1 dec r2d jnz .loop ; calculate sum and return HADDD m0, m1 movd eax, xm0 RET