Mercurial > hg > forks > libbpg
view x265/source/common/x86/sad-a.asm @ 0:772086c29cc7
Initial import.
| author | Matti Hamalainen <ccr@tnsp.org> |
|---|---|
| date | Wed, 16 Nov 2016 11:16:33 +0200 |
| parents | |
| children |
line wrap: on
line source
;***************************************************************************** ;* sad-a.asm: x86 sad functions ;***************************************************************************** ;* Copyright (C) 2003-2013 x264 project ;* ;* Authors: Loren Merritt <lorenm@u.washington.edu> ;* Fiona Glaser <fiona@x264.com> ;* Laurent Aimar <fenrir@via.ecp.fr> ;* Alex Izvorski <aizvorksi@gmail.com> ;* Min Chen <chenm003@163.com> ;* ;* This program is free software; you can redistribute it and/or modify ;* it under the terms of the GNU General Public License as published by ;* the Free Software Foundation; either version 2 of the License, or ;* (at your option) any later version. ;* ;* This program is distributed in the hope that it will be useful, ;* but WITHOUT ANY WARRANTY; without even the implied warranty of ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the ;* GNU General Public License for more details. ;* ;* You should have received a copy of the GNU General Public License ;* along with this program; if not, write to the Free Software ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111, USA. ;* ;* This program is also available under a commercial proprietary license. ;* For more information, contact us at license @ x265.com. ;***************************************************************************** %include "x86inc.asm" %include "x86util.asm" SECTION_RODATA 32 MSK: db 255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0 SECTION .text cextern pb_3 cextern pb_shuf8x8c cextern pw_8 cextern pd_64 ;============================================================================= ; SAD MMX ;============================================================================= %macro SAD_INC_2x16P 0 movq mm1, [r0] movq mm2, [r0+8] movq mm3, [r0+r1] movq mm4, [r0+r1+8] psadbw mm1, [r2] psadbw mm2, [r2+8] psadbw mm3, [r2+r3] psadbw mm4, [r2+r3+8] lea r0, [r0+2*r1] paddw mm1, mm2 paddw mm3, mm4 lea r2, [r2+2*r3] paddw mm0, mm1 paddw mm0, mm3 %endmacro %macro SAD_INC_2x8P 0 movq mm1, [r0] movq mm2, [r0+r1] psadbw mm1, [r2] psadbw mm2, [r2+r3] lea r0, [r0+2*r1] paddw mm0, mm1 paddw mm0, mm2 lea r2, [r2+2*r3] %endmacro %macro SAD_INC_2x4P 0 movd mm1, [r0] movd mm2, [r2] punpckldq mm1, [r0+r1] punpckldq mm2, [r2+r3] psadbw mm1, mm2 paddw mm0, mm1 lea r0, [r0+2*r1] lea r2, [r2+2*r3] %endmacro ;----------------------------------------------------------------------------- ; int pixel_sad_16x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- %macro SAD 2 cglobal pixel_sad_%1x%2_mmx2, 4,4 pxor mm0, mm0 %rep %2/2 SAD_INC_2x%1P %endrep movd eax, mm0 RET %endmacro SAD 16, 16 SAD 16, 8 SAD 8, 16 SAD 8, 8 SAD 8, 4 SAD 4, 16 SAD 4, 8 SAD 4, 4 ;============================================================================= ; SAD XMM ;============================================================================= %macro SAD_END_SSE2 0 movhlps m1, m0 paddw m0, m1 movd eax, m0 RET %endmacro %macro PROCESS_SAD_12x4 0 movu m1, [r2] movu m2, [r0] pand m1, m4 pand m2, m4 psadbw m1, m2 paddd m0, m1 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r0] pand m1, m4 pand m2, m4 psadbw m1, m2 paddd m0, m1 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r0] pand m1, m4 pand m2, m4 psadbw m1, m2 paddd m0, m1 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r0] pand m1, m4 pand m2, m4 psadbw m1, m2 paddd m0, m1 %endmacro %macro PROCESS_SAD_16x4 0 movu m1, [r2] movu m2, [r2 + r3] psadbw m1, [r0] psadbw m2, [r0 + r1] paddd m1, m2 paddd m0, m1 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] movu m1, [r2] movu m2, [r2 + r3] psadbw m1, [r0] psadbw m2, [r0 + r1] paddd m1, m2 paddd m0, m1 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] %endmacro %macro PROCESS_SAD_24x4 0 movu m1, [r2] movq m2, [r2 + 16] lea r2, [r2 + r3] movu m3, [r2] movq m4, [r2 + 16] psadbw m1, [r0] psadbw m3, [r0 + r1] paddd m0, m1 paddd m0, m3 movq m1, [r0 + 16] lea r0, [r0 + r1] movq m3, [r0 + 16] punpcklqdq m2, m4 punpcklqdq m1, m3 psadbw m2, m1 paddd m0, m2 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movq m2, [r2 + 16] lea r2, [r2 + r3] movu m3, [r2] movq m4, [r2 + 16] psadbw m1, [r0] psadbw m3, [r0 + r1] paddd m0, m1 paddd m0, m3 movq m1, [r0 + 16] lea r0, [r0 + r1] movq m3, [r0 + 16] punpcklqdq m2, m4 punpcklqdq m1, m3 psadbw m2, m1 paddd m0, m2 %endmacro %macro PROCESS_SAD_32x4 0 movu m1, [r2] movu m2, [r2 + 16] psadbw m1, [r0] psadbw m2, [r0 + 16] paddd m1, m2 paddd m0, m1 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] psadbw m1, [r0] psadbw m2, [r0 + 16] paddd m1, m2 paddd m0, m1 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] psadbw m1, [r0] psadbw m2, [r0 + 16] paddd m1, m2 paddd m0, m1 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] psadbw m1, [r0] psadbw m2, [r0 + 16] paddd m1, m2 paddd m0, m1 lea r2, [r2 + r3] lea r0, [r0 + r1] %endmacro %macro PROCESS_SAD_48x4 0 movu m1, [r2] movu m2, [r2 + 16] movu m3, [r2 + 32] psadbw m1, [r0] psadbw m2, [r0 + 16] psadbw m3, [r0 + 32] paddd m1, m2 paddd m0, m1 paddd m0, m3 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] movu m3, [r2 + 32] psadbw m1, [r0] psadbw m2, [r0 + 16] psadbw m3, [r0 + 32] paddd m1, m2 paddd m0, m1 paddd m0, m3 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] movu m3, [r2 + 32] psadbw m1, [r0] psadbw m2, [r0 + 16] psadbw m3, [r0 + 32] paddd m1, m2 paddd m0, m1 paddd m0, m3 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] movu m3, [r2 + 32] psadbw m1, [r0] psadbw m2, [r0 + 16] psadbw m3, [r0 + 32] paddd m1, m2 paddd m0, m1 paddd m0, m3 %endmacro %macro PROCESS_SAD_8x4 0 movq m1, [r2] movq m2, [r2 + r3] lea r2, [r2 + 2 * r3] movq m3, [r0] movq m4, [r0 + r1] lea r0, [r0 + 2 * r1] punpcklqdq m1, m2 punpcklqdq m3, m4 psadbw m1, m3 paddd m0, m1 movq m1, [r2] movq m2, [r2 + r3] lea r2, [r2 + 2 * r3] movq m3, [r0] movq m4, [r0 + r1] lea r0, [r0 + 2 * r1] punpcklqdq m1, m2 punpcklqdq m3, m4 psadbw m1, m3 paddd m0, m1 %endmacro %macro PROCESS_SAD_64x4 0 movu m1, [r2] movu m2, [r2 + 16] movu m3, [r2 + 32] movu m4, [r2 + 48] psadbw m1, [r0] psadbw m2, [r0 + 16] psadbw m3, [r0 + 32] psadbw m4, [r0 + 48] paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] movu m3, [r2 + 32] movu m4, [r2 + 48] psadbw m1, [r0] psadbw m2, [r0 + 16] psadbw m3, [r0 + 32] psadbw m4, [r0 + 48] paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] movu m3, [r2 + 32] movu m4, [r2 + 48] psadbw m1, [r0] psadbw m2, [r0 + 16] psadbw m3, [r0 + 32] psadbw m4, [r0 + 48] paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 lea r2, [r2 + r3] lea r0, [r0 + r1] movu m1, [r2] movu m2, [r2 + 16] movu m3, [r2 + 32] movu m4, [r2 + 48] psadbw m1, [r0] psadbw m2, [r0 + 16] psadbw m3, [r0 + 32] psadbw m4, [r0 + 48] paddd m1, m2 paddd m3, m4 paddd m0, m1 paddd m0, m3 lea r2, [r2 + r3] lea r0, [r0 + r1] %endmacro %macro SAD_W16 0 ;----------------------------------------------------------------------------- ; int pixel_sad_16x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_16x16, 4,4,8 movu m0, [r2] movu m1, [r2+r3] lea r2, [r2+2*r3] movu m2, [r2] movu m3, [r2+r3] lea r2, [r2+2*r3] psadbw m0, [r0] psadbw m1, [r0+r1] lea r0, [r0+2*r1] movu m4, [r2] paddw m0, m1 psadbw m2, [r0] psadbw m3, [r0+r1] lea r0, [r0+2*r1] movu m5, [r2+r3] lea r2, [r2+2*r3] paddw m2, m3 movu m6, [r2] movu m7, [r2+r3] lea r2, [r2+2*r3] paddw m0, m2 psadbw m4, [r0] psadbw m5, [r0+r1] lea r0, [r0+2*r1] movu m1, [r2] paddw m4, m5 psadbw m6, [r0] psadbw m7, [r0+r1] lea r0, [r0+2*r1] movu m2, [r2+r3] lea r2, [r2+2*r3] paddw m6, m7 movu m3, [r2] paddw m0, m4 movu m4, [r2+r3] lea r2, [r2+2*r3] paddw m0, m6 psadbw m1, [r0] psadbw m2, [r0+r1] lea r0, [r0+2*r1] movu m5, [r2] paddw m1, m2 psadbw m3, [r0] psadbw m4, [r0+r1] lea r0, [r0+2*r1] movu m6, [r2+r3] lea r2, [r2+2*r3] paddw m3, m4 movu m7, [r2] paddw m0, m1 movu m1, [r2+r3] paddw m0, m3 psadbw m5, [r0] psadbw m6, [r0+r1] lea r0, [r0+2*r1] paddw m5, m6 psadbw m7, [r0] psadbw m1, [r0+r1] paddw m7, m1 paddw m0, m5 paddw m0, m7 SAD_END_SSE2 ;----------------------------------------------------------------------------- ; int pixel_sad_16x8( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_16x8, 4,4 movu m0, [r2] movu m2, [r2+r3] lea r2, [r2+2*r3] movu m3, [r2] movu m4, [r2+r3] psadbw m0, [r0] psadbw m2, [r0+r1] lea r0, [r0+2*r1] psadbw m3, [r0] psadbw m4, [r0+r1] lea r0, [r0+2*r1] lea r2, [r2+2*r3] paddw m0, m2 paddw m3, m4 paddw m0, m3 movu m1, [r2] movu m2, [r2+r3] lea r2, [r2+2*r3] movu m3, [r2] movu m4, [r2+r3] psadbw m1, [r0] psadbw m2, [r0+r1] lea r0, [r0+2*r1] psadbw m3, [r0] psadbw m4, [r0+r1] lea r0, [r0+2*r1] lea r2, [r2+2*r3] paddw m1, m2 paddw m3, m4 paddw m0, m1 paddw m0, m3 SAD_END_SSE2 ;----------------------------------------------------------------------------- ; int pixel_sad_16x12( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_16x12, 4,4,3 pxor m0, m0 PROCESS_SAD_16x4 PROCESS_SAD_16x4 PROCESS_SAD_16x4 movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_16x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_16x32, 4,5,3 pxor m0, m0 mov r4d, 4 .loop: PROCESS_SAD_16x4 PROCESS_SAD_16x4 dec r4d jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_16x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_16x64, 4,5,3 pxor m0, m0 mov r4d, 8 .loop: PROCESS_SAD_16x4 PROCESS_SAD_16x4 dec r4d jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_16x4( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_16x4, 4,4,3 movu m0, [r2] movu m1, [r2 + r3] psadbw m0, [r0] psadbw m1, [r0 + r1] paddd m0, m1 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] movu m1, [r2] movu m2, [r2 + r3] psadbw m1, [r0] psadbw m2, [r0 + r1] paddd m1, m2 paddd m0, m1 movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_32x8( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_32x8, 4,4,3 pxor m0, m0 PROCESS_SAD_32x4 PROCESS_SAD_32x4 movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_32x24( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_32x24, 4,5,3 pxor m0, m0 mov r4d, 3 .loop: PROCESS_SAD_32x4 PROCESS_SAD_32x4 dec r4d jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_32x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_32x32, 4,5,3 pxor m0, m0 mov r4d, 4 .loop: PROCESS_SAD_32x4 PROCESS_SAD_32x4 dec r4d jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_32x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_32x16, 4,4,3 pxor m0, m0 PROCESS_SAD_32x4 PROCESS_SAD_32x4 PROCESS_SAD_32x4 PROCESS_SAD_32x4 movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_32x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_32x64, 4,5,3 pxor m0, m0 mov r4d, 8 .loop: PROCESS_SAD_32x4 PROCESS_SAD_32x4 dec r4d jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_8x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_8x32, 4,5,3 pxor m0, m0 mov r4d, 4 .loop: PROCESS_SAD_8x4 PROCESS_SAD_8x4 dec r4d jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_64x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_64x16, 4,4,5 pxor m0, m0 PROCESS_SAD_64x4 PROCESS_SAD_64x4 PROCESS_SAD_64x4 PROCESS_SAD_64x4 movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_64x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_64x32, 4,5,5 pxor m0, m0 mov r4, 4 .loop: PROCESS_SAD_64x4 PROCESS_SAD_64x4 dec r4 jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_64x48( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_64x48, 4,5,5 pxor m0, m0 mov r4, 6 .loop: PROCESS_SAD_64x4 PROCESS_SAD_64x4 dec r4d jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_64x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_64x64, 4,5,5 pxor m0, m0 mov r4, 8 .loop: PROCESS_SAD_64x4 PROCESS_SAD_64x4 dec r4 jnz .loop movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_48x64( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_48x64, 4,5,5 pxor m0, m0 mov r4, 64 .loop: PROCESS_SAD_48x4 lea r2, [r2 + r3] lea r0, [r0 + r1] PROCESS_SAD_48x4 lea r2, [r2 + r3] lea r0, [r0 + r1] sub r4, 8 cmp r4, 8 jnz .loop PROCESS_SAD_48x4 lea r2, [r2 + r3] lea r0, [r0 + r1] PROCESS_SAD_48x4 movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_24x32( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_24x32, 4,5,4 pxor m0, m0 mov r4, 32 .loop: PROCESS_SAD_24x4 lea r2, [r2 + r3] lea r0, [r0 + r1] PROCESS_SAD_24x4 lea r2, [r2 + r3] lea r0, [r0 + r1] sub r4, 8 cmp r4, 8 jnz .loop PROCESS_SAD_24x4 lea r2, [r2 + r3] lea r0, [r0 + r1] PROCESS_SAD_24x4 movhlps m1, m0 paddd m0, m1 movd eax, m0 RET ;----------------------------------------------------------------------------- ; int pixel_sad_12x16( uint8_t *, intptr_t, uint8_t *, intptr_t ) ;----------------------------------------------------------------------------- cglobal pixel_sad_12x16, 4,4,4 mova m4, [MSK] pxor m0, m0 PROCESS_SAD_12x4 lea r2, [r2 + r3] lea r0, [r0 + r1] PROCESS_SAD_12x4 lea r2, [r2 + r3] lea r0, [r0 + r1] PROCESS_SAD_12x4 lea r2, [r2 + r3] lea r0, [r0 + r1] PROCESS_SAD_12x4 movhlps m1, m0 paddd m0, m1 movd eax, m0 RET %endmacro INIT_XMM sse2 SAD_W16 INIT_XMM sse3 SAD_W16 INIT_XMM sse2, aligned SAD_W16 %macro SAD_INC_4x8P_SSE 1 movq m1, [r0] movq m2, [r0+r1] lea r0, [r0+2*r1] movq m3, [r2] movq m4, [r2+r3] lea r2, [r2+2*r3] movhps m1, [r0] movhps m2, [r0+r1] movhps m3, [r2] movhps m4, [r2+r3] lea r0, [r0+2*r1] psadbw m1, m3 psadbw m2, m4 lea r2, [r2+2*r3] ACCUM paddw, 0, 1, %1 paddw m0, m2 %endmacro INIT_XMM ;Even on Nehalem, no sizes other than 8x16 benefit from this method. cglobal pixel_sad_8x16_sse2, 4,4 SAD_INC_4x8P_SSE 0 SAD_INC_4x8P_SSE 1 SAD_INC_4x8P_SSE 1 SAD_INC_4x8P_SSE 1 SAD_END_SSE2 RET ;============================================================================= ; SAD x3/x4 MMX ;============================================================================= %macro SAD_X3_START_1x8P 0 movq mm3, [r0] movq mm0, [r1] movq mm1, [r2] movq mm2, [r3] psadbw mm0, mm3 psadbw mm1, mm3 psadbw mm2, mm3 %endmacro %macro SAD_X3_1x8P 2 movq mm3, [r0+%1] movq mm4, [r1+%2] movq mm5, [r2+%2] movq mm6, [r3+%2] psadbw mm4, mm3 psadbw mm5, mm3 psadbw mm6, mm3 paddw mm0, mm4 paddw mm1, mm5 paddw mm2, mm6 %endmacro %macro SAD_X3_START_2x4P 3 movd mm3, [r0] movd %1, [r1] movd %2, [r2] movd %3, [r3] punpckldq mm3, [r0+FENC_STRIDE] punpckldq %1, [r1+r4] punpckldq %2, [r2+r4] punpckldq %3, [r3+r4] psadbw %1, mm3 psadbw %2, mm3 psadbw %3, mm3 %endmacro %macro SAD_X3_2x16P 1 %if %1 SAD_X3_START_1x8P %else SAD_X3_1x8P 0, 0 %endif SAD_X3_1x8P 8, 8 SAD_X3_1x8P FENC_STRIDE, r4 SAD_X3_1x8P FENC_STRIDE+8, r4+8 add r0, 2*FENC_STRIDE lea r1, [r1+2*r4] lea r2, [r2+2*r4] lea r3, [r3+2*r4] %endmacro %macro SAD_X3_2x8P 1 %if %1 SAD_X3_START_1x8P %else SAD_X3_1x8P 0, 0 %endif SAD_X3_1x8P FENC_STRIDE, r4 add r0, 2*FENC_STRIDE lea r1, [r1+2*r4] lea r2, [r2+2*r4] lea r3, [r3+2*r4] %endmacro %macro SAD_X3_2x4P 1 %if %1 SAD_X3_START_2x4P mm0, mm1, mm2 %else SAD_X3_START_2x4P mm4, mm5, mm6 paddw mm0, mm4 paddw mm1, mm5 paddw mm2, mm6 %endif add r0, 2*FENC_STRIDE lea r1, [r1+2*r4] lea r2, [r2+2*r4] lea r3, [r3+2*r4] %endmacro %macro SAD_X4_START_1x8P 0 movq mm7, [r0] movq mm0, [r1] movq mm1, [r2] movq mm2, [r3] movq mm3, [r4] psadbw mm0, mm7 psadbw mm1, mm7 psadbw mm2, mm7 psadbw mm3, mm7 %endmacro %macro SAD_X4_1x8P 2 movq mm7, [r0+%1] movq mm4, [r1+%2] movq mm5, [r2+%2] movq mm6, [r3+%2] psadbw mm4, mm7 psadbw mm5, mm7 psadbw mm6, mm7 psadbw mm7, [r4+%2] paddw mm0, mm4 paddw mm1, mm5 paddw mm2, mm6 paddw mm3, mm7 %endmacro %macro SAD_X4_START_2x4P 0 movd mm7, [r0] movd mm0, [r1] movd mm1, [r2] movd mm2, [r3] movd mm3, [r4] punpckldq mm7, [r0+FENC_STRIDE] punpckldq mm0, [r1+r5] punpckldq mm1, [r2+r5] punpckldq mm2, [r3+r5] punpckldq mm3, [r4+r5] psadbw mm0, mm7 psadbw mm1, mm7 psadbw mm2, mm7 psadbw mm3, mm7 %endmacro %macro SAD_X4_INC_2x4P 0 movd mm7, [r0] movd mm4, [r1] movd mm5, [r2] punpckldq mm7, [r0+FENC_STRIDE] punpckldq mm4, [r1+r5] punpckldq mm5, [r2+r5] psadbw mm4, mm7 psadbw mm5, mm7 paddw mm0, mm4 paddw mm1, mm5 movd mm4, [r3] movd mm5, [r4] punpckldq mm4, [r3+r5] punpckldq mm5, [r4+r5] psadbw mm4, mm7 psadbw mm5, mm7 paddw mm2, mm4 paddw mm3, mm5 %endmacro %macro SAD_X4_2x16P 1 %if %1 SAD_X4_START_1x8P %else SAD_X4_1x8P 0, 0 %endif SAD_X4_1x8P 8, 8 SAD_X4_1x8P FENC_STRIDE, r5 SAD_X4_1x8P FENC_STRIDE+8, r5+8 add r0, 2*FENC_STRIDE lea r1, [r1+2*r5] lea r2, [r2+2*r5] lea r3, [r3+2*r5] lea r4, [r4+2*r5] %endmacro %macro SAD_X4_2x8P 1 %if %1 SAD_X4_START_1x8P %else SAD_X4_1x8P 0, 0 %endif SAD_X4_1x8P FENC_STRIDE, r5 add r0, 2*FENC_STRIDE lea r1, [r1+2*r5] lea r2, [r2+2*r5] lea r3, [r3+2*r5] lea r4, [r4+2*r5] %endmacro %macro SAD_X4_2x4P 1 %if %1 SAD_X4_START_2x4P %else SAD_X4_INC_2x4P %endif add r0, 2*FENC_STRIDE lea r1, [r1+2*r5] lea r2, [r2+2*r5] lea r3, [r3+2*r5] lea r4, [r4+2*r5] %endmacro %macro SAD_X3_END 0 %if UNIX64 movd [r5+0], mm0 movd [r5+4], mm1 movd [r5+8], mm2 %else mov r0, r5mp movd [r0+0], mm0 movd [r0+4], mm1 movd [r0+8], mm2 %endif RET %endmacro %macro SAD_X4_END 0 mov r0, r6mp movd [r0+0], mm0 movd [r0+4], mm1 movd [r0+8], mm2 movd [r0+12], mm3 RET %endmacro %macro SAD_X3_12x4 0 mova m3, [r0] movu m5, [r1] pand m3, m4 pand m5, m4 psadbw m5, m3 paddd m0, m5 movu m5, [r2] pand m5, m4 psadbw m5, m3 paddd m1, m5 movu m5, [r3] pand m5, m4 psadbw m5, m3 paddd m2, m5 mova m3, [r0 + FENC_STRIDE] movu m5, [r1 + r4] pand m3, m4 pand m5, m4 psadbw m5, m3 paddd m0, m5 movu m5, [r2 + r4] pand m5, m4 psadbw m5, m3 paddd m1, m5 movu m5, [r3 + r4] pand m5, m4 psadbw m5, m3 paddd m2, m5 mova m3, [r0 + FENC_STRIDE * 2] movu m5, [r1 + r4 * 2] pand m3, m4 pand m5, m4 psadbw m5, m3 paddd m0, m5 movu m5, [r2 + r4 * 2] pand m5, m4 psadbw m5, m3 paddd m1, m5 movu m5, [r3 + r4 * 2] pand m5, m4 psadbw m5, m3 paddd m2, m5 lea r1, [r1 + r4 * 2] lea r2, [r2 + r4 * 2] lea r3, [r3 + r4 * 2] mova m3, [r0 + FENC_STRIDE + FENC_STRIDE * 2] movu m5, [r1 + r4] pand m3, m4 pand m5, m4 psadbw m5, m3 paddd m0, m5 movu m5, [r2 + r4] pand m5, m4 psadbw m5, m3 paddd m1, m5 movu m5, [r3 + r4] pand m5, m4 psadbw m5, m3 paddd m2, m5 lea r0, [r0 + FENC_STRIDE * 4] lea r1, [r1 + r4 * 2] lea r2, [r2 + r4 * 2] lea r3, [r3 + r4 * 2] %endmacro %macro SAD_X4_12x4 0 mova m4, [r0] movu m5, [r1] pand m4, m6 pand m5, m6 psadbw m5, m4 paddd m0, m5 movu m5, [r2] pand m5, m6 psadbw m5, m4 paddd m1, m5 movu m5, [r3] pand m5, m6 psadbw m5, m4 paddd m2, m5 movu m5, [r4] pand m5, m6 psadbw m5, m4 paddd m3, m5 mova m4, [r0 + FENC_STRIDE] movu m5, [r1 + r5] pand m4, m6 pand m5, m6 psadbw m5, m4 paddd m0, m5 movu m5, [r2 + r5] pand m5, m6 psadbw m5, m4 paddd m1, m5 movu m5, [r3 + r5] pand m5, m6 psadbw m5, m4 paddd m2, m5 movu m5, [r4 + r5] pand m5, m6 psadbw m5, m4 paddd m3, m5 mova m4, [r0 + FENC_STRIDE * 2] movu m5, [r1 + r5 * 2] pand m4, m6 pand m5, m6 psadbw m5, m4 paddd m0, m5 movu m5, [r2 + r5 * 2] pand m5, m6 psadbw m5, m4 paddd m1, m5 movu m5, [r3 + r5 * 2] pand m5, m6 psadbw m5, m4 paddd m2, m5 movu m5, [r4 + r5 * 2] pand m5, m6 psadbw m5, m4 paddd m3, m5 lea r1, [r1 + r5 * 2] lea r2, [r2 + r5 * 2] lea r3, [r3 + r5 * 2] lea r4, [r4 + r5 * 2] mova m4, [r0 + FENC_STRIDE + FENC_STRIDE * 2] movu m5, [r1 + r5] pand m4, m6 pand m5, m6 psadbw m5, m4 paddd m0, m5 movu m5, [r2 + r5] pand m5, m6 psadbw m5, m4 paddd m1, m5 movu m5, [r3 + r5] pand m5, m6 psadbw m5, m4 paddd m2, m5 movu m5, [r4 + r5] pand m5, m6 psadbw m5, m4 paddd m3, m5 lea r0, [r0 + FENC_STRIDE * 4] lea r1, [r1 + r5 * 2] lea r2, [r2 + r5 * 2] lea r3, [r3 + r5 * 2] lea r4, [r4 + r5 * 2] %endmacro %macro SAD_X3_24x4 0 mova m3, [r0] mova m4, [r0 + 16] movu m5, [r1] movu m6, [r1 + 16] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m0, m5 movu m5, [r2] movu m6, [r2 + 16] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m1, m5 movu m5, [r3] movu m6, [r3 + 16] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m2, m5 mova m3, [r0 + FENC_STRIDE] mova m4, [r0 + 16 + FENC_STRIDE] movu m5, [r1 + r4] movu m6, [r1 + 16 + r4] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m0, m5 movu m5, [r2 + r4] movu m6, [r2 + 16 + r4] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m1, m5 movu m5, [r3 + r4] movu m6, [r3 + 16 + r4] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m2, m5 mova m3, [r0 + FENC_STRIDE * 2] mova m4, [r0 + 16 + FENC_STRIDE * 2] movu m5, [r1 + r4 * 2] movu m6, [r1 + 16 + r4 * 2] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m0, m5 movu m5, [r2 + r4 * 2] movu m6, [r2 + 16 + r4 * 2] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m1, m5 movu m5, [r3 + r4 * 2] movu m6, [r3 + 16 + r4 * 2] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m2, m5 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r4 * 2] lea r2, [r2 + r4 * 2] lea r3, [r3 + r4 * 2] mova m3, [r0 + FENC_STRIDE] mova m4, [r0 + 16 + FENC_STRIDE] movu m5, [r1 + r4] movu m6, [r1 + 16 + r4] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m0, m5 movu m5, [r2 + r4] movu m6, [r2 + 16 + r4] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m1, m5 movu m5, [r3 + r4] movu m6, [r3 + 16 + r4] psadbw m5, m3 psadbw m6, m4 pshufd m6, m6, 84 paddd m5, m6 paddd m2, m5 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r4 * 2] lea r2, [r2 + r4 * 2] lea r3, [r3 + r4 * 2] %endmacro %macro SAD_X4_24x4 0 mova m4, [r0] mova m5, [r0 + 16] movu m6, [r1] movu m7, [r1 + 16] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m0, m6 movu m6, [r2] movu m7, [r2 + 16] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m1, m6 movu m6, [r3] movu m7, [r3 + 16] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m2, m6 movu m6, [r4] movu m7, [r4 + 16] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m3, m6 mova m4, [r0 + FENC_STRIDE] mova m5, [r0 + 16 + FENC_STRIDE] movu m6, [r1 + r5] movu m7, [r1 + 16 + r5] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m0, m6 movu m6, [r2 + r5] movu m7, [r2 + 16 + r5] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m1, m6 movu m6, [r3 + r5] movu m7, [r3 + 16 + r5] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m2, m6 movu m6, [r4 + r5] movu m7, [r4 + 16 + r5] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m3, m6 mova m4, [r0 + FENC_STRIDE * 2] mova m5, [r0 + 16 + FENC_STRIDE * 2] movu m6, [r1 + r5 * 2] movu m7, [r1 + 16 + r5 * 2] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m0, m6 movu m6, [r2 + r5 * 2] movu m7, [r2 + 16 + r5 * 2] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m1, m6 movu m6, [r3 + r5 * 2] movu m7, [r3 + 16 + r5 * 2] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m2, m6 movu m6, [r4 + r5 * 2] movu m7, [r4 + 16 + r5 * 2] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m3, m6 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r5 * 2] lea r2, [r2 + r5 * 2] lea r3, [r3 + r5 * 2] lea r4, [r4 + r5 * 2] mova m4, [r0 + FENC_STRIDE] mova m5, [r0 + 16 + FENC_STRIDE] movu m6, [r1 + r5] movu m7, [r1 + 16 + r5] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m0, m6 movu m6, [r2 + r5] movu m7, [r2 + 16 + r5] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m1, m6 movu m6, [r3 + r5] movu m7, [r3 + 16 + r5] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m2, m6 movu m6, [r4 + r5] movu m7, [r4 + 16 + r5] psadbw m6, m4 psadbw m7, m5 pshufd m7, m7, 84 paddd m6, m7 paddd m3, m6 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r5 * 2] lea r2, [r2 + r5 * 2] lea r3, [r3 + r5 * 2] lea r4, [r4 + r5 * 2] %endmacro %macro SAD_X3_32x4 0 mova m3, [r0] mova m4, [r0 + 16] movu m5, [r1] movu m6, [r1 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m0, m5 movu m5, [r2] movu m6, [r2 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m1, m5 movu m5, [r3] movu m6, [r3 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m2, m5 lea r0, [r0 + FENC_STRIDE] lea r1, [r1 + r4] lea r2, [r2 + r4] lea r3, [r3 + r4] mova m3, [r0] mova m4, [r0 + 16] movu m5, [r1] movu m6, [r1 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m0, m5 movu m5, [r2] movu m6, [r2 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m1, m5 movu m5, [r3] movu m6, [r3 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m2, m5 lea r0, [r0 + FENC_STRIDE] lea r1, [r1 + r4] lea r2, [r2 + r4] lea r3, [r3 + r4] mova m3, [r0] mova m4, [r0 + 16] movu m5, [r1] movu m6, [r1 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m0, m5 movu m5, [r2] movu m6, [r2 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m1, m5 movu m5, [r3] movu m6, [r3 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m2, m5 lea r0, [r0 + FENC_STRIDE] lea r1, [r1 + r4] lea r2, [r2 + r4] lea r3, [r3 + r4] mova m3, [r0] mova m4, [r0 + 16] movu m5, [r1] movu m6, [r1 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m0, m5 movu m5, [r2] movu m6, [r2 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m1, m5 movu m5, [r3] movu m6, [r3 + 16] psadbw m5, m3 psadbw m6, m4 paddd m5, m6 paddd m2, m5 lea r0, [r0 + FENC_STRIDE] lea r1, [r1 + r4] lea r2, [r2 + r4] lea r3, [r3 + r4] %endmacro %macro SAD_X4_32x4 0 mova m4, [r0] mova m5, [r0 + 16] movu m6, [r1] movu m7, [r1 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m0, m6 movu m6, [r2] movu m7, [r2 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m1, m6 movu m6, [r3] movu m7, [r3 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m2, m6 movu m6, [r4] movu m7, [r4 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m3, m6 lea r0, [r0 + FENC_STRIDE] lea r1, [r1 + r5] lea r2, [r2 + r5] lea r3, [r3 + r5] lea r4, [r4 + r5] mova m4, [r0] mova m5, [r0 + 16] movu m6, [r1] movu m7, [r1 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m0, m6 movu m6, [r2] movu m7, [r2 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m1, m6 movu m6, [r3] movu m7, [r3 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m2, m6 movu m6, [r4] movu m7, [r4 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m3, m6 lea r0, [r0 + FENC_STRIDE] lea r1, [r1 + r5] lea r2, [r2 + r5] lea r3, [r3 + r5] lea r4, [r4 + r5] mova m4, [r0] mova m5, [r0 + 16] movu m6, [r1] movu m7, [r1 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m0, m6 movu m6, [r2] movu m7, [r2 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m1, m6 movu m6, [r3] movu m7, [r3 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m2, m6 movu m6, [r4] movu m7, [r4 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m3, m6 lea r0, [r0 + FENC_STRIDE] lea r1, [r1 + r5] lea r2, [r2 + r5] lea r3, [r3 + r5] lea r4, [r4 + r5] mova m4, [r0] mova m5, [r0 + 16] movu m6, [r1] movu m7, [r1 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m0, m6 movu m6, [r2] movu m7, [r2 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m1, m6 movu m6, [r3] movu m7, [r3 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m2, m6 movu m6, [r4] movu m7, [r4 + 16] psadbw m6, m4 psadbw m7, m5 paddd m6, m7 paddd m3, m6 lea r0, [r0 + FENC_STRIDE] lea r1, [r1 + r5] lea r2, [r2 + r5] lea r3, [r3 + r5] lea r4, [r4 + r5] %endmacro %macro SAD_X3_48x4 0 mova m3, [r0] mova m4, [r0 + 16] mova m5, [r0 + 32] movu m6, [r1] psadbw m6, m3 paddd m0, m6 movu m6, [r1 + 16] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 32] psadbw m6, m5 paddd m0, m6 movu m6, [r2] psadbw m6, m3 paddd m1, m6 movu m6, [r2 + 16] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 32] psadbw m6, m5 paddd m1, m6 movu m6, [r3] psadbw m6, m3 paddd m2, m6 movu m6, [r3 + 16] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 32] psadbw m6, m5 paddd m2, m6 mova m3, [r0 + FENC_STRIDE] mova m4, [r0 + 16 + FENC_STRIDE] mova m5, [r0 + 32 + FENC_STRIDE] movu m6, [r1 + r4] psadbw m6, m3 paddd m0, m6 movu m6, [r1 + 16 + r4] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 32 + r4] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + r4] psadbw m6, m3 paddd m1, m6 movu m6, [r2 + 16 + r4] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 32 + r4] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + r4] psadbw m6, m3 paddd m2, m6 movu m6, [r3 + 16 + r4] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 32 + r4] psadbw m6, m5 paddd m2, m6 mova m3, [r0 + FENC_STRIDE * 2] mova m4, [r0 + 16 + FENC_STRIDE * 2] mova m5, [r0 + 32 + FENC_STRIDE * 2] movu m6, [r1 + r4 * 2] psadbw m6, m3 paddd m0, m6 movu m6, [r1 + 16 + r4 * 2] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 32 + r4 * 2] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + r4 * 2] psadbw m6, m3 paddd m1, m6 movu m6, [r2 + 16 + r4 * 2] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 32 + r4 * 2] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + r4 * 2] psadbw m6, m3 paddd m2, m6 movu m6, [r3 + 16 + r4 * 2] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 32 + r4 * 2] psadbw m6, m5 paddd m2, m6 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r4 * 2] lea r2, [r2 + r4 * 2] lea r3, [r3 + r4 * 2] mova m3, [r0 + FENC_STRIDE] mova m4, [r0 + 16 + FENC_STRIDE] mova m5, [r0 + 32 + FENC_STRIDE] movu m6, [r1 + r4] psadbw m6, m3 paddd m0, m6 movu m6, [r1 + 16 + r4] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 32 + r4] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + r4] psadbw m6, m3 paddd m1, m6 movu m6, [r2 + 16 + r4] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 32 + r4] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + r4] psadbw m6, m3 paddd m2, m6 movu m6, [r3 + 16 + r4] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 32 + r4] psadbw m6, m5 paddd m2, m6 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r4 * 2] lea r2, [r2 + r4 * 2] lea r3, [r3 + r4 * 2] %endmacro %macro SAD_X4_48x4 0 mova m4, [r0] mova m5, [r0 + 16] mova m6, [r0 + 32] movu m7, [r1] psadbw m7, m4 paddd m0, m7 movu m7, [r1 + 16] psadbw m7, m5 paddd m0, m7 movu m7, [r1 + 32] psadbw m7, m6 paddd m0, m7 movu m7, [r2] psadbw m7, m4 paddd m1, m7 movu m7, [r2 + 16] psadbw m7, m5 paddd m1, m7 movu m7, [r2 + 32] psadbw m7, m6 paddd m1, m7 movu m7, [r3] psadbw m7, m4 paddd m2, m7 movu m7, [r3 + 16] psadbw m7, m5 paddd m2, m7 movu m7, [r3 + 32] psadbw m7, m6 paddd m2, m7 movu m7, [r4] psadbw m7, m4 paddd m3, m7 movu m7, [r4 + 16] psadbw m7, m5 paddd m3, m7 movu m7, [r4 + 32] psadbw m7, m6 paddd m3, m7 mova m4, [r0 + FENC_STRIDE] mova m5, [r0 + 16 + FENC_STRIDE] mova m6, [r0 + 32 + FENC_STRIDE] movu m7, [r1 + r5] psadbw m7, m4 paddd m0, m7 movu m7, [r1 + 16 + r5] psadbw m7, m5 paddd m0, m7 movu m7, [r1 + 32 + r5] psadbw m7, m6 paddd m0, m7 movu m7, [r2 + r5] psadbw m7, m4 paddd m1, m7 movu m7, [r2 + 16 + r5] psadbw m7, m5 paddd m1, m7 movu m7, [r2 + 32 + r5] psadbw m7, m6 paddd m1, m7 movu m7, [r3 + r5] psadbw m7, m4 paddd m2, m7 movu m7, [r3 + 16 + r5] psadbw m7, m5 paddd m2, m7 movu m7, [r3 + 32 + r5] psadbw m7, m6 paddd m2, m7 movu m7, [r4 + r5] psadbw m7, m4 paddd m3, m7 movu m7, [r4 + 16 + r5] psadbw m7, m5 paddd m3, m7 movu m7, [r4 + 32 + r5] psadbw m7, m6 paddd m3, m7 mova m4, [r0 + FENC_STRIDE * 2] mova m5, [r0 + 16 + FENC_STRIDE * 2] mova m6, [r0 + 32 + FENC_STRIDE * 2] movu m7, [r1 + r5 * 2] psadbw m7, m4 paddd m0, m7 movu m7, [r1 + 16 + r5 * 2] psadbw m7, m5 paddd m0, m7 movu m7, [r1 + 32 + r5 * 2] psadbw m7, m6 paddd m0, m7 movu m7, [r2 + r5 * 2] psadbw m7, m4 paddd m1, m7 movu m7, [r2 + 16 + r5 * 2] psadbw m7, m5 paddd m1, m7 movu m7, [r2 + 32 + r5 * 2] psadbw m7, m6 paddd m1, m7 movu m7, [r3 + r5 * 2] psadbw m7, m4 paddd m2, m7 movu m7, [r3 + 16 + r5 * 2] psadbw m7, m5 paddd m2, m7 movu m7, [r3 + 32 + r5 * 2] psadbw m7, m6 paddd m2, m7 movu m7, [r4 + r5 * 2] psadbw m7, m4 paddd m3, m7 movu m7, [r4 + 16 + r5 * 2] psadbw m7, m5 paddd m3, m7 movu m7, [r4 + 32 + r5 * 2] psadbw m7, m6 paddd m3, m7 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r5 * 2] lea r2, [r2 + r5 * 2] lea r3, [r3 + r5 * 2] lea r4, [r4 + r5 * 2] mova m4, [r0 + FENC_STRIDE] mova m5, [r0 + 16 + FENC_STRIDE] mova m6, [r0 + 32 + FENC_STRIDE] movu m7, [r1 + r5] psadbw m7, m4 paddd m0, m7 movu m7, [r1 + 16 + r5] psadbw m7, m5 paddd m0, m7 movu m7, [r1 + 32 + r5] psadbw m7, m6 paddd m0, m7 movu m7, [r2 + r5] psadbw m7, m4 paddd m1, m7 movu m7, [r2 + 16 + r5] psadbw m7, m5 paddd m1, m7 movu m7, [r2 + 32 + r5] psadbw m7, m6 paddd m1, m7 movu m7, [r3 + r5] psadbw m7, m4 paddd m2, m7 movu m7, [r3 + 16 + r5] psadbw m7, m5 paddd m2, m7 movu m7, [r3 + 32 + r5] psadbw m7, m6 paddd m2, m7 movu m7, [r4 + r5] psadbw m7, m4 paddd m3, m7 movu m7, [r4 + 16 + r5] psadbw m7, m5 paddd m3, m7 movu m7, [r4 + 32 + r5] psadbw m7, m6 paddd m3, m7 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r5 * 2] lea r2, [r2 + r5 * 2] lea r3, [r3 + r5 * 2] lea r4, [r4 + r5 * 2] %endmacro %macro SAD_X3_64x4 0 mova m3, [r0] mova m4, [r0 + 16] movu m5, [r1] psadbw m5, m3 paddd m0, m5 movu m5, [r1 + 16] psadbw m5, m4 paddd m0, m5 movu m5, [r2] psadbw m5, m3 paddd m1, m5 movu m5, [r2 + 16] psadbw m5, m4 paddd m1, m5 movu m5, [r3] psadbw m5, m3 paddd m2, m5 movu m5, [r3 + 16] psadbw m5, m4 paddd m2, m5 mova m3, [r0 + 32] mova m4, [r0 + 48] movu m5, [r1 + 32] psadbw m5, m3 paddd m0, m5 movu m5, [r1 + 48] psadbw m5, m4 paddd m0, m5 movu m5, [r2 + 32] psadbw m5, m3 paddd m1, m5 movu m5, [r2 + 48] psadbw m5, m4 paddd m1, m5 movu m5, [r3 + 32] psadbw m5, m3 paddd m2, m5 movu m5, [r3 + 48] psadbw m5, m4 paddd m2, m5 mova m3, [r0 + FENC_STRIDE] mova m4, [r0 + 16 + FENC_STRIDE] movu m5, [r1 + r4] psadbw m5, m3 paddd m0, m5 movu m5, [r1 + 16 + r4] psadbw m5, m4 paddd m0, m5 movu m5, [r2 + r4] psadbw m5, m3 paddd m1, m5 movu m5, [r2 + 16 + r4] psadbw m5, m4 paddd m1, m5 movu m5, [r3 + r4] psadbw m5, m3 paddd m2, m5 movu m5, [r3 + 16 + r4] psadbw m5, m4 paddd m2, m5 mova m3, [r0 + 32 + FENC_STRIDE] mova m4, [r0 + 48 + FENC_STRIDE] movu m5, [r1 + 32 + r4] psadbw m5, m3 paddd m0, m5 movu m5, [r1 + 48 + r4] psadbw m5, m4 paddd m0, m5 movu m5, [r2 + 32 + r4] psadbw m5, m3 paddd m1, m5 movu m5, [r2 + 48 + r4] psadbw m5, m4 paddd m1, m5 movu m5, [r3 + 32 + r4] psadbw m5, m3 paddd m2, m5 movu m5, [r3 + 48 + r4] psadbw m5, m4 paddd m2, m5 mova m3, [r0 + FENC_STRIDE * 2] mova m4, [r0 + 16 + FENC_STRIDE * 2] movu m5, [r1 + r4 * 2] psadbw m5, m3 paddd m0, m5 movu m5, [r1 + 16 + r4 * 2] psadbw m5, m4 paddd m0, m5 movu m5, [r2 + r4 * 2] psadbw m5, m3 paddd m1, m5 movu m5, [r2 + 16 + r4 * 2] psadbw m5, m4 paddd m1, m5 movu m5, [r3 + r4 * 2] psadbw m5, m3 paddd m2, m5 movu m5, [r3 + 16 + r4 * 2] psadbw m5, m4 paddd m2, m5 mova m3, [r0 + 32 + FENC_STRIDE * 2] mova m4, [r0 + 48 + FENC_STRIDE * 2] movu m5, [r1 + 32 + r4 * 2] psadbw m5, m3 paddd m0, m5 movu m5, [r1 + 48 + r4 * 2] psadbw m5, m4 paddd m0, m5 movu m5, [r2 + 32 + r4 * 2] psadbw m5, m3 paddd m1, m5 movu m5, [r2 + 48 + r4 * 2] psadbw m5, m4 paddd m1, m5 movu m5, [r3 + 32 + r4 * 2] psadbw m5, m3 paddd m2, m5 movu m5, [r3 + 48 + r4 * 2] psadbw m5, m4 paddd m2, m5 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r4 * 2] lea r2, [r2 + r4 * 2] lea r3, [r3 + r4 * 2] mova m3, [r0 + FENC_STRIDE] mova m4, [r0 + 16 + FENC_STRIDE] movu m5, [r1 + r4] psadbw m5, m3 paddd m0, m5 movu m5, [r1 + 16 + r4] psadbw m5, m4 paddd m0, m5 movu m5, [r2 + r4] psadbw m5, m3 paddd m1, m5 movu m5, [r2 + 16 + r4] psadbw m5, m4 paddd m1, m5 movu m5, [r3 + r4] psadbw m5, m3 paddd m2, m5 movu m5, [r3 + 16 + r4] psadbw m5, m4 paddd m2, m5 mova m3, [r0 + 32 + FENC_STRIDE] mova m4, [r0 + 48 + FENC_STRIDE] movu m5, [r1 + 32 + r4] psadbw m5, m3 paddd m0, m5 movu m5, [r1 + 48 + r4] psadbw m5, m4 paddd m0, m5 movu m5, [r2 + 32 + r4] psadbw m5, m3 paddd m1, m5 movu m5, [r2 + 48 + r4] psadbw m5, m4 paddd m1, m5 movu m5, [r3 + 32 + r4] psadbw m5, m3 paddd m2, m5 movu m5, [r3 + 48 + r4] psadbw m5, m4 paddd m2, m5 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r4 * 2] lea r2, [r2 + r4 * 2] lea r3, [r3 + r4 * 2] %endmacro %macro SAD_X4_64x4 0 mova m4, [r0] mova m5, [r0 + 16] movu m6, [r1] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 16] psadbw m6, m5 paddd m0, m6 movu m6, [r2] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 16] psadbw m6, m5 paddd m1, m6 movu m6, [r3] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 16] psadbw m6, m5 paddd m2, m6 movu m6, [r4] psadbw m6, m4 paddd m3, m6 movu m6, [r4 + 16] psadbw m6, m5 paddd m3, m6 mova m4, [r0 + 32] mova m5, [r0 + 48] movu m6, [r1 + 32] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 48] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + 32] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 48] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + 32] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 48] psadbw m6, m5 paddd m2, m6 movu m6, [r4 + 32] psadbw m6, m4 paddd m3, m6 movu m6, [r4 + 48] psadbw m6, m5 paddd m3, m6 mova m4, [r0 + FENC_STRIDE] mova m5, [r0 + 16 + FENC_STRIDE] movu m6, [r1 + r5] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 16 + r5] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + r5] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 16 + r5] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + r5] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 16 + r5] psadbw m6, m5 paddd m2, m6 movu m6, [r4 + r5] psadbw m6, m4 paddd m3, m6 movu m6, [r4 + 16 + r5] psadbw m6, m5 paddd m3, m6 mova m4, [r0 + 32 + FENC_STRIDE] mova m5, [r0 + 48 + FENC_STRIDE] movu m6, [r1 + 32 + r5] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 48 + r5] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + 32 + r5] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 48 + r5] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + 32 + r5] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 48 + r5] psadbw m6, m5 paddd m2, m6 movu m6, [r4 + 32 + r5] psadbw m6, m4 paddd m3, m6 movu m6, [r4 + 48 + r5] psadbw m6, m5 paddd m3, m6 mova m4, [r0 + FENC_STRIDE * 2] mova m5, [r0 + 16 + FENC_STRIDE * 2] movu m6, [r1 + r5 * 2] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 16 + r5 * 2] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + r5 * 2] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 16 + r5 * 2] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + r5 * 2] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 16 + r5 * 2] psadbw m6, m5 paddd m2, m6 movu m6, [r4 + r5 * 2] psadbw m6, m4 paddd m3, m6 movu m6, [r4 + 16 + r5 * 2] psadbw m6, m5 paddd m3, m6 mova m4, [r0 + 32 + FENC_STRIDE * 2] mova m5, [r0 + 48 + FENC_STRIDE * 2] movu m6, [r1 + 32 + r5 * 2] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 48 + r5 * 2] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + 32 + r5 * 2] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 48 + r5 * 2] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + 32 + r5 * 2] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 48 + r5 * 2] psadbw m6, m5 paddd m2, m6 movu m6, [r4 + 32 + r5 * 2] psadbw m6, m4 paddd m3, m6 movu m6, [r4 + 48 + r5 * 2] psadbw m6, m5 paddd m3, m6 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r5 * 2] lea r2, [r2 + r5 * 2] lea r3, [r3 + r5 * 2] lea r4, [r4 + r5 * 2] mova m4, [r0 + FENC_STRIDE] mova m5, [r0 + 16 + FENC_STRIDE] movu m6, [r1 + r5] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 16 + r5] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + r5] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 16 + r5] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + r5] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 16 + r5] psadbw m6, m5 paddd m2, m6 movu m6, [r4 + r5] psadbw m6, m4 paddd m3, m6 movu m6, [r4 + 16 + r5] psadbw m6, m5 paddd m3, m6 mova m4, [r0 + 32 + FENC_STRIDE] mova m5, [r0 + 48 + FENC_STRIDE] movu m6, [r1 + 32 + r5] psadbw m6, m4 paddd m0, m6 movu m6, [r1 + 48 + r5] psadbw m6, m5 paddd m0, m6 movu m6, [r2 + 32 + r5] psadbw m6, m4 paddd m1, m6 movu m6, [r2 + 48 + r5] psadbw m6, m5 paddd m1, m6 movu m6, [r3 + 32 + r5] psadbw m6, m4 paddd m2, m6 movu m6, [r3 + 48 + r5] psadbw m6, m5 paddd m2, m6 movu m6, [r4 + 32 + r5] psadbw m6, m4 paddd m3, m6 movu m6, [r4 + 48 + r5] psadbw m6, m5 paddd m3, m6 lea r0, [r0 + FENC_STRIDE * 2] lea r1, [r1 + r5 * 2] lea r2, [r2 + r5 * 2] lea r3, [r3 + r5 * 2] lea r4, [r4 + r5 * 2] %endmacro ;----------------------------------------------------------------------------- ; void pixel_sad_x3_16x16( uint8_t *fenc, uint8_t *pix0, uint8_t *pix1, ; uint8_t *pix2, intptr_t i_stride, int scores[3] ) ;----------------------------------------------------------------------------- %macro SAD_X 3 cglobal pixel_sad_x%1_%2x%3_mmx2, %1+2, %1+2 SAD_X%1_2x%2P 1 %rep %3/2-1 SAD_X%1_2x%2P 0 %endrep SAD_X%1_END %endmacro INIT_MMX SAD_X 3, 16, 16 SAD_X 3, 16, 8 SAD_X 3, 8, 16 SAD_X 3, 8, 8 SAD_X 3, 8, 4 SAD_X 3, 4, 16 SAD_X 3, 4, 8 SAD_X 3, 4, 4 SAD_X 4, 16, 16 SAD_X 4, 16, 8 SAD_X 4, 8, 16 SAD_X 4, 8, 8 SAD_X 4, 8, 4 SAD_X 4, 4, 16 SAD_X 4, 4, 8 SAD_X 4, 4, 4 ;============================================================================= ; SAD x3/x4 XMM ;============================================================================= %macro SAD_X3_START_1x16P_SSE2 0 mova m2, [r0] %if cpuflag(avx) psadbw m0, m2, [r1] psadbw m1, m2, [r2] psadbw m2, [r3] %else movu m0, [r1] movu m1, [r2] movu m3, [r3] psadbw m0, m2 psadbw m1, m2 psadbw m2, m3 %endif %endmacro %macro SAD_X3_1x16P_SSE2 2 mova m3, [r0+%1] %if cpuflag(avx) psadbw m4, m3, [r1+%2] psadbw m5, m3, [r2+%2] psadbw m3, [r3+%2] %else movu m4, [r1+%2] movu m5, [r2+%2] movu m6, [r3+%2] psadbw m4, m3 psadbw m5, m3 psadbw m3, m6 %endif paddd m0, m4 paddd m1, m5 paddd m2, m3 %endmacro %if ARCH_X86_64 DECLARE_REG_TMP 6 %else DECLARE_REG_TMP 5 %endif %macro SAD_X3_4x16P_SSE2 2 %if %1==0 lea t0, [r4*3] SAD_X3_START_1x16P_SSE2 %else SAD_X3_1x16P_SSE2 FENC_STRIDE*(0+(%1&1)*4), r4*0 %endif SAD_X3_1x16P_SSE2 FENC_STRIDE*(1+(%1&1)*4), r4*1 SAD_X3_1x16P_SSE2 FENC_STRIDE*(2+(%1&1)*4), r4*2 SAD_X3_1x16P_SSE2 FENC_STRIDE*(3+(%1&1)*4), t0 %if %1 != %2-1 %if (%1&1) != 0 add r0, 8*FENC_STRIDE %endif lea r1, [r1+4*r4] lea r2, [r2+4*r4] lea r3, [r3+4*r4] %endif %endmacro %macro SAD_X3_START_2x8P_SSE2 0 movq m3, [r0] movq m0, [r1] movq m1, [r2] movq m2, [r3] movhps m3, [r0+FENC_STRIDE] movhps m0, [r1+r4] movhps m1, [r2+r4] movhps m2, [r3+r4] psadbw m0, m3 psadbw m1, m3 psadbw m2, m3 %endmacro %macro SAD_X3_2x8P_SSE2 4 movq m6, [r0+%1] movq m3, [r1+%2] movq m4, [r2+%2] movq m5, [r3+%2] movhps m6, [r0+%3] movhps m3, [r1+%4] movhps m4, [r2+%4] movhps m5, [r3+%4] psadbw m3, m6 psadbw m4, m6 psadbw m5, m6 paddd m0, m3 paddd m1, m4 paddd m2, m5 %endmacro %macro SAD_X4_START_2x8P_SSE2 0 movq m4, [r0] movq m0, [r1] movq m1, [r2] movq m2, [r3] movq m3, [r4] movhps m4, [r0+FENC_STRIDE] movhps m0, [r1+r5] movhps m1, [r2+r5] movhps m2, [r3+r5] movhps m3, [r4+r5] psadbw m0, m4 psadbw m1, m4 psadbw m2, m4 psadbw m3, m4 %endmacro %macro SAD_X4_2x8P_SSE2 4 movq m6, [r0+%1] movq m4, [r1+%2] movq m5, [r2+%2] movhps m6, [r0+%3] movhps m4, [r1+%4] movhps m5, [r2+%4] psadbw m4, m6 psadbw m5, m6 paddd m0, m4 paddd m1, m5 movq m4, [r3+%2] movq m5, [r4+%2] movhps m4, [r3+%4] movhps m5, [r4+%4] psadbw m4, m6 psadbw m5, m6 paddd m2, m4 paddd m3, m5 %endmacro %macro SAD_X4_START_1x16P_SSE2 0 mova m3, [r0] %if cpuflag(avx) psadbw m0, m3, [r1] psadbw m1, m3, [r2] psadbw m2, m3, [r3] psadbw m3, [r4] %else movu m0, [r1] movu m1, [r2] movu m2, [r3] movu m4, [r4] psadbw m0, m3 psadbw m1, m3 psadbw m2, m3 psadbw m3, m4 %endif %endmacro %macro SAD_X4_1x16P_SSE2 2 mova m6, [r0+%1] %if cpuflag(avx) psadbw m4, m6, [r1+%2] psadbw m5, m6, [r2+%2] %else movu m4, [r1+%2] movu m5, [r2+%2] psadbw m4, m6 psadbw m5, m6 %endif paddd m0, m4 paddd m1, m5 %if cpuflag(avx) psadbw m4, m6, [r3+%2] psadbw m5, m6, [r4+%2] %else movu m4, [r3+%2] movu m5, [r4+%2] psadbw m4, m6 psadbw m5, m6 %endif paddd m2, m4 paddd m3, m5 %endmacro %macro SAD_X4_4x16P_SSE2 2 %if %1==0 lea r6, [r5*3] SAD_X4_START_1x16P_SSE2 %else SAD_X4_1x16P_SSE2 FENC_STRIDE*(0+(%1&1)*4), r5*0 %endif SAD_X4_1x16P_SSE2 FENC_STRIDE*(1+(%1&1)*4), r5*1 SAD_X4_1x16P_SSE2 FENC_STRIDE*(2+(%1&1)*4), r5*2 SAD_X4_1x16P_SSE2 FENC_STRIDE*(3+(%1&1)*4), r6 %if %1 != %2-1 %if (%1&1) != 0 add r0, 8*FENC_STRIDE %endif lea r1, [r1+4*r5] lea r2, [r2+4*r5] lea r3, [r3+4*r5] lea r4, [r4+4*r5] %endif %endmacro %macro SAD_X3_4x8P_SSE2 2 %if %1==0 lea t0, [r4*3] SAD_X3_START_2x8P_SSE2 %else SAD_X3_2x8P_SSE2 FENC_STRIDE*(0+(%1&1)*4), r4*0, FENC_STRIDE*(1+(%1&1)*4), r4*1 %endif SAD_X3_2x8P_SSE2 FENC_STRIDE*(2+(%1&1)*4), r4*2, FENC_STRIDE*(3+(%1&1)*4), t0 %if %1 != %2-1 %if (%1&1) != 0 add r0, 8*FENC_STRIDE %endif lea r1, [r1+4*r4] lea r2, [r2+4*r4] lea r3, [r3+4*r4] %endif %endmacro %macro SAD_X4_4x8P_SSE2 2 %if %1==0 lea r6, [r5*3] SAD_X4_START_2x8P_SSE2 %else SAD_X4_2x8P_SSE2 FENC_STRIDE*(0+(%1&1)*4), r5*0, FENC_STRIDE*(1+(%1&1)*4), r5*1 %endif SAD_X4_2x8P_SSE2 FENC_STRIDE*(2+(%1&1)*4), r5*2, FENC_STRIDE*(3+(%1&1)*4), r6 %if %1 != %2-1 %if (%1&1) != 0 add r0, 8*FENC_STRIDE %endif lea r1, [r1+4*r5] lea r2, [r2+4*r5] lea r3, [r3+4*r5] lea r4, [r4+4*r5] %endif %endmacro %macro SAD_X3_END_SSE2 1 movifnidn r5, r5mp movhlps m3, m0 movhlps m4, m1 movhlps m5, m2 paddd m0, m3 paddd m1, m4 paddd m2, m5 movd [r5+0], m0 movd [r5+4], m1 movd [r5+8], m2 RET %endmacro %macro SAD_X4_END_SSE2 1 mov r0, r6mp psllq m1, 32 psllq m3, 32 paddd m0, m1 paddd m2, m3 movhlps m1, m0 movhlps m3, m2 paddd m0, m1 paddd m2, m3 movq [r0+0], m0 movq [r0+8], m2 RET %endmacro %macro SAD_X3_START_2x16P_AVX2 0 movu m3, [r0] ; assumes FENC_STRIDE == 16 movu xm0, [r1] movu xm1, [r2] movu xm2, [r3] vinserti128 m0, m0, [r1+r4], 1 vinserti128 m1, m1, [r2+r4], 1 vinserti128 m2, m2, [r3+r4], 1 psadbw m0, m3 psadbw m1, m3 psadbw m2, m3 %endmacro %macro SAD_X3_2x16P_AVX2 3 movu m3, [r0+%1] ; assumes FENC_STRIDE == 16 movu xm4, [r1+%2] movu xm5, [r2+%2] movu xm6, [r3+%2] vinserti128 m4, m4, [r1+%3], 1 vinserti128 m5, m5, [r2+%3], 1 vinserti128 m6, m6, [r3+%3], 1 psadbw m4, m3 psadbw m5, m3 psadbw m6, m3 paddw m0, m4 paddw m1, m5 paddw m2, m6 %endmacro %macro SAD_X3_4x16P_AVX2 2 %if %1==0 lea t0, [r4*3] SAD_X3_START_2x16P_AVX2 %else SAD_X3_2x16P_AVX2 FENC_STRIDE*(0+(%1&1)*4), r4*0, r4*1 %endif SAD_X3_2x16P_AVX2 FENC_STRIDE*(2+(%1&1)*4), r4*2, t0 %if %1 != %2-1 %if (%1&1) != 0 add r0, 8*FENC_STRIDE %endif lea r1, [r1+4*r4] lea r2, [r2+4*r4] lea r3, [r3+4*r4] %endif %endmacro %macro SAD_X4_START_2x16P_AVX2 0 vbroadcasti128 m4, [r0] vbroadcasti128 m5, [r0+FENC_STRIDE] movu xm0, [r1] movu xm1, [r2] movu xm2, [r1+r5] movu xm3, [r2+r5] vinserti128 m0, m0, [r3], 1 vinserti128 m1, m1, [r4], 1 vinserti128 m2, m2, [r3+r5], 1 vinserti128 m3, m3, [r4+r5], 1 psadbw m0, m4 psadbw m1, m4 psadbw m2, m5 psadbw m3, m5 paddw m0, m2 paddw m1, m3 %endmacro %macro SAD_X4_2x16P_AVX2 4 vbroadcasti128 m6, [r0+%1] vbroadcasti128 m7, [r0+%3] movu xm2, [r1+%2] movu xm3, [r2+%2] movu xm4, [r1+%4] movu xm5, [r2+%4] vinserti128 m2, m2, [r3+%2], 1 vinserti128 m3, m3, [r4+%2], 1 vinserti128 m4, m4, [r3+%4], 1 vinserti128 m5, m5, [r4+%4], 1 psadbw m2, m6 psadbw m3, m6 psadbw m4, m7 psadbw m5, m7 paddd m0, m2 paddd m1, m3 paddd m0, m4 paddd m1, m5 %endmacro %macro SAD_X4_4x16P_AVX2 2 %if %1==0 lea r6, [r5*3] SAD_X4_START_2x16P_AVX2 %else SAD_X4_2x16P_AVX2 FENC_STRIDE*(0+(%1&1)*4), r5*0, FENC_STRIDE*(1+(%1&1)*4), r5*1 %endif SAD_X4_2x16P_AVX2 FENC_STRIDE*(2+(%1&1)*4), r5*2, FENC_STRIDE*(3+(%1&1)*4), r6 %if %1 != %2-1 %if (%1&1) != 0 add r0, 8*FENC_STRIDE %endif lea r1, [r1+4*r5] lea r2, [r2+4*r5] lea r3, [r3+4*r5] lea r4, [r4+4*r5] %endif %endmacro %macro SAD_X4_START_2x32P_AVX2 0 mova m4, [r0] movu m0, [r1] movu m2, [r2] movu m1, [r3] movu m3, [r4] psadbw m0, m4 psadbw m2, m4 psadbw m1, m4 psadbw m3, m4 packusdw m0, m2 packusdw m1, m3 mova m6, [r0+FENC_STRIDE] movu m2, [r1+r5] movu m4, [r2+r5] movu m3, [r3+r5] movu m5, [r4+r5] psadbw m2, m6 psadbw m4, m6 psadbw m3, m6 psadbw m5, m6 packusdw m2, m4 packusdw m3, m5 paddd m0, m2 paddd m1, m3 %endmacro %macro SAD_X4_2x32P_AVX2 4 mova m6, [r0+%1] movu m2, [r1+%2] movu m4, [r2+%2] movu m3, [r3+%2] movu m5, [r4+%2] psadbw m2, m6 psadbw m4, m6 psadbw m3, m6 psadbw m5, m6 packusdw m2, m4 packusdw m3, m5 paddd m0, m2 paddd m1, m3 mova m6, [r0+%3] movu m2, [r1+%4] movu m4, [r2+%4] movu m3, [r3+%4] movu m5, [r4+%4] psadbw m2, m6 psadbw m4, m6 psadbw m3, m6 psadbw m5, m6 packusdw m2, m4 packusdw m3, m5 paddd m0, m2 paddd m1, m3 %endmacro %macro SAD_X4_4x32P_AVX2 2 %if %1==0 lea r6, [r5*3] SAD_X4_START_2x32P_AVX2 %else SAD_X4_2x32P_AVX2 FENC_STRIDE*(0+(%1&1)*4), r5*0, FENC_STRIDE*(1+(%1&1)*4), r5*1 %endif SAD_X4_2x32P_AVX2 FENC_STRIDE*(2+(%1&1)*4), r5*2, FENC_STRIDE*(3+(%1&1)*4), r6 %if %1 != %2-1 %if (%1&1) != 0 add r0, 8*FENC_STRIDE %endif lea r1, [r1+4*r5] lea r2, [r2+4*r5] lea r3, [r3+4*r5] lea r4, [r4+4*r5] %endif %endmacro %macro SAD_X3_END_AVX2 0 movifnidn r5, r5mp packssdw m0, m1 ; 0 0 1 1 0 0 1 1 packssdw m2, m2 ; 2 2 _ _ 2 2 _ _ phaddd m0, m2 ; 0 1 2 _ 0 1 2 _ vextracti128 xm1, m0, 1 paddd xm0, xm1 ; 0 1 2 _ mova [r5], xm0 RET %endmacro %macro SAD_X4_END_AVX2 0 mov r0, r6mp pshufd m0, m0, 0x8 pshufd m1, m1, 0x8 vextracti128 xm2, m0, 1 vextracti128 xm3, m1, 1 punpcklqdq xm0, xm1 punpcklqdq xm2, xm3 phaddd xm0, xm2 ; 0 1 2 3 mova [r0], xm0 RET %endmacro %macro SAD_X4_32P_END_AVX2 0 mov r0, r6mp vextracti128 xm2, m0, 1 vextracti128 xm3, m1, 1 paddd xm0, xm2 paddd xm1, xm3 phaddd xm0, xm1 mova [r0], xm0 RET %endmacro ;----------------------------------------------------------------------------- ; void pixel_sad_x3_16x16( uint8_t *fenc, uint8_t *pix0, uint8_t *pix1, ; uint8_t *pix2, intptr_t i_stride, int scores[3] ) ;----------------------------------------------------------------------------- %macro SAD_X_SSE2 4 cglobal pixel_sad_x%1_%2x%3, 2+%1,3+%1,%4 %assign x 0 %rep %3/4 SAD_X%1_4x%2P_SSE2 x, %3/4 %assign x x+1 %endrep %if %3 == 64 SAD_X%1_END_SSE2 1 %else SAD_X%1_END_SSE2 0 %endif %endmacro %macro SAD_X3_W12 0 cglobal pixel_sad_x3_12x16, 5, 7, 8 mova m4, [MSK] pxor m0, m0 pxor m1, m1 pxor m2, m2 SAD_X3_12x4 SAD_X3_12x4 SAD_X3_12x4 SAD_X3_12x4 SAD_X3_END_SSE2 1 %endmacro %macro SAD_X4_W12 0 cglobal pixel_sad_x4_12x16, 6, 8, 8 mova m6, [MSK] pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 SAD_X4_12x4 SAD_X4_12x4 SAD_X4_12x4 SAD_X4_12x4 SAD_X4_END_SSE2 1 %endmacro %macro SAD_X3_W24 0 cglobal pixel_sad_x3_24x32, 5, 7, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 mov r6, 32 .loop: SAD_X3_24x4 SAD_X3_24x4 SAD_X3_24x4 SAD_X3_24x4 sub r6, 16 cmp r6, 0 jnz .loop SAD_X3_END_SSE2 1 %endmacro %macro SAD_X4_W24 0 %if ARCH_X86_64 == 1 cglobal pixel_sad_x4_24x32, 6, 8, 8 %define count r7 %else cglobal pixel_sad_x4_24x32, 6, 7, 8, 0-4 %define count dword [rsp] %endif pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 mov count, 32 .loop: SAD_X4_24x4 SAD_X4_24x4 SAD_X4_24x4 SAD_X4_24x4 sub count, 16 jnz .loop SAD_X4_END_SSE2 1 %endmacro %macro SAD_X3_W32 0 cglobal pixel_sad_x3_32x8, 5, 6, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_END_SSE2 1 cglobal pixel_sad_x3_32x16, 5, 6, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_END_SSE2 1 cglobal pixel_sad_x3_32x24, 5, 6, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_END_SSE2 1 cglobal pixel_sad_x3_32x32, 5, 7, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 mov r6, 32 .loop: SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 sub r6, 16 cmp r6, 0 jnz .loop SAD_X3_END_SSE2 1 cglobal pixel_sad_x3_32x64, 5, 7, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 mov r6, 64 .loop1: SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 SAD_X3_32x4 sub r6, 16 cmp r6, 0 jnz .loop1 SAD_X3_END_SSE2 1 %endmacro %macro SAD_X4_W32 0 cglobal pixel_sad_x4_32x8, 6, 7, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_END_SSE2 1 cglobal pixel_sad_x4_32x16, 6, 7, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_END_SSE2 1 cglobal pixel_sad_x4_32x24, 6, 7, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_END_SSE2 1 %if ARCH_X86_64 == 1 cglobal pixel_sad_x4_32x32, 6, 8, 8 %define count r7 %else cglobal pixel_sad_x4_32x32, 6, 7, 8, 0-4 %define count dword [rsp] %endif pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 mov count, 32 .loop: SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 sub count, 16 jnz .loop SAD_X4_END_SSE2 1 %if ARCH_X86_64 == 1 cglobal pixel_sad_x4_32x64, 6, 8, 8 %define count r7 %else cglobal pixel_sad_x4_32x64, 6, 7, 8, 0-4 %define count dword [rsp] %endif pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 mov count, 64 .loop: SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 SAD_X4_32x4 sub count, 16 jnz .loop SAD_X4_END_SSE2 1 %endmacro %macro SAD_X3_W48 0 cglobal pixel_sad_x3_48x64, 5, 7, 8 pxor m0, m0 pxor m1, m1 pxor m2, m2 mov r6, 64 .loop: SAD_X3_48x4 SAD_X3_48x4 SAD_X3_48x4 SAD_X3_48x4 sub r6, 16 jnz .loop SAD_X3_END_SSE2 1 %endmacro %macro SAD_X4_W48 0 %if ARCH_X86_64 == 1 cglobal pixel_sad_x4_48x64, 6, 8, 8 %define count r7 %else cglobal pixel_sad_x4_48x64, 6, 7, 8, 0-4 %define count dword [rsp] %endif pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 mov count, 64 .loop: SAD_X4_48x4 SAD_X4_48x4 SAD_X4_48x4 SAD_X4_48x4 sub count, 16 jnz .loop SAD_X4_END_SSE2 1 %endmacro %macro SAD_X3_W64 0 cglobal pixel_sad_x3_64x16, 5, 7, 7 pxor m0, m0 pxor m1, m1 pxor m2, m2 mov r6, 16 .loop: SAD_X3_64x4 SAD_X3_64x4 sub r6, 8 jnz .loop SAD_X3_END_SSE2 1 cglobal pixel_sad_x3_64x32, 5, 7, 7 pxor m0, m0 pxor m1, m1 pxor m2, m2 mov r6, 32 .loop: SAD_X3_64x4 SAD_X3_64x4 sub r6, 8 jnz .loop SAD_X3_END_SSE2 1 cglobal pixel_sad_x3_64x48, 5, 7, 7 pxor m0, m0 pxor m1, m1 pxor m2, m2 mov r6, 48 .loop: SAD_X3_64x4 SAD_X3_64x4 sub r6, 8 jnz .loop SAD_X3_END_SSE2 1 cglobal pixel_sad_x3_64x64, 5, 7, 7 pxor m0, m0 pxor m1, m1 pxor m2, m2 mov r6, 64 .loop: SAD_X3_64x4 SAD_X3_64x4 sub r6, 8 jnz .loop SAD_X3_END_SSE2 1 %endmacro %macro SAD_X4_W64 0 %if ARCH_X86_64 == 1 cglobal pixel_sad_x4_64x16, 6, 8, 8 %define count r7 %else cglobal pixel_sad_x4_64x16, 6, 7, 8, 0-4 %define count dword [rsp] %endif pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 mov count, 16 .loop: SAD_X4_64x4 SAD_X4_64x4 sub count, 8 jnz .loop SAD_X4_END_SSE2 1 %if ARCH_X86_64 == 1 cglobal pixel_sad_x4_64x32, 6, 8, 8 %define count r7 %else cglobal pixel_sad_x4_64x32, 6, 7, 8, 0-4 %define count dword [rsp] %endif pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 mov count, 32 .loop: SAD_X4_64x4 SAD_X4_64x4 sub count, 8 jnz .loop SAD_X4_END_SSE2 1 %if ARCH_X86_64 == 1 cglobal pixel_sad_x4_64x48, 6, 8, 8 %define count r7 %else cglobal pixel_sad_x4_64x48, 6, 7, 8, 0-4 %define count dword [rsp] %endif pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 mov count, 48 .loop: SAD_X4_64x4 SAD_X4_64x4 sub count, 8 jnz .loop SAD_X4_END_SSE2 1 %if ARCH_X86_64 == 1 cglobal pixel_sad_x4_64x64, 6, 8, 8 %define count r7 %else cglobal pixel_sad_x4_64x64, 6, 7, 8, 0-4 %define count dword [rsp] %endif pxor m0, m0 pxor m1, m1 pxor m2, m2 pxor m3, m3 mov count, 64 .loop: SAD_X4_64x4 SAD_X4_64x4 sub count, 8 jnz .loop SAD_X4_END_SSE2 1 %endmacro INIT_XMM sse2 SAD_X_SSE2 3, 16, 16, 7 SAD_X_SSE2 3, 16, 8, 7 SAD_X_SSE2 3, 8, 16, 7 SAD_X_SSE2 3, 8, 8, 7 SAD_X_SSE2 3, 8, 4, 7 SAD_X_SSE2 4, 16, 16, 7 SAD_X_SSE2 4, 16, 8, 7 SAD_X_SSE2 4, 8, 16, 7 SAD_X_SSE2 4, 8, 8, 7 SAD_X_SSE2 4, 8, 4, 7 INIT_XMM sse3 SAD_X_SSE2 3, 16, 16, 7 SAD_X_SSE2 3, 16, 8, 7 SAD_X_SSE2 3, 16, 4, 7 SAD_X_SSE2 4, 16, 16, 7 SAD_X_SSE2 4, 16, 8, 7 SAD_X_SSE2 4, 16, 4, 7 INIT_XMM ssse3 SAD_X3_W12 SAD_X3_W32 SAD_X3_W24 SAD_X3_W48 SAD_X3_W64 SAD_X_SSE2 3, 16, 64, 7 SAD_X_SSE2 3, 16, 32, 7 SAD_X_SSE2 3, 16, 16, 7 SAD_X_SSE2 3, 16, 12, 7 SAD_X_SSE2 3, 16, 8, 7 SAD_X_SSE2 3, 8, 32, 7 SAD_X_SSE2 3, 8, 16, 7 SAD_X4_W12 SAD_X4_W24 SAD_X4_W32 SAD_X4_W48 SAD_X4_W64 SAD_X_SSE2 4, 16, 64, 7 SAD_X_SSE2 4, 16, 32, 7 SAD_X_SSE2 4, 16, 16, 7 SAD_X_SSE2 4, 16, 12, 7 SAD_X_SSE2 4, 16, 8, 7 SAD_X_SSE2 4, 8, 32, 7 SAD_X_SSE2 4, 8, 16, 7 SAD_X_SSE2 4, 8, 8, 7 SAD_X_SSE2 4, 8, 4, 7 INIT_XMM avx SAD_X3_W12 SAD_X3_W32 SAD_X3_W24 SAD_X3_W48 SAD_X3_W64 SAD_X_SSE2 3, 16, 64, 7 SAD_X_SSE2 3, 16, 32, 6 SAD_X_SSE2 3, 16, 16, 6 SAD_X_SSE2 3, 16, 12, 6 SAD_X_SSE2 3, 16, 8, 6 SAD_X_SSE2 3, 16, 4, 6 SAD_X4_W12 SAD_X4_W24 SAD_X4_W32 SAD_X4_W48 SAD_X4_W64 SAD_X_SSE2 4, 16, 64, 7 SAD_X_SSE2 4, 16, 32, 7 SAD_X_SSE2 4, 16, 16, 7 SAD_X_SSE2 4, 16, 12, 7 SAD_X_SSE2 4, 16, 8, 7 SAD_X_SSE2 4, 16, 4, 7 %macro SAD_X_AVX2 4 cglobal pixel_sad_x%1_%2x%3, 2+%1,3+%1,%4 %assign x 0 %rep %3/4 SAD_X%1_4x%2P_AVX2 x, %3/4 %assign x x+1 %endrep %if (%1==4) && (%2==32) SAD_X%1_32P_END_AVX2 %else SAD_X%1_END_AVX2 %endif %endmacro INIT_YMM avx2 SAD_X_AVX2 3, 16, 32, 7 SAD_X_AVX2 3, 16, 16, 7 SAD_X_AVX2 3, 16, 12, 7 SAD_X_AVX2 3, 16, 8, 7 SAD_X_AVX2 4, 16, 32, 8 SAD_X_AVX2 4, 16, 16, 8 SAD_X_AVX2 4, 16, 12, 8 SAD_X_AVX2 4, 16, 8, 8 SAD_X_AVX2 4, 32, 8, 8 SAD_X_AVX2 4, 32, 16, 8 SAD_X_AVX2 4, 32, 24, 8 SAD_X_AVX2 4, 32, 32, 8 SAD_X_AVX2 4, 32, 64, 8 ;============================================================================= ; SAD cacheline split ;============================================================================= ; Core2 (Conroe) can load unaligned data just as quickly as aligned data... ; unless the unaligned data spans the border between 2 cachelines, in which ; case it's really slow. The exact numbers may differ, but all Intel cpus prior ; to Nehalem have a large penalty for cacheline splits. ; (8-byte alignment exactly half way between two cachelines is ok though.) ; LDDQU was supposed to fix this, but it only works on Pentium 4. ; So in the split case we load aligned data and explicitly perform the ; alignment between registers. Like on archs that have only aligned loads, ; except complicated by the fact that PALIGNR takes only an immediate, not ; a variable alignment. ; It is also possible to hoist the realignment to the macroblock level (keep ; 2 copies of the reference frame, offset by 32 bytes), but the extra memory ; needed for that method makes it often slower. ; sad 16x16 costs on Core2: ; good offsets: 49 cycles (50/64 of all mvs) ; cacheline split: 234 cycles (14/64 of all mvs. ammortized: +40 cycles) ; page split: 3600 cycles (14/4096 of all mvs. ammortized: +11.5 cycles) ; cache or page split with palignr: 57 cycles (ammortized: +2 cycles) ; computed jump assumes this loop is exactly 80 bytes %macro SAD16_CACHELINE_LOOP_SSE2 1 ; alignment ALIGN 16 sad_w16_align%1_sse2: movdqa xmm1, [r2+16] movdqa xmm2, [r2+r3+16] movdqa xmm3, [r2] movdqa xmm4, [r2+r3] pslldq xmm1, 16-%1 pslldq xmm2, 16-%1 psrldq xmm3, %1 psrldq xmm4, %1 por xmm1, xmm3 por xmm2, xmm4 psadbw xmm1, [r0] psadbw xmm2, [r0+r1] paddw xmm0, xmm1 paddw xmm0, xmm2 lea r0, [r0+2*r1] lea r2, [r2+2*r3] dec r4 jg sad_w16_align%1_sse2 ret %endmacro ; computed jump assumes this loop is exactly 64 bytes %macro SAD16_CACHELINE_LOOP_SSSE3 1 ; alignment ALIGN 16 sad_w16_align%1_ssse3: movdqa xmm1, [r2+16] movdqa xmm2, [r2+r3+16] palignr xmm1, [r2], %1 palignr xmm2, [r2+r3], %1 psadbw xmm1, [r0] psadbw xmm2, [r0+r1] paddw xmm0, xmm1 paddw xmm0, xmm2 lea r0, [r0+2*r1] lea r2, [r2+2*r3] dec r4 jg sad_w16_align%1_ssse3 ret %endmacro %macro SAD16_CACHELINE_FUNC 2 ; cpu, height cglobal pixel_sad_16x%2_cache64_%1 mov eax, r2m and eax, 0x37 cmp eax, 0x30 jle pixel_sad_16x%2_sse2 PROLOGUE 4,6 mov r4d, r2d and r4d, 15 %ifidn %1, ssse3 shl r4d, 6 ; code size = 64 %else lea r4, [r4*5] shl r4d, 4 ; code size = 80 %endif %define sad_w16_addr (sad_w16_align1_%1 + (sad_w16_align1_%1 - sad_w16_align2_%1)) %ifdef PIC lea r5, [sad_w16_addr] add r5, r4 %else lea r5, [sad_w16_addr + r4] %endif and r2, ~15 mov r4d, %2/2 pxor xmm0, xmm0 call r5 movhlps xmm1, xmm0 paddw xmm0, xmm1 movd eax, xmm0 RET %endmacro %macro SAD_CACHELINE_START_MMX2 4 ; width, height, iterations, cacheline mov eax, r2m and eax, 0x17|%1|(%4>>1) cmp eax, 0x10|%1|(%4>>1) jle pixel_sad_%1x%2_mmx2 and eax, 7 shl eax, 3 movd mm6, [pd_64] movd mm7, eax psubw mm6, mm7 PROLOGUE 4,5 and r2, ~7 mov r4d, %3 pxor mm0, mm0 %endmacro %macro SAD16_CACHELINE_FUNC_MMX2 2 ; height, cacheline cglobal pixel_sad_16x%1_cache%2_mmx2 SAD_CACHELINE_START_MMX2 16, %1, %1, %2 .loop: movq mm1, [r2] movq mm2, [r2+8] movq mm3, [r2+16] movq mm4, mm2 psrlq mm1, mm7 psllq mm2, mm6 psllq mm3, mm6 psrlq mm4, mm7 por mm1, mm2 por mm3, mm4 psadbw mm1, [r0] psadbw mm3, [r0+8] paddw mm0, mm1 paddw mm0, mm3 add r2, r3 add r0, r1 dec r4 jg .loop movd eax, mm0 RET %endmacro %macro SAD8_CACHELINE_FUNC_MMX2 2 ; height, cacheline cglobal pixel_sad_8x%1_cache%2_mmx2 SAD_CACHELINE_START_MMX2 8, %1, %1/2, %2 .loop: movq mm1, [r2+8] movq mm2, [r2+r3+8] movq mm3, [r2] movq mm4, [r2+r3] psllq mm1, mm6 psllq mm2, mm6 psrlq mm3, mm7 psrlq mm4, mm7 por mm1, mm3 por mm2, mm4 psadbw mm1, [r0] psadbw mm2, [r0+r1] paddw mm0, mm1 paddw mm0, mm2 lea r2, [r2+2*r3] lea r0, [r0+2*r1] dec r4 jg .loop movd eax, mm0 RET %endmacro ; sad_x3/x4_cache64: check each mv. ; if they're all within a cacheline, use normal sad_x3/x4. ; otherwise, send them individually to sad_cache64. %macro CHECK_SPLIT 3 ; pix, width, cacheline mov eax, %1 and eax, 0x17|%2|(%3>>1) cmp eax, 0x10|%2|(%3>>1) jg .split %endmacro %macro SADX3_CACHELINE_FUNC 6 ; width, height, cacheline, normal_ver, split_ver, name cglobal pixel_sad_x3_%1x%2_cache%3_%6 CHECK_SPLIT r1m, %1, %3 CHECK_SPLIT r2m, %1, %3 CHECK_SPLIT r3m, %1, %3 jmp pixel_sad_x3_%1x%2_%4 .split: %if ARCH_X86_64 PROLOGUE 6,9 push r3 push r2 %if WIN64 movsxd r4, r4d sub rsp, 40 ; shadow space and alignment %endif mov r2, r1 mov r1, FENC_STRIDE mov r3, r4 mov r7, r0 mov r8, r5 call pixel_sad_%1x%2_cache%3_%5 mov [r8], eax %if WIN64 mov r2, [rsp+40+0*8] %else pop r2 %endif mov r0, r7 call pixel_sad_%1x%2_cache%3_%5 mov [r8+4], eax %if WIN64 mov r2, [rsp+40+1*8] %else pop r2 %endif mov r0, r7 call pixel_sad_%1x%2_cache%3_%5 mov [r8+8], eax %if WIN64 add rsp, 40+2*8 %endif RET %else push edi mov edi, [esp+28] push dword [esp+24] push dword [esp+16] push dword 16 push dword [esp+20] call pixel_sad_%1x%2_cache%3_%5 mov ecx, [esp+32] mov [edi], eax mov [esp+8], ecx call pixel_sad_%1x%2_cache%3_%5 mov ecx, [esp+36] mov [edi+4], eax mov [esp+8], ecx call pixel_sad_%1x%2_cache%3_%5 mov [edi+8], eax add esp, 16 pop edi ret %endif %endmacro %macro SADX4_CACHELINE_FUNC 6 ; width, height, cacheline, normal_ver, split_ver, name cglobal pixel_sad_x4_%1x%2_cache%3_%6 CHECK_SPLIT r1m, %1, %3 CHECK_SPLIT r2m, %1, %3 CHECK_SPLIT r3m, %1, %3 CHECK_SPLIT r4m, %1, %3 jmp pixel_sad_x4_%1x%2_%4 .split: %if ARCH_X86_64 PROLOGUE 6,9 mov r8, r6mp push r4 push r3 push r2 %if WIN64 sub rsp, 32 ; shadow space %endif mov r2, r1 mov r1, FENC_STRIDE mov r3, r5 mov r7, r0 call pixel_sad_%1x%2_cache%3_%5 mov [r8], eax %if WIN64 mov r2, [rsp+32+0*8] %else pop r2 %endif mov r0, r7 call pixel_sad_%1x%2_cache%3_%5 mov [r8+4], eax %if WIN64 mov r2, [rsp+32+1*8] %else pop r2 %endif mov r0, r7 call pixel_sad_%1x%2_cache%3_%5 mov [r8+8], eax %if WIN64 mov r2, [rsp+32+2*8] %else pop r2 %endif mov r0, r7 call pixel_sad_%1x%2_cache%3_%5 mov [r8+12], eax %if WIN64 add rsp, 32+3*8 %endif RET %else push edi mov edi, [esp+32] push dword [esp+28] push dword [esp+16] push dword 16 push dword [esp+20] call pixel_sad_%1x%2_cache%3_%5 mov ecx, [esp+32] mov [edi], eax mov [esp+8], ecx call pixel_sad_%1x%2_cache%3_%5 mov ecx, [esp+36] mov [edi+4], eax mov [esp+8], ecx call pixel_sad_%1x%2_cache%3_%5 mov ecx, [esp+40] mov [edi+8], eax mov [esp+8], ecx call pixel_sad_%1x%2_cache%3_%5 mov [edi+12], eax add esp, 16 pop edi ret %endif %endmacro %macro SADX34_CACHELINE_FUNC 1+ SADX3_CACHELINE_FUNC %1 SADX4_CACHELINE_FUNC %1 %endmacro ; instantiate the aligned sads INIT_MMX %if ARCH_X86_64 == 0 SAD16_CACHELINE_FUNC_MMX2 8, 32 SAD16_CACHELINE_FUNC_MMX2 16, 32 SAD8_CACHELINE_FUNC_MMX2 4, 32 SAD8_CACHELINE_FUNC_MMX2 8, 32 SAD8_CACHELINE_FUNC_MMX2 16, 32 SAD16_CACHELINE_FUNC_MMX2 8, 64 SAD16_CACHELINE_FUNC_MMX2 16, 64 %endif ; !ARCH_X86_64 SAD8_CACHELINE_FUNC_MMX2 4, 64 SAD8_CACHELINE_FUNC_MMX2 8, 64 SAD8_CACHELINE_FUNC_MMX2 16, 64 %if ARCH_X86_64 == 0 SADX34_CACHELINE_FUNC 16, 16, 32, mmx2, mmx2, mmx2 SADX34_CACHELINE_FUNC 16, 8, 32, mmx2, mmx2, mmx2 SADX34_CACHELINE_FUNC 8, 16, 32, mmx2, mmx2, mmx2 SADX34_CACHELINE_FUNC 8, 8, 32, mmx2, mmx2, mmx2 SADX34_CACHELINE_FUNC 16, 16, 64, mmx2, mmx2, mmx2 SADX34_CACHELINE_FUNC 16, 8, 64, mmx2, mmx2, mmx2 %endif ; !ARCH_X86_64 SADX34_CACHELINE_FUNC 8, 16, 64, mmx2, mmx2, mmx2 SADX34_CACHELINE_FUNC 8, 8, 64, mmx2, mmx2, mmx2 %if ARCH_X86_64 == 0 SAD16_CACHELINE_FUNC sse2, 8 SAD16_CACHELINE_FUNC sse2, 16 %assign i 1 %rep 15 SAD16_CACHELINE_LOOP_SSE2 i %assign i i+1 %endrep SADX34_CACHELINE_FUNC 16, 16, 64, sse2, sse2, sse2 SADX34_CACHELINE_FUNC 16, 8, 64, sse2, sse2, sse2 %endif ; !ARCH_X86_64 SADX34_CACHELINE_FUNC 8, 16, 64, sse2, mmx2, sse2 SAD16_CACHELINE_FUNC ssse3, 8 SAD16_CACHELINE_FUNC ssse3, 16 %assign i 1 %rep 15 SAD16_CACHELINE_LOOP_SSSE3 i %assign i i+1 %endrep SADX34_CACHELINE_FUNC 16, 16, 64, sse2, ssse3, ssse3 SADX34_CACHELINE_FUNC 16, 8, 64, sse2, ssse3, ssse3 %if HIGH_BIT_DEPTH==0 INIT_YMM avx2 cglobal pixel_sad_x3_8x4, 6,6,5 xorps m0, m0 xorps m1, m1 sub r2, r1 ; rebase on pointer r1 sub r3, r1 ; row 0 vpbroadcastq xm2, [r0 + 0 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 add r1, r4 ; row 1 vpbroadcastq xm2, [r0 + 1 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 add r1, r4 ; row 2 vpbroadcastq xm2, [r0 + 2 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 add r1, r4 ; row 3 vpbroadcastq xm2, [r0 + 3 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 pshufd xm0, xm0, q0020 movq [r5 + 0], xm0 movd [r5 + 8], xm1 RET INIT_YMM avx2 cglobal pixel_sad_x3_8x8, 6,6,5 xorps m0, m0 xorps m1, m1 sub r2, r1 ; rebase on pointer r1 sub r3, r1 %assign x 0 %rep 4 ; row 0 vpbroadcastq xm2, [r0 + 0 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 add r1, r4 ; row 1 vpbroadcastq xm2, [r0 + 1 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 %assign x x+1 %if x < 4 add r1, r4 add r0, 2 * FENC_STRIDE %endif %endrep pshufd xm0, xm0, q0020 movq [r5 + 0], xm0 movd [r5 + 8], xm1 RET INIT_YMM avx2 cglobal pixel_sad_x3_8x16, 6,6,5 xorps m0, m0 xorps m1, m1 sub r2, r1 ; rebase on pointer r1 sub r3, r1 %assign x 0 %rep 8 ; row 0 vpbroadcastq xm2, [r0 + 0 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 add r1, r4 ; row 1 vpbroadcastq xm2, [r0 + 1 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 %assign x x+1 %if x < 8 add r1, r4 add r0, 2 * FENC_STRIDE %endif %endrep pshufd xm0, xm0, q0020 movq [r5 + 0], xm0 movd [r5 + 8], xm1 RET INIT_YMM avx2 cglobal pixel_sad_x4_8x8, 7,7,5 xorps m0, m0 xorps m1, m1 sub r2, r1 ; rebase on pointer r1 sub r3, r1 sub r4, r1 %assign x 0 %rep 4 ; row 0 vpbroadcastq xm2, [r0 + 0 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] movhps xm4, [r1 + r4] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 add r1, r5 ; row 1 vpbroadcastq xm2, [r0 + 1 * FENC_STRIDE] movq xm3, [r1] movhps xm3, [r1 + r2] movq xm4, [r1 + r3] movhps xm4, [r1 + r4] psadbw xm3, xm2 psadbw xm4, xm2 paddd xm0, xm3 paddd xm1, xm4 %assign x x+1 %if x < 4 add r1, r5 add r0, 2 * FENC_STRIDE %endif %endrep pshufd xm0, xm0, q0020 pshufd xm1, xm1, q0020 movq [r6 + 0], xm0 movq [r6 + 8], xm1 RET INIT_YMM avx2 cglobal pixel_sad_32x8, 4,4,6 xorps m0, m0 xorps m5, m5 movu m1, [r0] ; row 0 of pix0 movu m2, [r2] ; row 0 of pix1 movu m3, [r0 + r1] ; row 1 of pix0 movu m4, [r2 + r3] ; row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] movu m1, [r0] ; row 2 of pix0 movu m2, [r2] ; row 2 of pix1 movu m3, [r0 + r1] ; row 3 of pix0 movu m4, [r2 + r3] ; row 3 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] movu m1, [r0] ; row 4 of pix0 movu m2, [r2] ; row 4 of pix1 movu m3, [r0 + r1] ; row 5 of pix0 movu m4, [r2 + r3] ; row 5 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] movu m1, [r0] ; row 6 of pix0 movu m2, [r2] ; row 6 of pix1 movu m3, [r0 + r1] ; row 7 of pix0 movu m4, [r2 + r3] ; row 7 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 paddd m0, m5 vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_32x16, 4,5,6 xorps m0, m0 xorps m5, m5 mov r4d, 4 .loop movu m1, [r0] ; row 0 of pix0 movu m2, [r2] ; row 0 of pix1 movu m3, [r0 + r1] ; row 1 of pix0 movu m4, [r2 + r3] ; row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] movu m1, [r0] ; row 2 of pix0 movu m2, [r2] ; row 2 of pix1 movu m3, [r0 + r1] ; row 3 of pix0 movu m4, [r2 + r3] ; row 3 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] dec r4d jnz .loop paddd m0, m5 vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_32x24, 4,7,6 xorps m0, m0 xorps m5, m5 mov r4d, 6 lea r5, [r1 * 3] lea r6, [r3 * 3] .loop movu m1, [r0] ; row 0 of pix0 movu m2, [r2] ; row 0 of pix1 movu m3, [r0 + r1] ; row 1 of pix0 movu m4, [r2 + r3] ; row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + 2 * r1] ; row 2 of pix0 movu m2, [r2 + 2 * r3] ; row 2 of pix1 movu m3, [r0 + r5] ; row 3 of pix0 movu m4, [r2 + r6] ; row 3 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 4 * r3] lea r0, [r0 + 4 * r1] dec r4d jnz .loop paddd m0, m5 vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_32x32, 4,7,5 xorps m0, m0 mov r4d, 32/4 lea r5, [r1 * 3] lea r6, [r3 * 3] .loop movu m1, [r0] ; row 0 of pix0 movu m2, [r2] ; row 0 of pix1 movu m3, [r0 + r1] ; row 1 of pix0 movu m4, [r2 + r3] ; row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m0, m3 movu m1, [r0 + 2 * r1] ; row 2 of pix0 movu m2, [r2 + 2 * r3] ; row 2 of pix1 movu m3, [r0 + r5] ; row 3 of pix0 movu m4, [r2 + r6] ; row 3 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m0, m3 lea r2, [r2 + 4 * r3] lea r0, [r0 + 4 * r1] dec r4d jnz .loop vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_32x64, 4,7,5 xorps m0, m0 mov r4d, 64/8 lea r5, [r1 * 3] lea r6, [r3 * 3] .loop movu m1, [r0] ; row 0 of pix0 movu m2, [r2] ; row 0 of pix1 movu m3, [r0 + r1] ; row 1 of pix0 movu m4, [r2 + r3] ; row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m0, m3 movu m1, [r0 + 2 * r1] ; row 2 of pix0 movu m2, [r2 + 2 * r3] ; row 2 of pix1 movu m3, [r0 + r5] ; row 3 of pix0 movu m4, [r2 + r6] ; row 3 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m0, m3 lea r2, [r2 + 4 * r3] lea r0, [r0 + 4 * r1] movu m1, [r0] ; row 4 of pix0 movu m2, [r2] ; row 4 of pix1 movu m3, [r0 + r1] ; row 5 of pix0 movu m4, [r2 + r3] ; row 5 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m0, m3 movu m1, [r0 + 2 * r1] ; row 6 of pix0 movu m2, [r2 + 2 * r3] ; row 6 of pix1 movu m3, [r0 + r5] ; row 7 of pix0 movu m4, [r2 + r6] ; row 7 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m0, m3 lea r2, [r2 + 4 * r3] lea r0, [r0 + 4 * r1] dec r4d jnz .loop vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_48x64, 4,7,7 xorps m0, m0 mov r4d, 64/4 lea r5, [r1 * 3] lea r6, [r3 * 3] .loop movu m1, [r0] ; row 0 of pix0 movu m2, [r2] ; row 0 of pix1 movu m3, [r0 + r1] ; row 1 of pix0 movu m4, [r2 + r3] ; row 1 of pix1 movu xm5, [r0 +32] ; last 16 of row 0 of pix0 vinserti128 m5, m5, [r0 + r1 + 32], 1 movu xm6, [r2 +32] ; last 16 of row 0 of pix1 vinserti128 m6, m6, [r2 + r3 + 32], 1 psadbw m1, m2 psadbw m3, m4 psadbw m5, m6 paddd m0, m1 paddd m0, m3 paddd m0, m5 movu m1, [r0 + 2 * r1] ; row 2 of pix0 movu m2, [r2 + 2 * r3] ; row 2 of pix1 movu m3, [r0 + r5] ; row 3 of pix0 movu m4, [r2 + r6] ; row 3 of pix1 movu xm5, [r0 +32 + 2 * r1] vinserti128 m5, m5, [r0 + r5 + 32], 1 movu xm6, [r2 +32 + 2 * r3] vinserti128 m6, m6, [r2 + r6 + 32], 1 psadbw m1, m2 psadbw m3, m4 psadbw m5, m6 paddd m0, m1 paddd m0, m3 paddd m0, m5 lea r2, [r2 + 4 * r3] lea r0, [r0 + 4 * r1] dec r4d jnz .loop vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_64x16, 4,5,6 xorps m0, m0 xorps m5, m5 mov r4d, 4 .loop movu m1, [r0] ; first 32 of row 0 of pix0 movu m2, [r2] ; first 32 of row 0 of pix1 movu m3, [r0 + 32] ; second 32 of row 0 of pix0 movu m4, [r2 + 32] ; second 32 of row 0 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r1] ; first 32 of row 1 of pix0 movu m2, [r2 + r3] ; first 32 of row 1 of pix1 movu m3, [r0 + 32 + r1] ; second 32 of row 1 of pix0 movu m4, [r2 + 32 + r3] ; second 32 of row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] movu m1, [r0] ; first 32 of row 2 of pix0 movu m2, [r2] ; first 32 of row 2 of pix1 movu m3, [r0 + 32] ; second 32 of row 2 of pix0 movu m4, [r2 + 32] ; second 32 of row 2 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r1] ; first 32 of row 3 of pix0 movu m2, [r2 + r3] ; first 32 of row 3 of pix1 movu m3, [r0 + 32 + r1] ; second 32 of row 3 of pix0 movu m4, [r2 + 32 + r3] ; second 32 of row 3 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] dec r4d jnz .loop paddd m0, m5 vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_64x32, 4,5,6 xorps m0, m0 xorps m5, m5 mov r4d, 16 .loop movu m1, [r0] ; first 32 of row 0 of pix0 movu m2, [r2] ; first 32 of row 0 of pix1 movu m3, [r0 + 32] ; second 32 of row 0 of pix0 movu m4, [r2 + 32] ; second 32 of row 0 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r1] ; first 32 of row 1 of pix0 movu m2, [r2 + r3] ; first 32 of row 1 of pix1 movu m3, [r0 + 32 + r1] ; second 32 of row 1 of pix0 movu m4, [r2 + 32 + r3] ; second 32 of row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 2 * r3] lea r0, [r0 + 2 * r1] dec r4d jnz .loop paddd m0, m5 vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_64x48, 4,7,6 xorps m0, m0 xorps m5, m5 mov r4d, 12 lea r5, [r1 * 3] lea r6, [r3 * 3] .loop movu m1, [r0] ; first 32 of row 0 of pix0 movu m2, [r2] ; first 32 of row 0 of pix1 movu m3, [r0 + 32] ; second 32 of row 0 of pix0 movu m4, [r2 + 32] ; second 32 of row 0 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r1] ; first 32 of row 1 of pix0 movu m2, [r2 + r3] ; first 32 of row 1 of pix1 movu m3, [r0 + 32 + r1] ; second 32 of row 1 of pix0 movu m4, [r2 + 32 + r3] ; second 32 of row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + 2 * r1] ; first 32 of row 0 of pix0 movu m2, [r2 + 2 * r3] ; first 32 of row 0 of pix1 movu m3, [r0 + 2 * r1 + 32] ; second 32 of row 0 of pix0 movu m4, [r2 + 2 * r3 + 32] ; second 32 of row 0 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r5] ; first 32 of row 1 of pix0 movu m2, [r2 + r6] ; first 32 of row 1 of pix1 movu m3, [r0 + 32 + r5] ; second 32 of row 1 of pix0 movu m4, [r2 + 32 + r6] ; second 32 of row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 4 * r3] lea r0, [r0 + 4 * r1] dec r4d jnz .loop paddd m0, m5 vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET INIT_YMM avx2 cglobal pixel_sad_64x64, 4,7,6 xorps m0, m0 xorps m5, m5 mov r4d, 8 lea r5, [r1 * 3] lea r6, [r3 * 3] .loop movu m1, [r0] ; first 32 of row 0 of pix0 movu m2, [r2] ; first 32 of row 0 of pix1 movu m3, [r0 + 32] ; second 32 of row 0 of pix0 movu m4, [r2 + 32] ; second 32 of row 0 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r1] ; first 32 of row 1 of pix0 movu m2, [r2 + r3] ; first 32 of row 1 of pix1 movu m3, [r0 + 32 + r1] ; second 32 of row 1 of pix0 movu m4, [r2 + 32 + r3] ; second 32 of row 1 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + 2 * r1] ; first 32 of row 2 of pix0 movu m2, [r2 + 2 * r3] ; first 32 of row 2 of pix1 movu m3, [r0 + 2 * r1 + 32] ; second 32 of row 2 of pix0 movu m4, [r2 + 2 * r3 + 32] ; second 32 of row 2 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r5] ; first 32 of row 3 of pix0 movu m2, [r2 + r6] ; first 32 of row 3 of pix1 movu m3, [r0 + 32 + r5] ; second 32 of row 3 of pix0 movu m4, [r2 + 32 + r6] ; second 32 of row 3 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 4 * r3] lea r0, [r0 + 4 * r1] movu m1, [r0] ; first 32 of row 4 of pix0 movu m2, [r2] ; first 32 of row 4 of pix1 movu m3, [r0 + 32] ; second 32 of row 4 of pix0 movu m4, [r2 + 32] ; second 32 of row 4 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r1] ; first 32 of row 5 of pix0 movu m2, [r2 + r3] ; first 32 of row 5 of pix1 movu m3, [r0 + 32 + r1] ; second 32 of row 5 of pix0 movu m4, [r2 + 32 + r3] ; second 32 of row 5 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + 2 * r1] ; first 32 of row 6 of pix0 movu m2, [r2 + 2 * r3] ; first 32 of row 6 of pix1 movu m3, [r0 + 2 * r1 + 32] ; second 32 of row 6 of pix0 movu m4, [r2 + 2 * r3 + 32] ; second 32 of row 6 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 movu m1, [r0 + r5] ; first 32 of row 7 of pix0 movu m2, [r2 + r6] ; first 32 of row 7 of pix1 movu m3, [r0 + 32 + r5] ; second 32 of row 7 of pix0 movu m4, [r2 + 32 + r6] ; second 32 of row 7 of pix1 psadbw m1, m2 psadbw m3, m4 paddd m0, m1 paddd m5, m3 lea r2, [r2 + 4 * r3] lea r0, [r0 + 4 * r1] dec r4d jnz .loop paddd m0, m5 vextracti128 xm1, m0, 1 paddd xm0, xm1 pshufd xm1, xm0, 2 paddd xm0,xm1 movd eax, xm0 RET %endif