| Message ID | 20220410004250.1448020-3-goldstein.w.n@gmail.com |
|---|---|
| State | New |
| Headers | show |
| Series | None | expand |
Disregard this patch. It's from the wrong patchset. On Sat, Apr 9, 2022 at 7:46 PM Noah Goldstein <goldstein.w.n@gmail.com> wrote: > > The goal is to remove most SSSE3 function as SSE4, AVX2, and EVEX are > generally preferable. memcpy/memmove is one exception where avoiding > unaligned loads with `palignr` is important for some targets. > > This commit replaces memmove-ssse3 with a better optimized are lower > code footprint verion. As well it aliases memcpy to memmove. > > Aside from this function all other SSSE3 functions should be safe to > remove. > > The performance is not changed drastically although shows overall > improvements without any major regressions or gains. > > bench-memcpy geometric_mean(N=50) New / Original: 0.962 > > bench-memcpy-random geometric_mean(N=50) New / Original: 0.895 > > bench-memcpy-large geometric_mean(N=50) New / Original: 0.894 > > Benchmarks where run on Zhaoxin KX-6840@2000MHz See attached numbers > for all results. > > More important this saves 7246 bytes of code size in memmove an > additional 10741 bytes by reusing memmove code for memcpy (total 17987 > bytes saves). As well an additional 896 bytes of rodata for the jump > table entries. > --- > sysdeps/x86_64/multiarch/Makefile | 1 - > sysdeps/x86_64/multiarch/memcpy-ssse3.S | 3151 ---------------------- > sysdeps/x86_64/multiarch/memmove-ssse3.S | 386 ++- > 3 files changed, 382 insertions(+), 3156 deletions(-) > delete mode 100644 sysdeps/x86_64/multiarch/memcpy-ssse3.S > > diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile > index 303fb5d734..e7ea963fc0 100644 > --- a/sysdeps/x86_64/multiarch/Makefile > +++ b/sysdeps/x86_64/multiarch/Makefile > @@ -16,7 +16,6 @@ sysdep_routines += \ > memcmpeq-avx2-rtm \ > memcmpeq-evex \ > memcmpeq-sse2 \ > - memcpy-ssse3 \ > memmove-avx-unaligned-erms \ > memmove-avx-unaligned-erms-rtm \ > memmove-avx512-no-vzeroupper \ > diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3.S b/sysdeps/x86_64/multiarch/memcpy-ssse3.S > deleted file mode 100644 > index 65644d3a09..0000000000 > --- a/sysdeps/x86_64/multiarch/memcpy-ssse3.S > +++ /dev/null > @@ -1,3151 +0,0 @@ > -/* memcpy with SSSE3 > - Copyright (C) 2010-2022 Free Software Foundation, Inc. > - This file is part of the GNU C Library. > - > - The GNU C Library is free software; you can redistribute it and/or > - modify it under the terms of the GNU Lesser General Public > - License as published by the Free Software Foundation; either > - version 2.1 of the License, or (at your option) any later version. > - > - The GNU C Library is distributed in the hope that it will be useful, > - but WITHOUT ANY WARRANTY; without even the implied warranty of > - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU > - Lesser General Public License for more details. > - > - You should have received a copy of the GNU Lesser General Public > - License along with the GNU C Library; if not, see > - <https://www.gnu.org/licenses/>. */ > - > -#include <sysdep.h> > - > -#if IS_IN (libc) > - > -#include "asm-syntax.h" > - > -#ifndef MEMCPY > -# define MEMCPY __memcpy_ssse3 > -# define MEMCPY_CHK __memcpy_chk_ssse3 > -# define MEMPCPY __mempcpy_ssse3 > -# define MEMPCPY_CHK __mempcpy_chk_ssse3 > -#endif > - > -#define JMPTBL(I, B) I - B > - > -/* Branch to an entry in a jump table. TABLE is a jump table with > - relative offsets. INDEX is a register contains the index into the > - jump table. SCALE is the scale of INDEX. */ > -#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ > - lea TABLE(%rip), %r11; \ > - movslq (%r11, INDEX, SCALE), INDEX; \ > - lea (%r11, INDEX), INDEX; \ > - _CET_NOTRACK jmp *INDEX; \ > - ud2 > - > - .section .text.ssse3,"ax",@progbits > -#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE > -ENTRY (MEMPCPY_CHK) > - cmp %RDX_LP, %RCX_LP > - jb HIDDEN_JUMPTARGET (__chk_fail) > -END (MEMPCPY_CHK) > - > -ENTRY (MEMPCPY) > - mov %RDI_LP, %RAX_LP > - add %RDX_LP, %RAX_LP > - jmp L(start) > -END (MEMPCPY) > -#endif > - > -#if !defined USE_AS_BCOPY > -ENTRY (MEMCPY_CHK) > - cmp %RDX_LP, %RCX_LP > - jb HIDDEN_JUMPTARGET (__chk_fail) > -END (MEMCPY_CHK) > -#endif > - > -ENTRY (MEMCPY) > - mov %RDI_LP, %RAX_LP > -#ifdef USE_AS_MEMPCPY > - add %RDX_LP, %RAX_LP > -#endif > - > -#ifdef __ILP32__ > - /* Clear the upper 32 bits. */ > - mov %edx, %edx > -#endif > - > -#ifdef USE_AS_MEMMOVE > - cmp %rsi, %rdi > - jb L(copy_forward) > - je L(write_0bytes) > - cmp $79, %rdx > - jbe L(copy_forward) > - jmp L(copy_backward) > -L(copy_forward): > -#endif > -L(start): > - cmp $79, %rdx > - lea L(table_less_80bytes)(%rip), %r11 > - ja L(80bytesormore) > - movslq (%r11, %rdx, 4), %r9 > - add %rdx, %rsi > - add %rdx, %rdi > - add %r11, %r9 > - _CET_NOTRACK jmp *%r9 > - ud2 > - > - .p2align 4 > -L(80bytesormore): > -#ifndef USE_AS_MEMMOVE > - cmp %dil, %sil > - jle L(copy_backward) > -#endif > - > - movdqu (%rsi), %xmm0 > - mov %rdi, %rcx > - and $-16, %rdi > - add $16, %rdi > - mov %rcx, %r8 > - sub %rdi, %rcx > - add %rcx, %rdx > - sub %rcx, %rsi > - > -#ifdef SHARED_CACHE_SIZE_HALF > - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP > -#else > - mov __x86_shared_cache_size_half(%rip), %RCX_LP > -#endif > - cmp %rcx, %rdx > - mov %rsi, %r9 > - ja L(large_page_fwd) > - and $0xf, %r9 > - jz L(shl_0) > -#ifdef DATA_CACHE_SIZE_HALF > - mov $DATA_CACHE_SIZE_HALF, %RCX_LP > -#else > - mov __x86_data_cache_size_half(%rip), %RCX_LP > -#endif > - BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %r9, 4) > - > - .p2align 4 > -L(copy_backward): > - movdqu -16(%rsi, %rdx), %xmm0 > - add %rdx, %rsi > - lea -16(%rdi, %rdx), %r8 > - add %rdx, %rdi > - > - mov %rdi, %rcx > - and $0xf, %rcx > - xor %rcx, %rdi > - sub %rcx, %rdx > - sub %rcx, %rsi > - > -#ifdef SHARED_CACHE_SIZE_HALF > - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP > -#else > - mov __x86_shared_cache_size_half(%rip), %RCX_LP > -#endif > - > - cmp %rcx, %rdx > - mov %rsi, %r9 > - ja L(large_page_bwd) > - and $0xf, %r9 > - jz L(shl_0_bwd) > -#ifdef DATA_CACHE_SIZE_HALF > - mov $DATA_CACHE_SIZE_HALF, %RCX_LP > -#else > - mov __x86_data_cache_size_half(%rip), %RCX_LP > -#endif > - BRANCH_TO_JMPTBL_ENTRY (L(shl_table_bwd), %r9, 4) > - > - .p2align 4 > -L(shl_0): > - sub $16, %rdx > - movdqa (%rsi), %xmm1 > - add $16, %rsi > - movdqa %xmm1, (%rdi) > - add $16, %rdi > - cmp $128, %rdx > - movdqu %xmm0, (%r8) > - ja L(shl_0_gobble) > - cmp $64, %rdx > - jb L(shl_0_less_64bytes) > - movaps (%rsi), %xmm4 > - movaps 16(%rsi), %xmm1 > - movaps 32(%rsi), %xmm2 > - movaps 48(%rsi), %xmm3 > - movaps %xmm4, (%rdi) > - movaps %xmm1, 16(%rdi) > - movaps %xmm2, 32(%rdi) > - movaps %xmm3, 48(%rdi) > - sub $64, %rdx > - add $64, %rsi > - add $64, %rdi > -L(shl_0_less_64bytes): > - add %rdx, %rsi > - add %rdx, %rdi > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_0_gobble): > -#ifdef DATA_CACHE_SIZE_HALF > - cmp $DATA_CACHE_SIZE_HALF, %RDX_LP > -#else > - cmp __x86_data_cache_size_half(%rip), %RDX_LP > -#endif > - lea -128(%rdx), %rdx > - jae L(shl_0_gobble_mem_loop) > -L(shl_0_gobble_cache_loop): > - movdqa (%rsi), %xmm4 > - movaps 0x10(%rsi), %xmm1 > - movaps 0x20(%rsi), %xmm2 > - movaps 0x30(%rsi), %xmm3 > - > - movdqa %xmm4, (%rdi) > - movaps %xmm1, 0x10(%rdi) > - movaps %xmm2, 0x20(%rdi) > - movaps %xmm3, 0x30(%rdi) > - > - sub $128, %rdx > - movaps 0x40(%rsi), %xmm4 > - movaps 0x50(%rsi), %xmm5 > - movaps 0x60(%rsi), %xmm6 > - movaps 0x70(%rsi), %xmm7 > - lea 0x80(%rsi), %rsi > - movaps %xmm4, 0x40(%rdi) > - movaps %xmm5, 0x50(%rdi) > - movaps %xmm6, 0x60(%rdi) > - movaps %xmm7, 0x70(%rdi) > - lea 0x80(%rdi), %rdi > - > - jae L(shl_0_gobble_cache_loop) > - cmp $-0x40, %rdx > - lea 0x80(%rdx), %rdx > - jl L(shl_0_cache_less_64bytes) > - > - movdqa (%rsi), %xmm4 > - sub $0x40, %rdx > - movdqa 0x10(%rsi), %xmm1 > - > - movdqa %xmm4, (%rdi) > - movdqa %xmm1, 0x10(%rdi) > - > - movdqa 0x20(%rsi), %xmm4 > - movdqa 0x30(%rsi), %xmm1 > - add $0x40, %rsi > - > - movdqa %xmm4, 0x20(%rdi) > - movdqa %xmm1, 0x30(%rdi) > - add $0x40, %rdi > -L(shl_0_cache_less_64bytes): > - add %rdx, %rsi > - add %rdx, %rdi > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_0_gobble_mem_loop): > - prefetcht0 0x1c0(%rsi) > - prefetcht0 0x280(%rsi) > - > - movdqa (%rsi), %xmm0 > - movdqa 0x10(%rsi), %xmm1 > - movdqa 0x20(%rsi), %xmm2 > - movdqa 0x30(%rsi), %xmm3 > - movdqa 0x40(%rsi), %xmm4 > - movdqa 0x50(%rsi), %xmm5 > - movdqa 0x60(%rsi), %xmm6 > - movdqa 0x70(%rsi), %xmm7 > - lea 0x80(%rsi), %rsi > - sub $0x80, %rdx > - movdqa %xmm0, (%rdi) > - movdqa %xmm1, 0x10(%rdi) > - movdqa %xmm2, 0x20(%rdi) > - movdqa %xmm3, 0x30(%rdi) > - movdqa %xmm4, 0x40(%rdi) > - movdqa %xmm5, 0x50(%rdi) > - movdqa %xmm6, 0x60(%rdi) > - movdqa %xmm7, 0x70(%rdi) > - lea 0x80(%rdi), %rdi > - > - jae L(shl_0_gobble_mem_loop) > - cmp $-0x40, %rdx > - lea 0x80(%rdx), %rdx > - jl L(shl_0_mem_less_64bytes) > - > - movdqa (%rsi), %xmm0 > - sub $0x40, %rdx > - movdqa 0x10(%rsi), %xmm1 > - > - movdqa %xmm0, (%rdi) > - movdqa %xmm1, 0x10(%rdi) > - > - movdqa 0x20(%rsi), %xmm0 > - movdqa 0x30(%rsi), %xmm1 > - add $0x40, %rsi > - > - movdqa %xmm0, 0x20(%rdi) > - movdqa %xmm1, 0x30(%rdi) > - add $0x40, %rdi > -L(shl_0_mem_less_64bytes): > - cmp $0x20, %rdx > - jb L(shl_0_mem_less_32bytes) > - movdqa (%rsi), %xmm0 > - sub $0x20, %rdx > - movdqa 0x10(%rsi), %xmm1 > - add $0x20, %rsi > - movdqa %xmm0, (%rdi) > - movdqa %xmm1, 0x10(%rdi) > - add $0x20, %rdi > -L(shl_0_mem_less_32bytes): > - add %rdx, %rdi > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_0_bwd): > - sub $16, %rdx > - movdqa -0x10(%rsi), %xmm1 > - sub $16, %rsi > - movdqa %xmm1, -0x10(%rdi) > - sub $16, %rdi > - cmp $0x80, %rdx > - movdqu %xmm0, (%r8) > - ja L(shl_0_gobble_bwd) > - cmp $64, %rdx > - jb L(shl_0_less_64bytes_bwd) > - movaps -0x10(%rsi), %xmm0 > - movaps -0x20(%rsi), %xmm1 > - movaps -0x30(%rsi), %xmm2 > - movaps -0x40(%rsi), %xmm3 > - movaps %xmm0, -0x10(%rdi) > - movaps %xmm1, -0x20(%rdi) > - movaps %xmm2, -0x30(%rdi) > - movaps %xmm3, -0x40(%rdi) > - sub $64, %rdx > - sub $0x40, %rsi > - sub $0x40, %rdi > -L(shl_0_less_64bytes_bwd): > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_0_gobble_bwd): > -#ifdef DATA_CACHE_SIZE_HALF > - cmp $DATA_CACHE_SIZE_HALF, %RDX_LP > -#else > - cmp __x86_data_cache_size_half(%rip), %RDX_LP > -#endif > - lea -128(%rdx), %rdx > - jae L(shl_0_gobble_mem_bwd_loop) > -L(shl_0_gobble_bwd_loop): > - movdqa -0x10(%rsi), %xmm0 > - movaps -0x20(%rsi), %xmm1 > - movaps -0x30(%rsi), %xmm2 > - movaps -0x40(%rsi), %xmm3 > - > - movdqa %xmm0, -0x10(%rdi) > - movaps %xmm1, -0x20(%rdi) > - movaps %xmm2, -0x30(%rdi) > - movaps %xmm3, -0x40(%rdi) > - > - sub $0x80, %rdx > - movaps -0x50(%rsi), %xmm4 > - movaps -0x60(%rsi), %xmm5 > - movaps -0x70(%rsi), %xmm6 > - movaps -0x80(%rsi), %xmm7 > - lea -0x80(%rsi), %rsi > - movaps %xmm4, -0x50(%rdi) > - movaps %xmm5, -0x60(%rdi) > - movaps %xmm6, -0x70(%rdi) > - movaps %xmm7, -0x80(%rdi) > - lea -0x80(%rdi), %rdi > - > - jae L(shl_0_gobble_bwd_loop) > - cmp $-0x40, %rdx > - lea 0x80(%rdx), %rdx > - jl L(shl_0_gobble_bwd_less_64bytes) > - > - movdqa -0x10(%rsi), %xmm0 > - sub $0x40, %rdx > - movdqa -0x20(%rsi), %xmm1 > - > - movdqa %xmm0, -0x10(%rdi) > - movdqa %xmm1, -0x20(%rdi) > - > - movdqa -0x30(%rsi), %xmm0 > - movdqa -0x40(%rsi), %xmm1 > - sub $0x40, %rsi > - > - movdqa %xmm0, -0x30(%rdi) > - movdqa %xmm1, -0x40(%rdi) > - sub $0x40, %rdi > -L(shl_0_gobble_bwd_less_64bytes): > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_0_gobble_mem_bwd_loop): > - prefetcht0 -0x1c0(%rsi) > - prefetcht0 -0x280(%rsi) > - movdqa -0x10(%rsi), %xmm0 > - movdqa -0x20(%rsi), %xmm1 > - movdqa -0x30(%rsi), %xmm2 > - movdqa -0x40(%rsi), %xmm3 > - movdqa -0x50(%rsi), %xmm4 > - movdqa -0x60(%rsi), %xmm5 > - movdqa -0x70(%rsi), %xmm6 > - movdqa -0x80(%rsi), %xmm7 > - lea -0x80(%rsi), %rsi > - sub $0x80, %rdx > - movdqa %xmm0, -0x10(%rdi) > - movdqa %xmm1, -0x20(%rdi) > - movdqa %xmm2, -0x30(%rdi) > - movdqa %xmm3, -0x40(%rdi) > - movdqa %xmm4, -0x50(%rdi) > - movdqa %xmm5, -0x60(%rdi) > - movdqa %xmm6, -0x70(%rdi) > - movdqa %xmm7, -0x80(%rdi) > - lea -0x80(%rdi), %rdi > - > - jae L(shl_0_gobble_mem_bwd_loop) > - cmp $-0x40, %rdx > - lea 0x80(%rdx), %rdx > - jl L(shl_0_mem_bwd_less_64bytes) > - > - movdqa -0x10(%rsi), %xmm0 > - sub $0x40, %rdx > - movdqa -0x20(%rsi), %xmm1 > - > - movdqa %xmm0, -0x10(%rdi) > - movdqa %xmm1, -0x20(%rdi) > - > - movdqa -0x30(%rsi), %xmm0 > - movdqa -0x40(%rsi), %xmm1 > - sub $0x40, %rsi > - > - movdqa %xmm0, -0x30(%rdi) > - movdqa %xmm1, -0x40(%rdi) > - sub $0x40, %rdi > -L(shl_0_mem_bwd_less_64bytes): > - cmp $0x20, %rdx > - jb L(shl_0_mem_bwd_less_32bytes) > - movdqa -0x10(%rsi), %xmm0 > - sub $0x20, %rdx > - movdqa -0x20(%rsi), %xmm1 > - sub $0x20, %rsi > - movdqa %xmm0, -0x10(%rdi) > - movdqa %xmm1, -0x20(%rdi) > - sub $0x20, %rdi > -L(shl_0_mem_bwd_less_32bytes): > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_1): > - lea (L(shl_1_loop_L1)-L(shl_1))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x01(%rsi), %xmm1 > - jb L(L1_fwd) > - lea (L(shl_1_loop_L2)-L(shl_1_loop_L1))(%r9), %r9 > -L(L1_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_1_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_1_loop_L1): > - sub $64, %rdx > - movaps 0x0f(%rsi), %xmm2 > - movaps 0x1f(%rsi), %xmm3 > - movaps 0x2f(%rsi), %xmm4 > - movaps 0x3f(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $1, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $1, %xmm3, %xmm4 > - palignr $1, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $1, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_1_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_1_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_1_bwd): > - lea (L(shl_1_bwd_loop_L1)-L(shl_1_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x01(%rsi), %xmm1 > - jb L(L1_bwd) > - lea (L(shl_1_bwd_loop_L2)-L(shl_1_bwd_loop_L1))(%r9), %r9 > -L(L1_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_1_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_1_bwd_loop_L1): > - movaps -0x11(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x21(%rsi), %xmm3 > - movaps -0x31(%rsi), %xmm4 > - movaps -0x41(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $1, %xmm2, %xmm1 > - palignr $1, %xmm3, %xmm2 > - palignr $1, %xmm4, %xmm3 > - palignr $1, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_1_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_1_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_2): > - lea (L(shl_2_loop_L1)-L(shl_2))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x02(%rsi), %xmm1 > - jb L(L2_fwd) > - lea (L(shl_2_loop_L2)-L(shl_2_loop_L1))(%r9), %r9 > -L(L2_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_2_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_2_loop_L1): > - sub $64, %rdx > - movaps 0x0e(%rsi), %xmm2 > - movaps 0x1e(%rsi), %xmm3 > - movaps 0x2e(%rsi), %xmm4 > - movaps 0x3e(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $2, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $2, %xmm3, %xmm4 > - palignr $2, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $2, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_2_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_2_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_2_bwd): > - lea (L(shl_2_bwd_loop_L1)-L(shl_2_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x02(%rsi), %xmm1 > - jb L(L2_bwd) > - lea (L(shl_2_bwd_loop_L2)-L(shl_2_bwd_loop_L1))(%r9), %r9 > -L(L2_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_2_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_2_bwd_loop_L1): > - movaps -0x12(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x22(%rsi), %xmm3 > - movaps -0x32(%rsi), %xmm4 > - movaps -0x42(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $2, %xmm2, %xmm1 > - palignr $2, %xmm3, %xmm2 > - palignr $2, %xmm4, %xmm3 > - palignr $2, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_2_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_2_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_3): > - lea (L(shl_3_loop_L1)-L(shl_3))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x03(%rsi), %xmm1 > - jb L(L3_fwd) > - lea (L(shl_3_loop_L2)-L(shl_3_loop_L1))(%r9), %r9 > -L(L3_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_3_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_3_loop_L1): > - sub $64, %rdx > - movaps 0x0d(%rsi), %xmm2 > - movaps 0x1d(%rsi), %xmm3 > - movaps 0x2d(%rsi), %xmm4 > - movaps 0x3d(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $3, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $3, %xmm3, %xmm4 > - palignr $3, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $3, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_3_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_3_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_3_bwd): > - lea (L(shl_3_bwd_loop_L1)-L(shl_3_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x03(%rsi), %xmm1 > - jb L(L3_bwd) > - lea (L(shl_3_bwd_loop_L2)-L(shl_3_bwd_loop_L1))(%r9), %r9 > -L(L3_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_3_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_3_bwd_loop_L1): > - movaps -0x13(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x23(%rsi), %xmm3 > - movaps -0x33(%rsi), %xmm4 > - movaps -0x43(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $3, %xmm2, %xmm1 > - palignr $3, %xmm3, %xmm2 > - palignr $3, %xmm4, %xmm3 > - palignr $3, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_3_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_3_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_4): > - lea (L(shl_4_loop_L1)-L(shl_4))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x04(%rsi), %xmm1 > - jb L(L4_fwd) > - lea (L(shl_4_loop_L2)-L(shl_4_loop_L1))(%r9), %r9 > -L(L4_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_4_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_4_loop_L1): > - sub $64, %rdx > - movaps 0x0c(%rsi), %xmm2 > - movaps 0x1c(%rsi), %xmm3 > - movaps 0x2c(%rsi), %xmm4 > - movaps 0x3c(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $4, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $4, %xmm3, %xmm4 > - palignr $4, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $4, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_4_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_4_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_4_bwd): > - lea (L(shl_4_bwd_loop_L1)-L(shl_4_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x04(%rsi), %xmm1 > - jb L(L4_bwd) > - lea (L(shl_4_bwd_loop_L2)-L(shl_4_bwd_loop_L1))(%r9), %r9 > -L(L4_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_4_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_4_bwd_loop_L1): > - movaps -0x14(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x24(%rsi), %xmm3 > - movaps -0x34(%rsi), %xmm4 > - movaps -0x44(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $4, %xmm2, %xmm1 > - palignr $4, %xmm3, %xmm2 > - palignr $4, %xmm4, %xmm3 > - palignr $4, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_4_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_4_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_5): > - lea (L(shl_5_loop_L1)-L(shl_5))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x05(%rsi), %xmm1 > - jb L(L5_fwd) > - lea (L(shl_5_loop_L2)-L(shl_5_loop_L1))(%r9), %r9 > -L(L5_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_5_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_5_loop_L1): > - sub $64, %rdx > - movaps 0x0b(%rsi), %xmm2 > - movaps 0x1b(%rsi), %xmm3 > - movaps 0x2b(%rsi), %xmm4 > - movaps 0x3b(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $5, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $5, %xmm3, %xmm4 > - palignr $5, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $5, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_5_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_5_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_5_bwd): > - lea (L(shl_5_bwd_loop_L1)-L(shl_5_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x05(%rsi), %xmm1 > - jb L(L5_bwd) > - lea (L(shl_5_bwd_loop_L2)-L(shl_5_bwd_loop_L1))(%r9), %r9 > -L(L5_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_5_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_5_bwd_loop_L1): > - movaps -0x15(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x25(%rsi), %xmm3 > - movaps -0x35(%rsi), %xmm4 > - movaps -0x45(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $5, %xmm2, %xmm1 > - palignr $5, %xmm3, %xmm2 > - palignr $5, %xmm4, %xmm3 > - palignr $5, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_5_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_5_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_6): > - lea (L(shl_6_loop_L1)-L(shl_6))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x06(%rsi), %xmm1 > - jb L(L6_fwd) > - lea (L(shl_6_loop_L2)-L(shl_6_loop_L1))(%r9), %r9 > -L(L6_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_6_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_6_loop_L1): > - sub $64, %rdx > - movaps 0x0a(%rsi), %xmm2 > - movaps 0x1a(%rsi), %xmm3 > - movaps 0x2a(%rsi), %xmm4 > - movaps 0x3a(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $6, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $6, %xmm3, %xmm4 > - palignr $6, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $6, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_6_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_6_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_6_bwd): > - lea (L(shl_6_bwd_loop_L1)-L(shl_6_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x06(%rsi), %xmm1 > - jb L(L6_bwd) > - lea (L(shl_6_bwd_loop_L2)-L(shl_6_bwd_loop_L1))(%r9), %r9 > -L(L6_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_6_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_6_bwd_loop_L1): > - movaps -0x16(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x26(%rsi), %xmm3 > - movaps -0x36(%rsi), %xmm4 > - movaps -0x46(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $6, %xmm2, %xmm1 > - palignr $6, %xmm3, %xmm2 > - palignr $6, %xmm4, %xmm3 > - palignr $6, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_6_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_6_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_7): > - lea (L(shl_7_loop_L1)-L(shl_7))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x07(%rsi), %xmm1 > - jb L(L7_fwd) > - lea (L(shl_7_loop_L2)-L(shl_7_loop_L1))(%r9), %r9 > -L(L7_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_7_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_7_loop_L1): > - sub $64, %rdx > - movaps 0x09(%rsi), %xmm2 > - movaps 0x19(%rsi), %xmm3 > - movaps 0x29(%rsi), %xmm4 > - movaps 0x39(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $7, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $7, %xmm3, %xmm4 > - palignr $7, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $7, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_7_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_7_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_7_bwd): > - lea (L(shl_7_bwd_loop_L1)-L(shl_7_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x07(%rsi), %xmm1 > - jb L(L7_bwd) > - lea (L(shl_7_bwd_loop_L2)-L(shl_7_bwd_loop_L1))(%r9), %r9 > -L(L7_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_7_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_7_bwd_loop_L1): > - movaps -0x17(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x27(%rsi), %xmm3 > - movaps -0x37(%rsi), %xmm4 > - movaps -0x47(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $7, %xmm2, %xmm1 > - palignr $7, %xmm3, %xmm2 > - palignr $7, %xmm4, %xmm3 > - palignr $7, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_7_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_7_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_8): > - lea (L(shl_8_loop_L1)-L(shl_8))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x08(%rsi), %xmm1 > - jb L(L8_fwd) > - lea (L(shl_8_loop_L2)-L(shl_8_loop_L1))(%r9), %r9 > -L(L8_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > -L(shl_8_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_8_loop_L1): > - sub $64, %rdx > - movaps 0x08(%rsi), %xmm2 > - movaps 0x18(%rsi), %xmm3 > - movaps 0x28(%rsi), %xmm4 > - movaps 0x38(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $8, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $8, %xmm3, %xmm4 > - palignr $8, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $8, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_8_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > - .p2align 4 > -L(shl_8_end): > - lea 64(%rdx), %rdx > - movaps %xmm4, -0x20(%rdi) > - add %rdx, %rsi > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_8_bwd): > - lea (L(shl_8_bwd_loop_L1)-L(shl_8_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x08(%rsi), %xmm1 > - jb L(L8_bwd) > - lea (L(shl_8_bwd_loop_L2)-L(shl_8_bwd_loop_L1))(%r9), %r9 > -L(L8_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_8_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_8_bwd_loop_L1): > - movaps -0x18(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x28(%rsi), %xmm3 > - movaps -0x38(%rsi), %xmm4 > - movaps -0x48(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $8, %xmm2, %xmm1 > - palignr $8, %xmm3, %xmm2 > - palignr $8, %xmm4, %xmm3 > - palignr $8, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_8_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_8_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_9): > - lea (L(shl_9_loop_L1)-L(shl_9))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x09(%rsi), %xmm1 > - jb L(L9_fwd) > - lea (L(shl_9_loop_L2)-L(shl_9_loop_L1))(%r9), %r9 > -L(L9_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_9_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_9_loop_L1): > - sub $64, %rdx > - movaps 0x07(%rsi), %xmm2 > - movaps 0x17(%rsi), %xmm3 > - movaps 0x27(%rsi), %xmm4 > - movaps 0x37(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $9, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $9, %xmm3, %xmm4 > - palignr $9, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $9, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_9_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_9_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_9_bwd): > - lea (L(shl_9_bwd_loop_L1)-L(shl_9_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x09(%rsi), %xmm1 > - jb L(L9_bwd) > - lea (L(shl_9_bwd_loop_L2)-L(shl_9_bwd_loop_L1))(%r9), %r9 > -L(L9_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_9_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_9_bwd_loop_L1): > - movaps -0x19(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x29(%rsi), %xmm3 > - movaps -0x39(%rsi), %xmm4 > - movaps -0x49(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $9, %xmm2, %xmm1 > - palignr $9, %xmm3, %xmm2 > - palignr $9, %xmm4, %xmm3 > - palignr $9, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_9_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_9_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_10): > - lea (L(shl_10_loop_L1)-L(shl_10))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0a(%rsi), %xmm1 > - jb L(L10_fwd) > - lea (L(shl_10_loop_L2)-L(shl_10_loop_L1))(%r9), %r9 > -L(L10_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_10_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_10_loop_L1): > - sub $64, %rdx > - movaps 0x06(%rsi), %xmm2 > - movaps 0x16(%rsi), %xmm3 > - movaps 0x26(%rsi), %xmm4 > - movaps 0x36(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $10, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $10, %xmm3, %xmm4 > - palignr $10, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $10, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_10_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_10_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_10_bwd): > - lea (L(shl_10_bwd_loop_L1)-L(shl_10_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0a(%rsi), %xmm1 > - jb L(L10_bwd) > - lea (L(shl_10_bwd_loop_L2)-L(shl_10_bwd_loop_L1))(%r9), %r9 > -L(L10_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_10_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_10_bwd_loop_L1): > - movaps -0x1a(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x2a(%rsi), %xmm3 > - movaps -0x3a(%rsi), %xmm4 > - movaps -0x4a(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $10, %xmm2, %xmm1 > - palignr $10, %xmm3, %xmm2 > - palignr $10, %xmm4, %xmm3 > - palignr $10, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_10_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_10_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_11): > - lea (L(shl_11_loop_L1)-L(shl_11))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0b(%rsi), %xmm1 > - jb L(L11_fwd) > - lea (L(shl_11_loop_L2)-L(shl_11_loop_L1))(%r9), %r9 > -L(L11_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_11_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_11_loop_L1): > - sub $64, %rdx > - movaps 0x05(%rsi), %xmm2 > - movaps 0x15(%rsi), %xmm3 > - movaps 0x25(%rsi), %xmm4 > - movaps 0x35(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $11, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $11, %xmm3, %xmm4 > - palignr $11, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $11, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_11_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_11_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_11_bwd): > - lea (L(shl_11_bwd_loop_L1)-L(shl_11_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0b(%rsi), %xmm1 > - jb L(L11_bwd) > - lea (L(shl_11_bwd_loop_L2)-L(shl_11_bwd_loop_L1))(%r9), %r9 > -L(L11_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_11_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_11_bwd_loop_L1): > - movaps -0x1b(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x2b(%rsi), %xmm3 > - movaps -0x3b(%rsi), %xmm4 > - movaps -0x4b(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $11, %xmm2, %xmm1 > - palignr $11, %xmm3, %xmm2 > - palignr $11, %xmm4, %xmm3 > - palignr $11, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_11_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_11_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_12): > - lea (L(shl_12_loop_L1)-L(shl_12))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0c(%rsi), %xmm1 > - jb L(L12_fwd) > - lea (L(shl_12_loop_L2)-L(shl_12_loop_L1))(%r9), %r9 > -L(L12_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_12_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_12_loop_L1): > - sub $64, %rdx > - movaps 0x04(%rsi), %xmm2 > - movaps 0x14(%rsi), %xmm3 > - movaps 0x24(%rsi), %xmm4 > - movaps 0x34(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $12, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $12, %xmm3, %xmm4 > - palignr $12, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $12, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_12_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_12_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_12_bwd): > - lea (L(shl_12_bwd_loop_L1)-L(shl_12_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0c(%rsi), %xmm1 > - jb L(L12_bwd) > - lea (L(shl_12_bwd_loop_L2)-L(shl_12_bwd_loop_L1))(%r9), %r9 > -L(L12_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_12_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_12_bwd_loop_L1): > - movaps -0x1c(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x2c(%rsi), %xmm3 > - movaps -0x3c(%rsi), %xmm4 > - movaps -0x4c(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $12, %xmm2, %xmm1 > - palignr $12, %xmm3, %xmm2 > - palignr $12, %xmm4, %xmm3 > - palignr $12, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_12_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_12_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_13): > - lea (L(shl_13_loop_L1)-L(shl_13))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0d(%rsi), %xmm1 > - jb L(L13_fwd) > - lea (L(shl_13_loop_L2)-L(shl_13_loop_L1))(%r9), %r9 > -L(L13_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_13_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_13_loop_L1): > - sub $64, %rdx > - movaps 0x03(%rsi), %xmm2 > - movaps 0x13(%rsi), %xmm3 > - movaps 0x23(%rsi), %xmm4 > - movaps 0x33(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $13, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $13, %xmm3, %xmm4 > - palignr $13, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $13, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_13_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_13_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_13_bwd): > - lea (L(shl_13_bwd_loop_L1)-L(shl_13_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0d(%rsi), %xmm1 > - jb L(L13_bwd) > - lea (L(shl_13_bwd_loop_L2)-L(shl_13_bwd_loop_L1))(%r9), %r9 > -L(L13_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_13_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_13_bwd_loop_L1): > - movaps -0x1d(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x2d(%rsi), %xmm3 > - movaps -0x3d(%rsi), %xmm4 > - movaps -0x4d(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $13, %xmm2, %xmm1 > - palignr $13, %xmm3, %xmm2 > - palignr $13, %xmm4, %xmm3 > - palignr $13, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_13_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_13_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_14): > - lea (L(shl_14_loop_L1)-L(shl_14))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0e(%rsi), %xmm1 > - jb L(L14_fwd) > - lea (L(shl_14_loop_L2)-L(shl_14_loop_L1))(%r9), %r9 > -L(L14_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_14_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_14_loop_L1): > - sub $64, %rdx > - movaps 0x02(%rsi), %xmm2 > - movaps 0x12(%rsi), %xmm3 > - movaps 0x22(%rsi), %xmm4 > - movaps 0x32(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $14, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $14, %xmm3, %xmm4 > - palignr $14, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $14, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_14_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_14_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_14_bwd): > - lea (L(shl_14_bwd_loop_L1)-L(shl_14_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0e(%rsi), %xmm1 > - jb L(L14_bwd) > - lea (L(shl_14_bwd_loop_L2)-L(shl_14_bwd_loop_L1))(%r9), %r9 > -L(L14_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_14_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_14_bwd_loop_L1): > - movaps -0x1e(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x2e(%rsi), %xmm3 > - movaps -0x3e(%rsi), %xmm4 > - movaps -0x4e(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $14, %xmm2, %xmm1 > - palignr $14, %xmm3, %xmm2 > - palignr $14, %xmm4, %xmm3 > - palignr $14, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_14_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_14_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_15): > - lea (L(shl_15_loop_L1)-L(shl_15))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0f(%rsi), %xmm1 > - jb L(L15_fwd) > - lea (L(shl_15_loop_L2)-L(shl_15_loop_L1))(%r9), %r9 > -L(L15_fwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_15_loop_L2): > - prefetchnta 0x1c0(%rsi) > -L(shl_15_loop_L1): > - sub $64, %rdx > - movaps 0x01(%rsi), %xmm2 > - movaps 0x11(%rsi), %xmm3 > - movaps 0x21(%rsi), %xmm4 > - movaps 0x31(%rsi), %xmm5 > - movdqa %xmm5, %xmm6 > - palignr $15, %xmm4, %xmm5 > - lea 64(%rsi), %rsi > - palignr $15, %xmm3, %xmm4 > - palignr $15, %xmm2, %xmm3 > - lea 64(%rdi), %rdi > - palignr $15, %xmm1, %xmm2 > - movdqa %xmm6, %xmm1 > - movdqa %xmm2, -0x40(%rdi) > - movaps %xmm3, -0x30(%rdi) > - jb L(shl_15_end) > - movaps %xmm4, -0x20(%rdi) > - movaps %xmm5, -0x10(%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_15_end): > - movaps %xmm4, -0x20(%rdi) > - lea 64(%rdx), %rdx > - movaps %xmm5, -0x10(%rdi) > - add %rdx, %rdi > - movdqu %xmm0, (%r8) > - add %rdx, %rsi > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(shl_15_bwd): > - lea (L(shl_15_bwd_loop_L1)-L(shl_15_bwd))(%r9), %r9 > - cmp %rcx, %rdx > - movaps -0x0f(%rsi), %xmm1 > - jb L(L15_bwd) > - lea (L(shl_15_bwd_loop_L2)-L(shl_15_bwd_loop_L1))(%r9), %r9 > -L(L15_bwd): > - lea -64(%rdx), %rdx > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_15_bwd_loop_L2): > - prefetchnta -0x1c0(%rsi) > -L(shl_15_bwd_loop_L1): > - movaps -0x1f(%rsi), %xmm2 > - sub $0x40, %rdx > - movaps -0x2f(%rsi), %xmm3 > - movaps -0x3f(%rsi), %xmm4 > - movaps -0x4f(%rsi), %xmm5 > - lea -0x40(%rsi), %rsi > - palignr $15, %xmm2, %xmm1 > - palignr $15, %xmm3, %xmm2 > - palignr $15, %xmm4, %xmm3 > - palignr $15, %xmm5, %xmm4 > - > - movaps %xmm1, -0x10(%rdi) > - movaps %xmm5, %xmm1 > - > - movaps %xmm2, -0x20(%rdi) > - lea -0x40(%rdi), %rdi > - > - movaps %xmm3, 0x10(%rdi) > - jb L(shl_15_bwd_end) > - movaps %xmm4, (%rdi) > - _CET_NOTRACK jmp *%r9 > - ud2 > -L(shl_15_bwd_end): > - movaps %xmm4, (%rdi) > - lea 64(%rdx), %rdx > - movdqu %xmm0, (%r8) > - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) > - > - .p2align 4 > -L(write_72bytes): > - movdqu -72(%rsi), %xmm0 > - movdqu -56(%rsi), %xmm1 > - mov -40(%rsi), %r8 > - mov -32(%rsi), %r9 > - mov -24(%rsi), %r10 > - mov -16(%rsi), %r11 > - mov -8(%rsi), %rcx > - movdqu %xmm0, -72(%rdi) > - movdqu %xmm1, -56(%rdi) > - mov %r8, -40(%rdi) > - mov %r9, -32(%rdi) > - mov %r10, -24(%rdi) > - mov %r11, -16(%rdi) > - mov %rcx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_64bytes): > - movdqu -64(%rsi), %xmm0 > - mov -48(%rsi), %rcx > - mov -40(%rsi), %r8 > - mov -32(%rsi), %r9 > - mov -24(%rsi), %r10 > - mov -16(%rsi), %r11 > - mov -8(%rsi), %rdx > - movdqu %xmm0, -64(%rdi) > - mov %rcx, -48(%rdi) > - mov %r8, -40(%rdi) > - mov %r9, -32(%rdi) > - mov %r10, -24(%rdi) > - mov %r11, -16(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_56bytes): > - movdqu -56(%rsi), %xmm0 > - mov -40(%rsi), %r8 > - mov -32(%rsi), %r9 > - mov -24(%rsi), %r10 > - mov -16(%rsi), %r11 > - mov -8(%rsi), %rcx > - movdqu %xmm0, -56(%rdi) > - mov %r8, -40(%rdi) > - mov %r9, -32(%rdi) > - mov %r10, -24(%rdi) > - mov %r11, -16(%rdi) > - mov %rcx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_48bytes): > - mov -48(%rsi), %rcx > - mov -40(%rsi), %r8 > - mov -32(%rsi), %r9 > - mov -24(%rsi), %r10 > - mov -16(%rsi), %r11 > - mov -8(%rsi), %rdx > - mov %rcx, -48(%rdi) > - mov %r8, -40(%rdi) > - mov %r9, -32(%rdi) > - mov %r10, -24(%rdi) > - mov %r11, -16(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_40bytes): > - mov -40(%rsi), %r8 > - mov -32(%rsi), %r9 > - mov -24(%rsi), %r10 > - mov -16(%rsi), %r11 > - mov -8(%rsi), %rdx > - mov %r8, -40(%rdi) > - mov %r9, -32(%rdi) > - mov %r10, -24(%rdi) > - mov %r11, -16(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_32bytes): > - mov -32(%rsi), %r9 > - mov -24(%rsi), %r10 > - mov -16(%rsi), %r11 > - mov -8(%rsi), %rdx > - mov %r9, -32(%rdi) > - mov %r10, -24(%rdi) > - mov %r11, -16(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_24bytes): > - mov -24(%rsi), %r10 > - mov -16(%rsi), %r11 > - mov -8(%rsi), %rdx > - mov %r10, -24(%rdi) > - mov %r11, -16(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_16bytes): > - mov -16(%rsi), %r11 > - mov -8(%rsi), %rdx > - mov %r11, -16(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_8bytes): > - mov -8(%rsi), %rdx > - mov %rdx, -8(%rdi) > -L(write_0bytes): > - ret > - > - .p2align 4 > -L(write_73bytes): > - movdqu -73(%rsi), %xmm0 > - movdqu -57(%rsi), %xmm1 > - mov -41(%rsi), %rcx > - mov -33(%rsi), %r9 > - mov -25(%rsi), %r10 > - mov -17(%rsi), %r11 > - mov -9(%rsi), %r8 > - mov -4(%rsi), %edx > - movdqu %xmm0, -73(%rdi) > - movdqu %xmm1, -57(%rdi) > - mov %rcx, -41(%rdi) > - mov %r9, -33(%rdi) > - mov %r10, -25(%rdi) > - mov %r11, -17(%rdi) > - mov %r8, -9(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_65bytes): > - movdqu -65(%rsi), %xmm0 > - movdqu -49(%rsi), %xmm1 > - mov -33(%rsi), %r9 > - mov -25(%rsi), %r10 > - mov -17(%rsi), %r11 > - mov -9(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -65(%rdi) > - movdqu %xmm1, -49(%rdi) > - mov %r9, -33(%rdi) > - mov %r10, -25(%rdi) > - mov %r11, -17(%rdi) > - mov %rcx, -9(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_57bytes): > - movdqu -57(%rsi), %xmm0 > - mov -41(%rsi), %r8 > - mov -33(%rsi), %r9 > - mov -25(%rsi), %r10 > - mov -17(%rsi), %r11 > - mov -9(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -57(%rdi) > - mov %r8, -41(%rdi) > - mov %r9, -33(%rdi) > - mov %r10, -25(%rdi) > - mov %r11, -17(%rdi) > - mov %rcx, -9(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_49bytes): > - movdqu -49(%rsi), %xmm0 > - mov -33(%rsi), %r9 > - mov -25(%rsi), %r10 > - mov -17(%rsi), %r11 > - mov -9(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -49(%rdi) > - mov %r9, -33(%rdi) > - mov %r10, -25(%rdi) > - mov %r11, -17(%rdi) > - mov %rcx, -9(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_41bytes): > - mov -41(%rsi), %r8 > - mov -33(%rsi), %r9 > - mov -25(%rsi), %r10 > - mov -17(%rsi), %r11 > - mov -9(%rsi), %rcx > - mov -1(%rsi), %dl > - mov %r8, -41(%rdi) > - mov %r9, -33(%rdi) > - mov %r10, -25(%rdi) > - mov %r11, -17(%rdi) > - mov %rcx, -9(%rdi) > - mov %dl, -1(%rdi) > - ret > - > - .p2align 4 > -L(write_33bytes): > - mov -33(%rsi), %r9 > - mov -25(%rsi), %r10 > - mov -17(%rsi), %r11 > - mov -9(%rsi), %rcx > - mov -1(%rsi), %dl > - mov %r9, -33(%rdi) > - mov %r10, -25(%rdi) > - mov %r11, -17(%rdi) > - mov %rcx, -9(%rdi) > - mov %dl, -1(%rdi) > - ret > - > - .p2align 4 > -L(write_25bytes): > - mov -25(%rsi), %r10 > - mov -17(%rsi), %r11 > - mov -9(%rsi), %rcx > - mov -1(%rsi), %dl > - mov %r10, -25(%rdi) > - mov %r11, -17(%rdi) > - mov %rcx, -9(%rdi) > - mov %dl, -1(%rdi) > - ret > - > - .p2align 4 > -L(write_17bytes): > - mov -17(%rsi), %r11 > - mov -9(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r11, -17(%rdi) > - mov %rcx, -9(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_9bytes): > - mov -9(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %rcx, -9(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_1bytes): > - mov -1(%rsi), %dl > - mov %dl, -1(%rdi) > - ret > - > - .p2align 4 > -L(write_74bytes): > - movdqu -74(%rsi), %xmm0 > - movdqu -58(%rsi), %xmm1 > - mov -42(%rsi), %r8 > - mov -34(%rsi), %r9 > - mov -26(%rsi), %r10 > - mov -18(%rsi), %r11 > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -74(%rdi) > - movdqu %xmm1, -58(%rdi) > - mov %r8, -42(%rdi) > - mov %r9, -34(%rdi) > - mov %r10, -26(%rdi) > - mov %r11, -18(%rdi) > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_66bytes): > - movdqu -66(%rsi), %xmm0 > - movdqu -50(%rsi), %xmm1 > - mov -42(%rsi), %r8 > - mov -34(%rsi), %r9 > - mov -26(%rsi), %r10 > - mov -18(%rsi), %r11 > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -66(%rdi) > - movdqu %xmm1, -50(%rdi) > - mov %r8, -42(%rdi) > - mov %r9, -34(%rdi) > - mov %r10, -26(%rdi) > - mov %r11, -18(%rdi) > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_58bytes): > - movdqu -58(%rsi), %xmm1 > - mov -42(%rsi), %r8 > - mov -34(%rsi), %r9 > - mov -26(%rsi), %r10 > - mov -18(%rsi), %r11 > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm1, -58(%rdi) > - mov %r8, -42(%rdi) > - mov %r9, -34(%rdi) > - mov %r10, -26(%rdi) > - mov %r11, -18(%rdi) > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_50bytes): > - movdqu -50(%rsi), %xmm0 > - mov -34(%rsi), %r9 > - mov -26(%rsi), %r10 > - mov -18(%rsi), %r11 > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -50(%rdi) > - mov %r9, -34(%rdi) > - mov %r10, -26(%rdi) > - mov %r11, -18(%rdi) > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_42bytes): > - mov -42(%rsi), %r8 > - mov -34(%rsi), %r9 > - mov -26(%rsi), %r10 > - mov -18(%rsi), %r11 > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r8, -42(%rdi) > - mov %r9, -34(%rdi) > - mov %r10, -26(%rdi) > - mov %r11, -18(%rdi) > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_34bytes): > - mov -34(%rsi), %r9 > - mov -26(%rsi), %r10 > - mov -18(%rsi), %r11 > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r9, -34(%rdi) > - mov %r10, -26(%rdi) > - mov %r11, -18(%rdi) > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_26bytes): > - mov -26(%rsi), %r10 > - mov -18(%rsi), %r11 > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r10, -26(%rdi) > - mov %r11, -18(%rdi) > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_18bytes): > - mov -18(%rsi), %r11 > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r11, -18(%rdi) > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_10bytes): > - mov -10(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %rcx, -10(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_2bytes): > - mov -2(%rsi), %dx > - mov %dx, -2(%rdi) > - ret > - > - .p2align 4 > -L(write_75bytes): > - movdqu -75(%rsi), %xmm0 > - movdqu -59(%rsi), %xmm1 > - mov -43(%rsi), %r8 > - mov -35(%rsi), %r9 > - mov -27(%rsi), %r10 > - mov -19(%rsi), %r11 > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -75(%rdi) > - movdqu %xmm1, -59(%rdi) > - mov %r8, -43(%rdi) > - mov %r9, -35(%rdi) > - mov %r10, -27(%rdi) > - mov %r11, -19(%rdi) > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_67bytes): > - movdqu -67(%rsi), %xmm0 > - movdqu -59(%rsi), %xmm1 > - mov -43(%rsi), %r8 > - mov -35(%rsi), %r9 > - mov -27(%rsi), %r10 > - mov -19(%rsi), %r11 > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -67(%rdi) > - movdqu %xmm1, -59(%rdi) > - mov %r8, -43(%rdi) > - mov %r9, -35(%rdi) > - mov %r10, -27(%rdi) > - mov %r11, -19(%rdi) > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_59bytes): > - movdqu -59(%rsi), %xmm0 > - mov -43(%rsi), %r8 > - mov -35(%rsi), %r9 > - mov -27(%rsi), %r10 > - mov -19(%rsi), %r11 > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -59(%rdi) > - mov %r8, -43(%rdi) > - mov %r9, -35(%rdi) > - mov %r10, -27(%rdi) > - mov %r11, -19(%rdi) > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_51bytes): > - movdqu -51(%rsi), %xmm0 > - mov -35(%rsi), %r9 > - mov -27(%rsi), %r10 > - mov -19(%rsi), %r11 > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -51(%rdi) > - mov %r9, -35(%rdi) > - mov %r10, -27(%rdi) > - mov %r11, -19(%rdi) > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_43bytes): > - mov -43(%rsi), %r8 > - mov -35(%rsi), %r9 > - mov -27(%rsi), %r10 > - mov -19(%rsi), %r11 > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r8, -43(%rdi) > - mov %r9, -35(%rdi) > - mov %r10, -27(%rdi) > - mov %r11, -19(%rdi) > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_35bytes): > - mov -35(%rsi), %r9 > - mov -27(%rsi), %r10 > - mov -19(%rsi), %r11 > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r9, -35(%rdi) > - mov %r10, -27(%rdi) > - mov %r11, -19(%rdi) > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_27bytes): > - mov -27(%rsi), %r10 > - mov -19(%rsi), %r11 > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r10, -27(%rdi) > - mov %r11, -19(%rdi) > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_19bytes): > - mov -19(%rsi), %r11 > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r11, -19(%rdi) > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_11bytes): > - mov -11(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %rcx, -11(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_3bytes): > - mov -3(%rsi), %dx > - mov -2(%rsi), %cx > - mov %dx, -3(%rdi) > - mov %cx, -2(%rdi) > - ret > - > - .p2align 4 > -L(write_76bytes): > - movdqu -76(%rsi), %xmm0 > - movdqu -60(%rsi), %xmm1 > - mov -44(%rsi), %r8 > - mov -36(%rsi), %r9 > - mov -28(%rsi), %r10 > - mov -20(%rsi), %r11 > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -76(%rdi) > - movdqu %xmm1, -60(%rdi) > - mov %r8, -44(%rdi) > - mov %r9, -36(%rdi) > - mov %r10, -28(%rdi) > - mov %r11, -20(%rdi) > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_68bytes): > - movdqu -68(%rsi), %xmm0 > - movdqu -52(%rsi), %xmm1 > - mov -36(%rsi), %r9 > - mov -28(%rsi), %r10 > - mov -20(%rsi), %r11 > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -68(%rdi) > - movdqu %xmm1, -52(%rdi) > - mov %r9, -36(%rdi) > - mov %r10, -28(%rdi) > - mov %r11, -20(%rdi) > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_60bytes): > - movdqu -60(%rsi), %xmm0 > - mov -44(%rsi), %r8 > - mov -36(%rsi), %r9 > - mov -28(%rsi), %r10 > - mov -20(%rsi), %r11 > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -60(%rdi) > - mov %r8, -44(%rdi) > - mov %r9, -36(%rdi) > - mov %r10, -28(%rdi) > - mov %r11, -20(%rdi) > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_52bytes): > - movdqu -52(%rsi), %xmm0 > - mov -36(%rsi), %r9 > - mov -28(%rsi), %r10 > - mov -20(%rsi), %r11 > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - movdqu %xmm0, -52(%rdi) > - mov %r9, -36(%rdi) > - mov %r10, -28(%rdi) > - mov %r11, -20(%rdi) > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_44bytes): > - mov -44(%rsi), %r8 > - mov -36(%rsi), %r9 > - mov -28(%rsi), %r10 > - mov -20(%rsi), %r11 > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r8, -44(%rdi) > - mov %r9, -36(%rdi) > - mov %r10, -28(%rdi) > - mov %r11, -20(%rdi) > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_36bytes): > - mov -36(%rsi), %r9 > - mov -28(%rsi), %r10 > - mov -20(%rsi), %r11 > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r9, -36(%rdi) > - mov %r10, -28(%rdi) > - mov %r11, -20(%rdi) > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_28bytes): > - mov -28(%rsi), %r10 > - mov -20(%rsi), %r11 > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r10, -28(%rdi) > - mov %r11, -20(%rdi) > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_20bytes): > - mov -20(%rsi), %r11 > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %r11, -20(%rdi) > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_12bytes): > - mov -12(%rsi), %rcx > - mov -4(%rsi), %edx > - mov %rcx, -12(%rdi) > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_4bytes): > - mov -4(%rsi), %edx > - mov %edx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_77bytes): > - movdqu -77(%rsi), %xmm0 > - movdqu -61(%rsi), %xmm1 > - mov -45(%rsi), %r8 > - mov -37(%rsi), %r9 > - mov -29(%rsi), %r10 > - mov -21(%rsi), %r11 > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -77(%rdi) > - movdqu %xmm1, -61(%rdi) > - mov %r8, -45(%rdi) > - mov %r9, -37(%rdi) > - mov %r10, -29(%rdi) > - mov %r11, -21(%rdi) > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_69bytes): > - movdqu -69(%rsi), %xmm0 > - movdqu -53(%rsi), %xmm1 > - mov -37(%rsi), %r9 > - mov -29(%rsi), %r10 > - mov -21(%rsi), %r11 > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -69(%rdi) > - movdqu %xmm1, -53(%rdi) > - mov %r9, -37(%rdi) > - mov %r10, -29(%rdi) > - mov %r11, -21(%rdi) > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_61bytes): > - movdqu -61(%rsi), %xmm0 > - mov -45(%rsi), %r8 > - mov -37(%rsi), %r9 > - mov -29(%rsi), %r10 > - mov -21(%rsi), %r11 > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -61(%rdi) > - mov %r8, -45(%rdi) > - mov %r9, -37(%rdi) > - mov %r10, -29(%rdi) > - mov %r11, -21(%rdi) > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_53bytes): > - movdqu -53(%rsi), %xmm0 > - mov -45(%rsi), %r8 > - mov -37(%rsi), %r9 > - mov -29(%rsi), %r10 > - mov -21(%rsi), %r11 > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -53(%rdi) > - mov %r9, -37(%rdi) > - mov %r10, -29(%rdi) > - mov %r11, -21(%rdi) > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_45bytes): > - mov -45(%rsi), %r8 > - mov -37(%rsi), %r9 > - mov -29(%rsi), %r10 > - mov -21(%rsi), %r11 > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r8, -45(%rdi) > - mov %r9, -37(%rdi) > - mov %r10, -29(%rdi) > - mov %r11, -21(%rdi) > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_37bytes): > - mov -37(%rsi), %r9 > - mov -29(%rsi), %r10 > - mov -21(%rsi), %r11 > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r9, -37(%rdi) > - mov %r10, -29(%rdi) > - mov %r11, -21(%rdi) > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_29bytes): > - mov -29(%rsi), %r10 > - mov -21(%rsi), %r11 > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r10, -29(%rdi) > - mov %r11, -21(%rdi) > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_21bytes): > - mov -21(%rsi), %r11 > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r11, -21(%rdi) > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_13bytes): > - mov -13(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %rcx, -13(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_5bytes): > - mov -5(%rsi), %edx > - mov -4(%rsi), %ecx > - mov %edx, -5(%rdi) > - mov %ecx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_78bytes): > - movdqu -78(%rsi), %xmm0 > - movdqu -62(%rsi), %xmm1 > - mov -46(%rsi), %r8 > - mov -38(%rsi), %r9 > - mov -30(%rsi), %r10 > - mov -22(%rsi), %r11 > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -78(%rdi) > - movdqu %xmm1, -62(%rdi) > - mov %r8, -46(%rdi) > - mov %r9, -38(%rdi) > - mov %r10, -30(%rdi) > - mov %r11, -22(%rdi) > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_70bytes): > - movdqu -70(%rsi), %xmm0 > - movdqu -54(%rsi), %xmm1 > - mov -38(%rsi), %r9 > - mov -30(%rsi), %r10 > - mov -22(%rsi), %r11 > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -70(%rdi) > - movdqu %xmm1, -54(%rdi) > - mov %r9, -38(%rdi) > - mov %r10, -30(%rdi) > - mov %r11, -22(%rdi) > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_62bytes): > - movdqu -62(%rsi), %xmm0 > - mov -46(%rsi), %r8 > - mov -38(%rsi), %r9 > - mov -30(%rsi), %r10 > - mov -22(%rsi), %r11 > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -62(%rdi) > - mov %r8, -46(%rdi) > - mov %r9, -38(%rdi) > - mov %r10, -30(%rdi) > - mov %r11, -22(%rdi) > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_54bytes): > - movdqu -54(%rsi), %xmm0 > - mov -38(%rsi), %r9 > - mov -30(%rsi), %r10 > - mov -22(%rsi), %r11 > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -54(%rdi) > - mov %r9, -38(%rdi) > - mov %r10, -30(%rdi) > - mov %r11, -22(%rdi) > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_46bytes): > - mov -46(%rsi), %r8 > - mov -38(%rsi), %r9 > - mov -30(%rsi), %r10 > - mov -22(%rsi), %r11 > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r8, -46(%rdi) > - mov %r9, -38(%rdi) > - mov %r10, -30(%rdi) > - mov %r11, -22(%rdi) > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_38bytes): > - mov -38(%rsi), %r9 > - mov -30(%rsi), %r10 > - mov -22(%rsi), %r11 > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r9, -38(%rdi) > - mov %r10, -30(%rdi) > - mov %r11, -22(%rdi) > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_30bytes): > - mov -30(%rsi), %r10 > - mov -22(%rsi), %r11 > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r10, -30(%rdi) > - mov %r11, -22(%rdi) > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_22bytes): > - mov -22(%rsi), %r11 > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r11, -22(%rdi) > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_14bytes): > - mov -14(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %rcx, -14(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_6bytes): > - mov -6(%rsi), %edx > - mov -4(%rsi), %ecx > - mov %edx, -6(%rdi) > - mov %ecx, -4(%rdi) > - ret > - > - .p2align 4 > -L(write_79bytes): > - movdqu -79(%rsi), %xmm0 > - movdqu -63(%rsi), %xmm1 > - mov -47(%rsi), %r8 > - mov -39(%rsi), %r9 > - mov -31(%rsi), %r10 > - mov -23(%rsi), %r11 > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -79(%rdi) > - movdqu %xmm1, -63(%rdi) > - mov %r8, -47(%rdi) > - mov %r9, -39(%rdi) > - mov %r10, -31(%rdi) > - mov %r11, -23(%rdi) > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_71bytes): > - movdqu -71(%rsi), %xmm0 > - movdqu -55(%rsi), %xmm1 > - mov -39(%rsi), %r9 > - mov -31(%rsi), %r10 > - mov -23(%rsi), %r11 > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -71(%rdi) > - movdqu %xmm1, -55(%rdi) > - mov %r9, -39(%rdi) > - mov %r10, -31(%rdi) > - mov %r11, -23(%rdi) > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_63bytes): > - movdqu -63(%rsi), %xmm0 > - mov -47(%rsi), %r8 > - mov -39(%rsi), %r9 > - mov -31(%rsi), %r10 > - mov -23(%rsi), %r11 > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -63(%rdi) > - mov %r8, -47(%rdi) > - mov %r9, -39(%rdi) > - mov %r10, -31(%rdi) > - mov %r11, -23(%rdi) > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_55bytes): > - movdqu -55(%rsi), %xmm0 > - mov -39(%rsi), %r9 > - mov -31(%rsi), %r10 > - mov -23(%rsi), %r11 > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - movdqu %xmm0, -55(%rdi) > - mov %r9, -39(%rdi) > - mov %r10, -31(%rdi) > - mov %r11, -23(%rdi) > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_47bytes): > - mov -47(%rsi), %r8 > - mov -39(%rsi), %r9 > - mov -31(%rsi), %r10 > - mov -23(%rsi), %r11 > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r8, -47(%rdi) > - mov %r9, -39(%rdi) > - mov %r10, -31(%rdi) > - mov %r11, -23(%rdi) > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_39bytes): > - mov -39(%rsi), %r9 > - mov -31(%rsi), %r10 > - mov -23(%rsi), %r11 > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r9, -39(%rdi) > - mov %r10, -31(%rdi) > - mov %r11, -23(%rdi) > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_31bytes): > - mov -31(%rsi), %r10 > - mov -23(%rsi), %r11 > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r10, -31(%rdi) > - mov %r11, -23(%rdi) > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_23bytes): > - mov -23(%rsi), %r11 > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %r11, -23(%rdi) > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_15bytes): > - mov -15(%rsi), %rcx > - mov -8(%rsi), %rdx > - mov %rcx, -15(%rdi) > - mov %rdx, -8(%rdi) > - ret > - > - .p2align 4 > -L(write_7bytes): > - mov -7(%rsi), %edx > - mov -4(%rsi), %ecx > - mov %edx, -7(%rdi) > - mov %ecx, -4(%rdi) > - ret > - > - .p2align 4 > -L(large_page_fwd): > - movdqu (%rsi), %xmm1 > - lea 16(%rsi), %rsi > - movdqu %xmm0, (%r8) > - movntdq %xmm1, (%rdi) > - lea 16(%rdi), %rdi > - lea -0x90(%rdx), %rdx > -#ifdef USE_AS_MEMMOVE > - mov %rsi, %r9 > - sub %rdi, %r9 > - cmp %rdx, %r9 > - jae L(memmove_is_memcpy_fwd) > - shl $2, %rcx > - cmp %rcx, %rdx > - jb L(ll_cache_copy_fwd_start) > -L(memmove_is_memcpy_fwd): > -#endif > -L(large_page_loop): > - movdqu (%rsi), %xmm0 > - movdqu 0x10(%rsi), %xmm1 > - movdqu 0x20(%rsi), %xmm2 > - movdqu 0x30(%rsi), %xmm3 > - movdqu 0x40(%rsi), %xmm4 > - movdqu 0x50(%rsi), %xmm5 > - movdqu 0x60(%rsi), %xmm6 > - movdqu 0x70(%rsi), %xmm7 > - lea 0x80(%rsi), %rsi > - > - sub $0x80, %rdx > - movntdq %xmm0, (%rdi) > - movntdq %xmm1, 0x10(%rdi) > - movntdq %xmm2, 0x20(%rdi) > - movntdq %xmm3, 0x30(%rdi) > - movntdq %xmm4, 0x40(%rdi) > - movntdq %xmm5, 0x50(%rdi) > - movntdq %xmm6, 0x60(%rdi) > - movntdq %xmm7, 0x70(%rdi) > - lea 0x80(%rdi), %rdi > - jae L(large_page_loop) > - cmp $-0x40, %rdx > - lea 0x80(%rdx), %rdx > - jl L(large_page_less_64bytes) > - > - movdqu (%rsi), %xmm0 > - movdqu 0x10(%rsi), %xmm1 > - movdqu 0x20(%rsi), %xmm2 > - movdqu 0x30(%rsi), %xmm3 > - lea 0x40(%rsi), %rsi > - > - movntdq %xmm0, (%rdi) > - movntdq %xmm1, 0x10(%rdi) > - movntdq %xmm2, 0x20(%rdi) > - movntdq %xmm3, 0x30(%rdi) > - lea 0x40(%rdi), %rdi > - sub $0x40, %rdx > -L(large_page_less_64bytes): > - add %rdx, %rsi > - add %rdx, %rdi > - sfence > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > -#ifdef USE_AS_MEMMOVE > - .p2align 4 > -L(ll_cache_copy_fwd_start): > - prefetcht0 0x1c0(%rsi) > - prefetcht0 0x200(%rsi) > - movdqu (%rsi), %xmm0 > - movdqu 0x10(%rsi), %xmm1 > - movdqu 0x20(%rsi), %xmm2 > - movdqu 0x30(%rsi), %xmm3 > - movdqu 0x40(%rsi), %xmm4 > - movdqu 0x50(%rsi), %xmm5 > - movdqu 0x60(%rsi), %xmm6 > - movdqu 0x70(%rsi), %xmm7 > - lea 0x80(%rsi), %rsi > - > - sub $0x80, %rdx > - movaps %xmm0, (%rdi) > - movaps %xmm1, 0x10(%rdi) > - movaps %xmm2, 0x20(%rdi) > - movaps %xmm3, 0x30(%rdi) > - movaps %xmm4, 0x40(%rdi) > - movaps %xmm5, 0x50(%rdi) > - movaps %xmm6, 0x60(%rdi) > - movaps %xmm7, 0x70(%rdi) > - lea 0x80(%rdi), %rdi > - jae L(ll_cache_copy_fwd_start) > - cmp $-0x40, %rdx > - lea 0x80(%rdx), %rdx > - jl L(large_page_ll_less_fwd_64bytes) > - > - movdqu (%rsi), %xmm0 > - movdqu 0x10(%rsi), %xmm1 > - movdqu 0x20(%rsi), %xmm2 > - movdqu 0x30(%rsi), %xmm3 > - lea 0x40(%rsi), %rsi > - > - movaps %xmm0, (%rdi) > - movaps %xmm1, 0x10(%rdi) > - movaps %xmm2, 0x20(%rdi) > - movaps %xmm3, 0x30(%rdi) > - lea 0x40(%rdi), %rdi > - sub $0x40, %rdx > -L(large_page_ll_less_fwd_64bytes): > - add %rdx, %rsi > - add %rdx, %rdi > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > -#endif > - .p2align 4 > -L(large_page_bwd): > - movdqu -0x10(%rsi), %xmm1 > - lea -16(%rsi), %rsi > - movdqu %xmm0, (%r8) > - movdqa %xmm1, -0x10(%rdi) > - lea -16(%rdi), %rdi > - lea -0x90(%rdx), %rdx > -#ifdef USE_AS_MEMMOVE > - mov %rdi, %r9 > - sub %rsi, %r9 > - cmp %rdx, %r9 > - jae L(memmove_is_memcpy_bwd) > - cmp %rcx, %r9 > - jb L(ll_cache_copy_bwd_start) > -L(memmove_is_memcpy_bwd): > -#endif > -L(large_page_bwd_loop): > - movdqu -0x10(%rsi), %xmm0 > - movdqu -0x20(%rsi), %xmm1 > - movdqu -0x30(%rsi), %xmm2 > - movdqu -0x40(%rsi), %xmm3 > - movdqu -0x50(%rsi), %xmm4 > - movdqu -0x60(%rsi), %xmm5 > - movdqu -0x70(%rsi), %xmm6 > - movdqu -0x80(%rsi), %xmm7 > - lea -0x80(%rsi), %rsi > - > - sub $0x80, %rdx > - movntdq %xmm0, -0x10(%rdi) > - movntdq %xmm1, -0x20(%rdi) > - movntdq %xmm2, -0x30(%rdi) > - movntdq %xmm3, -0x40(%rdi) > - movntdq %xmm4, -0x50(%rdi) > - movntdq %xmm5, -0x60(%rdi) > - movntdq %xmm6, -0x70(%rdi) > - movntdq %xmm7, -0x80(%rdi) > - lea -0x80(%rdi), %rdi > - jae L(large_page_bwd_loop) > - cmp $-0x40, %rdx > - lea 0x80(%rdx), %rdx > - jl L(large_page_less_bwd_64bytes) > - > - movdqu -0x10(%rsi), %xmm0 > - movdqu -0x20(%rsi), %xmm1 > - movdqu -0x30(%rsi), %xmm2 > - movdqu -0x40(%rsi), %xmm3 > - lea -0x40(%rsi), %rsi > - > - movntdq %xmm0, -0x10(%rdi) > - movntdq %xmm1, -0x20(%rdi) > - movntdq %xmm2, -0x30(%rdi) > - movntdq %xmm3, -0x40(%rdi) > - lea -0x40(%rdi), %rdi > - sub $0x40, %rdx > -L(large_page_less_bwd_64bytes): > - sfence > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > - > -#ifdef USE_AS_MEMMOVE > - .p2align 4 > -L(ll_cache_copy_bwd_start): > - prefetcht0 -0x1c0(%rsi) > - prefetcht0 -0x200(%rsi) > - movdqu -0x10(%rsi), %xmm0 > - movdqu -0x20(%rsi), %xmm1 > - movdqu -0x30(%rsi), %xmm2 > - movdqu -0x40(%rsi), %xmm3 > - movdqu -0x50(%rsi), %xmm4 > - movdqu -0x60(%rsi), %xmm5 > - movdqu -0x70(%rsi), %xmm6 > - movdqu -0x80(%rsi), %xmm7 > - lea -0x80(%rsi), %rsi > - > - sub $0x80, %rdx > - movaps %xmm0, -0x10(%rdi) > - movaps %xmm1, -0x20(%rdi) > - movaps %xmm2, -0x30(%rdi) > - movaps %xmm3, -0x40(%rdi) > - movaps %xmm4, -0x50(%rdi) > - movaps %xmm5, -0x60(%rdi) > - movaps %xmm6, -0x70(%rdi) > - movaps %xmm7, -0x80(%rdi) > - lea -0x80(%rdi), %rdi > - jae L(ll_cache_copy_bwd_start) > - cmp $-0x40, %rdx > - lea 0x80(%rdx), %rdx > - jl L(large_page_ll_less_bwd_64bytes) > - > - movdqu -0x10(%rsi), %xmm0 > - movdqu -0x20(%rsi), %xmm1 > - movdqu -0x30(%rsi), %xmm2 > - movdqu -0x40(%rsi), %xmm3 > - lea -0x40(%rsi), %rsi > - > - movaps %xmm0, -0x10(%rdi) > - movaps %xmm1, -0x20(%rdi) > - movaps %xmm2, -0x30(%rdi) > - movaps %xmm3, -0x40(%rdi) > - lea -0x40(%rdi), %rdi > - sub $0x40, %rdx > -L(large_page_ll_less_bwd_64bytes): > - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) > -#endif > - > -END (MEMCPY) > - > - .section .rodata.ssse3,"a",@progbits > - .p2align 3 > -L(table_less_80bytes): > - .int JMPTBL (L(write_0bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_1bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_2bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_3bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_4bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_5bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_6bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_7bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_8bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_9bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_10bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_11bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_12bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_13bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_14bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_15bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_16bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_17bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_18bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_19bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_20bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_21bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_22bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_23bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_24bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_25bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_26bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_27bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_28bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_29bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_30bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_31bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_32bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_33bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_34bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_35bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_36bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_37bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_38bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_39bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_40bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_41bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_42bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_43bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_44bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_45bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_46bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_47bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_48bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_49bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_50bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_51bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_52bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_53bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_54bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_55bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_56bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_57bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_58bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_59bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_60bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_61bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_62bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_63bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_64bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_65bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_66bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_67bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_68bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_69bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_70bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_71bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_72bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_73bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_74bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_75bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_76bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_77bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_78bytes), L(table_less_80bytes)) > - .int JMPTBL (L(write_79bytes), L(table_less_80bytes)) > - > - .p2align 3 > -L(shl_table): > - .int JMPTBL (L(shl_0), L(shl_table)) > - .int JMPTBL (L(shl_1), L(shl_table)) > - .int JMPTBL (L(shl_2), L(shl_table)) > - .int JMPTBL (L(shl_3), L(shl_table)) > - .int JMPTBL (L(shl_4), L(shl_table)) > - .int JMPTBL (L(shl_5), L(shl_table)) > - .int JMPTBL (L(shl_6), L(shl_table)) > - .int JMPTBL (L(shl_7), L(shl_table)) > - .int JMPTBL (L(shl_8), L(shl_table)) > - .int JMPTBL (L(shl_9), L(shl_table)) > - .int JMPTBL (L(shl_10), L(shl_table)) > - .int JMPTBL (L(shl_11), L(shl_table)) > - .int JMPTBL (L(shl_12), L(shl_table)) > - .int JMPTBL (L(shl_13), L(shl_table)) > - .int JMPTBL (L(shl_14), L(shl_table)) > - .int JMPTBL (L(shl_15), L(shl_table)) > - > - .p2align 3 > -L(shl_table_bwd): > - .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd)) > - .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd)) > - > -#endif > diff --git a/sysdeps/x86_64/multiarch/memmove-ssse3.S b/sysdeps/x86_64/multiarch/memmove-ssse3.S > index 295430b1ef..84e4e0f6cb 100644 > --- a/sysdeps/x86_64/multiarch/memmove-ssse3.S > +++ b/sysdeps/x86_64/multiarch/memmove-ssse3.S > @@ -1,4 +1,382 @@ > -#define USE_AS_MEMMOVE > -#define MEMCPY __memmove_ssse3 > -#define MEMCPY_CHK __memmove_chk_ssse3 > -#include "memcpy-ssse3.S" > +#include <sysdep.h> > + > +#ifndef MEMMOVE > +# define MEMMOVE __memmove_ssse3 > +# define MEMMOVE_CHK __memmove_chk_ssse3 > +# define MEMCPY __memcpy_ssse3 > +# define MEMCPY_CHK __memcpy_chk_ssse3 > +# define MEMPCPY __mempcpy_ssse3 > +# define MEMPCPY_CHK __mempcpy_chk_ssse3 > +#endif > + > + .section .text.ssse3, "ax", @progbits > +ENTRY(MEMPCPY_CHK) > + cmp %RDX_LP, %RCX_LP > + jb HIDDEN_JUMPTARGET(__chk_fail) > +END(MEMPCPY_CHK) > + > +ENTRY(MEMPCPY) > + mov %RDI_LP, %RAX_LP > + add %RDX_LP, %RAX_LP > + jmp L(start) > +END(MEMPCPY) > + > +ENTRY(MEMMOVE_CHK) > + cmp %RDX_LP, %RCX_LP > + jb HIDDEN_JUMPTARGET(__chk_fail) > +END(MEMMOVE_CHK) > + > +ENTRY_P2ALIGN(MEMMOVE, 6) > + movq %rdi, %rax > +L(start): > + cmpq $16, %rdx > + jb L(copy_0_15) > + > + /* These loads are always useful. */ > + movups 0(%rsi), %xmm0 > + movups -16(%rsi, %rdx), %xmm7 > + cmpq $32, %rdx > + ja L(more_2x_vec) > + > + movups %xmm0, 0(%rdi) > + movups %xmm7, -16(%rdi, %rdx) > + ret > + > + .p2align 4,, 8 > +L(copy_4x_vec): > + movups 16(%rsi), %xmm1 > + movups -32(%rsi, %rdx), %xmm2 > + > + movups %xmm0, 0(%rdi) > + movups %xmm1, 16(%rdi) > + movups %xmm2, -32(%rdi, %rdx) > + movups %xmm7, -16(%rdi, %rdx) > + ret > + > + .p2align 4,, 8 > +L(copy_0_15): > + cmpl $8, %edx > + ja L(copy_9_15) > + > + cmpl $4, %edx > + jb L(copy_0_3) > + > + movl 0(%rsi), %ecx > + movl -4(%rsi, %rdx), %esi > + movl %ecx, 0(%rdi) > + movl %esi, -4(%rdi, %rdx) > + ret > + > + .p2align 4,, 8 > +L(copy_9_15): > + movq 0(%rsi), %rcx > + movq -8(%rsi, %rdx), %rsi > + movq %rcx, 0(%rdi) > + movq %rsi, -8(%rdi, %rdx) > + ret > + > + .p2align 4,, 4 > +L(copy_0_3): > + cmpl $1, %edx > + jl L(copy_0_0) > + movzbl (%rsi), %ecx > + je L(copy_0_1) > + > + movzwl -2(%rsi, %rdx), %esi > + movw %si, -2(%rdi, %rdx) > +L(copy_0_1): > + movb %cl, (%rdi) > +L(copy_0_0): > +L(nop): > + ret > + > + .p2align 4 > +L(more_2x_vec): > + cmpq $64, %rdx > + jbe L(copy_4x_vec) > + > + /* We use rcx later to get alignr value. */ > + movq %rdi, %rcx > + > + /* Backward copy for overlap + dst > src for memmove safety. */ > + subq %rsi, %rcx > + cmpq %rdx, %rcx > + jb L(copy_backward) > + > + /* Load tail. */ > + > + /* -16(%rsi, %rdx) already loaded into xmm7. */ > + movups -32(%rsi, %rdx), %xmm8 > + movups -48(%rsi, %rdx), %xmm9 > + > + /* Get misalignment. */ > + andl $0xf, %ecx > + > + movq %rsi, %r9 > + addq %rcx, %rsi > + andq $-16, %rsi > + /* Get first vec for `palignr`. */ > + movaps (%rsi), %xmm1 > + > + /* We have loaded (%rsi) so safe to do this store before the > + loop. */ > + movups %xmm0, (%rdi) > + > +#ifdef SHARED_CACHE_SIZE_HALF > + cmp $SHARED_CACHE_SIZE_HALF, %RDX_LP > +#else > + cmp __x86_shared_cache_size_half(%rip), %rdx > +#endif > + ja L(large_memcpy) > + > + leaq -64(%rdi, %rdx), %r8 > + andq $-16, %rdi > + movl $48, %edx > + > + leaq L(loop_fwd_start)(%rip), %r9 > + sall $6, %ecx > + addq %r9, %rcx > + jmp * %rcx > + > + .p2align 4,, 8 > +L(copy_backward): > + testq %rcx, %rcx > + jz L(nop) > + > + /* Preload tail. */ > + > + /* (%rsi) already loaded into xmm0. */ > + movups 16(%rsi), %xmm4 > + movups 32(%rsi), %xmm5 > + > + movq %rdi, %r8 > + subq %rdi, %rsi > + leaq -49(%rdi, %rdx), %rdi > + andq $-16, %rdi > + addq %rdi, %rsi > + andq $-16, %rsi > + > + movaps 48(%rsi), %xmm6 > + > + > + leaq L(loop_bkwd_start)(%rip), %r9 > + andl $0xf, %ecx > + sall $6, %ecx > + addq %r9, %rcx > + jmp * %rcx > + > + .p2align 4,, 8 > +L(large_memcpy): > + movups -64(%r9, %rdx), %xmm10 > + movups -80(%r9, %rdx), %xmm11 > + > + sall $5, %ecx > + leal (%rcx, %rcx, 2), %r8d > + leaq -96(%rdi, %rdx), %rcx > + andq $-16, %rdi > + leaq L(large_loop_fwd_start)(%rip), %rdx > + addq %r8, %rdx > + jmp * %rdx > + > + > + /* Instead of a typical jump table all 16 loops are exactly > + 64-bytes in size. So, we can just jump to first loop + r8 * > + 64. Before modifying any loop ensure all their sizes match! > + */ > + .p2align 6 > +L(loop_fwd_start): > +L(loop_fwd_0x0): > + movaps 16(%rsi), %xmm1 > + movaps 32(%rsi), %xmm2 > + movaps 48(%rsi), %xmm3 > + movaps %xmm1, 16(%rdi) > + movaps %xmm2, 32(%rdi) > + movaps %xmm3, 48(%rdi) > + addq %rdx, %rdi > + addq %rdx, %rsi > + cmpq %rdi, %r8 > + ja L(loop_fwd_0x0) > +L(end_loop_fwd): > + movups %xmm9, 16(%r8) > + movups %xmm8, 32(%r8) > + movups %xmm7, 48(%r8) > + ret > + > + /* Extactly 64 bytes if `jmp L(end_loop_fwd)` is long encoding. > + 60 bytes otherwise. */ > +#define ALIGNED_LOOP_FWD(align_by); \ > + .p2align 6; \ > +L(loop_fwd_ ## align_by): \ > + movaps 16(%rsi), %xmm0; \ > + movaps 32(%rsi), %xmm2; \ > + movaps 48(%rsi), %xmm3; \ > + movaps %xmm3, %xmm4; \ > + palignr $align_by, %xmm2, %xmm3; \ > + palignr $align_by, %xmm0, %xmm2; \ > + palignr $align_by, %xmm1, %xmm0; \ > + movaps %xmm4, %xmm1; \ > + movaps %xmm0, 16(%rdi); \ > + movaps %xmm2, 32(%rdi); \ > + movaps %xmm3, 48(%rdi); \ > + addq %rdx, %rdi; \ > + addq %rdx, %rsi; \ > + cmpq %rdi, %r8; \ > + ja L(loop_fwd_ ## align_by); \ > + jmp L(end_loop_fwd); > + > + /* Must be in descending order. */ > + ALIGNED_LOOP_FWD (0xf) > + ALIGNED_LOOP_FWD (0xe) > + ALIGNED_LOOP_FWD (0xd) > + ALIGNED_LOOP_FWD (0xc) > + ALIGNED_LOOP_FWD (0xb) > + ALIGNED_LOOP_FWD (0xa) > + ALIGNED_LOOP_FWD (0x9) > + ALIGNED_LOOP_FWD (0x8) > + ALIGNED_LOOP_FWD (0x7) > + ALIGNED_LOOP_FWD (0x6) > + ALIGNED_LOOP_FWD (0x5) > + ALIGNED_LOOP_FWD (0x4) > + ALIGNED_LOOP_FWD (0x3) > + ALIGNED_LOOP_FWD (0x2) > + ALIGNED_LOOP_FWD (0x1) > + > + .p2align 6 > +L(large_loop_fwd_start): > +L(large_loop_fwd_0x0): > + movaps 16(%rsi), %xmm1 > + movaps 32(%rsi), %xmm2 > + movaps 48(%rsi), %xmm3 > + movaps 64(%rsi), %xmm4 > + movaps 80(%rsi), %xmm5 > + movntps %xmm1, 16(%rdi) > + movntps %xmm2, 32(%rdi) > + movntps %xmm3, 48(%rdi) > + movntps %xmm4, 64(%rdi) > + movntps %xmm5, 80(%rdi) > + addq $80, %rdi > + addq $80, %rsi > + cmpq %rdi, %rcx > + ja L(large_loop_fwd_0x0) > + > + /* Ensure no icache line split on tail. */ > + .p2align 4 > +L(end_large_loop_fwd): > + sfence > + movups %xmm11, 16(%rcx) > + movups %xmm10, 32(%rcx) > + movups %xmm9, 48(%rcx) > + movups %xmm8, 64(%rcx) > + movups %xmm7, 80(%rcx) > + ret > + > + > + /* Size > 64 bytes and <= 96 bytes. 32-byte align between ensure > + 96-byte spacing between each. */ > +#define ALIGNED_LARGE_LOOP_FWD(align_by); \ > + .p2align 5; \ > +L(large_loop_fwd_ ## align_by): \ > + movaps 16(%rsi), %xmm0; \ > + movaps 32(%rsi), %xmm2; \ > + movaps 48(%rsi), %xmm3; \ > + movaps 64(%rsi), %xmm4; \ > + movaps 80(%rsi), %xmm5; \ > + movaps %xmm5, %xmm6; \ > + palignr $align_by, %xmm4, %xmm5; \ > + palignr $align_by, %xmm3, %xmm4; \ > + palignr $align_by, %xmm2, %xmm3; \ > + palignr $align_by, %xmm0, %xmm2; \ > + palignr $align_by, %xmm1, %xmm0; \ > + movaps %xmm6, %xmm1; \ > + movntps %xmm0, 16(%rdi); \ > + movntps %xmm2, 32(%rdi); \ > + movntps %xmm3, 48(%rdi); \ > + movntps %xmm4, 64(%rdi); \ > + movntps %xmm5, 80(%rdi); \ > + addq $80, %rdi; \ > + addq $80, %rsi; \ > + cmpq %rdi, %rcx; \ > + ja L(large_loop_fwd_ ## align_by); \ > + jmp L(end_large_loop_fwd); > + > + /* Must be in descending order. */ > + ALIGNED_LARGE_LOOP_FWD (0xf) > + ALIGNED_LARGE_LOOP_FWD (0xe) > + ALIGNED_LARGE_LOOP_FWD (0xd) > + ALIGNED_LARGE_LOOP_FWD (0xc) > + ALIGNED_LARGE_LOOP_FWD (0xb) > + ALIGNED_LARGE_LOOP_FWD (0xa) > + ALIGNED_LARGE_LOOP_FWD (0x9) > + ALIGNED_LARGE_LOOP_FWD (0x8) > + ALIGNED_LARGE_LOOP_FWD (0x7) > + ALIGNED_LARGE_LOOP_FWD (0x6) > + ALIGNED_LARGE_LOOP_FWD (0x5) > + ALIGNED_LARGE_LOOP_FWD (0x4) > + ALIGNED_LARGE_LOOP_FWD (0x3) > + ALIGNED_LARGE_LOOP_FWD (0x2) > + ALIGNED_LARGE_LOOP_FWD (0x1) > + > + > + .p2align 6 > +L(loop_bkwd_start): > +L(loop_bkwd_0x0): > + movaps 32(%rsi), %xmm1 > + movaps 16(%rsi), %xmm2 > + movaps 0(%rsi), %xmm3 > + movaps %xmm1, 32(%rdi) > + movaps %xmm2, 16(%rdi) > + movaps %xmm3, 0(%rdi) > + subq $48, %rdi > + subq $48, %rsi > + cmpq %rdi, %r8 > + jb L(loop_bkwd_0x0) > +L(end_loop_bkwd): > + movups %xmm7, -16(%r8, %rdx) > + movups %xmm0, 0(%r8) > + movups %xmm4, 16(%r8) > + movups %xmm5, 32(%r8) > + > + ret > + > + > + /* Extactly 64 bytes if `jmp L(end_loop_bkwd)` is long encoding. > + 60 bytes otherwise. */ > +#define ALIGNED_LOOP_BKWD(align_by); \ > + .p2align 6; \ > +L(loop_bkwd_ ## align_by): \ > + movaps 32(%rsi), %xmm1; \ > + movaps 16(%rsi), %xmm2; \ > + movaps 0(%rsi), %xmm3; \ > + palignr $align_by, %xmm1, %xmm6; \ > + palignr $align_by, %xmm2, %xmm1; \ > + palignr $align_by, %xmm3, %xmm2; \ > + movaps %xmm6, 32(%rdi); \ > + movaps %xmm1, 16(%rdi); \ > + movaps %xmm2, 0(%rdi); \ > + subq $48, %rdi; \ > + subq $48, %rsi; \ > + movaps %xmm3, %xmm6; \ > + cmpq %rdi, %r8; \ > + jb L(loop_bkwd_ ## align_by); \ > + jmp L(end_loop_bkwd); > + > + /* Must be in descending order. */ > + ALIGNED_LOOP_BKWD (0xf) > + ALIGNED_LOOP_BKWD (0xe) > + ALIGNED_LOOP_BKWD (0xd) > + ALIGNED_LOOP_BKWD (0xc) > + ALIGNED_LOOP_BKWD (0xb) > + ALIGNED_LOOP_BKWD (0xa) > + ALIGNED_LOOP_BKWD (0x9) > + ALIGNED_LOOP_BKWD (0x8) > + ALIGNED_LOOP_BKWD (0x7) > + ALIGNED_LOOP_BKWD (0x6) > + ALIGNED_LOOP_BKWD (0x5) > + ALIGNED_LOOP_BKWD (0x4) > + ALIGNED_LOOP_BKWD (0x3) > + ALIGNED_LOOP_BKWD (0x2) > + ALIGNED_LOOP_BKWD (0x1) > +END(MEMMOVE) > + > +strong_alias (MEMMOVE, MEMCPY) > +strong_alias (MEMMOVE_CHK, MEMCPY_CHK) > -- > 2.25.1 >
diff --git a/sysdeps/x86_64/multiarch/Makefile b/sysdeps/x86_64/multiarch/Makefile index 303fb5d734..e7ea963fc0 100644 --- a/sysdeps/x86_64/multiarch/Makefile +++ b/sysdeps/x86_64/multiarch/Makefile @@ -16,7 +16,6 @@ sysdep_routines += \ memcmpeq-avx2-rtm \ memcmpeq-evex \ memcmpeq-sse2 \ - memcpy-ssse3 \ memmove-avx-unaligned-erms \ memmove-avx-unaligned-erms-rtm \ memmove-avx512-no-vzeroupper \ diff --git a/sysdeps/x86_64/multiarch/memcpy-ssse3.S b/sysdeps/x86_64/multiarch/memcpy-ssse3.S deleted file mode 100644 index 65644d3a09..0000000000 --- a/sysdeps/x86_64/multiarch/memcpy-ssse3.S +++ /dev/null @@ -1,3151 +0,0 @@ -/* memcpy with SSSE3 - Copyright (C) 2010-2022 Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, see - <https://www.gnu.org/licenses/>. */ - -#include <sysdep.h> - -#if IS_IN (libc) - -#include "asm-syntax.h" - -#ifndef MEMCPY -# define MEMCPY __memcpy_ssse3 -# define MEMCPY_CHK __memcpy_chk_ssse3 -# define MEMPCPY __mempcpy_ssse3 -# define MEMPCPY_CHK __mempcpy_chk_ssse3 -#endif - -#define JMPTBL(I, B) I - B - -/* Branch to an entry in a jump table. TABLE is a jump table with - relative offsets. INDEX is a register contains the index into the - jump table. SCALE is the scale of INDEX. */ -#define BRANCH_TO_JMPTBL_ENTRY(TABLE, INDEX, SCALE) \ - lea TABLE(%rip), %r11; \ - movslq (%r11, INDEX, SCALE), INDEX; \ - lea (%r11, INDEX), INDEX; \ - _CET_NOTRACK jmp *INDEX; \ - ud2 - - .section .text.ssse3,"ax",@progbits -#if !defined USE_AS_MEMPCPY && !defined USE_AS_MEMMOVE -ENTRY (MEMPCPY_CHK) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) -END (MEMPCPY_CHK) - -ENTRY (MEMPCPY) - mov %RDI_LP, %RAX_LP - add %RDX_LP, %RAX_LP - jmp L(start) -END (MEMPCPY) -#endif - -#if !defined USE_AS_BCOPY -ENTRY (MEMCPY_CHK) - cmp %RDX_LP, %RCX_LP - jb HIDDEN_JUMPTARGET (__chk_fail) -END (MEMCPY_CHK) -#endif - -ENTRY (MEMCPY) - mov %RDI_LP, %RAX_LP -#ifdef USE_AS_MEMPCPY - add %RDX_LP, %RAX_LP -#endif - -#ifdef __ILP32__ - /* Clear the upper 32 bits. */ - mov %edx, %edx -#endif - -#ifdef USE_AS_MEMMOVE - cmp %rsi, %rdi - jb L(copy_forward) - je L(write_0bytes) - cmp $79, %rdx - jbe L(copy_forward) - jmp L(copy_backward) -L(copy_forward): -#endif -L(start): - cmp $79, %rdx - lea L(table_less_80bytes)(%rip), %r11 - ja L(80bytesormore) - movslq (%r11, %rdx, 4), %r9 - add %rdx, %rsi - add %rdx, %rdi - add %r11, %r9 - _CET_NOTRACK jmp *%r9 - ud2 - - .p2align 4 -L(80bytesormore): -#ifndef USE_AS_MEMMOVE - cmp %dil, %sil - jle L(copy_backward) -#endif - - movdqu (%rsi), %xmm0 - mov %rdi, %rcx - and $-16, %rdi - add $16, %rdi - mov %rcx, %r8 - sub %rdi, %rcx - add %rcx, %rdx - sub %rcx, %rsi - -#ifdef SHARED_CACHE_SIZE_HALF - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP -#else - mov __x86_shared_cache_size_half(%rip), %RCX_LP -#endif - cmp %rcx, %rdx - mov %rsi, %r9 - ja L(large_page_fwd) - and $0xf, %r9 - jz L(shl_0) -#ifdef DATA_CACHE_SIZE_HALF - mov $DATA_CACHE_SIZE_HALF, %RCX_LP -#else - mov __x86_data_cache_size_half(%rip), %RCX_LP -#endif - BRANCH_TO_JMPTBL_ENTRY (L(shl_table), %r9, 4) - - .p2align 4 -L(copy_backward): - movdqu -16(%rsi, %rdx), %xmm0 - add %rdx, %rsi - lea -16(%rdi, %rdx), %r8 - add %rdx, %rdi - - mov %rdi, %rcx - and $0xf, %rcx - xor %rcx, %rdi - sub %rcx, %rdx - sub %rcx, %rsi - -#ifdef SHARED_CACHE_SIZE_HALF - mov $SHARED_CACHE_SIZE_HALF, %RCX_LP -#else - mov __x86_shared_cache_size_half(%rip), %RCX_LP -#endif - - cmp %rcx, %rdx - mov %rsi, %r9 - ja L(large_page_bwd) - and $0xf, %r9 - jz L(shl_0_bwd) -#ifdef DATA_CACHE_SIZE_HALF - mov $DATA_CACHE_SIZE_HALF, %RCX_LP -#else - mov __x86_data_cache_size_half(%rip), %RCX_LP -#endif - BRANCH_TO_JMPTBL_ENTRY (L(shl_table_bwd), %r9, 4) - - .p2align 4 -L(shl_0): - sub $16, %rdx - movdqa (%rsi), %xmm1 - add $16, %rsi - movdqa %xmm1, (%rdi) - add $16, %rdi - cmp $128, %rdx - movdqu %xmm0, (%r8) - ja L(shl_0_gobble) - cmp $64, %rdx - jb L(shl_0_less_64bytes) - movaps (%rsi), %xmm4 - movaps 16(%rsi), %xmm1 - movaps 32(%rsi), %xmm2 - movaps 48(%rsi), %xmm3 - movaps %xmm4, (%rdi) - movaps %xmm1, 16(%rdi) - movaps %xmm2, 32(%rdi) - movaps %xmm3, 48(%rdi) - sub $64, %rdx - add $64, %rsi - add $64, %rdi -L(shl_0_less_64bytes): - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_0_gobble): -#ifdef DATA_CACHE_SIZE_HALF - cmp $DATA_CACHE_SIZE_HALF, %RDX_LP -#else - cmp __x86_data_cache_size_half(%rip), %RDX_LP -#endif - lea -128(%rdx), %rdx - jae L(shl_0_gobble_mem_loop) -L(shl_0_gobble_cache_loop): - movdqa (%rsi), %xmm4 - movaps 0x10(%rsi), %xmm1 - movaps 0x20(%rsi), %xmm2 - movaps 0x30(%rsi), %xmm3 - - movdqa %xmm4, (%rdi) - movaps %xmm1, 0x10(%rdi) - movaps %xmm2, 0x20(%rdi) - movaps %xmm3, 0x30(%rdi) - - sub $128, %rdx - movaps 0x40(%rsi), %xmm4 - movaps 0x50(%rsi), %xmm5 - movaps 0x60(%rsi), %xmm6 - movaps 0x70(%rsi), %xmm7 - lea 0x80(%rsi), %rsi - movaps %xmm4, 0x40(%rdi) - movaps %xmm5, 0x50(%rdi) - movaps %xmm6, 0x60(%rdi) - movaps %xmm7, 0x70(%rdi) - lea 0x80(%rdi), %rdi - - jae L(shl_0_gobble_cache_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(shl_0_cache_less_64bytes) - - movdqa (%rsi), %xmm4 - sub $0x40, %rdx - movdqa 0x10(%rsi), %xmm1 - - movdqa %xmm4, (%rdi) - movdqa %xmm1, 0x10(%rdi) - - movdqa 0x20(%rsi), %xmm4 - movdqa 0x30(%rsi), %xmm1 - add $0x40, %rsi - - movdqa %xmm4, 0x20(%rdi) - movdqa %xmm1, 0x30(%rdi) - add $0x40, %rdi -L(shl_0_cache_less_64bytes): - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_0_gobble_mem_loop): - prefetcht0 0x1c0(%rsi) - prefetcht0 0x280(%rsi) - - movdqa (%rsi), %xmm0 - movdqa 0x10(%rsi), %xmm1 - movdqa 0x20(%rsi), %xmm2 - movdqa 0x30(%rsi), %xmm3 - movdqa 0x40(%rsi), %xmm4 - movdqa 0x50(%rsi), %xmm5 - movdqa 0x60(%rsi), %xmm6 - movdqa 0x70(%rsi), %xmm7 - lea 0x80(%rsi), %rsi - sub $0x80, %rdx - movdqa %xmm0, (%rdi) - movdqa %xmm1, 0x10(%rdi) - movdqa %xmm2, 0x20(%rdi) - movdqa %xmm3, 0x30(%rdi) - movdqa %xmm4, 0x40(%rdi) - movdqa %xmm5, 0x50(%rdi) - movdqa %xmm6, 0x60(%rdi) - movdqa %xmm7, 0x70(%rdi) - lea 0x80(%rdi), %rdi - - jae L(shl_0_gobble_mem_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(shl_0_mem_less_64bytes) - - movdqa (%rsi), %xmm0 - sub $0x40, %rdx - movdqa 0x10(%rsi), %xmm1 - - movdqa %xmm0, (%rdi) - movdqa %xmm1, 0x10(%rdi) - - movdqa 0x20(%rsi), %xmm0 - movdqa 0x30(%rsi), %xmm1 - add $0x40, %rsi - - movdqa %xmm0, 0x20(%rdi) - movdqa %xmm1, 0x30(%rdi) - add $0x40, %rdi -L(shl_0_mem_less_64bytes): - cmp $0x20, %rdx - jb L(shl_0_mem_less_32bytes) - movdqa (%rsi), %xmm0 - sub $0x20, %rdx - movdqa 0x10(%rsi), %xmm1 - add $0x20, %rsi - movdqa %xmm0, (%rdi) - movdqa %xmm1, 0x10(%rdi) - add $0x20, %rdi -L(shl_0_mem_less_32bytes): - add %rdx, %rdi - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_0_bwd): - sub $16, %rdx - movdqa -0x10(%rsi), %xmm1 - sub $16, %rsi - movdqa %xmm1, -0x10(%rdi) - sub $16, %rdi - cmp $0x80, %rdx - movdqu %xmm0, (%r8) - ja L(shl_0_gobble_bwd) - cmp $64, %rdx - jb L(shl_0_less_64bytes_bwd) - movaps -0x10(%rsi), %xmm0 - movaps -0x20(%rsi), %xmm1 - movaps -0x30(%rsi), %xmm2 - movaps -0x40(%rsi), %xmm3 - movaps %xmm0, -0x10(%rdi) - movaps %xmm1, -0x20(%rdi) - movaps %xmm2, -0x30(%rdi) - movaps %xmm3, -0x40(%rdi) - sub $64, %rdx - sub $0x40, %rsi - sub $0x40, %rdi -L(shl_0_less_64bytes_bwd): - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_0_gobble_bwd): -#ifdef DATA_CACHE_SIZE_HALF - cmp $DATA_CACHE_SIZE_HALF, %RDX_LP -#else - cmp __x86_data_cache_size_half(%rip), %RDX_LP -#endif - lea -128(%rdx), %rdx - jae L(shl_0_gobble_mem_bwd_loop) -L(shl_0_gobble_bwd_loop): - movdqa -0x10(%rsi), %xmm0 - movaps -0x20(%rsi), %xmm1 - movaps -0x30(%rsi), %xmm2 - movaps -0x40(%rsi), %xmm3 - - movdqa %xmm0, -0x10(%rdi) - movaps %xmm1, -0x20(%rdi) - movaps %xmm2, -0x30(%rdi) - movaps %xmm3, -0x40(%rdi) - - sub $0x80, %rdx - movaps -0x50(%rsi), %xmm4 - movaps -0x60(%rsi), %xmm5 - movaps -0x70(%rsi), %xmm6 - movaps -0x80(%rsi), %xmm7 - lea -0x80(%rsi), %rsi - movaps %xmm4, -0x50(%rdi) - movaps %xmm5, -0x60(%rdi) - movaps %xmm6, -0x70(%rdi) - movaps %xmm7, -0x80(%rdi) - lea -0x80(%rdi), %rdi - - jae L(shl_0_gobble_bwd_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(shl_0_gobble_bwd_less_64bytes) - - movdqa -0x10(%rsi), %xmm0 - sub $0x40, %rdx - movdqa -0x20(%rsi), %xmm1 - - movdqa %xmm0, -0x10(%rdi) - movdqa %xmm1, -0x20(%rdi) - - movdqa -0x30(%rsi), %xmm0 - movdqa -0x40(%rsi), %xmm1 - sub $0x40, %rsi - - movdqa %xmm0, -0x30(%rdi) - movdqa %xmm1, -0x40(%rdi) - sub $0x40, %rdi -L(shl_0_gobble_bwd_less_64bytes): - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_0_gobble_mem_bwd_loop): - prefetcht0 -0x1c0(%rsi) - prefetcht0 -0x280(%rsi) - movdqa -0x10(%rsi), %xmm0 - movdqa -0x20(%rsi), %xmm1 - movdqa -0x30(%rsi), %xmm2 - movdqa -0x40(%rsi), %xmm3 - movdqa -0x50(%rsi), %xmm4 - movdqa -0x60(%rsi), %xmm5 - movdqa -0x70(%rsi), %xmm6 - movdqa -0x80(%rsi), %xmm7 - lea -0x80(%rsi), %rsi - sub $0x80, %rdx - movdqa %xmm0, -0x10(%rdi) - movdqa %xmm1, -0x20(%rdi) - movdqa %xmm2, -0x30(%rdi) - movdqa %xmm3, -0x40(%rdi) - movdqa %xmm4, -0x50(%rdi) - movdqa %xmm5, -0x60(%rdi) - movdqa %xmm6, -0x70(%rdi) - movdqa %xmm7, -0x80(%rdi) - lea -0x80(%rdi), %rdi - - jae L(shl_0_gobble_mem_bwd_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(shl_0_mem_bwd_less_64bytes) - - movdqa -0x10(%rsi), %xmm0 - sub $0x40, %rdx - movdqa -0x20(%rsi), %xmm1 - - movdqa %xmm0, -0x10(%rdi) - movdqa %xmm1, -0x20(%rdi) - - movdqa -0x30(%rsi), %xmm0 - movdqa -0x40(%rsi), %xmm1 - sub $0x40, %rsi - - movdqa %xmm0, -0x30(%rdi) - movdqa %xmm1, -0x40(%rdi) - sub $0x40, %rdi -L(shl_0_mem_bwd_less_64bytes): - cmp $0x20, %rdx - jb L(shl_0_mem_bwd_less_32bytes) - movdqa -0x10(%rsi), %xmm0 - sub $0x20, %rdx - movdqa -0x20(%rsi), %xmm1 - sub $0x20, %rsi - movdqa %xmm0, -0x10(%rdi) - movdqa %xmm1, -0x20(%rdi) - sub $0x20, %rdi -L(shl_0_mem_bwd_less_32bytes): - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_1): - lea (L(shl_1_loop_L1)-L(shl_1))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x01(%rsi), %xmm1 - jb L(L1_fwd) - lea (L(shl_1_loop_L2)-L(shl_1_loop_L1))(%r9), %r9 -L(L1_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_1_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_1_loop_L1): - sub $64, %rdx - movaps 0x0f(%rsi), %xmm2 - movaps 0x1f(%rsi), %xmm3 - movaps 0x2f(%rsi), %xmm4 - movaps 0x3f(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $1, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $1, %xmm3, %xmm4 - palignr $1, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $1, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_1_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_1_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_1_bwd): - lea (L(shl_1_bwd_loop_L1)-L(shl_1_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x01(%rsi), %xmm1 - jb L(L1_bwd) - lea (L(shl_1_bwd_loop_L2)-L(shl_1_bwd_loop_L1))(%r9), %r9 -L(L1_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_1_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_1_bwd_loop_L1): - movaps -0x11(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x21(%rsi), %xmm3 - movaps -0x31(%rsi), %xmm4 - movaps -0x41(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $1, %xmm2, %xmm1 - palignr $1, %xmm3, %xmm2 - palignr $1, %xmm4, %xmm3 - palignr $1, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_1_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_1_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_2): - lea (L(shl_2_loop_L1)-L(shl_2))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x02(%rsi), %xmm1 - jb L(L2_fwd) - lea (L(shl_2_loop_L2)-L(shl_2_loop_L1))(%r9), %r9 -L(L2_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_2_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_2_loop_L1): - sub $64, %rdx - movaps 0x0e(%rsi), %xmm2 - movaps 0x1e(%rsi), %xmm3 - movaps 0x2e(%rsi), %xmm4 - movaps 0x3e(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $2, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $2, %xmm3, %xmm4 - palignr $2, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $2, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_2_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_2_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_2_bwd): - lea (L(shl_2_bwd_loop_L1)-L(shl_2_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x02(%rsi), %xmm1 - jb L(L2_bwd) - lea (L(shl_2_bwd_loop_L2)-L(shl_2_bwd_loop_L1))(%r9), %r9 -L(L2_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_2_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_2_bwd_loop_L1): - movaps -0x12(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x22(%rsi), %xmm3 - movaps -0x32(%rsi), %xmm4 - movaps -0x42(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $2, %xmm2, %xmm1 - palignr $2, %xmm3, %xmm2 - palignr $2, %xmm4, %xmm3 - palignr $2, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_2_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_2_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_3): - lea (L(shl_3_loop_L1)-L(shl_3))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x03(%rsi), %xmm1 - jb L(L3_fwd) - lea (L(shl_3_loop_L2)-L(shl_3_loop_L1))(%r9), %r9 -L(L3_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_3_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_3_loop_L1): - sub $64, %rdx - movaps 0x0d(%rsi), %xmm2 - movaps 0x1d(%rsi), %xmm3 - movaps 0x2d(%rsi), %xmm4 - movaps 0x3d(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $3, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $3, %xmm3, %xmm4 - palignr $3, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $3, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_3_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_3_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_3_bwd): - lea (L(shl_3_bwd_loop_L1)-L(shl_3_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x03(%rsi), %xmm1 - jb L(L3_bwd) - lea (L(shl_3_bwd_loop_L2)-L(shl_3_bwd_loop_L1))(%r9), %r9 -L(L3_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_3_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_3_bwd_loop_L1): - movaps -0x13(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x23(%rsi), %xmm3 - movaps -0x33(%rsi), %xmm4 - movaps -0x43(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $3, %xmm2, %xmm1 - palignr $3, %xmm3, %xmm2 - palignr $3, %xmm4, %xmm3 - palignr $3, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_3_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_3_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_4): - lea (L(shl_4_loop_L1)-L(shl_4))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x04(%rsi), %xmm1 - jb L(L4_fwd) - lea (L(shl_4_loop_L2)-L(shl_4_loop_L1))(%r9), %r9 -L(L4_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_4_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_4_loop_L1): - sub $64, %rdx - movaps 0x0c(%rsi), %xmm2 - movaps 0x1c(%rsi), %xmm3 - movaps 0x2c(%rsi), %xmm4 - movaps 0x3c(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $4, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $4, %xmm3, %xmm4 - palignr $4, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $4, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_4_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_4_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_4_bwd): - lea (L(shl_4_bwd_loop_L1)-L(shl_4_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x04(%rsi), %xmm1 - jb L(L4_bwd) - lea (L(shl_4_bwd_loop_L2)-L(shl_4_bwd_loop_L1))(%r9), %r9 -L(L4_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_4_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_4_bwd_loop_L1): - movaps -0x14(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x24(%rsi), %xmm3 - movaps -0x34(%rsi), %xmm4 - movaps -0x44(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $4, %xmm2, %xmm1 - palignr $4, %xmm3, %xmm2 - palignr $4, %xmm4, %xmm3 - palignr $4, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_4_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_4_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_5): - lea (L(shl_5_loop_L1)-L(shl_5))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x05(%rsi), %xmm1 - jb L(L5_fwd) - lea (L(shl_5_loop_L2)-L(shl_5_loop_L1))(%r9), %r9 -L(L5_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_5_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_5_loop_L1): - sub $64, %rdx - movaps 0x0b(%rsi), %xmm2 - movaps 0x1b(%rsi), %xmm3 - movaps 0x2b(%rsi), %xmm4 - movaps 0x3b(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $5, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $5, %xmm3, %xmm4 - palignr $5, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $5, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_5_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_5_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_5_bwd): - lea (L(shl_5_bwd_loop_L1)-L(shl_5_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x05(%rsi), %xmm1 - jb L(L5_bwd) - lea (L(shl_5_bwd_loop_L2)-L(shl_5_bwd_loop_L1))(%r9), %r9 -L(L5_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_5_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_5_bwd_loop_L1): - movaps -0x15(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x25(%rsi), %xmm3 - movaps -0x35(%rsi), %xmm4 - movaps -0x45(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $5, %xmm2, %xmm1 - palignr $5, %xmm3, %xmm2 - palignr $5, %xmm4, %xmm3 - palignr $5, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_5_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_5_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_6): - lea (L(shl_6_loop_L1)-L(shl_6))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x06(%rsi), %xmm1 - jb L(L6_fwd) - lea (L(shl_6_loop_L2)-L(shl_6_loop_L1))(%r9), %r9 -L(L6_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_6_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_6_loop_L1): - sub $64, %rdx - movaps 0x0a(%rsi), %xmm2 - movaps 0x1a(%rsi), %xmm3 - movaps 0x2a(%rsi), %xmm4 - movaps 0x3a(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $6, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $6, %xmm3, %xmm4 - palignr $6, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $6, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_6_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_6_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_6_bwd): - lea (L(shl_6_bwd_loop_L1)-L(shl_6_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x06(%rsi), %xmm1 - jb L(L6_bwd) - lea (L(shl_6_bwd_loop_L2)-L(shl_6_bwd_loop_L1))(%r9), %r9 -L(L6_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_6_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_6_bwd_loop_L1): - movaps -0x16(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x26(%rsi), %xmm3 - movaps -0x36(%rsi), %xmm4 - movaps -0x46(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $6, %xmm2, %xmm1 - palignr $6, %xmm3, %xmm2 - palignr $6, %xmm4, %xmm3 - palignr $6, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_6_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_6_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_7): - lea (L(shl_7_loop_L1)-L(shl_7))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x07(%rsi), %xmm1 - jb L(L7_fwd) - lea (L(shl_7_loop_L2)-L(shl_7_loop_L1))(%r9), %r9 -L(L7_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_7_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_7_loop_L1): - sub $64, %rdx - movaps 0x09(%rsi), %xmm2 - movaps 0x19(%rsi), %xmm3 - movaps 0x29(%rsi), %xmm4 - movaps 0x39(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $7, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $7, %xmm3, %xmm4 - palignr $7, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $7, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_7_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_7_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_7_bwd): - lea (L(shl_7_bwd_loop_L1)-L(shl_7_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x07(%rsi), %xmm1 - jb L(L7_bwd) - lea (L(shl_7_bwd_loop_L2)-L(shl_7_bwd_loop_L1))(%r9), %r9 -L(L7_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_7_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_7_bwd_loop_L1): - movaps -0x17(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x27(%rsi), %xmm3 - movaps -0x37(%rsi), %xmm4 - movaps -0x47(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $7, %xmm2, %xmm1 - palignr $7, %xmm3, %xmm2 - palignr $7, %xmm4, %xmm3 - palignr $7, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_7_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_7_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_8): - lea (L(shl_8_loop_L1)-L(shl_8))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x08(%rsi), %xmm1 - jb L(L8_fwd) - lea (L(shl_8_loop_L2)-L(shl_8_loop_L1))(%r9), %r9 -L(L8_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 -L(shl_8_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_8_loop_L1): - sub $64, %rdx - movaps 0x08(%rsi), %xmm2 - movaps 0x18(%rsi), %xmm3 - movaps 0x28(%rsi), %xmm4 - movaps 0x38(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $8, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $8, %xmm3, %xmm4 - palignr $8, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $8, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_8_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 - .p2align 4 -L(shl_8_end): - lea 64(%rdx), %rdx - movaps %xmm4, -0x20(%rdi) - add %rdx, %rsi - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_8_bwd): - lea (L(shl_8_bwd_loop_L1)-L(shl_8_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x08(%rsi), %xmm1 - jb L(L8_bwd) - lea (L(shl_8_bwd_loop_L2)-L(shl_8_bwd_loop_L1))(%r9), %r9 -L(L8_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_8_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_8_bwd_loop_L1): - movaps -0x18(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x28(%rsi), %xmm3 - movaps -0x38(%rsi), %xmm4 - movaps -0x48(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $8, %xmm2, %xmm1 - palignr $8, %xmm3, %xmm2 - palignr $8, %xmm4, %xmm3 - palignr $8, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_8_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_8_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_9): - lea (L(shl_9_loop_L1)-L(shl_9))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x09(%rsi), %xmm1 - jb L(L9_fwd) - lea (L(shl_9_loop_L2)-L(shl_9_loop_L1))(%r9), %r9 -L(L9_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_9_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_9_loop_L1): - sub $64, %rdx - movaps 0x07(%rsi), %xmm2 - movaps 0x17(%rsi), %xmm3 - movaps 0x27(%rsi), %xmm4 - movaps 0x37(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $9, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $9, %xmm3, %xmm4 - palignr $9, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $9, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_9_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_9_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_9_bwd): - lea (L(shl_9_bwd_loop_L1)-L(shl_9_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x09(%rsi), %xmm1 - jb L(L9_bwd) - lea (L(shl_9_bwd_loop_L2)-L(shl_9_bwd_loop_L1))(%r9), %r9 -L(L9_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_9_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_9_bwd_loop_L1): - movaps -0x19(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x29(%rsi), %xmm3 - movaps -0x39(%rsi), %xmm4 - movaps -0x49(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $9, %xmm2, %xmm1 - palignr $9, %xmm3, %xmm2 - palignr $9, %xmm4, %xmm3 - palignr $9, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_9_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_9_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_10): - lea (L(shl_10_loop_L1)-L(shl_10))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0a(%rsi), %xmm1 - jb L(L10_fwd) - lea (L(shl_10_loop_L2)-L(shl_10_loop_L1))(%r9), %r9 -L(L10_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_10_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_10_loop_L1): - sub $64, %rdx - movaps 0x06(%rsi), %xmm2 - movaps 0x16(%rsi), %xmm3 - movaps 0x26(%rsi), %xmm4 - movaps 0x36(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $10, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $10, %xmm3, %xmm4 - palignr $10, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $10, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_10_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_10_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_10_bwd): - lea (L(shl_10_bwd_loop_L1)-L(shl_10_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0a(%rsi), %xmm1 - jb L(L10_bwd) - lea (L(shl_10_bwd_loop_L2)-L(shl_10_bwd_loop_L1))(%r9), %r9 -L(L10_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_10_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_10_bwd_loop_L1): - movaps -0x1a(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2a(%rsi), %xmm3 - movaps -0x3a(%rsi), %xmm4 - movaps -0x4a(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $10, %xmm2, %xmm1 - palignr $10, %xmm3, %xmm2 - palignr $10, %xmm4, %xmm3 - palignr $10, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_10_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_10_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_11): - lea (L(shl_11_loop_L1)-L(shl_11))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0b(%rsi), %xmm1 - jb L(L11_fwd) - lea (L(shl_11_loop_L2)-L(shl_11_loop_L1))(%r9), %r9 -L(L11_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_11_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_11_loop_L1): - sub $64, %rdx - movaps 0x05(%rsi), %xmm2 - movaps 0x15(%rsi), %xmm3 - movaps 0x25(%rsi), %xmm4 - movaps 0x35(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $11, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $11, %xmm3, %xmm4 - palignr $11, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $11, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_11_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_11_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_11_bwd): - lea (L(shl_11_bwd_loop_L1)-L(shl_11_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0b(%rsi), %xmm1 - jb L(L11_bwd) - lea (L(shl_11_bwd_loop_L2)-L(shl_11_bwd_loop_L1))(%r9), %r9 -L(L11_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_11_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_11_bwd_loop_L1): - movaps -0x1b(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2b(%rsi), %xmm3 - movaps -0x3b(%rsi), %xmm4 - movaps -0x4b(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $11, %xmm2, %xmm1 - palignr $11, %xmm3, %xmm2 - palignr $11, %xmm4, %xmm3 - palignr $11, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_11_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_11_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_12): - lea (L(shl_12_loop_L1)-L(shl_12))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0c(%rsi), %xmm1 - jb L(L12_fwd) - lea (L(shl_12_loop_L2)-L(shl_12_loop_L1))(%r9), %r9 -L(L12_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_12_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_12_loop_L1): - sub $64, %rdx - movaps 0x04(%rsi), %xmm2 - movaps 0x14(%rsi), %xmm3 - movaps 0x24(%rsi), %xmm4 - movaps 0x34(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $12, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $12, %xmm3, %xmm4 - palignr $12, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $12, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_12_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_12_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_12_bwd): - lea (L(shl_12_bwd_loop_L1)-L(shl_12_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0c(%rsi), %xmm1 - jb L(L12_bwd) - lea (L(shl_12_bwd_loop_L2)-L(shl_12_bwd_loop_L1))(%r9), %r9 -L(L12_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_12_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_12_bwd_loop_L1): - movaps -0x1c(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2c(%rsi), %xmm3 - movaps -0x3c(%rsi), %xmm4 - movaps -0x4c(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $12, %xmm2, %xmm1 - palignr $12, %xmm3, %xmm2 - palignr $12, %xmm4, %xmm3 - palignr $12, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_12_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_12_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_13): - lea (L(shl_13_loop_L1)-L(shl_13))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0d(%rsi), %xmm1 - jb L(L13_fwd) - lea (L(shl_13_loop_L2)-L(shl_13_loop_L1))(%r9), %r9 -L(L13_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_13_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_13_loop_L1): - sub $64, %rdx - movaps 0x03(%rsi), %xmm2 - movaps 0x13(%rsi), %xmm3 - movaps 0x23(%rsi), %xmm4 - movaps 0x33(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $13, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $13, %xmm3, %xmm4 - palignr $13, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $13, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_13_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_13_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_13_bwd): - lea (L(shl_13_bwd_loop_L1)-L(shl_13_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0d(%rsi), %xmm1 - jb L(L13_bwd) - lea (L(shl_13_bwd_loop_L2)-L(shl_13_bwd_loop_L1))(%r9), %r9 -L(L13_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_13_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_13_bwd_loop_L1): - movaps -0x1d(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2d(%rsi), %xmm3 - movaps -0x3d(%rsi), %xmm4 - movaps -0x4d(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $13, %xmm2, %xmm1 - palignr $13, %xmm3, %xmm2 - palignr $13, %xmm4, %xmm3 - palignr $13, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_13_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_13_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_14): - lea (L(shl_14_loop_L1)-L(shl_14))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0e(%rsi), %xmm1 - jb L(L14_fwd) - lea (L(shl_14_loop_L2)-L(shl_14_loop_L1))(%r9), %r9 -L(L14_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_14_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_14_loop_L1): - sub $64, %rdx - movaps 0x02(%rsi), %xmm2 - movaps 0x12(%rsi), %xmm3 - movaps 0x22(%rsi), %xmm4 - movaps 0x32(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $14, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $14, %xmm3, %xmm4 - palignr $14, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $14, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_14_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_14_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_14_bwd): - lea (L(shl_14_bwd_loop_L1)-L(shl_14_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0e(%rsi), %xmm1 - jb L(L14_bwd) - lea (L(shl_14_bwd_loop_L2)-L(shl_14_bwd_loop_L1))(%r9), %r9 -L(L14_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_14_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_14_bwd_loop_L1): - movaps -0x1e(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2e(%rsi), %xmm3 - movaps -0x3e(%rsi), %xmm4 - movaps -0x4e(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $14, %xmm2, %xmm1 - palignr $14, %xmm3, %xmm2 - palignr $14, %xmm4, %xmm3 - palignr $14, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_14_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_14_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_15): - lea (L(shl_15_loop_L1)-L(shl_15))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0f(%rsi), %xmm1 - jb L(L15_fwd) - lea (L(shl_15_loop_L2)-L(shl_15_loop_L1))(%r9), %r9 -L(L15_fwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_15_loop_L2): - prefetchnta 0x1c0(%rsi) -L(shl_15_loop_L1): - sub $64, %rdx - movaps 0x01(%rsi), %xmm2 - movaps 0x11(%rsi), %xmm3 - movaps 0x21(%rsi), %xmm4 - movaps 0x31(%rsi), %xmm5 - movdqa %xmm5, %xmm6 - palignr $15, %xmm4, %xmm5 - lea 64(%rsi), %rsi - palignr $15, %xmm3, %xmm4 - palignr $15, %xmm2, %xmm3 - lea 64(%rdi), %rdi - palignr $15, %xmm1, %xmm2 - movdqa %xmm6, %xmm1 - movdqa %xmm2, -0x40(%rdi) - movaps %xmm3, -0x30(%rdi) - jb L(shl_15_end) - movaps %xmm4, -0x20(%rdi) - movaps %xmm5, -0x10(%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_15_end): - movaps %xmm4, -0x20(%rdi) - lea 64(%rdx), %rdx - movaps %xmm5, -0x10(%rdi) - add %rdx, %rdi - movdqu %xmm0, (%r8) - add %rdx, %rsi - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(shl_15_bwd): - lea (L(shl_15_bwd_loop_L1)-L(shl_15_bwd))(%r9), %r9 - cmp %rcx, %rdx - movaps -0x0f(%rsi), %xmm1 - jb L(L15_bwd) - lea (L(shl_15_bwd_loop_L2)-L(shl_15_bwd_loop_L1))(%r9), %r9 -L(L15_bwd): - lea -64(%rdx), %rdx - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_15_bwd_loop_L2): - prefetchnta -0x1c0(%rsi) -L(shl_15_bwd_loop_L1): - movaps -0x1f(%rsi), %xmm2 - sub $0x40, %rdx - movaps -0x2f(%rsi), %xmm3 - movaps -0x3f(%rsi), %xmm4 - movaps -0x4f(%rsi), %xmm5 - lea -0x40(%rsi), %rsi - palignr $15, %xmm2, %xmm1 - palignr $15, %xmm3, %xmm2 - palignr $15, %xmm4, %xmm3 - palignr $15, %xmm5, %xmm4 - - movaps %xmm1, -0x10(%rdi) - movaps %xmm5, %xmm1 - - movaps %xmm2, -0x20(%rdi) - lea -0x40(%rdi), %rdi - - movaps %xmm3, 0x10(%rdi) - jb L(shl_15_bwd_end) - movaps %xmm4, (%rdi) - _CET_NOTRACK jmp *%r9 - ud2 -L(shl_15_bwd_end): - movaps %xmm4, (%rdi) - lea 64(%rdx), %rdx - movdqu %xmm0, (%r8) - BRANCH_TO_JMPTBL_ENTRY(L(table_less_80bytes), %rdx, 4) - - .p2align 4 -L(write_72bytes): - movdqu -72(%rsi), %xmm0 - movdqu -56(%rsi), %xmm1 - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rcx - movdqu %xmm0, -72(%rdi) - movdqu %xmm1, -56(%rdi) - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rcx, -8(%rdi) - ret - - .p2align 4 -L(write_64bytes): - movdqu -64(%rsi), %xmm0 - mov -48(%rsi), %rcx - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - movdqu %xmm0, -64(%rdi) - mov %rcx, -48(%rdi) - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_56bytes): - movdqu -56(%rsi), %xmm0 - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rcx - movdqu %xmm0, -56(%rdi) - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rcx, -8(%rdi) - ret - - .p2align 4 -L(write_48bytes): - mov -48(%rsi), %rcx - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %rcx, -48(%rdi) - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_40bytes): - mov -40(%rsi), %r8 - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %r8, -40(%rdi) - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_32bytes): - mov -32(%rsi), %r9 - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %r9, -32(%rdi) - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_24bytes): - mov -24(%rsi), %r10 - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %r10, -24(%rdi) - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_16bytes): - mov -16(%rsi), %r11 - mov -8(%rsi), %rdx - mov %r11, -16(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_8bytes): - mov -8(%rsi), %rdx - mov %rdx, -8(%rdi) -L(write_0bytes): - ret - - .p2align 4 -L(write_73bytes): - movdqu -73(%rsi), %xmm0 - movdqu -57(%rsi), %xmm1 - mov -41(%rsi), %rcx - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %r8 - mov -4(%rsi), %edx - movdqu %xmm0, -73(%rdi) - movdqu %xmm1, -57(%rdi) - mov %rcx, -41(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %r8, -9(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_65bytes): - movdqu -65(%rsi), %xmm0 - movdqu -49(%rsi), %xmm1 - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -65(%rdi) - movdqu %xmm1, -49(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_57bytes): - movdqu -57(%rsi), %xmm0 - mov -41(%rsi), %r8 - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -57(%rdi) - mov %r8, -41(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_49bytes): - movdqu -49(%rsi), %xmm0 - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -49(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_41bytes): - mov -41(%rsi), %r8 - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -1(%rsi), %dl - mov %r8, -41(%rdi) - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %dl, -1(%rdi) - ret - - .p2align 4 -L(write_33bytes): - mov -33(%rsi), %r9 - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -1(%rsi), %dl - mov %r9, -33(%rdi) - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %dl, -1(%rdi) - ret - - .p2align 4 -L(write_25bytes): - mov -25(%rsi), %r10 - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -1(%rsi), %dl - mov %r10, -25(%rdi) - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %dl, -1(%rdi) - ret - - .p2align 4 -L(write_17bytes): - mov -17(%rsi), %r11 - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - mov %r11, -17(%rdi) - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_9bytes): - mov -9(%rsi), %rcx - mov -4(%rsi), %edx - mov %rcx, -9(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_1bytes): - mov -1(%rsi), %dl - mov %dl, -1(%rdi) - ret - - .p2align 4 -L(write_74bytes): - movdqu -74(%rsi), %xmm0 - movdqu -58(%rsi), %xmm1 - mov -42(%rsi), %r8 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -74(%rdi) - movdqu %xmm1, -58(%rdi) - mov %r8, -42(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_66bytes): - movdqu -66(%rsi), %xmm0 - movdqu -50(%rsi), %xmm1 - mov -42(%rsi), %r8 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -66(%rdi) - movdqu %xmm1, -50(%rdi) - mov %r8, -42(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_58bytes): - movdqu -58(%rsi), %xmm1 - mov -42(%rsi), %r8 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm1, -58(%rdi) - mov %r8, -42(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_50bytes): - movdqu -50(%rsi), %xmm0 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -50(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_42bytes): - mov -42(%rsi), %r8 - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %r8, -42(%rdi) - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_34bytes): - mov -34(%rsi), %r9 - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %r9, -34(%rdi) - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_26bytes): - mov -26(%rsi), %r10 - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %r10, -26(%rdi) - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_18bytes): - mov -18(%rsi), %r11 - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %r11, -18(%rdi) - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_10bytes): - mov -10(%rsi), %rcx - mov -4(%rsi), %edx - mov %rcx, -10(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_2bytes): - mov -2(%rsi), %dx - mov %dx, -2(%rdi) - ret - - .p2align 4 -L(write_75bytes): - movdqu -75(%rsi), %xmm0 - movdqu -59(%rsi), %xmm1 - mov -43(%rsi), %r8 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -75(%rdi) - movdqu %xmm1, -59(%rdi) - mov %r8, -43(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_67bytes): - movdqu -67(%rsi), %xmm0 - movdqu -59(%rsi), %xmm1 - mov -43(%rsi), %r8 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -67(%rdi) - movdqu %xmm1, -59(%rdi) - mov %r8, -43(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_59bytes): - movdqu -59(%rsi), %xmm0 - mov -43(%rsi), %r8 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -59(%rdi) - mov %r8, -43(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_51bytes): - movdqu -51(%rsi), %xmm0 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -51(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_43bytes): - mov -43(%rsi), %r8 - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %r8, -43(%rdi) - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_35bytes): - mov -35(%rsi), %r9 - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %r9, -35(%rdi) - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_27bytes): - mov -27(%rsi), %r10 - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %r10, -27(%rdi) - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_19bytes): - mov -19(%rsi), %r11 - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %r11, -19(%rdi) - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_11bytes): - mov -11(%rsi), %rcx - mov -4(%rsi), %edx - mov %rcx, -11(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_3bytes): - mov -3(%rsi), %dx - mov -2(%rsi), %cx - mov %dx, -3(%rdi) - mov %cx, -2(%rdi) - ret - - .p2align 4 -L(write_76bytes): - movdqu -76(%rsi), %xmm0 - movdqu -60(%rsi), %xmm1 - mov -44(%rsi), %r8 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -76(%rdi) - movdqu %xmm1, -60(%rdi) - mov %r8, -44(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_68bytes): - movdqu -68(%rsi), %xmm0 - movdqu -52(%rsi), %xmm1 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -68(%rdi) - movdqu %xmm1, -52(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_60bytes): - movdqu -60(%rsi), %xmm0 - mov -44(%rsi), %r8 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -60(%rdi) - mov %r8, -44(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_52bytes): - movdqu -52(%rsi), %xmm0 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - movdqu %xmm0, -52(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_44bytes): - mov -44(%rsi), %r8 - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %r8, -44(%rdi) - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_36bytes): - mov -36(%rsi), %r9 - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %r9, -36(%rdi) - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_28bytes): - mov -28(%rsi), %r10 - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %r10, -28(%rdi) - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_20bytes): - mov -20(%rsi), %r11 - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %r11, -20(%rdi) - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_12bytes): - mov -12(%rsi), %rcx - mov -4(%rsi), %edx - mov %rcx, -12(%rdi) - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_4bytes): - mov -4(%rsi), %edx - mov %edx, -4(%rdi) - ret - - .p2align 4 -L(write_77bytes): - movdqu -77(%rsi), %xmm0 - movdqu -61(%rsi), %xmm1 - mov -45(%rsi), %r8 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -77(%rdi) - movdqu %xmm1, -61(%rdi) - mov %r8, -45(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_69bytes): - movdqu -69(%rsi), %xmm0 - movdqu -53(%rsi), %xmm1 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -69(%rdi) - movdqu %xmm1, -53(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_61bytes): - movdqu -61(%rsi), %xmm0 - mov -45(%rsi), %r8 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -61(%rdi) - mov %r8, -45(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_53bytes): - movdqu -53(%rsi), %xmm0 - mov -45(%rsi), %r8 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -53(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_45bytes): - mov -45(%rsi), %r8 - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r8, -45(%rdi) - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_37bytes): - mov -37(%rsi), %r9 - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r9, -37(%rdi) - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_29bytes): - mov -29(%rsi), %r10 - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r10, -29(%rdi) - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_21bytes): - mov -21(%rsi), %r11 - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r11, -21(%rdi) - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_13bytes): - mov -13(%rsi), %rcx - mov -8(%rsi), %rdx - mov %rcx, -13(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_5bytes): - mov -5(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -5(%rdi) - mov %ecx, -4(%rdi) - ret - - .p2align 4 -L(write_78bytes): - movdqu -78(%rsi), %xmm0 - movdqu -62(%rsi), %xmm1 - mov -46(%rsi), %r8 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -78(%rdi) - movdqu %xmm1, -62(%rdi) - mov %r8, -46(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_70bytes): - movdqu -70(%rsi), %xmm0 - movdqu -54(%rsi), %xmm1 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -70(%rdi) - movdqu %xmm1, -54(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_62bytes): - movdqu -62(%rsi), %xmm0 - mov -46(%rsi), %r8 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -62(%rdi) - mov %r8, -46(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_54bytes): - movdqu -54(%rsi), %xmm0 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -54(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_46bytes): - mov -46(%rsi), %r8 - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r8, -46(%rdi) - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_38bytes): - mov -38(%rsi), %r9 - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r9, -38(%rdi) - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_30bytes): - mov -30(%rsi), %r10 - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r10, -30(%rdi) - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_22bytes): - mov -22(%rsi), %r11 - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r11, -22(%rdi) - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_14bytes): - mov -14(%rsi), %rcx - mov -8(%rsi), %rdx - mov %rcx, -14(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_6bytes): - mov -6(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -6(%rdi) - mov %ecx, -4(%rdi) - ret - - .p2align 4 -L(write_79bytes): - movdqu -79(%rsi), %xmm0 - movdqu -63(%rsi), %xmm1 - mov -47(%rsi), %r8 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -79(%rdi) - movdqu %xmm1, -63(%rdi) - mov %r8, -47(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_71bytes): - movdqu -71(%rsi), %xmm0 - movdqu -55(%rsi), %xmm1 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -71(%rdi) - movdqu %xmm1, -55(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_63bytes): - movdqu -63(%rsi), %xmm0 - mov -47(%rsi), %r8 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -63(%rdi) - mov %r8, -47(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_55bytes): - movdqu -55(%rsi), %xmm0 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - movdqu %xmm0, -55(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_47bytes): - mov -47(%rsi), %r8 - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r8, -47(%rdi) - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_39bytes): - mov -39(%rsi), %r9 - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r9, -39(%rdi) - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_31bytes): - mov -31(%rsi), %r10 - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r10, -31(%rdi) - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_23bytes): - mov -23(%rsi), %r11 - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %r11, -23(%rdi) - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_15bytes): - mov -15(%rsi), %rcx - mov -8(%rsi), %rdx - mov %rcx, -15(%rdi) - mov %rdx, -8(%rdi) - ret - - .p2align 4 -L(write_7bytes): - mov -7(%rsi), %edx - mov -4(%rsi), %ecx - mov %edx, -7(%rdi) - mov %ecx, -4(%rdi) - ret - - .p2align 4 -L(large_page_fwd): - movdqu (%rsi), %xmm1 - lea 16(%rsi), %rsi - movdqu %xmm0, (%r8) - movntdq %xmm1, (%rdi) - lea 16(%rdi), %rdi - lea -0x90(%rdx), %rdx -#ifdef USE_AS_MEMMOVE - mov %rsi, %r9 - sub %rdi, %r9 - cmp %rdx, %r9 - jae L(memmove_is_memcpy_fwd) - shl $2, %rcx - cmp %rcx, %rdx - jb L(ll_cache_copy_fwd_start) -L(memmove_is_memcpy_fwd): -#endif -L(large_page_loop): - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - movdqu 0x40(%rsi), %xmm4 - movdqu 0x50(%rsi), %xmm5 - movdqu 0x60(%rsi), %xmm6 - movdqu 0x70(%rsi), %xmm7 - lea 0x80(%rsi), %rsi - - sub $0x80, %rdx - movntdq %xmm0, (%rdi) - movntdq %xmm1, 0x10(%rdi) - movntdq %xmm2, 0x20(%rdi) - movntdq %xmm3, 0x30(%rdi) - movntdq %xmm4, 0x40(%rdi) - movntdq %xmm5, 0x50(%rdi) - movntdq %xmm6, 0x60(%rdi) - movntdq %xmm7, 0x70(%rdi) - lea 0x80(%rdi), %rdi - jae L(large_page_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(large_page_less_64bytes) - - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - lea 0x40(%rsi), %rsi - - movntdq %xmm0, (%rdi) - movntdq %xmm1, 0x10(%rdi) - movntdq %xmm2, 0x20(%rdi) - movntdq %xmm3, 0x30(%rdi) - lea 0x40(%rdi), %rdi - sub $0x40, %rdx -L(large_page_less_64bytes): - add %rdx, %rsi - add %rdx, %rdi - sfence - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - -#ifdef USE_AS_MEMMOVE - .p2align 4 -L(ll_cache_copy_fwd_start): - prefetcht0 0x1c0(%rsi) - prefetcht0 0x200(%rsi) - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - movdqu 0x40(%rsi), %xmm4 - movdqu 0x50(%rsi), %xmm5 - movdqu 0x60(%rsi), %xmm6 - movdqu 0x70(%rsi), %xmm7 - lea 0x80(%rsi), %rsi - - sub $0x80, %rdx - movaps %xmm0, (%rdi) - movaps %xmm1, 0x10(%rdi) - movaps %xmm2, 0x20(%rdi) - movaps %xmm3, 0x30(%rdi) - movaps %xmm4, 0x40(%rdi) - movaps %xmm5, 0x50(%rdi) - movaps %xmm6, 0x60(%rdi) - movaps %xmm7, 0x70(%rdi) - lea 0x80(%rdi), %rdi - jae L(ll_cache_copy_fwd_start) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(large_page_ll_less_fwd_64bytes) - - movdqu (%rsi), %xmm0 - movdqu 0x10(%rsi), %xmm1 - movdqu 0x20(%rsi), %xmm2 - movdqu 0x30(%rsi), %xmm3 - lea 0x40(%rsi), %rsi - - movaps %xmm0, (%rdi) - movaps %xmm1, 0x10(%rdi) - movaps %xmm2, 0x20(%rdi) - movaps %xmm3, 0x30(%rdi) - lea 0x40(%rdi), %rdi - sub $0x40, %rdx -L(large_page_ll_less_fwd_64bytes): - add %rdx, %rsi - add %rdx, %rdi - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - -#endif - .p2align 4 -L(large_page_bwd): - movdqu -0x10(%rsi), %xmm1 - lea -16(%rsi), %rsi - movdqu %xmm0, (%r8) - movdqa %xmm1, -0x10(%rdi) - lea -16(%rdi), %rdi - lea -0x90(%rdx), %rdx -#ifdef USE_AS_MEMMOVE - mov %rdi, %r9 - sub %rsi, %r9 - cmp %rdx, %r9 - jae L(memmove_is_memcpy_bwd) - cmp %rcx, %r9 - jb L(ll_cache_copy_bwd_start) -L(memmove_is_memcpy_bwd): -#endif -L(large_page_bwd_loop): - movdqu -0x10(%rsi), %xmm0 - movdqu -0x20(%rsi), %xmm1 - movdqu -0x30(%rsi), %xmm2 - movdqu -0x40(%rsi), %xmm3 - movdqu -0x50(%rsi), %xmm4 - movdqu -0x60(%rsi), %xmm5 - movdqu -0x70(%rsi), %xmm6 - movdqu -0x80(%rsi), %xmm7 - lea -0x80(%rsi), %rsi - - sub $0x80, %rdx - movntdq %xmm0, -0x10(%rdi) - movntdq %xmm1, -0x20(%rdi) - movntdq %xmm2, -0x30(%rdi) - movntdq %xmm3, -0x40(%rdi) - movntdq %xmm4, -0x50(%rdi) - movntdq %xmm5, -0x60(%rdi) - movntdq %xmm6, -0x70(%rdi) - movntdq %xmm7, -0x80(%rdi) - lea -0x80(%rdi), %rdi - jae L(large_page_bwd_loop) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(large_page_less_bwd_64bytes) - - movdqu -0x10(%rsi), %xmm0 - movdqu -0x20(%rsi), %xmm1 - movdqu -0x30(%rsi), %xmm2 - movdqu -0x40(%rsi), %xmm3 - lea -0x40(%rsi), %rsi - - movntdq %xmm0, -0x10(%rdi) - movntdq %xmm1, -0x20(%rdi) - movntdq %xmm2, -0x30(%rdi) - movntdq %xmm3, -0x40(%rdi) - lea -0x40(%rdi), %rdi - sub $0x40, %rdx -L(large_page_less_bwd_64bytes): - sfence - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) - -#ifdef USE_AS_MEMMOVE - .p2align 4 -L(ll_cache_copy_bwd_start): - prefetcht0 -0x1c0(%rsi) - prefetcht0 -0x200(%rsi) - movdqu -0x10(%rsi), %xmm0 - movdqu -0x20(%rsi), %xmm1 - movdqu -0x30(%rsi), %xmm2 - movdqu -0x40(%rsi), %xmm3 - movdqu -0x50(%rsi), %xmm4 - movdqu -0x60(%rsi), %xmm5 - movdqu -0x70(%rsi), %xmm6 - movdqu -0x80(%rsi), %xmm7 - lea -0x80(%rsi), %rsi - - sub $0x80, %rdx - movaps %xmm0, -0x10(%rdi) - movaps %xmm1, -0x20(%rdi) - movaps %xmm2, -0x30(%rdi) - movaps %xmm3, -0x40(%rdi) - movaps %xmm4, -0x50(%rdi) - movaps %xmm5, -0x60(%rdi) - movaps %xmm6, -0x70(%rdi) - movaps %xmm7, -0x80(%rdi) - lea -0x80(%rdi), %rdi - jae L(ll_cache_copy_bwd_start) - cmp $-0x40, %rdx - lea 0x80(%rdx), %rdx - jl L(large_page_ll_less_bwd_64bytes) - - movdqu -0x10(%rsi), %xmm0 - movdqu -0x20(%rsi), %xmm1 - movdqu -0x30(%rsi), %xmm2 - movdqu -0x40(%rsi), %xmm3 - lea -0x40(%rsi), %rsi - - movaps %xmm0, -0x10(%rdi) - movaps %xmm1, -0x20(%rdi) - movaps %xmm2, -0x30(%rdi) - movaps %xmm3, -0x40(%rdi) - lea -0x40(%rdi), %rdi - sub $0x40, %rdx -L(large_page_ll_less_bwd_64bytes): - BRANCH_TO_JMPTBL_ENTRY (L(table_less_80bytes), %rdx, 4) -#endif - -END (MEMCPY) - - .section .rodata.ssse3,"a",@progbits - .p2align 3 -L(table_less_80bytes): - .int JMPTBL (L(write_0bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_1bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_2bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_3bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_4bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_5bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_6bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_7bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_8bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_9bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_10bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_11bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_12bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_13bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_14bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_15bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_16bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_17bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_18bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_19bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_20bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_21bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_22bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_23bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_24bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_25bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_26bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_27bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_28bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_29bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_30bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_31bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_32bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_33bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_34bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_35bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_36bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_37bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_38bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_39bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_40bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_41bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_42bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_43bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_44bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_45bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_46bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_47bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_48bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_49bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_50bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_51bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_52bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_53bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_54bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_55bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_56bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_57bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_58bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_59bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_60bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_61bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_62bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_63bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_64bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_65bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_66bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_67bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_68bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_69bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_70bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_71bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_72bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_73bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_74bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_75bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_76bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_77bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_78bytes), L(table_less_80bytes)) - .int JMPTBL (L(write_79bytes), L(table_less_80bytes)) - - .p2align 3 -L(shl_table): - .int JMPTBL (L(shl_0), L(shl_table)) - .int JMPTBL (L(shl_1), L(shl_table)) - .int JMPTBL (L(shl_2), L(shl_table)) - .int JMPTBL (L(shl_3), L(shl_table)) - .int JMPTBL (L(shl_4), L(shl_table)) - .int JMPTBL (L(shl_5), L(shl_table)) - .int JMPTBL (L(shl_6), L(shl_table)) - .int JMPTBL (L(shl_7), L(shl_table)) - .int JMPTBL (L(shl_8), L(shl_table)) - .int JMPTBL (L(shl_9), L(shl_table)) - .int JMPTBL (L(shl_10), L(shl_table)) - .int JMPTBL (L(shl_11), L(shl_table)) - .int JMPTBL (L(shl_12), L(shl_table)) - .int JMPTBL (L(shl_13), L(shl_table)) - .int JMPTBL (L(shl_14), L(shl_table)) - .int JMPTBL (L(shl_15), L(shl_table)) - - .p2align 3 -L(shl_table_bwd): - .int JMPTBL (L(shl_0_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_1_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_2_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_3_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_4_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_5_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_6_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_7_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_8_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_9_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_10_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_11_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_12_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_13_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_14_bwd), L(shl_table_bwd)) - .int JMPTBL (L(shl_15_bwd), L(shl_table_bwd)) - -#endif diff --git a/sysdeps/x86_64/multiarch/memmove-ssse3.S b/sysdeps/x86_64/multiarch/memmove-ssse3.S index 295430b1ef..84e4e0f6cb 100644 --- a/sysdeps/x86_64/multiarch/memmove-ssse3.S +++ b/sysdeps/x86_64/multiarch/memmove-ssse3.S @@ -1,4 +1,382 @@ -#define USE_AS_MEMMOVE -#define MEMCPY __memmove_ssse3 -#define MEMCPY_CHK __memmove_chk_ssse3 -#include "memcpy-ssse3.S" +#include <sysdep.h> + +#ifndef MEMMOVE +# define MEMMOVE __memmove_ssse3 +# define MEMMOVE_CHK __memmove_chk_ssse3 +# define MEMCPY __memcpy_ssse3 +# define MEMCPY_CHK __memcpy_chk_ssse3 +# define MEMPCPY __mempcpy_ssse3 +# define MEMPCPY_CHK __mempcpy_chk_ssse3 +#endif + + .section .text.ssse3, "ax", @progbits +ENTRY(MEMPCPY_CHK) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET(__chk_fail) +END(MEMPCPY_CHK) + +ENTRY(MEMPCPY) + mov %RDI_LP, %RAX_LP + add %RDX_LP, %RAX_LP + jmp L(start) +END(MEMPCPY) + +ENTRY(MEMMOVE_CHK) + cmp %RDX_LP, %RCX_LP + jb HIDDEN_JUMPTARGET(__chk_fail) +END(MEMMOVE_CHK) + +ENTRY_P2ALIGN(MEMMOVE, 6) + movq %rdi, %rax +L(start): + cmpq $16, %rdx + jb L(copy_0_15) + + /* These loads are always useful. */ + movups 0(%rsi), %xmm0 + movups -16(%rsi, %rdx), %xmm7 + cmpq $32, %rdx + ja L(more_2x_vec) + + movups %xmm0, 0(%rdi) + movups %xmm7, -16(%rdi, %rdx) + ret + + .p2align 4,, 8 +L(copy_4x_vec): + movups 16(%rsi), %xmm1 + movups -32(%rsi, %rdx), %xmm2 + + movups %xmm0, 0(%rdi) + movups %xmm1, 16(%rdi) + movups %xmm2, -32(%rdi, %rdx) + movups %xmm7, -16(%rdi, %rdx) + ret + + .p2align 4,, 8 +L(copy_0_15): + cmpl $8, %edx + ja L(copy_9_15) + + cmpl $4, %edx + jb L(copy_0_3) + + movl 0(%rsi), %ecx + movl -4(%rsi, %rdx), %esi + movl %ecx, 0(%rdi) + movl %esi, -4(%rdi, %rdx) + ret + + .p2align 4,, 8 +L(copy_9_15): + movq 0(%rsi), %rcx + movq -8(%rsi, %rdx), %rsi + movq %rcx, 0(%rdi) + movq %rsi, -8(%rdi, %rdx) + ret + + .p2align 4,, 4 +L(copy_0_3): + cmpl $1, %edx + jl L(copy_0_0) + movzbl (%rsi), %ecx + je L(copy_0_1) + + movzwl -2(%rsi, %rdx), %esi + movw %si, -2(%rdi, %rdx) +L(copy_0_1): + movb %cl, (%rdi) +L(copy_0_0): +L(nop): + ret + + .p2align 4 +L(more_2x_vec): + cmpq $64, %rdx + jbe L(copy_4x_vec) + + /* We use rcx later to get alignr value. */ + movq %rdi, %rcx + + /* Backward copy for overlap + dst > src for memmove safety. */ + subq %rsi, %rcx + cmpq %rdx, %rcx + jb L(copy_backward) + + /* Load tail. */ + + /* -16(%rsi, %rdx) already loaded into xmm7. */ + movups -32(%rsi, %rdx), %xmm8 + movups -48(%rsi, %rdx), %xmm9 + + /* Get misalignment. */ + andl $0xf, %ecx + + movq %rsi, %r9 + addq %rcx, %rsi + andq $-16, %rsi + /* Get first vec for `palignr`. */ + movaps (%rsi), %xmm1 + + /* We have loaded (%rsi) so safe to do this store before the + loop. */ + movups %xmm0, (%rdi) + +#ifdef SHARED_CACHE_SIZE_HALF + cmp $SHARED_CACHE_SIZE_HALF, %RDX_LP +#else + cmp __x86_shared_cache_size_half(%rip), %rdx +#endif + ja L(large_memcpy) + + leaq -64(%rdi, %rdx), %r8 + andq $-16, %rdi + movl $48, %edx + + leaq L(loop_fwd_start)(%rip), %r9 + sall $6, %ecx + addq %r9, %rcx + jmp * %rcx + + .p2align 4,, 8 +L(copy_backward): + testq %rcx, %rcx + jz L(nop) + + /* Preload tail. */ + + /* (%rsi) already loaded into xmm0. */ + movups 16(%rsi), %xmm4 + movups 32(%rsi), %xmm5 + + movq %rdi, %r8 + subq %rdi, %rsi + leaq -49(%rdi, %rdx), %rdi + andq $-16, %rdi + addq %rdi, %rsi + andq $-16, %rsi + + movaps 48(%rsi), %xmm6 + + + leaq L(loop_bkwd_start)(%rip), %r9 + andl $0xf, %ecx + sall $6, %ecx + addq %r9, %rcx + jmp * %rcx + + .p2align 4,, 8 +L(large_memcpy): + movups -64(%r9, %rdx), %xmm10 + movups -80(%r9, %rdx), %xmm11 + + sall $5, %ecx + leal (%rcx, %rcx, 2), %r8d + leaq -96(%rdi, %rdx), %rcx + andq $-16, %rdi + leaq L(large_loop_fwd_start)(%rip), %rdx + addq %r8, %rdx + jmp * %rdx + + + /* Instead of a typical jump table all 16 loops are exactly + 64-bytes in size. So, we can just jump to first loop + r8 * + 64. Before modifying any loop ensure all their sizes match! + */ + .p2align 6 +L(loop_fwd_start): +L(loop_fwd_0x0): + movaps 16(%rsi), %xmm1 + movaps 32(%rsi), %xmm2 + movaps 48(%rsi), %xmm3 + movaps %xmm1, 16(%rdi) + movaps %xmm2, 32(%rdi) + movaps %xmm3, 48(%rdi) + addq %rdx, %rdi + addq %rdx, %rsi + cmpq %rdi, %r8 + ja L(loop_fwd_0x0) +L(end_loop_fwd): + movups %xmm9, 16(%r8) + movups %xmm8, 32(%r8) + movups %xmm7, 48(%r8) + ret + + /* Extactly 64 bytes if `jmp L(end_loop_fwd)` is long encoding. + 60 bytes otherwise. */ +#define ALIGNED_LOOP_FWD(align_by); \ + .p2align 6; \ +L(loop_fwd_ ## align_by): \ + movaps 16(%rsi), %xmm0; \ + movaps 32(%rsi), %xmm2; \ + movaps 48(%rsi), %xmm3; \ + movaps %xmm3, %xmm4; \ + palignr $align_by, %xmm2, %xmm3; \ + palignr $align_by, %xmm0, %xmm2; \ + palignr $align_by, %xmm1, %xmm0; \ + movaps %xmm4, %xmm1; \ + movaps %xmm0, 16(%rdi); \ + movaps %xmm2, 32(%rdi); \ + movaps %xmm3, 48(%rdi); \ + addq %rdx, %rdi; \ + addq %rdx, %rsi; \ + cmpq %rdi, %r8; \ + ja L(loop_fwd_ ## align_by); \ + jmp L(end_loop_fwd); + + /* Must be in descending order. */ + ALIGNED_LOOP_FWD (0xf) + ALIGNED_LOOP_FWD (0xe) + ALIGNED_LOOP_FWD (0xd) + ALIGNED_LOOP_FWD (0xc) + ALIGNED_LOOP_FWD (0xb) + ALIGNED_LOOP_FWD (0xa) + ALIGNED_LOOP_FWD (0x9) + ALIGNED_LOOP_FWD (0x8) + ALIGNED_LOOP_FWD (0x7) + ALIGNED_LOOP_FWD (0x6) + ALIGNED_LOOP_FWD (0x5) + ALIGNED_LOOP_FWD (0x4) + ALIGNED_LOOP_FWD (0x3) + ALIGNED_LOOP_FWD (0x2) + ALIGNED_LOOP_FWD (0x1) + + .p2align 6 +L(large_loop_fwd_start): +L(large_loop_fwd_0x0): + movaps 16(%rsi), %xmm1 + movaps 32(%rsi), %xmm2 + movaps 48(%rsi), %xmm3 + movaps 64(%rsi), %xmm4 + movaps 80(%rsi), %xmm5 + movntps %xmm1, 16(%rdi) + movntps %xmm2, 32(%rdi) + movntps %xmm3, 48(%rdi) + movntps %xmm4, 64(%rdi) + movntps %xmm5, 80(%rdi) + addq $80, %rdi + addq $80, %rsi + cmpq %rdi, %rcx + ja L(large_loop_fwd_0x0) + + /* Ensure no icache line split on tail. */ + .p2align 4 +L(end_large_loop_fwd): + sfence + movups %xmm11, 16(%rcx) + movups %xmm10, 32(%rcx) + movups %xmm9, 48(%rcx) + movups %xmm8, 64(%rcx) + movups %xmm7, 80(%rcx) + ret + + + /* Size > 64 bytes and <= 96 bytes. 32-byte align between ensure + 96-byte spacing between each. */ +#define ALIGNED_LARGE_LOOP_FWD(align_by); \ + .p2align 5; \ +L(large_loop_fwd_ ## align_by): \ + movaps 16(%rsi), %xmm0; \ + movaps 32(%rsi), %xmm2; \ + movaps 48(%rsi), %xmm3; \ + movaps 64(%rsi), %xmm4; \ + movaps 80(%rsi), %xmm5; \ + movaps %xmm5, %xmm6; \ + palignr $align_by, %xmm4, %xmm5; \ + palignr $align_by, %xmm3, %xmm4; \ + palignr $align_by, %xmm2, %xmm3; \ + palignr $align_by, %xmm0, %xmm2; \ + palignr $align_by, %xmm1, %xmm0; \ + movaps %xmm6, %xmm1; \ + movntps %xmm0, 16(%rdi); \ + movntps %xmm2, 32(%rdi); \ + movntps %xmm3, 48(%rdi); \ + movntps %xmm4, 64(%rdi); \ + movntps %xmm5, 80(%rdi); \ + addq $80, %rdi; \ + addq $80, %rsi; \ + cmpq %rdi, %rcx; \ + ja L(large_loop_fwd_ ## align_by); \ + jmp L(end_large_loop_fwd); + + /* Must be in descending order. */ + ALIGNED_LARGE_LOOP_FWD (0xf) + ALIGNED_LARGE_LOOP_FWD (0xe) + ALIGNED_LARGE_LOOP_FWD (0xd) + ALIGNED_LARGE_LOOP_FWD (0xc) + ALIGNED_LARGE_LOOP_FWD (0xb) + ALIGNED_LARGE_LOOP_FWD (0xa) + ALIGNED_LARGE_LOOP_FWD (0x9) + ALIGNED_LARGE_LOOP_FWD (0x8) + ALIGNED_LARGE_LOOP_FWD (0x7) + ALIGNED_LARGE_LOOP_FWD (0x6) + ALIGNED_LARGE_LOOP_FWD (0x5) + ALIGNED_LARGE_LOOP_FWD (0x4) + ALIGNED_LARGE_LOOP_FWD (0x3) + ALIGNED_LARGE_LOOP_FWD (0x2) + ALIGNED_LARGE_LOOP_FWD (0x1) + + + .p2align 6 +L(loop_bkwd_start): +L(loop_bkwd_0x0): + movaps 32(%rsi), %xmm1 + movaps 16(%rsi), %xmm2 + movaps 0(%rsi), %xmm3 + movaps %xmm1, 32(%rdi) + movaps %xmm2, 16(%rdi) + movaps %xmm3, 0(%rdi) + subq $48, %rdi + subq $48, %rsi + cmpq %rdi, %r8 + jb L(loop_bkwd_0x0) +L(end_loop_bkwd): + movups %xmm7, -16(%r8, %rdx) + movups %xmm0, 0(%r8) + movups %xmm4, 16(%r8) + movups %xmm5, 32(%r8) + + ret + + + /* Extactly 64 bytes if `jmp L(end_loop_bkwd)` is long encoding. + 60 bytes otherwise. */ +#define ALIGNED_LOOP_BKWD(align_by); \ + .p2align 6; \ +L(loop_bkwd_ ## align_by): \ + movaps 32(%rsi), %xmm1; \ + movaps 16(%rsi), %xmm2; \ + movaps 0(%rsi), %xmm3; \ + palignr $align_by, %xmm1, %xmm6; \ + palignr $align_by, %xmm2, %xmm1; \ + palignr $align_by, %xmm3, %xmm2; \ + movaps %xmm6, 32(%rdi); \ + movaps %xmm1, 16(%rdi); \ + movaps %xmm2, 0(%rdi); \ + subq $48, %rdi; \ + subq $48, %rsi; \ + movaps %xmm3, %xmm6; \ + cmpq %rdi, %r8; \ + jb L(loop_bkwd_ ## align_by); \ + jmp L(end_loop_bkwd); + + /* Must be in descending order. */ + ALIGNED_LOOP_BKWD (0xf) + ALIGNED_LOOP_BKWD (0xe) + ALIGNED_LOOP_BKWD (0xd) + ALIGNED_LOOP_BKWD (0xc) + ALIGNED_LOOP_BKWD (0xb) + ALIGNED_LOOP_BKWD (0xa) + ALIGNED_LOOP_BKWD (0x9) + ALIGNED_LOOP_BKWD (0x8) + ALIGNED_LOOP_BKWD (0x7) + ALIGNED_LOOP_BKWD (0x6) + ALIGNED_LOOP_BKWD (0x5) + ALIGNED_LOOP_BKWD (0x4) + ALIGNED_LOOP_BKWD (0x3) + ALIGNED_LOOP_BKWD (0x2) + ALIGNED_LOOP_BKWD (0x1) +END(MEMMOVE) + +strong_alias (MEMMOVE, MEMCPY) +strong_alias (MEMMOVE_CHK, MEMCPY_CHK)