From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
Martin Willi <martin@strongswan.org>,
Antonio Quartulli <antonio@mandelbit.com>,
Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH] lib/crypto: x86/chacha: Avoid overly-narrow keystream XORs
Date: Tue, 3 Nov 2026 22:57:03 -0800 [thread overview]
Message-ID: <20261104065703.247316-1-ebiggers@kernel.org> (raw)
chacha_2block_xor_avx2(), chacha_4block_xor_avx2(),
chacha_2block_xor_avx512vl(), and chacha_4block_xor_avx512vl() currently
XOR the data with the keystream 128 bits at a time. This is
unnecessarily narrow; the same functions already use 256-bit vectors
elsewhere. Do it 256 bits at a time instead, which is more efficient.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/chacha-avx2-x86_64.S | 220 +++++++++-----------------
lib/crypto/x86/chacha-avx512-x86_64.S | 220 +++++++++-----------------
2 files changed, 150 insertions(+), 290 deletions(-)
diff --git a/lib/crypto/x86/chacha-avx2-x86_64.S b/lib/crypto/x86/chacha-avx2-x86_64.S
index 0b0cf92a022c..8179a04a3027 100644
--- a/lib/crypto/x86/chacha-avx2-x86_64.S
+++ b/lib/crypto/x86/chacha-avx2-x86_64.S
@@ -135,59 +135,37 @@ SYM_FUNC_START(chacha_2block_xor_avx2)
sub $2,%r8d
jnz .Ldoubleround
- # o0 = i0 ^ (x0 + s0)
- vpaddd %ymm8,%ymm0,%ymm7
- cmp $0x10,%rax
- jl .Lxorpart2
- vpxor 0x00(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x00(%rsi)
- vextracti128 $1,%ymm7,%xmm0
- # o1 = i1 ^ (x1 + s1)
- vpaddd %ymm9,%ymm1,%ymm7
- cmp $0x20,%rax
- jl .Lxorpart2
- vpxor 0x10(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x10(%rsi)
- vextracti128 $1,%ymm7,%xmm1
- # o2 = i2 ^ (x2 + s2)
- vpaddd %ymm10,%ymm2,%ymm7
- cmp $0x30,%rax
- jl .Lxorpart2
- vpxor 0x20(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x20(%rsi)
- vextracti128 $1,%ymm7,%xmm2
- # o3 = i3 ^ (x3 + s3)
- vpaddd %ymm11,%ymm3,%ymm7
- cmp $0x40,%rax
- jl .Lxorpart2
- vpxor 0x30(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x30(%rsi)
- vextracti128 $1,%ymm7,%xmm3
+ # x0 += s0, x1 += s1, x2 += s2, x3 += s3
+ vpaddd %ymm8,%ymm0,%ymm0
+ vpaddd %ymm9,%ymm1,%ymm1
+ vpaddd %ymm10,%ymm2,%ymm2
+ vpaddd %ymm11,%ymm3,%ymm3
- # xor and write second block
- vmovdqa %xmm0,%xmm7
- cmp $0x50,%rax
+ # xor the data with the keystream
+
+ vperm2i128 $0x20, %ymm1, %ymm0, %ymm4
+ cmp $1*32,%rax
jl .Lxorpart2
- vpxor 0x40(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x40(%rsi)
+ vpxor 0*32(%rdx),%ymm4,%ymm4
+ vmovdqu %ymm4,0*32(%rsi)
- vmovdqa %xmm1,%xmm7
- cmp $0x60,%rax
+ vperm2i128 $0x20, %ymm3, %ymm2, %ymm4
+ cmp $2*32,%rax
jl .Lxorpart2
- vpxor 0x50(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x50(%rsi)
+ vpxor 1*32(%rdx),%ymm4,%ymm4
+ vmovdqu %ymm4,1*32(%rsi)
- vmovdqa %xmm2,%xmm7
- cmp $0x70,%rax
+ vperm2i128 $0x31, %ymm1, %ymm0, %ymm4
+ cmp $3*32,%rax
jl .Lxorpart2
- vpxor 0x60(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x60(%rsi)
+ vpxor 2*32(%rdx),%ymm4,%ymm4
+ vmovdqu %ymm4,2*32(%rsi)
- vmovdqa %xmm3,%xmm7
- cmp $0x80,%rax
+ vperm2i128 $0x31, %ymm3, %ymm2, %ymm4
+ cmp $4*32,%rax
jl .Lxorpart2
- vpxor 0x70(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x70(%rsi)
+ vpxor 3*32(%rdx),%ymm4,%ymm4
+ vmovdqu %ymm4,3*32(%rsi)
.Ldone2:
vzeroupper
@@ -196,14 +174,14 @@ SYM_FUNC_START(chacha_2block_xor_avx2)
.Lxorpart2:
# xor remaining bytes from partial register into output
mov %rax,%r9
- and $0x0f,%r9
+ and $31,%r9
jz .Ldone2
- and $~0x0f,%rax
+ and $~31,%rax
mov %rsi,%r11
lea 8(%rsp),%r10
- sub $0x10,%rsp
+ sub $32,%rsp
and $~31,%rsp
lea (%rdx,%rax),%rsi
@@ -211,8 +189,8 @@ SYM_FUNC_START(chacha_2block_xor_avx2)
mov %r9,%rcx
rep movsb
- vpxor 0x00(%rsp),%xmm7,%xmm7
- vmovdqa %xmm7,0x00(%rsp)
+ vpxor 0x00(%rsp),%ymm4,%ymm4
+ vmovdqa %ymm4,0x00(%rsp)
mov %rsp,%rsi
lea (%r11,%rax),%rdi
@@ -384,113 +362,65 @@ SYM_FUNC_START(chacha_4block_xor_avx2)
sub $2,%r8d
jnz .Ldoubleround4
- # o0 = i0 ^ (x0 + s0), first block
- vpaddd %ymm11,%ymm0,%ymm10
- cmp $0x10,%rax
- jl .Lxorpart4
- vpxor 0x00(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x00(%rsi)
- vextracti128 $1,%ymm10,%xmm0
- # o1 = i1 ^ (x1 + s1), first block
- vpaddd %ymm12,%ymm1,%ymm10
- cmp $0x20,%rax
- jl .Lxorpart4
- vpxor 0x10(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x10(%rsi)
- vextracti128 $1,%ymm10,%xmm1
- # o2 = i2 ^ (x2 + s2), first block
- vpaddd %ymm13,%ymm2,%ymm10
- cmp $0x30,%rax
- jl .Lxorpart4
- vpxor 0x20(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x20(%rsi)
- vextracti128 $1,%ymm10,%xmm2
- # o3 = i3 ^ (x3 + s3), first block
- vpaddd %ymm14,%ymm3,%ymm10
- cmp $0x40,%rax
- jl .Lxorpart4
- vpxor 0x30(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x30(%rsi)
- vextracti128 $1,%ymm10,%xmm3
+ # x0 += s0, x1 += s1, x2 += s2, x3 += s3
+ vpaddd %ymm11,%ymm0,%ymm0
+ vpaddd %ymm12,%ymm1,%ymm1
+ vpaddd %ymm13,%ymm2,%ymm2
+ vpaddd %ymm14,%ymm3,%ymm3
+ vpaddd %ymm11,%ymm4,%ymm4
+ vpaddd %ymm12,%ymm5,%ymm5
+ vpaddd %ymm13,%ymm6,%ymm6
+ vpaddd %ymm15,%ymm7,%ymm7
- # xor and write second block
- vmovdqa %xmm0,%xmm10
- cmp $0x50,%rax
- jl .Lxorpart4
- vpxor 0x40(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x40(%rsi)
+ # xor the data with the keystream
- vmovdqa %xmm1,%xmm10
- cmp $0x60,%rax
+ vperm2i128 $0x20, %ymm1, %ymm0, %ymm8
+ cmp $1*32,%rax
jl .Lxorpart4
- vpxor 0x50(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x50(%rsi)
+ vpxor 0*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,0*32(%rsi)
- vmovdqa %xmm2,%xmm10
- cmp $0x70,%rax
+ vperm2i128 $0x20, %ymm3, %ymm2, %ymm8
+ cmp $2*32,%rax
jl .Lxorpart4
- vpxor 0x60(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x60(%rsi)
+ vpxor 1*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,1*32(%rsi)
- vmovdqa %xmm3,%xmm10
- cmp $0x80,%rax
+ vperm2i128 $0x31, %ymm1, %ymm0, %ymm8
+ cmp $3*32,%rax
jl .Lxorpart4
- vpxor 0x70(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x70(%rsi)
+ vpxor 2*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,2*32(%rsi)
- # o0 = i0 ^ (x0 + s0), third block
- vpaddd %ymm11,%ymm4,%ymm10
- cmp $0x90,%rax
- jl .Lxorpart4
- vpxor 0x80(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x80(%rsi)
- vextracti128 $1,%ymm10,%xmm4
- # o1 = i1 ^ (x1 + s1), third block
- vpaddd %ymm12,%ymm5,%ymm10
- cmp $0xa0,%rax
- jl .Lxorpart4
- vpxor 0x90(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x90(%rsi)
- vextracti128 $1,%ymm10,%xmm5
- # o2 = i2 ^ (x2 + s2), third block
- vpaddd %ymm13,%ymm6,%ymm10
- cmp $0xb0,%rax
- jl .Lxorpart4
- vpxor 0xa0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xa0(%rsi)
- vextracti128 $1,%ymm10,%xmm6
- # o3 = i3 ^ (x3 + s3), third block
- vpaddd %ymm15,%ymm7,%ymm10
- cmp $0xc0,%rax
+ vperm2i128 $0x31, %ymm3, %ymm2, %ymm8
+ cmp $4*32,%rax
jl .Lxorpart4
- vpxor 0xb0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xb0(%rsi)
- vextracti128 $1,%ymm10,%xmm7
+ vpxor 3*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,3*32(%rsi)
- # xor and write fourth block
- vmovdqa %xmm4,%xmm10
- cmp $0xd0,%rax
+ vperm2i128 $0x20, %ymm5, %ymm4, %ymm8
+ cmp $5*32,%rax
jl .Lxorpart4
- vpxor 0xc0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xc0(%rsi)
+ vpxor 4*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,4*32(%rsi)
- vmovdqa %xmm5,%xmm10
- cmp $0xe0,%rax
+ vperm2i128 $0x20, %ymm7, %ymm6, %ymm8
+ cmp $6*32,%rax
jl .Lxorpart4
- vpxor 0xd0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xd0(%rsi)
+ vpxor 5*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,5*32(%rsi)
- vmovdqa %xmm6,%xmm10
- cmp $0xf0,%rax
+ vperm2i128 $0x31, %ymm5, %ymm4, %ymm8
+ cmp $7*32,%rax
jl .Lxorpart4
- vpxor 0xe0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xe0(%rsi)
+ vpxor 6*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,6*32(%rsi)
- vmovdqa %xmm7,%xmm10
- cmp $0x100,%rax
+ vperm2i128 $0x31, %ymm7, %ymm6, %ymm8
+ cmp $8*32,%rax
jl .Lxorpart4
- vpxor 0xf0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xf0(%rsi)
+ vpxor 7*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,7*32(%rsi)
.Ldone4:
vzeroupper
@@ -499,14 +429,14 @@ SYM_FUNC_START(chacha_4block_xor_avx2)
.Lxorpart4:
# xor remaining bytes from partial register into output
mov %rax,%r9
- and $0x0f,%r9
+ and $31,%r9
jz .Ldone4
- and $~0x0f,%rax
+ and $~31,%rax
mov %rsi,%r11
lea 8(%rsp),%r10
- sub $0x10,%rsp
+ sub $32,%rsp
and $~31,%rsp
lea (%rdx,%rax),%rsi
@@ -514,8 +444,8 @@ SYM_FUNC_START(chacha_4block_xor_avx2)
mov %r9,%rcx
rep movsb
- vpxor 0x00(%rsp),%xmm10,%xmm10
- vmovdqa %xmm10,0x00(%rsp)
+ vpxor 0x00(%rsp),%ymm8,%ymm8
+ vmovdqa %ymm8,0x00(%rsp)
mov %rsp,%rsi
lea (%r11,%rax),%rdi
diff --git a/lib/crypto/x86/chacha-avx512-x86_64.S b/lib/crypto/x86/chacha-avx512-x86_64.S
index bfa708c5e3ee..a0553decd9c5 100644
--- a/lib/crypto/x86/chacha-avx512-x86_64.S
+++ b/lib/crypto/x86/chacha-avx512-x86_64.S
@@ -127,59 +127,37 @@ SYM_FUNC_START(chacha_2block_xor_avx512vl)
sub $2,%r8d
jnz .Ldoubleround
- # o0 = i0 ^ (x0 + s0)
- vpaddd %ymm8,%ymm0,%ymm7
- cmp $0x10,%rcx
- jl .Lxorpart2
- vpxord 0x00(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x00(%rsi)
- vextracti128 $1,%ymm7,%xmm0
- # o1 = i1 ^ (x1 + s1)
- vpaddd %ymm9,%ymm1,%ymm7
- cmp $0x20,%rcx
- jl .Lxorpart2
- vpxord 0x10(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x10(%rsi)
- vextracti128 $1,%ymm7,%xmm1
- # o2 = i2 ^ (x2 + s2)
- vpaddd %ymm10,%ymm2,%ymm7
- cmp $0x30,%rcx
- jl .Lxorpart2
- vpxord 0x20(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x20(%rsi)
- vextracti128 $1,%ymm7,%xmm2
- # o3 = i3 ^ (x3 + s3)
- vpaddd %ymm11,%ymm3,%ymm7
- cmp $0x40,%rcx
- jl .Lxorpart2
- vpxord 0x30(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x30(%rsi)
- vextracti128 $1,%ymm7,%xmm3
+ # x0 += s0, x1 += s1, x2 += s2, x3 += s3
+ vpaddd %ymm8,%ymm0,%ymm0
+ vpaddd %ymm9,%ymm1,%ymm1
+ vpaddd %ymm10,%ymm2,%ymm2
+ vpaddd %ymm11,%ymm3,%ymm3
- # xor and write second block
- vmovdqa %xmm0,%xmm7
- cmp $0x50,%rcx
+ # xor the data with the keystream
+
+ vperm2i128 $0x20, %ymm1, %ymm0, %ymm4
+ cmp $1*32,%rcx
jl .Lxorpart2
- vpxord 0x40(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x40(%rsi)
+ vpxor 0*32(%rdx),%ymm4,%ymm4
+ vmovdqu %ymm4,0*32(%rsi)
- vmovdqa %xmm1,%xmm7
- cmp $0x60,%rcx
+ vperm2i128 $0x20, %ymm3, %ymm2, %ymm4
+ cmp $2*32,%rcx
jl .Lxorpart2
- vpxord 0x50(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x50(%rsi)
+ vpxor 1*32(%rdx),%ymm4,%ymm4
+ vmovdqu %ymm4,1*32(%rsi)
- vmovdqa %xmm2,%xmm7
- cmp $0x70,%rcx
+ vperm2i128 $0x31, %ymm1, %ymm0, %ymm4
+ cmp $3*32,%rcx
jl .Lxorpart2
- vpxord 0x60(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x60(%rsi)
+ vpxor 2*32(%rdx),%ymm4,%ymm4
+ vmovdqu %ymm4,2*32(%rsi)
- vmovdqa %xmm3,%xmm7
- cmp $0x80,%rcx
+ vperm2i128 $0x31, %ymm3, %ymm2, %ymm4
+ cmp $4*32,%rcx
jl .Lxorpart2
- vpxord 0x70(%rdx),%xmm7,%xmm6
- vmovdqu %xmm6,0x70(%rsi)
+ vpxor 3*32(%rdx),%ymm4,%ymm4
+ vmovdqu %ymm4,3*32(%rsi)
.Ldone2:
vzeroupper
@@ -188,17 +166,17 @@ SYM_FUNC_START(chacha_2block_xor_avx512vl)
.Lxorpart2:
# xor remaining bytes from partial register into output
mov %rcx,%r9
- and $~0xf,%r9
- and $0xf,%rcx
+ and $~31,%r9
+ and $31,%rcx
jz .Ldone2
mov $-1,%rax
bzhi %rcx,%rax,%rax
kmovq %rax,%k1
- vmovdqu8 (%rdx,%r9),%xmm1{%k1}{z}
- vpxord %xmm7,%xmm1,%xmm1
- vmovdqu8 %xmm1,(%rsi,%r9){%k1}
+ vmovdqu8 (%rdx,%r9),%ymm0{%k1}{z}
+ vpxor %ymm4,%ymm0,%ymm0
+ vmovdqu8 %ymm0,(%rsi,%r9){%k1}
jmp .Ldone2
@@ -335,113 +313,65 @@ SYM_FUNC_START(chacha_4block_xor_avx512vl)
sub $2,%r8d
jnz .Ldoubleround4
- # o0 = i0 ^ (x0 + s0), first block
- vpaddd %ymm11,%ymm0,%ymm10
- cmp $0x10,%rcx
- jl .Lxorpart4
- vpxord 0x00(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x00(%rsi)
- vextracti128 $1,%ymm10,%xmm0
- # o1 = i1 ^ (x1 + s1), first block
- vpaddd %ymm12,%ymm1,%ymm10
- cmp $0x20,%rcx
- jl .Lxorpart4
- vpxord 0x10(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x10(%rsi)
- vextracti128 $1,%ymm10,%xmm1
- # o2 = i2 ^ (x2 + s2), first block
- vpaddd %ymm13,%ymm2,%ymm10
- cmp $0x30,%rcx
- jl .Lxorpart4
- vpxord 0x20(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x20(%rsi)
- vextracti128 $1,%ymm10,%xmm2
- # o3 = i3 ^ (x3 + s3), first block
- vpaddd %ymm14,%ymm3,%ymm10
- cmp $0x40,%rcx
- jl .Lxorpart4
- vpxord 0x30(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x30(%rsi)
- vextracti128 $1,%ymm10,%xmm3
+ # x0 += s0, x1 += s1, x2 += s2, x3 += s3
+ vpaddd %ymm11,%ymm0,%ymm0
+ vpaddd %ymm12,%ymm1,%ymm1
+ vpaddd %ymm13,%ymm2,%ymm2
+ vpaddd %ymm14,%ymm3,%ymm3
+ vpaddd %ymm11,%ymm4,%ymm4
+ vpaddd %ymm12,%ymm5,%ymm5
+ vpaddd %ymm13,%ymm6,%ymm6
+ vpaddd %ymm15,%ymm7,%ymm7
- # xor and write second block
- vmovdqa %xmm0,%xmm10
- cmp $0x50,%rcx
- jl .Lxorpart4
- vpxord 0x40(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x40(%rsi)
+ # xor the data with the keystream
- vmovdqa %xmm1,%xmm10
- cmp $0x60,%rcx
+ vperm2i128 $0x20, %ymm1, %ymm0, %ymm8
+ cmp $1*32,%rcx
jl .Lxorpart4
- vpxord 0x50(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x50(%rsi)
+ vpxor 0*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,0*32(%rsi)
- vmovdqa %xmm2,%xmm10
- cmp $0x70,%rcx
+ vperm2i128 $0x20, %ymm3, %ymm2, %ymm8
+ cmp $2*32,%rcx
jl .Lxorpart4
- vpxord 0x60(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x60(%rsi)
+ vpxor 1*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,1*32(%rsi)
- vmovdqa %xmm3,%xmm10
- cmp $0x80,%rcx
+ vperm2i128 $0x31, %ymm1, %ymm0, %ymm8
+ cmp $3*32,%rcx
jl .Lxorpart4
- vpxord 0x70(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x70(%rsi)
+ vpxor 2*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,2*32(%rsi)
- # o0 = i0 ^ (x0 + s0), third block
- vpaddd %ymm11,%ymm4,%ymm10
- cmp $0x90,%rcx
- jl .Lxorpart4
- vpxord 0x80(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x80(%rsi)
- vextracti128 $1,%ymm10,%xmm4
- # o1 = i1 ^ (x1 + s1), third block
- vpaddd %ymm12,%ymm5,%ymm10
- cmp $0xa0,%rcx
- jl .Lxorpart4
- vpxord 0x90(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0x90(%rsi)
- vextracti128 $1,%ymm10,%xmm5
- # o2 = i2 ^ (x2 + s2), third block
- vpaddd %ymm13,%ymm6,%ymm10
- cmp $0xb0,%rcx
- jl .Lxorpart4
- vpxord 0xa0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xa0(%rsi)
- vextracti128 $1,%ymm10,%xmm6
- # o3 = i3 ^ (x3 + s3), third block
- vpaddd %ymm15,%ymm7,%ymm10
- cmp $0xc0,%rcx
+ vperm2i128 $0x31, %ymm3, %ymm2, %ymm8
+ cmp $4*32,%rcx
jl .Lxorpart4
- vpxord 0xb0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xb0(%rsi)
- vextracti128 $1,%ymm10,%xmm7
+ vpxor 3*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,3*32(%rsi)
- # xor and write fourth block
- vmovdqa %xmm4,%xmm10
- cmp $0xd0,%rcx
+ vperm2i128 $0x20, %ymm5, %ymm4, %ymm8
+ cmp $5*32,%rcx
jl .Lxorpart4
- vpxord 0xc0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xc0(%rsi)
+ vpxor 4*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,4*32(%rsi)
- vmovdqa %xmm5,%xmm10
- cmp $0xe0,%rcx
+ vperm2i128 $0x20, %ymm7, %ymm6, %ymm8
+ cmp $6*32,%rcx
jl .Lxorpart4
- vpxord 0xd0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xd0(%rsi)
+ vpxor 5*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,5*32(%rsi)
- vmovdqa %xmm6,%xmm10
- cmp $0xf0,%rcx
+ vperm2i128 $0x31, %ymm5, %ymm4, %ymm8
+ cmp $7*32,%rcx
jl .Lxorpart4
- vpxord 0xe0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xe0(%rsi)
+ vpxor 6*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,6*32(%rsi)
- vmovdqa %xmm7,%xmm10
- cmp $0x100,%rcx
+ vperm2i128 $0x31, %ymm7, %ymm6, %ymm8
+ cmp $8*32,%rcx
jl .Lxorpart4
- vpxord 0xf0(%rdx),%xmm10,%xmm9
- vmovdqu %xmm9,0xf0(%rsi)
+ vpxor 7*32(%rdx),%ymm8,%ymm8
+ vmovdqu %ymm8,7*32(%rsi)
.Ldone4:
vzeroupper
@@ -450,17 +380,17 @@ SYM_FUNC_START(chacha_4block_xor_avx512vl)
.Lxorpart4:
# xor remaining bytes from partial register into output
mov %rcx,%r9
- and $~0xf,%r9
- and $0xf,%rcx
+ and $~31,%r9
+ and $31,%rcx
jz .Ldone4
mov $-1,%rax
bzhi %rcx,%rax,%rax
kmovq %rax,%k1
- vmovdqu8 (%rdx,%r9),%xmm1{%k1}{z}
- vpxord %xmm10,%xmm1,%xmm1
- vmovdqu8 %xmm1,(%rsi,%r9){%k1}
+ vmovdqu8 (%rdx,%r9),%ymm0{%k1}{z}
+ vpxor %ymm8,%ymm0,%ymm0
+ vmovdqu8 %ymm0,(%rsi,%r9){%k1}
jmp .Ldone4
base-commit: 9fbbbaefd5f95551da18426c200cf41e1b4255a1
prerequisite-patch-id: 890b33260038bff2e3adfe7867af5919666677fc
prerequisite-patch-id: 36cadf8c358a9778d745637c00368df82a93ce0d
prerequisite-patch-id: b9ef92f4abcdfe8946d645db12187f6293a46381
--
2.56.0
next reply other threads:[~2026-11-04 6:57 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-04 6:57 Eric Biggers [this message]
2026-11-05 9:44 ` [PATCH] lib/crypto: x86/chacha: Avoid overly-narrow keystream XORs Antonio Quartulli
2026-11-07 21:46 ` Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261104065703.247316-1-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=antonio@mandelbit.com \
--cc=ardb@kernel.org \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=martin@strongswan.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.