* [PATCH] lib/crypto: x86/chacha: Improve interleaving in chacha_8block_xor_avx512vl()
@ 2026-11-04 6:12 Eric Biggers
0 siblings, 0 replies; only message in thread
From: Eric Biggers @ 2026-11-04 6:12 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, Martin Willi,
Antonio Quartulli, Eric Biggers
The 'VL == 64' case has an optimization that eliminates some 'vmovdqa64'
instructions, but it was not carried over to the 'VL == 32' case. Do
so, and make most of the code that interleaves 64-bit words shared.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/chacha-avx512-x86_64.S | 77 ++++++++++-----------------
1 file changed, 27 insertions(+), 50 deletions(-)
diff --git a/lib/crypto/x86/chacha-avx512-x86_64.S b/lib/crypto/x86/chacha-avx512-x86_64.S
index ffdfe005cba6..bfa708c5e3ee 100644
--- a/lib/crypto/x86/chacha-avx512-x86_64.S
+++ b/lib/crypto/x86/chacha-avx512-x86_64.S
@@ -699,152 +699,129 @@ SYM_FUNC_END(chacha_4block_xor_avx512vl)
vpunpckldq V15,V14,V30
vpunpckhdq V15,V14,V31
-.if VL == 32
# interleave 64-bit words in state n, n+2
+.if VL == 32
+ vpunpckhqdq V18,V16,V1
+ vpunpcklqdq V18,V16,V16 # leave V0 free to use as temporary register
+.else
vpunpcklqdq V18,V16,V0
- vpunpcklqdq V19,V17,V1
- vpunpckhqdq V18,V16,V2
+ vpunpckhqdq V18,V16,V1
+.endif
+ vpunpcklqdq V19,V17,V2
vpunpckhqdq V19,V17,V3
vpunpcklqdq V22,V20,V4
- vpunpcklqdq V23,V21,V5
- vpunpckhqdq V22,V20,V6
+ vpunpckhqdq V22,V20,V5
+ vpunpcklqdq V23,V21,V6
vpunpckhqdq V23,V21,V7
vpunpcklqdq V26,V24,V8
- vpunpcklqdq V27,V25,V9
- vpunpckhqdq V26,V24,V10
+ vpunpckhqdq V26,V24,V9
+ vpunpcklqdq V27,V25,V10
vpunpckhqdq V27,V25,V11
vpunpcklqdq V30,V28,V12
- vpunpcklqdq V31,V29,V13
- vpunpckhqdq V30,V28,V14
+ vpunpckhqdq V30,V28,V13
+ vpunpcklqdq V31,V29,V14
vpunpckhqdq V31,V29,V15
+.if VL == 32
# interleave 128-bit words in state n, n+4
# xor/write first four blocks
- vmovdqa64 V0,V16
- vperm2i128 $0x20,V4,V0,V0
+ vshufi64x2 $0x0,V4,V16,V0
cmp $0x0020,%rcx
jl .Lxorpart\@
vpxord 0x0000(%rdx),V0,V0
vmovdqu64 V0,0x0000(%rsi)
- vmovdqa64 V16,V0
- vperm2i128 $0x31,V4,V0,V4
vperm2i128 $0x20,V12,V8,V0
cmp $0x0040,%rcx
jl .Lxorpart\@
vpxord 0x0020(%rdx),V0,V0
vmovdqu64 V0,0x0020(%rsi)
- vperm2i128 $0x31,V12,V8,V12
- vperm2i128 $0x20,V6,V2,V0
+ vperm2i128 $0x20,V5,V1,V0
cmp $0x0060,%rcx
jl .Lxorpart\@
vpxord 0x0040(%rdx),V0,V0
vmovdqu64 V0,0x0040(%rsi)
- vperm2i128 $0x31,V6,V2,V6
- vperm2i128 $0x20,V14,V10,V0
+ vperm2i128 $0x20,V13,V9,V0
cmp $0x0080,%rcx
jl .Lxorpart\@
vpxord 0x0060(%rdx),V0,V0
vmovdqu64 V0,0x0060(%rsi)
- vperm2i128 $0x31,V14,V10,V14
- vperm2i128 $0x20,V5,V1,V0
+ vperm2i128 $0x20,V6,V2,V0
cmp $0x00a0,%rcx
jl .Lxorpart\@
vpxord 0x0080(%rdx),V0,V0
vmovdqu64 V0,0x0080(%rsi)
- vperm2i128 $0x31,V5,V1,V5
- vperm2i128 $0x20,V13,V9,V0
+ vperm2i128 $0x20,V14,V10,V0
cmp $0x00c0,%rcx
jl .Lxorpart\@
vpxord 0x00a0(%rdx),V0,V0
vmovdqu64 V0,0x00a0(%rsi)
- vperm2i128 $0x31,V13,V9,V13
vperm2i128 $0x20,V7,V3,V0
cmp $0x00e0,%rcx
jl .Lxorpart\@
vpxord 0x00c0(%rdx),V0,V0
vmovdqu64 V0,0x00c0(%rsi)
- vperm2i128 $0x31,V7,V3,V7
vperm2i128 $0x20,V15,V11,V0
cmp $0x0100,%rcx
jl .Lxorpart\@
vpxord 0x00e0(%rdx),V0,V0
vmovdqu64 V0,0x00e0(%rsi)
- vperm2i128 $0x31,V15,V11,V15
# xor remaining blocks, write to output
- vmovdqa64 V4,V0
+ vshufi64x2 $0x3,V4,V16,V0
cmp $0x0120,%rcx
jl .Lxorpart\@
vpxord 0x0100(%rdx),V0,V0
vmovdqu64 V0,0x0100(%rsi)
- vmovdqa64 V12,V0
+ vperm2i128 $0x31,V12,V8,V0
cmp $0x0140,%rcx
jl .Lxorpart\@
vpxord 0x0120(%rdx),V0,V0
vmovdqu64 V0,0x0120(%rsi)
- vmovdqa64 V6,V0
+ vperm2i128 $0x31,V5,V1,V0
cmp $0x0160,%rcx
jl .Lxorpart\@
vpxord 0x0140(%rdx),V0,V0
vmovdqu64 V0,0x0140(%rsi)
- vmovdqa64 V14,V0
+ vperm2i128 $0x31,V13,V9,V0
cmp $0x0180,%rcx
jl .Lxorpart\@
vpxord 0x0160(%rdx),V0,V0
vmovdqu64 V0,0x0160(%rsi)
- vmovdqa64 V5,V0
+ vperm2i128 $0x31,V6,V2,V0
cmp $0x01a0,%rcx
jl .Lxorpart\@
vpxord 0x0180(%rdx),V0,V0
vmovdqu64 V0,0x0180(%rsi)
- vmovdqa64 V13,V0
+ vperm2i128 $0x31,V14,V10,V0
cmp $0x01c0,%rcx
jl .Lxorpart\@
vpxord 0x01a0(%rdx),V0,V0
vmovdqu64 V0,0x01a0(%rsi)
- vmovdqa64 V7,V0
+ vperm2i128 $0x31,V7,V3,V0
cmp $0x01e0,%rcx
jl .Lxorpart\@
vpxord 0x01c0(%rdx),V0,V0
vmovdqu64 V0,0x01c0(%rsi)
- vmovdqa64 V15,V0
+ vperm2i128 $0x31,V15,V11,V0
cmp $0x0200,%rcx
jl .Lxorpart\@
vpxord 0x01e0(%rdx),V0,V0
vmovdqu64 V0,0x01e0(%rsi)
.else
- # interleave 64-bit words in state n, n+2 -> zmm0..15
- vpunpcklqdq V18,V16,V0
- vpunpckhqdq V18,V16,V1
- vpunpcklqdq V19,V17,V2
- vpunpckhqdq V19,V17,V3
- vpunpcklqdq V22,V20,V4
- vpunpckhqdq V22,V20,V5
- vpunpcklqdq V23,V21,V6
- vpunpckhqdq V23,V21,V7
- vpunpcklqdq V26,V24,V8
- vpunpckhqdq V26,V24,V9
- vpunpcklqdq V27,V25,V10
- vpunpckhqdq V27,V25,V11
- vpunpcklqdq V30,V28,V12
- vpunpckhqdq V30,V28,V13
- vpunpcklqdq V31,V29,V14
- vpunpckhqdq V31,V29,V15
-
# interleave 128-bit lanes in state n, n+4 -> zmm16..31
vshufi64x2 $0x88,V4,V0,V16
vshufi64x2 $0x88,V12,V8,V20
base-commit: 9fbbbaefd5f95551da18426c200cf41e1b4255a1
prerequisite-patch-id: 890b33260038bff2e3adfe7867af5919666677fc
prerequisite-patch-id: 36cadf8c358a9778d745637c00368df82a93ce0d
--
2.56.0
^ permalink raw reply related [flat|nested] only message in thread
only message in thread, other threads:[~2026-11-04 6:13 UTC | newest]
Thread overview: (only message) (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-11-04 6:12 [PATCH] lib/crypto: x86/chacha: Improve interleaving in chacha_8block_xor_avx512vl() Eric Biggers
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox