From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
x86@kernel.org, Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 3/8] lib/crypto: x86/sha512: Keep W[t]+K[t] in %xmm5 instead of stack
Date: Sat, 21 Nov 2026 18:20:25 -0800 [thread overview]
Message-ID: <20261122022030.296468-4-ebiggers@kernel.org> (raw)
In-Reply-To: <20261122022030.296468-1-ebiggers@kernel.org>
Instead of bouncing each pair of precomputed W[t] + K[t] values from XMM
to GPRs through a 16-byte stack buffer ('frame_WK'), keep the pair in
%xmm5 and transfer each word directly to a GPR using movq and psrldq.
This avoids stack round-trips, and it removes the remaining unzeroized
stack buffer that held input-derived data. Moreover, it eliminates the
last use of the stack at all besides the (digest, msglen) pair which
don't require 16-byte alignment. Thus the dynamic alignment and need to
maintain %rbp go away too, and we can eliminate the frame abstraction
entirely and just handle (digest, msglen) using pushes.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 80 +++++++++++++------------------
1 file changed, 33 insertions(+), 47 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index fe72ceb6a2735..f2a43c4ac8bec 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -74,6 +74,7 @@ tmp0 = %rax
kptr = %rdi # Pointer to K512[72]
toff = %rdx # 8*(t - 80), where W[t] is the first QWORD scheduled in the iteration.
+XWK = %xmm5 # {W[t]+K[t], W[t+1]+K[t+1]} for the next two rounds
# Rolling window of the last 16 message schedule QWORDs:
# XW0 = {W[t-16], W[t-15]}, ..., XW7 = {W[t-2], W[t-1]}
XW0 = %xmm6
@@ -85,18 +86,8 @@ XW5 = %xmm11
XW6 = %xmm12
XW7 = %xmm13
-# Local variables (stack frame)
-
-WK_SIZE = 2*8
-
-frame_WK = 0
-frame_digest = frame_WK + WK_SIZE
-frame_msglen = frame_digest + 8
-frame_size = frame_msglen + 8
-
# Useful QWORD "arrays" for simpler memory references
# MSG, DIGEST, K_t are arrays
-# WK_2(t) points to 1 of 2 qwords at frame.WK depending on t being odd/even
# Input message (arg1)
#define MSG(i) 8*i(msg)
@@ -107,9 +98,6 @@ frame_size = frame_msglen + 8
# SHA Constants (static mem)
#define K_t(i) 8*i+K512(%rip)
-# W[t]+K[t] (stack frame)
-#define WK_2(i) 8*((i%2))+frame_WK(%rsp)
-
.macro RotateXW
# Rotate symbols XW0..XW7 left
XWTMP = XW0
@@ -139,6 +127,12 @@ frame_size = frame_msglen + 8
.macro SHA512_Round rnd
# Compute Round %%t
+ .if ((\rnd) % 2) == 0
+ movq XWK, T2
+ .else
+ psrldq $8, XWK
+ movq XWK, T2
+ .endif
mov f_64, T1 # T1 = f
mov e_64, tmp0 # tmp = e
xor g_64, T1 # T1 = f ^ g
@@ -146,8 +140,7 @@ frame_size = frame_msglen + 8
and e_64, T1 # T1 = (f ^ g) & e
xor e_64, tmp0 # tmp = (e ror 23) ^ e
xor g_64, T1 # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
- idx = \rnd
- add WK_2(idx), T1 # W[t] + K[t] from message scheduler
+ add T2, T1 # + W[t] + K[t] from message scheduler
ror $4, tmp0 # 18 # tmp = ((e ror 23) ^ e) ror 4
xor e_64, tmp0 # tmp = (((e ror 23) ^ e) ror 4) ^ e
mov a_64, T2 # T2 = a
@@ -178,30 +171,30 @@ frame_size = frame_msglen + 8
# where t = 80 + (toff + \offset) / 8
# Two rounds are computed based on the values for K[t-2]+W[t-2] and
- # K[t-1]+W[t-1] which were previously stored at WK_2 by the message
+ # K[t-1]+W[t-1] which were previously stored in XWK by the message
# scheduler.
# The two new schedule QWORDS are stored in XW0 (which previously held
# W[t-16] and W[t-15], and is rotated to XW7 at the end of the macro).
# They are then added to their respective SHA512 constants at
- # (\offset + 64)(kptr, toff) and (\offset + 72)(kptr, toff) and stored at
- # dqword WK_2(0).
+ # (\offset + 64)(kptr, toff) and (\offset + 72)(kptr, toff) and stored in XWK.
# For brevity, the comments following vectored instructions only refer to
# the first of a pair of QWORDS.
# Eg. XW7=W[t-2] really means XW7={W[t-2]|W[t-1]}
# The computation of the message schedule and the rounds are tightly
# stitched to take advantage of instruction-level parallelism.
+ movq XWK, T2
mov f_64, T1
- movdqa XW1, %xmm5 # XMM5 = W[t-14]
+ movdqa XW1, %xmm2 # XMM2 = W[t-14]
xor g_64, T1
and e_64, T1
movdqa XW7, %xmm0 # XMM0 = W[t-2]
xor g_64, T1
- add WK_2(0), T1
- palignr $8, XW0, %xmm5 # XMM5 = W[t-15]
+ add T2, T1
+ palignr $8, XW0, %xmm2 # XMM2 = W[t-15]
mov e_64, tmp0
ror $23, tmp0 # 41
- movdqa %xmm5, %xmm3 # XMM3 = W[t-15]
+ movdqa %xmm2, %xmm3 # XMM3 = W[t-15]
xor e_64, tmp0
ror $4, tmp0 # 18
psrlq $61-19, %xmm0 # XMM0 = W[t-2] >> 42
@@ -213,7 +206,7 @@ frame_size = frame_msglen + 8
pxor XW7, %xmm0 # XMM0 = (W[t-2] >> 42) ^ W[t-2]
mov a_64, T2
xor c_64, T2
- pxor %xmm5, %xmm3 # XMM3 = (W[t-15] >> 1) ^ W[t-15]
+ pxor %xmm2, %xmm3 # XMM3 = (W[t-15] >> 1) ^ W[t-15]
and b_64, T2
mov a_64, tmp0
psrlq $(19-6), %xmm0 # XMM0 = ((W[t-2]>>42)^W[t-2])>>13
@@ -225,7 +218,7 @@ frame_size = frame_msglen + 8
pxor XW7, %xmm0 # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
xor a_64, tmp0
ror $6, tmp0 # 34
- pxor %xmm5, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
+ pxor %xmm2, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
xor a_64, tmp0
ror $28, tmp0 # 28
psrlq $6, %xmm0 # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6
@@ -234,14 +227,16 @@ frame_size = frame_msglen + 8
psrlq $1, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
lea (T1, T2), h_64
RotateState
+ psrldq $8, XWK
+ movq XWK, T2
movdqa XW7, %xmm1 # XMM1 = W[t-2]
mov f_64, T1
xor g_64, T1
- movdqa %xmm5, %xmm4 # XMM4 = W[t-15]
+ movdqa %xmm2, %xmm4 # XMM4 = W[t-15]
and e_64, T1
xor g_64, T1
psllq $(64-19)-(64-61) , %xmm1 # XMM1 = W[t-2] << 42
- add WK_2(1), T1
+ add T2, T1
mov e_64, tmp0
psllq $(64-1)-(64-8), %xmm4 # XMM4 = W[t-15] << 7
ror $23, tmp0 # 41
@@ -249,7 +244,7 @@ frame_size = frame_msglen + 8
pxor XW7, %xmm1 # XMM1 = (W[t-2] << 42)^W[t-2]
ror $4, tmp0 # 18
xor e_64, tmp0
- pxor %xmm5, %xmm4 # XMM4 = (W[t-15]<<7)^W[t-15]
+ pxor %xmm2, %xmm4 # XMM4 = (W[t-15]<<7)^W[t-15]
ror $14, tmp0 # 14
add tmp0, T1
psllq $(64-61), %xmm1 # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
@@ -277,7 +272,7 @@ frame_size = frame_msglen + 8
ror $28, tmp0 # 28
paddq (\offset + 64)(kptr, toff), %xmm0 # Compute W[t]+K[t]
add tmp0, T2
- movdqa %xmm0, WK_2(0) # Store W[t]+K[t] for next rounds
+ movdqa %xmm0, XWK # Save W[t]+K[t] for next rounds
add T1, d_64
lea (T1, T2), h_64
RotateState
@@ -295,9 +290,6 @@ frame_size = frame_msglen + 8
########################################################################
SYM_FUNC_START(sha512_transform_ssse3)
- push %rbp
- mov %rsp, %rbp
-
# Save GPRs
push %rbx
push %r12
@@ -305,13 +297,9 @@ SYM_FUNC_START(sha512_transform_ssse3)
push %r14
push %r15
- # Allocate Stack Space
- sub $frame_size, %rsp
- and $~(0x20 - 1), %rsp
-
# Spill digest and msglen to free up %rdi and %rdx for kptr and toff.
- mov digest, frame_digest(%rsp)
- mov msglen, frame_msglen(%rsp)
+ push msglen
+ push digest
.Lupdateblock:
@@ -329,9 +317,8 @@ SYM_FUNC_START(sha512_transform_ssse3)
movdqa XMM_QWORD_BSWAP(%rip), %xmm1
movdqu MSG(0), XW0
pshufb %xmm1, XW0 # BSWAP
- movdqa XW0, %xmm0
- paddq K_t(0), %xmm0 # Compute W[t]+K[t]
- movdqa %xmm0, WK_2(0) # Store into WK for rounds
+ movdqa XW0, XWK
+ paddq K_t(0), XWK # Compute W[0..1]+K[0..1]
t = 2
.irp reg, XW1,XW2,XW3,XW4,XW5,XW6,XW7
movdqu MSG(t), \reg
@@ -340,7 +327,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
movdqa \reg, %xmm0
paddq K_t(t), %xmm0 # Compute W[t]+K[t]
SHA512_Round (t-1) # Round t-1
- movdqa %xmm0, WK_2(t) # Store W[t]+K[t] into WK
+ movdqa %xmm0, XWK # Save W[t]+K[t] for next rounds
t = t+2
.endr
@@ -385,7 +372,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
SHA512_Round 79
# Update digest
- mov frame_digest(%rsp), digest
+ mov 0(%rsp), digest
add a_64, DIGEST(0)
add b_64, DIGEST(1)
add c_64, DIGEST(2)
@@ -395,13 +382,13 @@ SYM_FUNC_START(sha512_transform_ssse3)
add g_64, DIGEST(6)
add h_64, DIGEST(7)
- # Advance to next message block
+ # Decrement msglen, and advance to next message block if needed
add $16*8, msg
- decq frame_msglen(%rsp)
+ decq 8(%rsp)
jnz .Lupdateblock
- # Deallocate Stack Space
- lea -40(%rbp), %rsp
+ # Skip over digest and msglen
+ add $16, %rsp
# Restore GPRs
pop %r15
@@ -409,7 +396,6 @@ SYM_FUNC_START(sha512_transform_ssse3)
pop %r13
pop %r12
pop %rbx
- pop %rbp
RET
SYM_FUNC_END(sha512_transform_ssse3)
--
2.56.1
next prev parent reply other threads:[~2026-11-22 2:21 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
2026-11-22 2:20 ` [PATCH 1/8] lib/crypto: x86/sha512: Partially roll up sha512_transform_ssse3() Eric Biggers
2026-11-22 2:20 ` [PATCH 2/8] lib/crypto: x86/sha512: Keep message schedule in XMM registers Eric Biggers
2026-11-22 2:20 ` Eric Biggers [this message]
2026-11-22 2:20 ` [PATCH 4/8] lib/crypto: x86/sha512: Keep XMM_QWORD_BSWAP in %xmm14 across blocks Eric Biggers
2026-11-22 2:20 ` [PATCH 5/8] lib/crypto: x86/sha512: Use pshufb for ror64(x, 8) in SSSE3 s0 Eric Biggers
2026-11-22 2:20 ` [PATCH 6/8] lib/crypto: x86/sha512: Keep state in registers across blocks in SSSE3 Eric Biggers
2026-11-22 2:20 ` [PATCH 7/8] lib/crypto: x86/sha512: Shorten critical-path additions " Eric Biggers
2026-11-22 2:20 ` [PATCH 8/8] lib/crypto: x86/sha512: Use local labels for data Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261122022030.296468-4-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=ardb@kernel.org \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox