Linux cryptographic layer development
 help / color / mirror / Atom feed
From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
	"Jason A . Donenfeld" <Jason@zx2c4.com>,
	Herbert Xu <herbert@gondor.apana.org.au>,
	x86@kernel.org, Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 3/8] lib/crypto: x86/sha512: Keep W[t]+K[t] in %xmm5 instead of stack
Date: Sat, 21 Nov 2026 18:20:25 -0800	[thread overview]
Message-ID: <20261122022030.296468-4-ebiggers@kernel.org> (raw)
In-Reply-To: <20261122022030.296468-1-ebiggers@kernel.org>

Instead of bouncing each pair of precomputed W[t] + K[t] values from XMM
to GPRs through a 16-byte stack buffer ('frame_WK'), keep the pair in
%xmm5 and transfer each word directly to a GPR using movq and psrldq.

This avoids stack round-trips, and it removes the remaining unzeroized
stack buffer that held input-derived data.  Moreover, it eliminates the
last use of the stack at all besides the (digest, msglen) pair which
don't require 16-byte alignment.  Thus the dynamic alignment and need to
maintain %rbp go away too, and we can eliminate the frame abstraction
entirely and just handle (digest, msglen) using pushes.

Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 lib/crypto/x86/sha512-ssse3-asm.S | 80 +++++++++++++------------------
 1 file changed, 33 insertions(+), 47 deletions(-)

diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index fe72ceb6a2735..f2a43c4ac8bec 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -74,6 +74,7 @@ tmp0 =		%rax
 kptr =		%rdi	# Pointer to K512[72]
 toff =		%rdx	# 8*(t - 80), where W[t] is the first QWORD scheduled in the iteration.
 
+XWK =		%xmm5	# {W[t]+K[t], W[t+1]+K[t+1]} for the next two rounds
 # Rolling window of the last 16 message schedule QWORDs:
 # XW0 = {W[t-16], W[t-15]}, ..., XW7 = {W[t-2], W[t-1]}
 XW0 =		%xmm6
@@ -85,18 +86,8 @@ XW5 =		%xmm11
 XW6 =		%xmm12
 XW7 =		%xmm13
 
-# Local variables (stack frame)
-
-WK_SIZE = 2*8
-
-frame_WK = 0
-frame_digest = frame_WK + WK_SIZE
-frame_msglen = frame_digest + 8
-frame_size = frame_msglen + 8
-
 # Useful QWORD "arrays" for simpler memory references
 # MSG, DIGEST, K_t are arrays
-# WK_2(t) points to 1 of 2 qwords at frame.WK depending on t being odd/even
 
 # Input message (arg1)
 #define MSG(i)    8*i(msg)
@@ -107,9 +98,6 @@ frame_size = frame_msglen + 8
 # SHA Constants (static mem)
 #define K_t(i)    8*i+K512(%rip)
 
-# W[t]+K[t] (stack frame)
-#define WK_2(i)   8*((i%2))+frame_WK(%rsp)
-
 .macro RotateXW
 	# Rotate symbols XW0..XW7 left
 	XWTMP = XW0
@@ -139,6 +127,12 @@ frame_size = frame_msglen + 8
 .macro SHA512_Round rnd
 
 	# Compute Round %%t
+	.if ((\rnd) % 2) == 0
+		movq	XWK, T2
+	.else
+		psrldq	$8, XWK
+		movq	XWK, T2
+	.endif
 	mov	f_64, T1          # T1 = f
 	mov	e_64, tmp0        # tmp = e
 	xor	g_64, T1          # T1 = f ^ g
@@ -146,8 +140,7 @@ frame_size = frame_msglen + 8
 	and	e_64, T1          # T1 = (f ^ g) & e
 	xor	e_64, tmp0        # tmp = (e ror 23) ^ e
 	xor	g_64, T1          # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
-	idx = \rnd
-	add	WK_2(idx), T1     # W[t] + K[t] from message scheduler
+	add	T2, T1            # + W[t] + K[t] from message scheduler
 	ror	$4, tmp0  # 18    # tmp = ((e ror 23) ^ e) ror 4
 	xor	e_64, tmp0        # tmp = (((e ror 23) ^ e) ror 4) ^ e
 	mov	a_64, T2          # T2 = a
@@ -178,30 +171,30 @@ frame_size = frame_msglen + 8
 	# where t = 80 + (toff + \offset) / 8
 
 	#   Two rounds are computed based on the values for K[t-2]+W[t-2] and
-	# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
+	# K[t-1]+W[t-1] which were previously stored in XWK by the message
 	# scheduler.
 	#   The two new schedule QWORDS are stored in XW0 (which previously held
 	# W[t-16] and W[t-15], and is rotated to XW7 at the end of the macro).
 	# They are then added to their respective SHA512 constants at
-	# (\offset + 64)(kptr, toff) and (\offset + 72)(kptr, toff) and stored at
-	# dqword WK_2(0).
+	# (\offset + 64)(kptr, toff) and (\offset + 72)(kptr, toff) and stored in XWK.
 	#   For brevity, the comments following vectored instructions only refer to
 	# the first of a pair of QWORDS.
 	# Eg. XW7=W[t-2] really means XW7={W[t-2]|W[t-1]}
 	#   The computation of the message schedule and the rounds are tightly
 	# stitched to take advantage of instruction-level parallelism.
 
+	movq	XWK, T2
 	mov	f_64, T1
-	movdqa	XW1, %xmm5		    # XMM5 = W[t-14]
+	movdqa	XW1, %xmm2		    # XMM2 = W[t-14]
 	xor	g_64, T1
 	and	e_64, T1
 	movdqa	XW7, %xmm0		    # XMM0 = W[t-2]
 	xor	g_64, T1
-	add	WK_2(0), T1
-	palignr	$8, XW0, %xmm5		    # XMM5 = W[t-15]
+	add	T2, T1
+	palignr	$8, XW0, %xmm2		    # XMM2 = W[t-15]
 	mov	e_64, tmp0
 	ror	$23, tmp0 # 41
-	movdqa	%xmm5, %xmm3	            # XMM3 = W[t-15]
+	movdqa	%xmm2, %xmm3	            # XMM3 = W[t-15]
 	xor	e_64, tmp0
 	ror	$4, tmp0 # 18
 	psrlq	$61-19, %xmm0		    # XMM0 = W[t-2] >> 42
@@ -213,7 +206,7 @@ frame_size = frame_msglen + 8
 	pxor	XW7, %xmm0		    # XMM0 = (W[t-2] >> 42) ^ W[t-2]
 	mov	a_64, T2
 	xor	c_64, T2
-	pxor	%xmm5, %xmm3                # XMM3 = (W[t-15] >> 1) ^ W[t-15]
+	pxor	%xmm2, %xmm3                # XMM3 = (W[t-15] >> 1) ^ W[t-15]
 	and	b_64, T2
 	mov	a_64, tmp0
 	psrlq	$(19-6), %xmm0		    # XMM0 = ((W[t-2]>>42)^W[t-2])>>13
@@ -225,7 +218,7 @@ frame_size = frame_msglen + 8
 	pxor	XW7, %xmm0		    # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
 	xor	a_64, tmp0
 	ror	$6, tmp0 # 34
-	pxor	%xmm5, %xmm3                # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
+	pxor	%xmm2, %xmm3                # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
 	xor	a_64, tmp0
 	ror	$28, tmp0 # 28
 	psrlq	$6, %xmm0                   # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6
@@ -234,14 +227,16 @@ frame_size = frame_msglen + 8
 	psrlq	$1, %xmm3                   # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
 	lea	(T1, T2), h_64
 	RotateState
+	psrldq	$8, XWK
+	movq	XWK, T2
 	movdqa	XW7, %xmm1		    # XMM1 = W[t-2]
 	mov	f_64, T1
 	xor	g_64, T1
-	movdqa	%xmm5, %xmm4		    # XMM4 = W[t-15]
+	movdqa	%xmm2, %xmm4		    # XMM4 = W[t-15]
 	and	e_64, T1
 	xor	g_64, T1
 	psllq	$(64-19)-(64-61) , %xmm1    # XMM1 = W[t-2] << 42
-	add	WK_2(1), T1
+	add	T2, T1
 	mov	e_64, tmp0
 	psllq	$(64-1)-(64-8), %xmm4	    # XMM4 = W[t-15] << 7
 	ror	$23, tmp0 # 41
@@ -249,7 +244,7 @@ frame_size = frame_msglen + 8
 	pxor	XW7, %xmm1		    # XMM1 = (W[t-2] << 42)^W[t-2]
 	ror	$4, tmp0 # 18
 	xor	e_64, tmp0
-	pxor	%xmm5, %xmm4		    # XMM4 = (W[t-15]<<7)^W[t-15]
+	pxor	%xmm2, %xmm4		    # XMM4 = (W[t-15]<<7)^W[t-15]
 	ror	$14, tmp0 # 14
 	add	tmp0, T1
 	psllq	$(64-61), %xmm1		    # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
@@ -277,7 +272,7 @@ frame_size = frame_msglen + 8
 	ror	$28, tmp0 # 28
 	paddq	(\offset + 64)(kptr, toff), %xmm0  # Compute W[t]+K[t]
 	add	tmp0, T2
-	movdqa	%xmm0, WK_2(0)		    # Store W[t]+K[t] for next rounds
+	movdqa	%xmm0, XWK		    # Save W[t]+K[t] for next rounds
 	add	T1, d_64
 	lea	(T1, T2), h_64
 	RotateState
@@ -295,9 +290,6 @@ frame_size = frame_msglen + 8
 ########################################################################
 SYM_FUNC_START(sha512_transform_ssse3)
 
-	push	%rbp
-	mov	%rsp, %rbp
-
 	# Save GPRs
 	push	%rbx
 	push	%r12
@@ -305,13 +297,9 @@ SYM_FUNC_START(sha512_transform_ssse3)
 	push	%r14
 	push	%r15
 
-	# Allocate Stack Space
-	sub	$frame_size, %rsp
-	and	$~(0x20 - 1), %rsp
-
 	# Spill digest and msglen to free up %rdi and %rdx for kptr and toff.
-	mov	digest, frame_digest(%rsp)
-	mov	msglen, frame_msglen(%rsp)
+	push	msglen
+	push	digest
 
 .Lupdateblock:
 
@@ -329,9 +317,8 @@ SYM_FUNC_START(sha512_transform_ssse3)
 	movdqa	XMM_QWORD_BSWAP(%rip), %xmm1
 	movdqu	MSG(0), XW0
 	pshufb	%xmm1, XW0	# BSWAP
-	movdqa	XW0, %xmm0
-	paddq	K_t(0), %xmm0	# Compute W[t]+K[t]
-	movdqa	%xmm0, WK_2(0)	# Store into WK for rounds
+	movdqa	XW0, XWK
+	paddq	K_t(0), XWK	# Compute W[0..1]+K[0..1]
 	t = 2
 	.irp reg, XW1,XW2,XW3,XW4,XW5,XW6,XW7
 		movdqu	MSG(t), \reg
@@ -340,7 +327,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
 		movdqa	\reg, %xmm0
 		paddq	K_t(t), %xmm0	# Compute W[t]+K[t]
 		SHA512_Round (t-1)	# Round t-1
-		movdqa	%xmm0, WK_2(t)	# Store W[t]+K[t] into WK
+		movdqa	%xmm0, XWK	# Save W[t]+K[t] for next rounds
 		t = t+2
 	.endr
 
@@ -385,7 +372,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
 	SHA512_Round 79
 
 	# Update digest
-	mov	frame_digest(%rsp), digest
+	mov	0(%rsp), digest
 	add	a_64, DIGEST(0)
 	add	b_64, DIGEST(1)
 	add	c_64, DIGEST(2)
@@ -395,13 +382,13 @@ SYM_FUNC_START(sha512_transform_ssse3)
 	add	g_64, DIGEST(6)
 	add	h_64, DIGEST(7)
 
-	# Advance to next message block
+	# Decrement msglen, and advance to next message block if needed
 	add	$16*8, msg
-	decq	frame_msglen(%rsp)
+	decq	8(%rsp)
 	jnz	.Lupdateblock
 
-	# Deallocate Stack Space
-	lea	-40(%rbp), %rsp
+	# Skip over digest and msglen
+	add	$16, %rsp
 
 	# Restore GPRs
 	pop	%r15
@@ -409,7 +396,6 @@ SYM_FUNC_START(sha512_transform_ssse3)
 	pop	%r13
 	pop	%r12
 	pop	%rbx
-	pop	%rbp
 
 	RET
 SYM_FUNC_END(sha512_transform_ssse3)
-- 
2.56.1


  parent reply	other threads:[~2026-11-22  2:21 UTC|newest]

Thread overview: 9+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-22  2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
2026-11-22  2:20 ` [PATCH 1/8] lib/crypto: x86/sha512: Partially roll up sha512_transform_ssse3() Eric Biggers
2026-11-22  2:20 ` [PATCH 2/8] lib/crypto: x86/sha512: Keep message schedule in XMM registers Eric Biggers
2026-11-22  2:20 ` Eric Biggers [this message]
2026-11-22  2:20 ` [PATCH 4/8] lib/crypto: x86/sha512: Keep XMM_QWORD_BSWAP in %xmm14 across blocks Eric Biggers
2026-11-22  2:20 ` [PATCH 5/8] lib/crypto: x86/sha512: Use pshufb for ror64(x, 8) in SSSE3 s0 Eric Biggers
2026-11-22  2:20 ` [PATCH 6/8] lib/crypto: x86/sha512: Keep state in registers across blocks in SSSE3 Eric Biggers
2026-11-22  2:20 ` [PATCH 7/8] lib/crypto: x86/sha512: Shorten critical-path additions " Eric Biggers
2026-11-22  2:20 ` [PATCH 8/8] lib/crypto: x86/sha512: Use local labels for data Eric Biggers

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261122022030.296468-4-ebiggers@kernel.org \
    --to=ebiggers@kernel.org \
    --cc=Jason@zx2c4.com \
    --cc=ardb@kernel.org \
    --cc=herbert@gondor.apana.org.au \
    --cc=linux-crypto@vger.kernel.org \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox