Linux cryptographic layer development
 help / color / mirror / Atom feed
From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
	"Jason A . Donenfeld" <Jason@zx2c4.com>,
	Herbert Xu <herbert@gondor.apana.org.au>,
	x86@kernel.org, Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 7/8] lib/crypto: x86/sha512: Shorten critical-path additions in SSSE3
Date: Sat, 21 Nov 2026 18:20:29 -0800	[thread overview]
Message-ID: <20261122022030.296468-8-ebiggers@kernel.org> (raw)
In-Reply-To: <20261122022030.296468-1-ebiggers@kernel.org>

In SHA512_2Sched_2Round_sse, T1 = Ch(e, f, g) + W[t] + K[t] is ready
well before S1(e) finishes in tmp0, yet 'add tmp0, T1' was executed
before 'add h_64, T1'.  Swap them so 'add h_64, T1' runs in parallel
with S1(e), matching SHA512_Round and removing 1 cycle per round from
the critical path.

Similarly, in SHA512_Round, T1 finishes later than both Maj(a, b, c)
and S0(a).  Add S0(a) into T2 before 'lea (T1, T2), h_64', matching
SHA512_2Sched_2Round_sse and removing 1 cycle from the T1 -> h_64 path.

Note that sha512-avx2-asm.S already structures its additions similarly
to avoid both of these serial dependencies on the critical path.

Also fix a few comments in the file.

Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 lib/crypto/x86/sha512-ssse3-asm.S | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index f7dade03c6527..62f1f5109f629 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -91,10 +91,10 @@ XROR8 =		%xmm15	# XMM_QWORD_ROR8
 # Useful QWORD "arrays" for simpler memory references
 # MSG, DIGEST, K_t are arrays
 
-# Input message (arg1)
+# Input message (arg2)
 #define MSG(i)    8*i(msg)
 
-# Output Digest (arg2)
+# Output Digest (arg1)
 #define DIGEST(i) 8*i(digest)
 
 # SHA Constants (static mem)
@@ -134,7 +134,7 @@ XROR8 =		%xmm15	# XMM_QWORD_ROR8
 
 .macro SHA512_Round rnd
 
-	# Compute Round %%t
+	# Compute Round \rnd
 	.if ((\rnd) % 2) == 0
 		movq	XWK, T2
 	.else
@@ -154,7 +154,7 @@ XROR8 =		%xmm15	# XMM_QWORD_ROR8
 	mov	a_64, T2          # T2 = a
 	add	h_64, T1          # T1 = CH(e,f,g) + W[t] + K[t] + h
 	ror	$14, tmp0 # 14    # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
-	add	tmp0, T1          # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
+	add	tmp0, T1          # T1 = CH(e,f,g) + W[t] + K[t] + h + S1(e)
 	mov	a_64, tmp0        # tmp = a
 	xor	c_64, T2          # T2 = a ^ c
 	and	c_64, tmp0        # tmp = a & c
@@ -166,9 +166,9 @@ XROR8 =		%xmm15	# XMM_QWORD_ROR8
 	add	T1, d_64          # e(next_state) = d + T1
 	ror	$6, tmp0 # 34     # tmp = ((a ror 5) ^ a) ror 6
 	xor	a_64, tmp0        # tmp = (((a ror 5) ^ a) ror 6) ^ a
-	lea	(T1, T2), h_64    # a(next_state) = T1 + Maj(a,b,c)
 	ror	$28, tmp0 # 28    # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
-	add	tmp0, h_64        # a(next_state) = T1 + Maj(a,b,c) S0(a)
+	add	tmp0, T2          # T2 = Maj(a,b,c) + S0(a)
+	lea	(T1, T2), h_64    # a(next_state) = T1 + Maj(a,b,c) + S0(a)
 	RotateState
 .endm
 
@@ -209,8 +209,8 @@ XROR8 =		%xmm15	# XMM_QWORD_ROR8
 	xor	e_64, tmp0
 	ror	$14, tmp0 # 14
 	pshufb	XROR8, %xmm3		    # XMM3 = W[t-15] ror 8
-	add	tmp0, T1
 	add	h_64, T1
+	add	tmp0, T1
 	pxor	XW7, %xmm0		    # XMM0 = (W[t-2] >> 42) ^ W[t-2]
 	mov	a_64, T2
 	xor	c_64, T2
@@ -254,9 +254,9 @@ XROR8 =		%xmm15	# XMM_QWORD_ROR8
 	xor	e_64, tmp0
 	pxor	%xmm2, %xmm3		    # XMM3 = s0(W[t-15])
 	ror	$14, tmp0 # 14
-	add	tmp0, T1
-	psllq	$(64-61), %xmm1		    # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
 	add	h_64, T1
+	psllq	$(64-61), %xmm1		    # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
+	add	tmp0, T1
 	mov	a_64, T2
 	xor	c_64, T2
 	and	b_64, T2
@@ -413,7 +413,7 @@ SYM_FUNC_END(sha512_transform_ssse3)
 
 .section	.rodata.cst16.XMM_QWORD_BSWAP, "aM", @progbits, 16
 .align 16
-# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
+# Mask for byte-swapping a couple of qwords in an XMM register using pshufb.
 XMM_QWORD_BSWAP:
 	.octa 0x08090a0b0c0d0e0f0001020304050607
 
-- 
2.56.1


  parent reply	other threads:[~2026-11-22  2:21 UTC|newest]

Thread overview: 9+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-22  2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
2026-11-22  2:20 ` [PATCH 1/8] lib/crypto: x86/sha512: Partially roll up sha512_transform_ssse3() Eric Biggers
2026-11-22  2:20 ` [PATCH 2/8] lib/crypto: x86/sha512: Keep message schedule in XMM registers Eric Biggers
2026-11-22  2:20 ` [PATCH 3/8] lib/crypto: x86/sha512: Keep W[t]+K[t] in %xmm5 instead of stack Eric Biggers
2026-11-22  2:20 ` [PATCH 4/8] lib/crypto: x86/sha512: Keep XMM_QWORD_BSWAP in %xmm14 across blocks Eric Biggers
2026-11-22  2:20 ` [PATCH 5/8] lib/crypto: x86/sha512: Use pshufb for ror64(x, 8) in SSSE3 s0 Eric Biggers
2026-11-22  2:20 ` [PATCH 6/8] lib/crypto: x86/sha512: Keep state in registers across blocks in SSSE3 Eric Biggers
2026-11-22  2:20 ` Eric Biggers [this message]
2026-11-22  2:20 ` [PATCH 8/8] lib/crypto: x86/sha512: Use local labels for data Eric Biggers

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261122022030.296468-8-ebiggers@kernel.org \
    --to=ebiggers@kernel.org \
    --cc=Jason@zx2c4.com \
    --cc=ardb@kernel.org \
    --cc=herbert@gondor.apana.org.au \
    --cc=linux-crypto@vger.kernel.org \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox