From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
x86@kernel.org, Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 7/8] lib/crypto: x86/sha512: Shorten critical-path additions in SSSE3
Date: Sat, 21 Nov 2026 18:20:29 -0800 [thread overview]
Message-ID: <20261122022030.296468-8-ebiggers@kernel.org> (raw)
In-Reply-To: <20261122022030.296468-1-ebiggers@kernel.org>
In SHA512_2Sched_2Round_sse, T1 = Ch(e, f, g) + W[t] + K[t] is ready
well before S1(e) finishes in tmp0, yet 'add tmp0, T1' was executed
before 'add h_64, T1'. Swap them so 'add h_64, T1' runs in parallel
with S1(e), matching SHA512_Round and removing 1 cycle per round from
the critical path.
Similarly, in SHA512_Round, T1 finishes later than both Maj(a, b, c)
and S0(a). Add S0(a) into T2 before 'lea (T1, T2), h_64', matching
SHA512_2Sched_2Round_sse and removing 1 cycle from the T1 -> h_64 path.
Note that sha512-avx2-asm.S already structures its additions similarly
to avoid both of these serial dependencies on the critical path.
Also fix a few comments in the file.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index f7dade03c6527..62f1f5109f629 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -91,10 +91,10 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
# Useful QWORD "arrays" for simpler memory references
# MSG, DIGEST, K_t are arrays
-# Input message (arg1)
+# Input message (arg2)
#define MSG(i) 8*i(msg)
-# Output Digest (arg2)
+# Output Digest (arg1)
#define DIGEST(i) 8*i(digest)
# SHA Constants (static mem)
@@ -134,7 +134,7 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
.macro SHA512_Round rnd
- # Compute Round %%t
+ # Compute Round \rnd
.if ((\rnd) % 2) == 0
movq XWK, T2
.else
@@ -154,7 +154,7 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
mov a_64, T2 # T2 = a
add h_64, T1 # T1 = CH(e,f,g) + W[t] + K[t] + h
ror $14, tmp0 # 14 # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
- add tmp0, T1 # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
+ add tmp0, T1 # T1 = CH(e,f,g) + W[t] + K[t] + h + S1(e)
mov a_64, tmp0 # tmp = a
xor c_64, T2 # T2 = a ^ c
and c_64, tmp0 # tmp = a & c
@@ -166,9 +166,9 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
add T1, d_64 # e(next_state) = d + T1
ror $6, tmp0 # 34 # tmp = ((a ror 5) ^ a) ror 6
xor a_64, tmp0 # tmp = (((a ror 5) ^ a) ror 6) ^ a
- lea (T1, T2), h_64 # a(next_state) = T1 + Maj(a,b,c)
ror $28, tmp0 # 28 # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
- add tmp0, h_64 # a(next_state) = T1 + Maj(a,b,c) S0(a)
+ add tmp0, T2 # T2 = Maj(a,b,c) + S0(a)
+ lea (T1, T2), h_64 # a(next_state) = T1 + Maj(a,b,c) + S0(a)
RotateState
.endm
@@ -209,8 +209,8 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
xor e_64, tmp0
ror $14, tmp0 # 14
pshufb XROR8, %xmm3 # XMM3 = W[t-15] ror 8
- add tmp0, T1
add h_64, T1
+ add tmp0, T1
pxor XW7, %xmm0 # XMM0 = (W[t-2] >> 42) ^ W[t-2]
mov a_64, T2
xor c_64, T2
@@ -254,9 +254,9 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
xor e_64, tmp0
pxor %xmm2, %xmm3 # XMM3 = s0(W[t-15])
ror $14, tmp0 # 14
- add tmp0, T1
- psllq $(64-61), %xmm1 # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
add h_64, T1
+ psllq $(64-61), %xmm1 # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
+ add tmp0, T1
mov a_64, T2
xor c_64, T2
and b_64, T2
@@ -413,7 +413,7 @@ SYM_FUNC_END(sha512_transform_ssse3)
.section .rodata.cst16.XMM_QWORD_BSWAP, "aM", @progbits, 16
.align 16
-# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
+# Mask for byte-swapping a couple of qwords in an XMM register using pshufb.
XMM_QWORD_BSWAP:
.octa 0x08090a0b0c0d0e0f0001020304050607
--
2.56.1
next prev parent reply other threads:[~2026-11-22 2:21 UTC|newest]
Thread overview: 9+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
2026-11-22 2:20 ` [PATCH 1/8] lib/crypto: x86/sha512: Partially roll up sha512_transform_ssse3() Eric Biggers
2026-11-22 2:20 ` [PATCH 2/8] lib/crypto: x86/sha512: Keep message schedule in XMM registers Eric Biggers
2026-11-22 2:20 ` [PATCH 3/8] lib/crypto: x86/sha512: Keep W[t]+K[t] in %xmm5 instead of stack Eric Biggers
2026-11-22 2:20 ` [PATCH 4/8] lib/crypto: x86/sha512: Keep XMM_QWORD_BSWAP in %xmm14 across blocks Eric Biggers
2026-11-22 2:20 ` [PATCH 5/8] lib/crypto: x86/sha512: Use pshufb for ror64(x, 8) in SSSE3 s0 Eric Biggers
2026-11-22 2:20 ` [PATCH 6/8] lib/crypto: x86/sha512: Keep state in registers across blocks in SSSE3 Eric Biggers
2026-11-22 2:20 ` Eric Biggers [this message]
2026-11-22 2:20 ` [PATCH 8/8] lib/crypto: x86/sha512: Use local labels for data Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261122022030.296468-8-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=ardb@kernel.org \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox