Linux cryptographic layer development
 help / color / mirror / Atom feed
From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
	"Jason A . Donenfeld" <Jason@zx2c4.com>,
	Herbert Xu <herbert@gondor.apana.org.au>,
	x86@kernel.org, Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 5/8] lib/crypto: x86/sha512: Use pshufb for ror64(x, 8) in SSSE3 s0
Date: Sat, 21 Nov 2026 18:20:27 -0800	[thread overview]
Message-ID: <20261122022030.296468-6-ebiggers@kernel.org> (raw)
In-Reply-To: <20261122022030.296468-1-ebiggers@kernel.org>

In the SHA-512 message schedule, s0(x) = ror64(x, 1) ^ ror64(x, 8) ^ (x
>> 7) was previously computed using two factored shift-and-XOR chains:

  - right-shift chain: ((((x >> 1) ^ x) >> 6) ^ x) >> 1
                       = (x >> 8) ^ (x >> 7) ^ (x >> 1)
  - left-shift chain:  ((x << 7) ^ x) << 56
                       = (x << 63) ^ (x << 56)

Instead, compute ror64(x, 8) directly with pshufb (keeping the shuffle
mask in the free register %xmm15 across blocks).  With the >> 8 and
<< 56 terms handled by pshufb, the factored chains are no longer needed,
so restructure the remaining terms directly as x << 63, x >> 1, and (x
>> 1) >> 6 (= x >> 7), and XOR them into the pshufb result.

Using pshufb is the standard way to implement 8-bit and other
byte-aligned rotations with SSSE3, as done elsewhere such as in the
ChaCha code.

This reduces s0 from 11 instructions to 9 instructions and shortens its
dependency chain.  This is likely to be a modest performance
improvement, or at worst neutral, on the CPUs that use this code.

Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 lib/crypto/x86/sha512-ssse3-asm.S | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index d40a3361aefeb..f96ce3c083ea7 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -86,6 +86,7 @@ XW5 =		%xmm11
 XW6 =		%xmm12
 XW7 =		%xmm13
 XBSWAP =	%xmm14	# XMM_QWORD_BSWAP
+XROR8 =		%xmm15	# XMM_QWORD_ROR8
 
 # Useful QWORD "arrays" for simpler memory references
 # MSG, DIGEST, K_t are arrays
@@ -201,31 +202,31 @@ XBSWAP =	%xmm14	# XMM_QWORD_BSWAP
 	psrlq	$61-19, %xmm0		    # XMM0 = W[t-2] >> 42
 	xor	e_64, tmp0
 	ror	$14, tmp0 # 14
-	psrlq	$(8-7), %xmm3		    # XMM3 = W[t-15] >> 1
+	pshufb	XROR8, %xmm3		    # XMM3 = W[t-15] ror 8
 	add	tmp0, T1
 	add	h_64, T1
 	pxor	XW7, %xmm0		    # XMM0 = (W[t-2] >> 42) ^ W[t-2]
 	mov	a_64, T2
 	xor	c_64, T2
-	pxor	%xmm2, %xmm3                # XMM3 = (W[t-15] >> 1) ^ W[t-15]
+	movdqa	%xmm2, %xmm4		    # XMM4 = W[t-15]
 	and	b_64, T2
 	mov	a_64, tmp0
 	psrlq	$(19-6), %xmm0		    # XMM0 = ((W[t-2]>>42)^W[t-2])>>13
 	and	c_64, tmp0
 	xor	tmp0, T2
-	psrlq	$(7-1), %xmm3		    # XMM3 = ((W[t-15]>>1)^W[t-15])>>6
+	psllq	$63, %xmm4		    # XMM4 = W[t-15] << 63
 	mov	a_64, tmp0
 	ror	$5, tmp0 # 39
 	pxor	XW7, %xmm0		    # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
 	xor	a_64, tmp0
 	ror	$6, tmp0 # 34
-	pxor	%xmm2, %xmm3                # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
+	psrlq	$1, %xmm2		    # XMM2 = W[t-15] >> 1
 	xor	a_64, tmp0
 	ror	$28, tmp0 # 28
 	psrlq	$6, %xmm0                   # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6
 	add	tmp0, T2
 	add	T1, d_64
-	psrlq	$1, %xmm3                   # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
+	pxor	%xmm4, %xmm3                # XMM3 = (W[t-15] ror 8) ^ (W[t-15] << 63)
 	lea	(T1, T2), h_64
 	RotateState
 	psrldq	$8, XWK
@@ -233,25 +234,24 @@ XBSWAP =	%xmm14	# XMM_QWORD_BSWAP
 	movdqa	XW7, %xmm1		    # XMM1 = W[t-2]
 	mov	f_64, T1
 	xor	g_64, T1
-	movdqa	%xmm2, %xmm4		    # XMM4 = W[t-15]
+	pxor	%xmm2, %xmm3		    # XMM3 = (W[t-15] ror 8) ^ (W[t-15] ror 1)
 	and	e_64, T1
 	xor	g_64, T1
 	psllq	$(64-19)-(64-61) , %xmm1    # XMM1 = W[t-2] << 42
 	add	T2, T1
 	mov	e_64, tmp0
-	psllq	$(64-1)-(64-8), %xmm4	    # XMM4 = W[t-15] << 7
+	psrlq	$6, %xmm2		    # XMM2 = W[t-15] >> 7
 	ror	$23, tmp0 # 41
 	xor	e_64, tmp0
 	pxor	XW7, %xmm1		    # XMM1 = (W[t-2] << 42)^W[t-2]
 	ror	$4, tmp0 # 18
 	xor	e_64, tmp0
-	pxor	%xmm2, %xmm4		    # XMM4 = (W[t-15]<<7)^W[t-15]
+	pxor	%xmm2, %xmm3		    # XMM3 = s0(W[t-15])
 	ror	$14, tmp0 # 14
 	add	tmp0, T1
 	psllq	$(64-61), %xmm1		    # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
 	add	h_64, T1
 	mov	a_64, T2
-	psllq	$(64-8), %xmm4		    # XMM4 = ((W[t-15]<<7)^W[t-15])<<56
 	xor	c_64, T2
 	and	b_64, T2
 	pxor	%xmm1, %xmm0		    # XMM0 = s1(W[t-2])
@@ -259,7 +259,6 @@ XBSWAP =	%xmm14	# XMM_QWORD_BSWAP
 	and	c_64, tmp0
 	movdqa	XW5, %xmm1		    # XMM1 = W[t-6]
 	xor	tmp0, T2
-	pxor	%xmm4, %xmm3                # XMM3 = s0(W[t-15])
 	mov	a_64, tmp0
 	paddq	%xmm3, %xmm0		    # XMM0 = s1(W[t-2]) + s0(W[t-15])
 	ror	$5, tmp0 # 39
@@ -303,6 +302,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
 	push	digest
 
 	movdqa	XMM_QWORD_BSWAP(%rip), XBSWAP
+	movdqa	XMM_QWORD_ROR8(%rip), XROR8
 
 .Lupdateblock:
 
@@ -411,6 +411,12 @@ SYM_FUNC_END(sha512_transform_ssse3)
 XMM_QWORD_BSWAP:
 	.octa 0x08090a0b0c0d0e0f0001020304050607
 
+.section	.rodata.cst16.XMM_QWORD_ROR8, "aM", @progbits, 16
+.align 16
+# Mask for rotating each qword in an XMM register right by 8 bits using pshufb.
+XMM_QWORD_ROR8:
+	.octa 0x080f0e0d0c0b0a090007060504030201
+
 # Mergeable 640-byte rodata section. This allows linker to merge the table
 # with other, exactly the same 640-byte fragment of another rodata section
 # (if such section exists).
-- 
2.56.1


  parent reply	other threads:[~2026-11-22  2:21 UTC|newest]

Thread overview: 9+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-22  2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
2026-11-22  2:20 ` [PATCH 1/8] lib/crypto: x86/sha512: Partially roll up sha512_transform_ssse3() Eric Biggers
2026-11-22  2:20 ` [PATCH 2/8] lib/crypto: x86/sha512: Keep message schedule in XMM registers Eric Biggers
2026-11-22  2:20 ` [PATCH 3/8] lib/crypto: x86/sha512: Keep W[t]+K[t] in %xmm5 instead of stack Eric Biggers
2026-11-22  2:20 ` [PATCH 4/8] lib/crypto: x86/sha512: Keep XMM_QWORD_BSWAP in %xmm14 across blocks Eric Biggers
2026-11-22  2:20 ` Eric Biggers [this message]
2026-11-22  2:20 ` [PATCH 6/8] lib/crypto: x86/sha512: Keep state in registers across blocks in SSSE3 Eric Biggers
2026-11-22  2:20 ` [PATCH 7/8] lib/crypto: x86/sha512: Shorten critical-path additions " Eric Biggers
2026-11-22  2:20 ` [PATCH 8/8] lib/crypto: x86/sha512: Use local labels for data Eric Biggers

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261122022030.296468-6-ebiggers@kernel.org \
    --to=ebiggers@kernel.org \
    --cc=Jason@zx2c4.com \
    --cc=ardb@kernel.org \
    --cc=herbert@gondor.apana.org.au \
    --cc=linux-crypto@vger.kernel.org \
    --cc=x86@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox