* [PATCH 1/8] lib/crypto: x86/sha512: Partially roll up sha512_transform_ssse3()
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
@ 2026-11-22 2:20 ` Eric Biggers
2026-11-22 2:20 ` [PATCH 2/8] lib/crypto: x86/sha512: Keep message schedule in XMM registers Eric Biggers
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Biggers @ 2026-11-22 2:20 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, x86,
Eric Biggers
Currently, sha512_transform_ssse3() unrolls all 80 SHA-512 rounds,
generating 13.4 KB of text. This is wasted on most systems, which
execute the AVX2 code anyway. And it's also likely to be harmful on
systems that do run this code, due to cache pollution.
Roll up rounds 14 through 77 into 8 groups of 8 rounds. This requires
some additional GPRs, as the offsets into the message schedule words and
round constants are no longer immediates. Therefore, free up two GPRs
by spilling the 'digest' and 'msglen' arguments to the stack; use them
to store a pointer to the round constants and an offset into it.
This reduces the text size from 13.4 KB to 3.3 KB. (For some
perspective, the 10.1 KB saved is greater than the size of all four
variants of the AES-XTS assembly code combined at 7.5 KB.)
I also ran the sha512 benchmark on a Haswell system (the oldest I could
easily get access to) with the AVX and AVX2 code disabled. SHA-512
throughput actually increased by 5% as a result of this commit, likely
as a result of the code fitting much better into caches.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 124 ++++++++++++++++--------------
1 file changed, 67 insertions(+), 57 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index 1a37b7a735cf9..0d24fec04d635 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -69,15 +69,20 @@ f_64 = %r14
g_64 = %r15
h_64 = %rbx
tmp0 = %rax
+# Used in the rounds 14-77 loop, while digest and msglen are spilled
+kptr = %rdi # Pointer to K512[72]
+toff = %rdx # 8*(t - 80), where W[t] is the first QWORD scheduled in the iteration.
# Local variables (stack frame)
W_SIZE = 80*8
WK_SIZE = 2*8
-frame_W = 0
-frame_WK = frame_W + W_SIZE
-frame_size = frame_WK + WK_SIZE
+frame_WK = 0
+frame_W = frame_WK + WK_SIZE
+frame_digest = frame_W + W_SIZE
+frame_msglen = frame_digest + 8
+frame_size = frame_msglen + 8
# Useful QWORD "arrays" for simpler memory references
# MSG, DIGEST, K_t, W_t are arrays
@@ -146,37 +151,35 @@ frame_size = frame_WK + WK_SIZE
RotateState
.endm
-.macro SHA512_2Sched_2Round_sse rnd
+.macro SHA512_2Sched_2Round_sse offset
# Compute rounds t-2 and t-1
# Compute message schedule QWORDS t and t+1
+ # where t = 80 + (toff + \offset) / 8
# Two rounds are computed based on the values for K[t-2]+W[t-2] and
# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
# scheduler.
- # The two new schedule QWORDS are stored at [W_t(%%t)] and [W_t(%%t+1)].
+ # The two new schedule QWORDS are stored at
+ # (frame_W + W_SIZE + \offset)(%rsp, toff) and
+ # (frame_W + W_SIZE + \offset + 8)(%rsp, toff).
# They are then added to their respective SHA512 constants at
- # [K_t(%%t)] and [K_t(%%t+1)] and stored at dqword [WK_2(%%t)]
- # For brievity, the comments following vectored instructions only refer to
+ # (\offset + 64)(kptr, toff) and (\offset + 72)(kptr, toff) and stored at
+ # dqword WK_2(0).
+ # For brevity, the comments following vectored instructions only refer to
# the first of a pair of QWORDS.
# Eg. XMM2=W[t-2] really means XMM2={W[t-2]|W[t-1]}
# The computation of the message schedule and the rounds are tightly
# stitched to take advantage of instruction-level parallelism.
- # For clarity, integer instructions (for the rounds calculation) are indented
- # by one tab. Vectored instructions (for the message scheduler) are indented
- # by two tabs.
mov f_64, T1
- idx = \rnd -2
- movdqa W_t(idx), %xmm2 # XMM2 = W[t-2]
+ movdqa (frame_W + W_SIZE + \offset - 16)(%rsp, toff), %xmm2 # XMM2 = W[t-2]
xor g_64, T1
and e_64, T1
movdqa %xmm2, %xmm0 # XMM0 = W[t-2]
xor g_64, T1
- idx = \rnd
- add WK_2(idx), T1
- idx = \rnd - 15
- movdqu W_t(idx), %xmm5 # XMM5 = W[t-15]
+ add WK_2(0), T1
+ movdqu (frame_W + W_SIZE + \offset - 120)(%rsp, toff), %xmm5 # XMM5 = W[t-15]
mov e_64, tmp0
ror $23, tmp0 # 41
movdqa %xmm5, %xmm3 # XMM3 = W[t-15]
@@ -219,8 +222,7 @@ frame_size = frame_WK + WK_SIZE
and e_64, T1
xor g_64, T1
psllq $(64-19)-(64-61) , %xmm1 # XMM1 = W[t-2] << 42
- idx = \rnd + 1
- add WK_2(idx), T1
+ add WK_2(1), T1
mov e_64, tmp0
psllq $(64-1)-(64-8), %xmm4 # XMM4 = W[t-15] << 7
ror $23, tmp0 # 41
@@ -240,24 +242,21 @@ frame_size = frame_WK + WK_SIZE
pxor %xmm1, %xmm0 # XMM0 = s1(W[t-2])
mov a_64, tmp0
and c_64, tmp0
- idx = \rnd - 7
- movdqu W_t(idx), %xmm1 # XMM1 = W[t-7]
+ movdqu (frame_W + W_SIZE + \offset - 56)(%rsp, toff), %xmm1 # XMM1 = W[t-7]
xor tmp0, T2
pxor %xmm4, %xmm3 # XMM3 = s0(W[t-15])
mov a_64, tmp0
paddq %xmm3, %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15])
ror $5, tmp0 # 39
- idx =\rnd-16
- paddq W_t(idx), %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15]) + W[t-16]
+ paddq (frame_W + W_SIZE + \offset - 128)(%rsp, toff), %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15]) + W[t-16]
xor a_64, tmp0
paddq %xmm1, %xmm0 # XMM0 = s1(W[t-2]) + W[t-7] + s0(W[t-15]) + W[t-16]
ror $6, tmp0 # 34
- movdqa %xmm0, W_t(\rnd) # Store scheduled qwords
+ movdqa %xmm0, (frame_W + W_SIZE + \offset)(%rsp, toff) # Store scheduled qwords
xor a_64, tmp0
- paddq K_t(\rnd), %xmm0 # Compute W[t]+K[t]
+ paddq (\offset + 64)(kptr, toff), %xmm0 # Compute W[t]+K[t]
ror $28, tmp0 # 28
- idx = \rnd
- movdqa %xmm0, WK_2(idx) # Store W[t]+K[t] for next rounds
+ movdqa %xmm0, WK_2(0) # Store W[t]+K[t] for next rounds
add tmp0, T2
add T1, d_64
lea (T1, T2), h_64
@@ -289,9 +288,13 @@ SYM_FUNC_START(sha512_transform_ssse3)
sub $frame_size, %rsp
and $~(0x20 - 1), %rsp
+ # Spill digest and msglen to free up %rdi and %rdx for kptr and toff.
+ mov digest, frame_digest(%rsp)
+ mov msglen, frame_msglen(%rsp)
+
.Lupdateblock:
-# Load state variables
+ # Load state variables
mov DIGEST(0), a_64
mov DIGEST(1), b_64
mov DIGEST(2), c_64
@@ -301,39 +304,46 @@ SYM_FUNC_START(sha512_transform_ssse3)
mov DIGEST(6), g_64
mov DIGEST(7), h_64
- t = 0
- .rept 80/2 + 1
- # (80 rounds) / (2 rounds/iteration) + (1 iteration)
- # +1 iteration because the scheduler leads hashing by 1 iteration
- .if t < 2
- # BSWAP 2 QWORDS
- movdqa XMM_QWORD_BSWAP(%rip), %xmm1
- movdqu MSG(t), %xmm0
- pshufb %xmm1, %xmm0 # BSWAP
- movdqa %xmm0, W_t(t) # Store Scheduled Pair
- paddq K_t(t), %xmm0 # Compute W[t]+K[t]
- movdqa %xmm0, WK_2(t) # Store into WK for rounds
- .elseif t < 16
- # BSWAP 2 QWORDS# Compute 2 Rounds
- movdqu MSG(t), %xmm0
- pshufb %xmm1, %xmm0 # BSWAP
- SHA512_Round t-2 # Round t-2
- movdqa %xmm0, W_t(t) # Store Scheduled Pair
- paddq K_t(t), %xmm0 # Compute W[t]+K[t]
- SHA512_Round t-1 # Round t-1
- movdqa %xmm0, WK_2(t) # Store W[t]+K[t] into WK
- .elseif t < 79
- # Schedule 2 QWORDS# Compute 2 Rounds
- SHA512_2Sched_2Round_sse t
- .else
- # Compute 2 Rounds
- SHA512_Round t-2
- SHA512_Round t-1
- .endif
+ # Rounds 0 - 13: BSWAP W[0..15] and compute rounds 0..13.
+ movdqa XMM_QWORD_BSWAP(%rip), %xmm1
+ movdqu MSG(0), %xmm0
+ pshufb %xmm1, %xmm0 # BSWAP
+ movdqa %xmm0, W_t(0) # Store Scheduled Pair
+ paddq K_t(0), %xmm0 # Compute W[t]+K[t]
+ movdqa %xmm0, WK_2(0) # Store into WK for rounds
+ t = 2
+ .rept 7
+ movdqu MSG(t), %xmm0
+ pshufb %xmm1, %xmm0 # BSWAP
+ SHA512_Round (t-2) # Round t-2
+ movdqa %xmm0, W_t(t) # Store Scheduled Pair
+ paddq K_t(t), %xmm0 # Compute W[t]+K[t]
+ SHA512_Round (t-1) # Round t-1
+ movdqa %xmm0, WK_2(t) # Store W[t]+K[t] into WK
t = t+2
.endr
+ # Rounds 14 - 77: Schedule W[16..79] and compute rounds 14..77,
+ # rolled into 8 iterations of 8 rounds each.
+ lea K512+72*8(%rip), kptr
+ mov $-(80-16)*8, toff
+
+.align 16
+.Lsched_loop:
+ SHA512_2Sched_2Round_sse 0
+ SHA512_2Sched_2Round_sse 16
+ SHA512_2Sched_2Round_sse 32
+ SHA512_2Sched_2Round_sse 48
+
+ add $8*8, toff
+ jnz .Lsched_loop
+
+ # Compute last 2 Rounds
+ SHA512_Round 78
+ SHA512_Round 79
+
# Update digest
+ mov frame_digest(%rsp), digest
add a_64, DIGEST(0)
add b_64, DIGEST(1)
add c_64, DIGEST(2)
@@ -345,7 +355,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
# Advance to next message block
add $16*8, msg
- dec msglen
+ decq frame_msglen(%rsp)
jnz .Lupdateblock
# Deallocate Stack Space
--
2.56.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 2/8] lib/crypto: x86/sha512: Keep message schedule in XMM registers
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
2026-11-22 2:20 ` [PATCH 1/8] lib/crypto: x86/sha512: Partially roll up sha512_transform_ssse3() Eric Biggers
@ 2026-11-22 2:20 ` Eric Biggers
2026-11-22 2:20 ` [PATCH 3/8] lib/crypto: x86/sha512: Keep W[t]+K[t] in %xmm5 instead of stack Eric Biggers
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Biggers @ 2026-11-22 2:20 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, x86,
Eric Biggers
sha512_transform_ssse3() stores all 80 message schedule words W[0..79]
in a 640-byte stack buffer. However, W[t] depends only on the previous
16 words, and enough XMM registers are free to hold 16 words.
Therefore, just keep a rolling window of W[t-16..t-1] in %xmm6-%xmm13
instead. These contain even-indexed word pairs, so when an odd-indexed
pair is needed use palignr to extract it.
Cycling through all 8 registers would take 16 rounds, whereas the
current loop does 8 per iteration. Prioritize code size (saving
~1.1 KB) by keeping it at 8 rounds per iteration and just doing the
needed series of moves at the end of each one.
This shrinks sha512_transform_ssse3()'s stack frame from a max of 744
bytes (which was large for a kernel function) to a max of 104 bytes.
This also is likely to be a modest performance improvement, or at worst
neutral, on the CPUs that use this code.
This also fixes the lack of zeroization of the message schedule buffer,
which included the input data itself, e.g. possibly an HMAC key.
('frame_WK' is not yet zeroized, but a later commit removes it.)
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 108 +++++++++++++++++++++---------
1 file changed, 75 insertions(+), 33 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index 0d24fec04d635..fe72ceb6a2735 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -2,6 +2,7 @@
# Implement fast SHA-512 with SSSE3 instructions. (x86_64)
#
# Copyright (C) 2013 Intel Corporation.
+# Copyright 2026 Google LLC
#
# Authors:
# James Guilford <james.guilford@intel.com>
@@ -73,19 +74,28 @@ tmp0 = %rax
kptr = %rdi # Pointer to K512[72]
toff = %rdx # 8*(t - 80), where W[t] is the first QWORD scheduled in the iteration.
+# Rolling window of the last 16 message schedule QWORDs:
+# XW0 = {W[t-16], W[t-15]}, ..., XW7 = {W[t-2], W[t-1]}
+XW0 = %xmm6
+XW1 = %xmm7
+XW2 = %xmm8
+XW3 = %xmm9
+XW4 = %xmm10
+XW5 = %xmm11
+XW6 = %xmm12
+XW7 = %xmm13
+
# Local variables (stack frame)
-W_SIZE = 80*8
WK_SIZE = 2*8
frame_WK = 0
-frame_W = frame_WK + WK_SIZE
-frame_digest = frame_W + W_SIZE
+frame_digest = frame_WK + WK_SIZE
frame_msglen = frame_digest + 8
frame_size = frame_msglen + 8
# Useful QWORD "arrays" for simpler memory references
-# MSG, DIGEST, K_t, W_t are arrays
+# MSG, DIGEST, K_t are arrays
# WK_2(t) points to 1 of 2 qwords at frame.WK depending on t being odd/even
# Input message (arg1)
@@ -97,12 +107,22 @@ frame_size = frame_msglen + 8
# SHA Constants (static mem)
#define K_t(i) 8*i+K512(%rip)
-# Message Schedule (stack frame)
-#define W_t(i) 8*i+frame_W(%rsp)
-
# W[t]+K[t] (stack frame)
#define WK_2(i) 8*((i%2))+frame_WK(%rsp)
+.macro RotateXW
+ # Rotate symbols XW0..XW7 left
+ XWTMP = XW0
+ XW0 = XW1
+ XW1 = XW2
+ XW2 = XW3
+ XW3 = XW4
+ XW4 = XW5
+ XW5 = XW6
+ XW6 = XW7
+ XW7 = XWTMP
+.endm
+
.macro RotateState
# Rotate symbols a..h right
TMP = h_64
@@ -160,26 +180,25 @@ frame_size = frame_msglen + 8
# Two rounds are computed based on the values for K[t-2]+W[t-2] and
# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
# scheduler.
- # The two new schedule QWORDS are stored at
- # (frame_W + W_SIZE + \offset)(%rsp, toff) and
- # (frame_W + W_SIZE + \offset + 8)(%rsp, toff).
+ # The two new schedule QWORDS are stored in XW0 (which previously held
+ # W[t-16] and W[t-15], and is rotated to XW7 at the end of the macro).
# They are then added to their respective SHA512 constants at
# (\offset + 64)(kptr, toff) and (\offset + 72)(kptr, toff) and stored at
# dqword WK_2(0).
# For brevity, the comments following vectored instructions only refer to
# the first of a pair of QWORDS.
- # Eg. XMM2=W[t-2] really means XMM2={W[t-2]|W[t-1]}
+ # Eg. XW7=W[t-2] really means XW7={W[t-2]|W[t-1]}
# The computation of the message schedule and the rounds are tightly
# stitched to take advantage of instruction-level parallelism.
mov f_64, T1
- movdqa (frame_W + W_SIZE + \offset - 16)(%rsp, toff), %xmm2 # XMM2 = W[t-2]
+ movdqa XW1, %xmm5 # XMM5 = W[t-14]
xor g_64, T1
and e_64, T1
- movdqa %xmm2, %xmm0 # XMM0 = W[t-2]
+ movdqa XW7, %xmm0 # XMM0 = W[t-2]
xor g_64, T1
add WK_2(0), T1
- movdqu (frame_W + W_SIZE + \offset - 120)(%rsp, toff), %xmm5 # XMM5 = W[t-15]
+ palignr $8, XW0, %xmm5 # XMM5 = W[t-15]
mov e_64, tmp0
ror $23, tmp0 # 41
movdqa %xmm5, %xmm3 # XMM3 = W[t-15]
@@ -191,7 +210,7 @@ frame_size = frame_msglen + 8
psrlq $(8-7), %xmm3 # XMM3 = W[t-15] >> 1
add tmp0, T1
add h_64, T1
- pxor %xmm2, %xmm0 # XMM0 = (W[t-2] >> 42) ^ W[t-2]
+ pxor XW7, %xmm0 # XMM0 = (W[t-2] >> 42) ^ W[t-2]
mov a_64, T2
xor c_64, T2
pxor %xmm5, %xmm3 # XMM3 = (W[t-15] >> 1) ^ W[t-15]
@@ -203,7 +222,7 @@ frame_size = frame_msglen + 8
psrlq $(7-1), %xmm3 # XMM3 = ((W[t-15]>>1)^W[t-15])>>6
mov a_64, tmp0
ror $5, tmp0 # 39
- pxor %xmm2, %xmm0 # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
+ pxor XW7, %xmm0 # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
xor a_64, tmp0
ror $6, tmp0 # 34
pxor %xmm5, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
@@ -215,7 +234,7 @@ frame_size = frame_msglen + 8
psrlq $1, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
lea (T1, T2), h_64
RotateState
- movdqa %xmm2, %xmm1 # XMM1 = W[t-2]
+ movdqa XW7, %xmm1 # XMM1 = W[t-2]
mov f_64, T1
xor g_64, T1
movdqa %xmm5, %xmm4 # XMM4 = W[t-15]
@@ -227,7 +246,7 @@ frame_size = frame_msglen + 8
psllq $(64-1)-(64-8), %xmm4 # XMM4 = W[t-15] << 7
ror $23, tmp0 # 41
xor e_64, tmp0
- pxor %xmm2, %xmm1 # XMM1 = (W[t-2] << 42)^W[t-2]
+ pxor XW7, %xmm1 # XMM1 = (W[t-2] << 42)^W[t-2]
ror $4, tmp0 # 18
xor e_64, tmp0
pxor %xmm5, %xmm4 # XMM4 = (W[t-15]<<7)^W[t-15]
@@ -242,25 +261,27 @@ frame_size = frame_msglen + 8
pxor %xmm1, %xmm0 # XMM0 = s1(W[t-2])
mov a_64, tmp0
and c_64, tmp0
- movdqu (frame_W + W_SIZE + \offset - 56)(%rsp, toff), %xmm1 # XMM1 = W[t-7]
+ movdqa XW5, %xmm1 # XMM1 = W[t-6]
xor tmp0, T2
pxor %xmm4, %xmm3 # XMM3 = s0(W[t-15])
mov a_64, tmp0
paddq %xmm3, %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15])
ror $5, tmp0 # 39
- paddq (frame_W + W_SIZE + \offset - 128)(%rsp, toff), %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15]) + W[t-16]
+ palignr $8, XW4, %xmm1 # XMM1 = W[t-7]
xor a_64, tmp0
- paddq %xmm1, %xmm0 # XMM0 = s1(W[t-2]) + W[t-7] + s0(W[t-15]) + W[t-16]
+ paddq XW0, %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15]) + W[t-16]
ror $6, tmp0 # 34
- movdqa %xmm0, (frame_W + W_SIZE + \offset)(%rsp, toff) # Store scheduled qwords
+ paddq %xmm1, %xmm0 # XMM0 = s1(W[t-2]) + W[t-7] + s0(W[t-15]) + W[t-16]
xor a_64, tmp0
- paddq (\offset + 64)(kptr, toff), %xmm0 # Compute W[t]+K[t]
+ movdqa %xmm0, XW0 # Store scheduled qwords
ror $28, tmp0 # 28
- movdqa %xmm0, WK_2(0) # Store W[t]+K[t] for next rounds
+ paddq (\offset + 64)(kptr, toff), %xmm0 # Compute W[t]+K[t]
add tmp0, T2
+ movdqa %xmm0, WK_2(0) # Store W[t]+K[t] for next rounds
add T1, d_64
lea (T1, T2), h_64
RotateState
+ RotateXW
.endm
########################################################################
@@ -304,19 +325,19 @@ SYM_FUNC_START(sha512_transform_ssse3)
mov DIGEST(6), g_64
mov DIGEST(7), h_64
- # Rounds 0 - 13: BSWAP W[0..15] and compute rounds 0..13.
+ # Rounds 0 - 13: BSWAP W[0..15] into XW0..XW7 and compute rounds 0..13.
movdqa XMM_QWORD_BSWAP(%rip), %xmm1
- movdqu MSG(0), %xmm0
- pshufb %xmm1, %xmm0 # BSWAP
- movdqa %xmm0, W_t(0) # Store Scheduled Pair
+ movdqu MSG(0), XW0
+ pshufb %xmm1, XW0 # BSWAP
+ movdqa XW0, %xmm0
paddq K_t(0), %xmm0 # Compute W[t]+K[t]
movdqa %xmm0, WK_2(0) # Store into WK for rounds
t = 2
- .rept 7
- movdqu MSG(t), %xmm0
- pshufb %xmm1, %xmm0 # BSWAP
+ .irp reg, XW1,XW2,XW3,XW4,XW5,XW6,XW7
+ movdqu MSG(t), \reg
+ pshufb %xmm1, \reg # BSWAP
SHA512_Round (t-2) # Round t-2
- movdqa %xmm0, W_t(t) # Store Scheduled Pair
+ movdqa \reg, %xmm0
paddq K_t(t), %xmm0 # Compute W[t]+K[t]
SHA512_Round (t-1) # Round t-1
movdqa %xmm0, WK_2(t) # Store W[t]+K[t] into WK
@@ -324,7 +345,10 @@ SYM_FUNC_START(sha512_transform_ssse3)
.endr
# Rounds 14 - 77: Schedule W[16..79] and compute rounds 14..77,
- # rolled into 8 iterations of 8 rounds each.
+ # rolled into 8 iterations of 8 rounds each. Unrolling to 16 rounds per
+ # iteration would avoid the register swap below, but 8 rounds per
+ # iteration saves ~1.1 KB of code size, which is worthwhile for this
+ # fallback implementation and improves I-cache residency.
lea K512+72*8(%rip), kptr
mov $-(80-16)*8, toff
@@ -335,6 +359,24 @@ SYM_FUNC_START(sha512_transform_ssse3)
SHA512_2Sched_2Round_sse 32
SHA512_2Sched_2Round_sse 48
+ # Swap XW0..XW3 (%xmm10..%xmm13) and XW4..XW7 (%xmm6..%xmm9) so the
+ # rolling window is back in %xmm6..%xmm13 for the next 8-round iteration.
+ movdqa %xmm6, %xmm0
+ movdqa %xmm7, %xmm1
+ movdqa %xmm8, %xmm2
+ movdqa %xmm9, %xmm3
+ movdqa %xmm10, %xmm6
+ movdqa %xmm11, %xmm7
+ movdqa %xmm12, %xmm8
+ movdqa %xmm13, %xmm9
+ movdqa %xmm0, %xmm10
+ movdqa %xmm1, %xmm11
+ movdqa %xmm2, %xmm12
+ movdqa %xmm3, %xmm13
+ .rept 4
+ RotateXW
+ .endr
+
add $8*8, toff
jnz .Lsched_loop
--
2.56.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 3/8] lib/crypto: x86/sha512: Keep W[t]+K[t] in %xmm5 instead of stack
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
2026-11-22 2:20 ` [PATCH 1/8] lib/crypto: x86/sha512: Partially roll up sha512_transform_ssse3() Eric Biggers
2026-11-22 2:20 ` [PATCH 2/8] lib/crypto: x86/sha512: Keep message schedule in XMM registers Eric Biggers
@ 2026-11-22 2:20 ` Eric Biggers
2026-11-22 2:20 ` [PATCH 4/8] lib/crypto: x86/sha512: Keep XMM_QWORD_BSWAP in %xmm14 across blocks Eric Biggers
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Biggers @ 2026-11-22 2:20 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, x86,
Eric Biggers
Instead of bouncing each pair of precomputed W[t] + K[t] values from XMM
to GPRs through a 16-byte stack buffer ('frame_WK'), keep the pair in
%xmm5 and transfer each word directly to a GPR using movq and psrldq.
This avoids stack round-trips, and it removes the remaining unzeroized
stack buffer that held input-derived data. Moreover, it eliminates the
last use of the stack at all besides the (digest, msglen) pair which
don't require 16-byte alignment. Thus the dynamic alignment and need to
maintain %rbp go away too, and we can eliminate the frame abstraction
entirely and just handle (digest, msglen) using pushes.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 80 +++++++++++++------------------
1 file changed, 33 insertions(+), 47 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index fe72ceb6a2735..f2a43c4ac8bec 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -74,6 +74,7 @@ tmp0 = %rax
kptr = %rdi # Pointer to K512[72]
toff = %rdx # 8*(t - 80), where W[t] is the first QWORD scheduled in the iteration.
+XWK = %xmm5 # {W[t]+K[t], W[t+1]+K[t+1]} for the next two rounds
# Rolling window of the last 16 message schedule QWORDs:
# XW0 = {W[t-16], W[t-15]}, ..., XW7 = {W[t-2], W[t-1]}
XW0 = %xmm6
@@ -85,18 +86,8 @@ XW5 = %xmm11
XW6 = %xmm12
XW7 = %xmm13
-# Local variables (stack frame)
-
-WK_SIZE = 2*8
-
-frame_WK = 0
-frame_digest = frame_WK + WK_SIZE
-frame_msglen = frame_digest + 8
-frame_size = frame_msglen + 8
-
# Useful QWORD "arrays" for simpler memory references
# MSG, DIGEST, K_t are arrays
-# WK_2(t) points to 1 of 2 qwords at frame.WK depending on t being odd/even
# Input message (arg1)
#define MSG(i) 8*i(msg)
@@ -107,9 +98,6 @@ frame_size = frame_msglen + 8
# SHA Constants (static mem)
#define K_t(i) 8*i+K512(%rip)
-# W[t]+K[t] (stack frame)
-#define WK_2(i) 8*((i%2))+frame_WK(%rsp)
-
.macro RotateXW
# Rotate symbols XW0..XW7 left
XWTMP = XW0
@@ -139,6 +127,12 @@ frame_size = frame_msglen + 8
.macro SHA512_Round rnd
# Compute Round %%t
+ .if ((\rnd) % 2) == 0
+ movq XWK, T2
+ .else
+ psrldq $8, XWK
+ movq XWK, T2
+ .endif
mov f_64, T1 # T1 = f
mov e_64, tmp0 # tmp = e
xor g_64, T1 # T1 = f ^ g
@@ -146,8 +140,7 @@ frame_size = frame_msglen + 8
and e_64, T1 # T1 = (f ^ g) & e
xor e_64, tmp0 # tmp = (e ror 23) ^ e
xor g_64, T1 # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
- idx = \rnd
- add WK_2(idx), T1 # W[t] + K[t] from message scheduler
+ add T2, T1 # + W[t] + K[t] from message scheduler
ror $4, tmp0 # 18 # tmp = ((e ror 23) ^ e) ror 4
xor e_64, tmp0 # tmp = (((e ror 23) ^ e) ror 4) ^ e
mov a_64, T2 # T2 = a
@@ -178,30 +171,30 @@ frame_size = frame_msglen + 8
# where t = 80 + (toff + \offset) / 8
# Two rounds are computed based on the values for K[t-2]+W[t-2] and
- # K[t-1]+W[t-1] which were previously stored at WK_2 by the message
+ # K[t-1]+W[t-1] which were previously stored in XWK by the message
# scheduler.
# The two new schedule QWORDS are stored in XW0 (which previously held
# W[t-16] and W[t-15], and is rotated to XW7 at the end of the macro).
# They are then added to their respective SHA512 constants at
- # (\offset + 64)(kptr, toff) and (\offset + 72)(kptr, toff) and stored at
- # dqword WK_2(0).
+ # (\offset + 64)(kptr, toff) and (\offset + 72)(kptr, toff) and stored in XWK.
# For brevity, the comments following vectored instructions only refer to
# the first of a pair of QWORDS.
# Eg. XW7=W[t-2] really means XW7={W[t-2]|W[t-1]}
# The computation of the message schedule and the rounds are tightly
# stitched to take advantage of instruction-level parallelism.
+ movq XWK, T2
mov f_64, T1
- movdqa XW1, %xmm5 # XMM5 = W[t-14]
+ movdqa XW1, %xmm2 # XMM2 = W[t-14]
xor g_64, T1
and e_64, T1
movdqa XW7, %xmm0 # XMM0 = W[t-2]
xor g_64, T1
- add WK_2(0), T1
- palignr $8, XW0, %xmm5 # XMM5 = W[t-15]
+ add T2, T1
+ palignr $8, XW0, %xmm2 # XMM2 = W[t-15]
mov e_64, tmp0
ror $23, tmp0 # 41
- movdqa %xmm5, %xmm3 # XMM3 = W[t-15]
+ movdqa %xmm2, %xmm3 # XMM3 = W[t-15]
xor e_64, tmp0
ror $4, tmp0 # 18
psrlq $61-19, %xmm0 # XMM0 = W[t-2] >> 42
@@ -213,7 +206,7 @@ frame_size = frame_msglen + 8
pxor XW7, %xmm0 # XMM0 = (W[t-2] >> 42) ^ W[t-2]
mov a_64, T2
xor c_64, T2
- pxor %xmm5, %xmm3 # XMM3 = (W[t-15] >> 1) ^ W[t-15]
+ pxor %xmm2, %xmm3 # XMM3 = (W[t-15] >> 1) ^ W[t-15]
and b_64, T2
mov a_64, tmp0
psrlq $(19-6), %xmm0 # XMM0 = ((W[t-2]>>42)^W[t-2])>>13
@@ -225,7 +218,7 @@ frame_size = frame_msglen + 8
pxor XW7, %xmm0 # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
xor a_64, tmp0
ror $6, tmp0 # 34
- pxor %xmm5, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
+ pxor %xmm2, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
xor a_64, tmp0
ror $28, tmp0 # 28
psrlq $6, %xmm0 # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6
@@ -234,14 +227,16 @@ frame_size = frame_msglen + 8
psrlq $1, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
lea (T1, T2), h_64
RotateState
+ psrldq $8, XWK
+ movq XWK, T2
movdqa XW7, %xmm1 # XMM1 = W[t-2]
mov f_64, T1
xor g_64, T1
- movdqa %xmm5, %xmm4 # XMM4 = W[t-15]
+ movdqa %xmm2, %xmm4 # XMM4 = W[t-15]
and e_64, T1
xor g_64, T1
psllq $(64-19)-(64-61) , %xmm1 # XMM1 = W[t-2] << 42
- add WK_2(1), T1
+ add T2, T1
mov e_64, tmp0
psllq $(64-1)-(64-8), %xmm4 # XMM4 = W[t-15] << 7
ror $23, tmp0 # 41
@@ -249,7 +244,7 @@ frame_size = frame_msglen + 8
pxor XW7, %xmm1 # XMM1 = (W[t-2] << 42)^W[t-2]
ror $4, tmp0 # 18
xor e_64, tmp0
- pxor %xmm5, %xmm4 # XMM4 = (W[t-15]<<7)^W[t-15]
+ pxor %xmm2, %xmm4 # XMM4 = (W[t-15]<<7)^W[t-15]
ror $14, tmp0 # 14
add tmp0, T1
psllq $(64-61), %xmm1 # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
@@ -277,7 +272,7 @@ frame_size = frame_msglen + 8
ror $28, tmp0 # 28
paddq (\offset + 64)(kptr, toff), %xmm0 # Compute W[t]+K[t]
add tmp0, T2
- movdqa %xmm0, WK_2(0) # Store W[t]+K[t] for next rounds
+ movdqa %xmm0, XWK # Save W[t]+K[t] for next rounds
add T1, d_64
lea (T1, T2), h_64
RotateState
@@ -295,9 +290,6 @@ frame_size = frame_msglen + 8
########################################################################
SYM_FUNC_START(sha512_transform_ssse3)
- push %rbp
- mov %rsp, %rbp
-
# Save GPRs
push %rbx
push %r12
@@ -305,13 +297,9 @@ SYM_FUNC_START(sha512_transform_ssse3)
push %r14
push %r15
- # Allocate Stack Space
- sub $frame_size, %rsp
- and $~(0x20 - 1), %rsp
-
# Spill digest and msglen to free up %rdi and %rdx for kptr and toff.
- mov digest, frame_digest(%rsp)
- mov msglen, frame_msglen(%rsp)
+ push msglen
+ push digest
.Lupdateblock:
@@ -329,9 +317,8 @@ SYM_FUNC_START(sha512_transform_ssse3)
movdqa XMM_QWORD_BSWAP(%rip), %xmm1
movdqu MSG(0), XW0
pshufb %xmm1, XW0 # BSWAP
- movdqa XW0, %xmm0
- paddq K_t(0), %xmm0 # Compute W[t]+K[t]
- movdqa %xmm0, WK_2(0) # Store into WK for rounds
+ movdqa XW0, XWK
+ paddq K_t(0), XWK # Compute W[0..1]+K[0..1]
t = 2
.irp reg, XW1,XW2,XW3,XW4,XW5,XW6,XW7
movdqu MSG(t), \reg
@@ -340,7 +327,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
movdqa \reg, %xmm0
paddq K_t(t), %xmm0 # Compute W[t]+K[t]
SHA512_Round (t-1) # Round t-1
- movdqa %xmm0, WK_2(t) # Store W[t]+K[t] into WK
+ movdqa %xmm0, XWK # Save W[t]+K[t] for next rounds
t = t+2
.endr
@@ -385,7 +372,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
SHA512_Round 79
# Update digest
- mov frame_digest(%rsp), digest
+ mov 0(%rsp), digest
add a_64, DIGEST(0)
add b_64, DIGEST(1)
add c_64, DIGEST(2)
@@ -395,13 +382,13 @@ SYM_FUNC_START(sha512_transform_ssse3)
add g_64, DIGEST(6)
add h_64, DIGEST(7)
- # Advance to next message block
+ # Decrement msglen, and advance to next message block if needed
add $16*8, msg
- decq frame_msglen(%rsp)
+ decq 8(%rsp)
jnz .Lupdateblock
- # Deallocate Stack Space
- lea -40(%rbp), %rsp
+ # Skip over digest and msglen
+ add $16, %rsp
# Restore GPRs
pop %r15
@@ -409,7 +396,6 @@ SYM_FUNC_START(sha512_transform_ssse3)
pop %r13
pop %r12
pop %rbx
- pop %rbp
RET
SYM_FUNC_END(sha512_transform_ssse3)
--
2.56.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 4/8] lib/crypto: x86/sha512: Keep XMM_QWORD_BSWAP in %xmm14 across blocks
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
` (2 preceding siblings ...)
2026-11-22 2:20 ` [PATCH 3/8] lib/crypto: x86/sha512: Keep W[t]+K[t] in %xmm5 instead of stack Eric Biggers
@ 2026-11-22 2:20 ` Eric Biggers
2026-11-22 2:20 ` [PATCH 5/8] lib/crypto: x86/sha512: Use pshufb for ror64(x, 8) in SSSE3 s0 Eric Biggers
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Biggers @ 2026-11-22 2:20 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, x86,
Eric Biggers
Load the byte-swap shuffle mask into a free XMM register once before the
block loop instead of reloading it from memory on every block.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index f2a43c4ac8bec..d40a3361aefeb 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -85,6 +85,7 @@ XW4 = %xmm10
XW5 = %xmm11
XW6 = %xmm12
XW7 = %xmm13
+XBSWAP = %xmm14 # XMM_QWORD_BSWAP
# Useful QWORD "arrays" for simpler memory references
# MSG, DIGEST, K_t are arrays
@@ -301,6 +302,8 @@ SYM_FUNC_START(sha512_transform_ssse3)
push msglen
push digest
+ movdqa XMM_QWORD_BSWAP(%rip), XBSWAP
+
.Lupdateblock:
# Load state variables
@@ -314,15 +317,14 @@ SYM_FUNC_START(sha512_transform_ssse3)
mov DIGEST(7), h_64
# Rounds 0 - 13: BSWAP W[0..15] into XW0..XW7 and compute rounds 0..13.
- movdqa XMM_QWORD_BSWAP(%rip), %xmm1
movdqu MSG(0), XW0
- pshufb %xmm1, XW0 # BSWAP
+ pshufb XBSWAP, XW0 # BSWAP
movdqa XW0, XWK
paddq K_t(0), XWK # Compute W[0..1]+K[0..1]
t = 2
.irp reg, XW1,XW2,XW3,XW4,XW5,XW6,XW7
movdqu MSG(t), \reg
- pshufb %xmm1, \reg # BSWAP
+ pshufb XBSWAP, \reg # BSWAP
SHA512_Round (t-2) # Round t-2
movdqa \reg, %xmm0
paddq K_t(t), %xmm0 # Compute W[t]+K[t]
--
2.56.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 5/8] lib/crypto: x86/sha512: Use pshufb for ror64(x, 8) in SSSE3 s0
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
` (3 preceding siblings ...)
2026-11-22 2:20 ` [PATCH 4/8] lib/crypto: x86/sha512: Keep XMM_QWORD_BSWAP in %xmm14 across blocks Eric Biggers
@ 2026-11-22 2:20 ` Eric Biggers
2026-11-22 2:20 ` [PATCH 6/8] lib/crypto: x86/sha512: Keep state in registers across blocks in SSSE3 Eric Biggers
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Eric Biggers @ 2026-11-22 2:20 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, x86,
Eric Biggers
In the SHA-512 message schedule, s0(x) = ror64(x, 1) ^ ror64(x, 8) ^ (x
>> 7) was previously computed using two factored shift-and-XOR chains:
- right-shift chain: ((((x >> 1) ^ x) >> 6) ^ x) >> 1
= (x >> 8) ^ (x >> 7) ^ (x >> 1)
- left-shift chain: ((x << 7) ^ x) << 56
= (x << 63) ^ (x << 56)
Instead, compute ror64(x, 8) directly with pshufb (keeping the shuffle
mask in the free register %xmm15 across blocks). With the >> 8 and
<< 56 terms handled by pshufb, the factored chains are no longer needed,
so restructure the remaining terms directly as x << 63, x >> 1, and (x
>> 1) >> 6 (= x >> 7), and XOR them into the pshufb result.
Using pshufb is the standard way to implement 8-bit and other
byte-aligned rotations with SSSE3, as done elsewhere such as in the
ChaCha code.
This reduces s0 from 11 instructions to 9 instructions and shortens its
dependency chain. This is likely to be a modest performance
improvement, or at worst neutral, on the CPUs that use this code.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 26 ++++++++++++++++----------
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index d40a3361aefeb..f96ce3c083ea7 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -86,6 +86,7 @@ XW5 = %xmm11
XW6 = %xmm12
XW7 = %xmm13
XBSWAP = %xmm14 # XMM_QWORD_BSWAP
+XROR8 = %xmm15 # XMM_QWORD_ROR8
# Useful QWORD "arrays" for simpler memory references
# MSG, DIGEST, K_t are arrays
@@ -201,31 +202,31 @@ XBSWAP = %xmm14 # XMM_QWORD_BSWAP
psrlq $61-19, %xmm0 # XMM0 = W[t-2] >> 42
xor e_64, tmp0
ror $14, tmp0 # 14
- psrlq $(8-7), %xmm3 # XMM3 = W[t-15] >> 1
+ pshufb XROR8, %xmm3 # XMM3 = W[t-15] ror 8
add tmp0, T1
add h_64, T1
pxor XW7, %xmm0 # XMM0 = (W[t-2] >> 42) ^ W[t-2]
mov a_64, T2
xor c_64, T2
- pxor %xmm2, %xmm3 # XMM3 = (W[t-15] >> 1) ^ W[t-15]
+ movdqa %xmm2, %xmm4 # XMM4 = W[t-15]
and b_64, T2
mov a_64, tmp0
psrlq $(19-6), %xmm0 # XMM0 = ((W[t-2]>>42)^W[t-2])>>13
and c_64, tmp0
xor tmp0, T2
- psrlq $(7-1), %xmm3 # XMM3 = ((W[t-15]>>1)^W[t-15])>>6
+ psllq $63, %xmm4 # XMM4 = W[t-15] << 63
mov a_64, tmp0
ror $5, tmp0 # 39
pxor XW7, %xmm0 # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
xor a_64, tmp0
ror $6, tmp0 # 34
- pxor %xmm2, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
+ psrlq $1, %xmm2 # XMM2 = W[t-15] >> 1
xor a_64, tmp0
ror $28, tmp0 # 28
psrlq $6, %xmm0 # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6
add tmp0, T2
add T1, d_64
- psrlq $1, %xmm3 # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
+ pxor %xmm4, %xmm3 # XMM3 = (W[t-15] ror 8) ^ (W[t-15] << 63)
lea (T1, T2), h_64
RotateState
psrldq $8, XWK
@@ -233,25 +234,24 @@ XBSWAP = %xmm14 # XMM_QWORD_BSWAP
movdqa XW7, %xmm1 # XMM1 = W[t-2]
mov f_64, T1
xor g_64, T1
- movdqa %xmm2, %xmm4 # XMM4 = W[t-15]
+ pxor %xmm2, %xmm3 # XMM3 = (W[t-15] ror 8) ^ (W[t-15] ror 1)
and e_64, T1
xor g_64, T1
psllq $(64-19)-(64-61) , %xmm1 # XMM1 = W[t-2] << 42
add T2, T1
mov e_64, tmp0
- psllq $(64-1)-(64-8), %xmm4 # XMM4 = W[t-15] << 7
+ psrlq $6, %xmm2 # XMM2 = W[t-15] >> 7
ror $23, tmp0 # 41
xor e_64, tmp0
pxor XW7, %xmm1 # XMM1 = (W[t-2] << 42)^W[t-2]
ror $4, tmp0 # 18
xor e_64, tmp0
- pxor %xmm2, %xmm4 # XMM4 = (W[t-15]<<7)^W[t-15]
+ pxor %xmm2, %xmm3 # XMM3 = s0(W[t-15])
ror $14, tmp0 # 14
add tmp0, T1
psllq $(64-61), %xmm1 # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
add h_64, T1
mov a_64, T2
- psllq $(64-8), %xmm4 # XMM4 = ((W[t-15]<<7)^W[t-15])<<56
xor c_64, T2
and b_64, T2
pxor %xmm1, %xmm0 # XMM0 = s1(W[t-2])
@@ -259,7 +259,6 @@ XBSWAP = %xmm14 # XMM_QWORD_BSWAP
and c_64, tmp0
movdqa XW5, %xmm1 # XMM1 = W[t-6]
xor tmp0, T2
- pxor %xmm4, %xmm3 # XMM3 = s0(W[t-15])
mov a_64, tmp0
paddq %xmm3, %xmm0 # XMM0 = s1(W[t-2]) + s0(W[t-15])
ror $5, tmp0 # 39
@@ -303,6 +302,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
push digest
movdqa XMM_QWORD_BSWAP(%rip), XBSWAP
+ movdqa XMM_QWORD_ROR8(%rip), XROR8
.Lupdateblock:
@@ -411,6 +411,12 @@ SYM_FUNC_END(sha512_transform_ssse3)
XMM_QWORD_BSWAP:
.octa 0x08090a0b0c0d0e0f0001020304050607
+.section .rodata.cst16.XMM_QWORD_ROR8, "aM", @progbits, 16
+.align 16
+# Mask for rotating each qword in an XMM register right by 8 bits using pshufb.
+XMM_QWORD_ROR8:
+ .octa 0x080f0e0d0c0b0a090007060504030201
+
# Mergeable 640-byte rodata section. This allows linker to merge the table
# with other, exactly the same 640-byte fragment of another rodata section
# (if such section exists).
--
2.56.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6/8] lib/crypto: x86/sha512: Keep state in registers across blocks in SSSE3
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
` (4 preceding siblings ...)
2026-11-22 2:20 ` [PATCH 5/8] lib/crypto: x86/sha512: Use pshufb for ror64(x, 8) in SSSE3 s0 Eric Biggers
@ 2026-11-22 2:20 ` Eric Biggers
2026-11-22 2:20 ` [PATCH 7/8] lib/crypto: x86/sha512: Shorten critical-path additions " Eric Biggers
2026-11-22 2:20 ` [PATCH 8/8] lib/crypto: x86/sha512: Use local labels for data Eric Biggers
7 siblings, 0 replies; 9+ messages in thread
From: Eric Biggers @ 2026-11-22 2:20 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, x86,
Eric Biggers
In sha512_transform_ssse3(), each iteration of .Lupdateblock updates the
state in memory using 'add reg, mem' and then immediately reloads all 8
state words from memory at the top of the next iteration.
Hoist the initial state load before .Lupdateblock, and update the state
at the end of each block using 'add mem, reg' followed by 'mov reg, mem'
(via a new 'addm' macro), as sha512-avx2-asm.S, sha256-ssse3-asm.S, and
sha256-avx2-asm.S already do.
This keeps the updated state in registers across blocks, eliminating 8
loads per additional block and avoiding store-to-load forwarding stalls.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 26 ++++++++++++++++----------
1 file changed, 16 insertions(+), 10 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index f96ce3c083ea7..f7dade03c6527 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -100,6 +100,12 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
# SHA Constants (static mem)
#define K_t(i) 8*i+K512(%rip)
+# Add \mem to \reg, then copy the result back to \mem.
+.macro addm mem reg
+ add \mem, \reg
+ mov \reg, \mem
+.endm
+
.macro RotateXW
# Rotate symbols XW0..XW7 left
XWTMP = XW0
@@ -304,8 +310,6 @@ SYM_FUNC_START(sha512_transform_ssse3)
movdqa XMM_QWORD_BSWAP(%rip), XBSWAP
movdqa XMM_QWORD_ROR8(%rip), XROR8
-.Lupdateblock:
-
# Load state variables
mov DIGEST(0), a_64
mov DIGEST(1), b_64
@@ -316,6 +320,8 @@ SYM_FUNC_START(sha512_transform_ssse3)
mov DIGEST(6), g_64
mov DIGEST(7), h_64
+.Lupdateblock:
+
# Rounds 0 - 13: BSWAP W[0..15] into XW0..XW7 and compute rounds 0..13.
movdqu MSG(0), XW0
pshufb XBSWAP, XW0 # BSWAP
@@ -375,14 +381,14 @@ SYM_FUNC_START(sha512_transform_ssse3)
# Update digest
mov 0(%rsp), digest
- add a_64, DIGEST(0)
- add b_64, DIGEST(1)
- add c_64, DIGEST(2)
- add d_64, DIGEST(3)
- add e_64, DIGEST(4)
- add f_64, DIGEST(5)
- add g_64, DIGEST(6)
- add h_64, DIGEST(7)
+ addm DIGEST(0), a_64
+ addm DIGEST(1), b_64
+ addm DIGEST(2), c_64
+ addm DIGEST(3), d_64
+ addm DIGEST(4), e_64
+ addm DIGEST(5), f_64
+ addm DIGEST(6), g_64
+ addm DIGEST(7), h_64
# Decrement msglen, and advance to next message block if needed
add $16*8, msg
--
2.56.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 7/8] lib/crypto: x86/sha512: Shorten critical-path additions in SSSE3
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
` (5 preceding siblings ...)
2026-11-22 2:20 ` [PATCH 6/8] lib/crypto: x86/sha512: Keep state in registers across blocks in SSSE3 Eric Biggers
@ 2026-11-22 2:20 ` Eric Biggers
2026-11-22 2:20 ` [PATCH 8/8] lib/crypto: x86/sha512: Use local labels for data Eric Biggers
7 siblings, 0 replies; 9+ messages in thread
From: Eric Biggers @ 2026-11-22 2:20 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, x86,
Eric Biggers
In SHA512_2Sched_2Round_sse, T1 = Ch(e, f, g) + W[t] + K[t] is ready
well before S1(e) finishes in tmp0, yet 'add tmp0, T1' was executed
before 'add h_64, T1'. Swap them so 'add h_64, T1' runs in parallel
with S1(e), matching SHA512_Round and removing 1 cycle per round from
the critical path.
Similarly, in SHA512_Round, T1 finishes later than both Maj(a, b, c)
and S0(a). Add S0(a) into T2 before 'lea (T1, T2), h_64', matching
SHA512_2Sched_2Round_sse and removing 1 cycle from the T1 -> h_64 path.
Note that sha512-avx2-asm.S already structures its additions similarly
to avoid both of these serial dependencies on the critical path.
Also fix a few comments in the file.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index f7dade03c6527..62f1f5109f629 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -91,10 +91,10 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
# Useful QWORD "arrays" for simpler memory references
# MSG, DIGEST, K_t are arrays
-# Input message (arg1)
+# Input message (arg2)
#define MSG(i) 8*i(msg)
-# Output Digest (arg2)
+# Output Digest (arg1)
#define DIGEST(i) 8*i(digest)
# SHA Constants (static mem)
@@ -134,7 +134,7 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
.macro SHA512_Round rnd
- # Compute Round %%t
+ # Compute Round \rnd
.if ((\rnd) % 2) == 0
movq XWK, T2
.else
@@ -154,7 +154,7 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
mov a_64, T2 # T2 = a
add h_64, T1 # T1 = CH(e,f,g) + W[t] + K[t] + h
ror $14, tmp0 # 14 # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
- add tmp0, T1 # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
+ add tmp0, T1 # T1 = CH(e,f,g) + W[t] + K[t] + h + S1(e)
mov a_64, tmp0 # tmp = a
xor c_64, T2 # T2 = a ^ c
and c_64, tmp0 # tmp = a & c
@@ -166,9 +166,9 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
add T1, d_64 # e(next_state) = d + T1
ror $6, tmp0 # 34 # tmp = ((a ror 5) ^ a) ror 6
xor a_64, tmp0 # tmp = (((a ror 5) ^ a) ror 6) ^ a
- lea (T1, T2), h_64 # a(next_state) = T1 + Maj(a,b,c)
ror $28, tmp0 # 28 # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
- add tmp0, h_64 # a(next_state) = T1 + Maj(a,b,c) S0(a)
+ add tmp0, T2 # T2 = Maj(a,b,c) + S0(a)
+ lea (T1, T2), h_64 # a(next_state) = T1 + Maj(a,b,c) + S0(a)
RotateState
.endm
@@ -209,8 +209,8 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
xor e_64, tmp0
ror $14, tmp0 # 14
pshufb XROR8, %xmm3 # XMM3 = W[t-15] ror 8
- add tmp0, T1
add h_64, T1
+ add tmp0, T1
pxor XW7, %xmm0 # XMM0 = (W[t-2] >> 42) ^ W[t-2]
mov a_64, T2
xor c_64, T2
@@ -254,9 +254,9 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
xor e_64, tmp0
pxor %xmm2, %xmm3 # XMM3 = s0(W[t-15])
ror $14, tmp0 # 14
- add tmp0, T1
- psllq $(64-61), %xmm1 # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
add h_64, T1
+ psllq $(64-61), %xmm1 # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
+ add tmp0, T1
mov a_64, T2
xor c_64, T2
and b_64, T2
@@ -413,7 +413,7 @@ SYM_FUNC_END(sha512_transform_ssse3)
.section .rodata.cst16.XMM_QWORD_BSWAP, "aM", @progbits, 16
.align 16
-# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
+# Mask for byte-swapping a couple of qwords in an XMM register using pshufb.
XMM_QWORD_BSWAP:
.octa 0x08090a0b0c0d0e0f0001020304050607
--
2.56.1
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 8/8] lib/crypto: x86/sha512: Use local labels for data
2026-11-22 2:20 [PATCH 0/8] lib/crypto: De-bloat and optimize sha512-ssse3-asm.S Eric Biggers
` (6 preceding siblings ...)
2026-11-22 2:20 ` [PATCH 7/8] lib/crypto: x86/sha512: Shorten critical-path additions " Eric Biggers
@ 2026-11-22 2:20 ` Eric Biggers
7 siblings, 0 replies; 9+ messages in thread
From: Eric Biggers @ 2026-11-22 2:20 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, x86,
Eric Biggers
Prefix XMM_QWORD_BSWAP, XMM_QWORD_ROR8, and K512 with ".L_" to make them
local labels and avoid emitting extra symbol table entries.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/sha512-ssse3-asm.S | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/lib/crypto/x86/sha512-ssse3-asm.S b/lib/crypto/x86/sha512-ssse3-asm.S
index 62f1f5109f629..42375adde01ef 100644
--- a/lib/crypto/x86/sha512-ssse3-asm.S
+++ b/lib/crypto/x86/sha512-ssse3-asm.S
@@ -71,7 +71,7 @@ g_64 = %r15
h_64 = %rbx
tmp0 = %rax
# Used in the rounds 14-77 loop, while digest and msglen are spilled
-kptr = %rdi # Pointer to K512[72]
+kptr = %rdi # Pointer to .L_K512[72]
toff = %rdx # 8*(t - 80), where W[t] is the first QWORD scheduled in the iteration.
XWK = %xmm5 # {W[t]+K[t], W[t+1]+K[t+1]} for the next two rounds
@@ -85,8 +85,8 @@ XW4 = %xmm10
XW5 = %xmm11
XW6 = %xmm12
XW7 = %xmm13
-XBSWAP = %xmm14 # XMM_QWORD_BSWAP
-XROR8 = %xmm15 # XMM_QWORD_ROR8
+XBSWAP = %xmm14 # .L_XMM_QWORD_BSWAP
+XROR8 = %xmm15 # .L_XMM_QWORD_ROR8
# Useful QWORD "arrays" for simpler memory references
# MSG, DIGEST, K_t are arrays
@@ -98,7 +98,7 @@ XROR8 = %xmm15 # XMM_QWORD_ROR8
#define DIGEST(i) 8*i(digest)
# SHA Constants (static mem)
-#define K_t(i) 8*i+K512(%rip)
+#define K_t(i) 8*i+.L_K512(%rip)
# Add \mem to \reg, then copy the result back to \mem.
.macro addm mem reg
@@ -307,8 +307,8 @@ SYM_FUNC_START(sha512_transform_ssse3)
push msglen
push digest
- movdqa XMM_QWORD_BSWAP(%rip), XBSWAP
- movdqa XMM_QWORD_ROR8(%rip), XROR8
+ movdqa .L_XMM_QWORD_BSWAP(%rip), XBSWAP
+ movdqa .L_XMM_QWORD_ROR8(%rip), XROR8
# Load state variables
mov DIGEST(0), a_64
@@ -344,7 +344,7 @@ SYM_FUNC_START(sha512_transform_ssse3)
# iteration would avoid the register swap below, but 8 rounds per
# iteration saves ~1.1 KB of code size, which is worthwhile for this
# fallback implementation and improves I-cache residency.
- lea K512+72*8(%rip), kptr
+ lea .L_K512+72*8(%rip), kptr
mov $-(80-16)*8, toff
.align 16
@@ -414,13 +414,13 @@ SYM_FUNC_END(sha512_transform_ssse3)
.section .rodata.cst16.XMM_QWORD_BSWAP, "aM", @progbits, 16
.align 16
# Mask for byte-swapping a couple of qwords in an XMM register using pshufb.
-XMM_QWORD_BSWAP:
+.L_XMM_QWORD_BSWAP:
.octa 0x08090a0b0c0d0e0f0001020304050607
.section .rodata.cst16.XMM_QWORD_ROR8, "aM", @progbits, 16
.align 16
# Mask for rotating each qword in an XMM register right by 8 bits using pshufb.
-XMM_QWORD_ROR8:
+.L_XMM_QWORD_ROR8:
.octa 0x080f0e0d0c0b0a090007060504030201
# Mergeable 640-byte rodata section. This allows linker to merge the table
@@ -429,7 +429,7 @@ XMM_QWORD_ROR8:
.section .rodata.cst640.K512, "aM", @progbits, 640
.align 64
# K[t] used in SHA512 hashing
-K512:
+.L_K512:
.quad 0x428a2f98d728ae22,0x7137449123ef65cd
.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
.quad 0x3956c25bf348b538,0x59f111f1b605d019
--
2.56.1
^ permalink raw reply related [flat|nested] 9+ messages in thread