From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
x86@kernel.org, Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 2/3] lib/crypto: x86: Opt out of initializing MXCSR and 387 state
Date: Wed, 4 Nov 2026 16:55:22 -0800 [thread overview]
Message-ID: <20261105005523.84925-3-ebiggers@kernel.org> (raw)
In-Reply-To: <20261105005523.84925-1-ebiggers@kernel.org>
The crypto code using "kernel-mode FPU" is integer code that does not
use any MMX or x87 instructions. To the extent that float-domain
instructions are used at all, they are only moves and shuffles, neither
of which raises SIMD floating-point exceptions or depends on MXCSR.
Therefore, there's no need to initialize MXCSR or the 387 FPU state.
Opt out of this initialization by using kernel_fpu_begin_mask(0) instead
of kernel_fpu_begin(). This eliminates an 'ldmxcsr' instruction in
every kernel-mode FPU section, reducing overhead, which matters most for
short input lengths. On 32-bit kernels it also eliminates 'fninit'.
A precedent for this optimization can be found in
commit a58db7ad80e8 ("netfilter: nft_set_pipapo_avx2: Skip LDMXCSR, we
don't need a valid MXCSR state").
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/x86/aes.h | 24 ++++++++++++------------
lib/crypto/x86/blake2s.h | 2 +-
lib/crypto/x86/chacha.h | 4 ++--
lib/crypto/x86/gf128hash.h | 8 ++++----
lib/crypto/x86/nh.h | 2 +-
lib/crypto/x86/poly1305.h | 2 +-
lib/crypto/x86/sha1.h | 4 ++--
lib/crypto/x86/sha256.h | 4 ++--
lib/crypto/x86/sha512.h | 2 +-
lib/crypto/x86/sm3.h | 2 +-
10 files changed, 27 insertions(+), 27 deletions(-)
diff --git a/lib/crypto/x86/aes.h b/lib/crypto/x86/aes.h
index b555290ab075..01af1a522b42 100644
--- a/lib/crypto/x86/aes.h
+++ b/lib/crypto/x86/aes.h
@@ -50,7 +50,7 @@ static void aes_preparekey_arch(union aes_enckey_arch *k,
if (static_branch_likely(&have_aesni) && key_len != AES_KEYSIZE_192 &&
irq_fpu_usable()) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (key_len == AES_KEYSIZE_128)
aes128_expandkey_aesni(rndkeys, inv_rndkeys, in_key);
else
@@ -66,7 +66,7 @@ static void aes_encrypt_arch(const struct aes_enckey *key,
const u8 in[AES_BLOCK_SIZE])
{
if (static_branch_likely(&have_aesni) && irq_fpu_usable()) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aes_encrypt_aesni(out, in, key);
kernel_fpu_end();
} else {
@@ -79,7 +79,7 @@ static void aes_decrypt_arch(const struct aes_key *key,
const u8 in[AES_BLOCK_SIZE])
{
if (static_branch_likely(&have_aesni) && irq_fpu_usable()) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aes_decrypt_aesni(out, in, key);
kernel_fpu_end();
} else {
@@ -101,7 +101,7 @@ static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aes_ecb_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key);
kernel_fpu_end();
return true;
@@ -114,7 +114,7 @@ static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aes_ecb_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, key);
kernel_fpu_end();
return true;
@@ -141,7 +141,7 @@ static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len,
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aes_cbc_encrypt_aesni(dst, src, len / AES_BLOCK_SIZE, iv, key);
kernel_fpu_end();
return true;
@@ -155,7 +155,7 @@ static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len,
{
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aes_cbc_decrypt_aesni(dst, src, len / AES_BLOCK_SIZE, iv, key);
kernel_fpu_end();
return true;
@@ -173,7 +173,7 @@ static bool aes_cbc_cts_encrypt_arch(u8 *dst, const u8 *src, size_t len,
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (cbc_blocks) {
aes_cbc_encrypt_aesni(dst, src, cbc_blocks, iv, key);
dst += cbc_blocks * AES_BLOCK_SIZE;
@@ -197,7 +197,7 @@ static bool aes_cbc_cts_decrypt_arch(u8 *dst, const u8 *src, size_t len,
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (cbc_blocks) {
aes_cbc_decrypt_aesni(dst, src, cbc_blocks, iv, key);
dst += cbc_blocks * AES_BLOCK_SIZE;
@@ -260,7 +260,7 @@ static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
ctr64 = le_ctr[0] = get_unaligned_be64(&ctr[8]);
le_ctr[1] = get_unaligned_be64(&ctr[0]);
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
nblocks = DIV_ROUND_UP(len, AES_BLOCK_SIZE);
ctr64 += nblocks;
@@ -296,7 +296,7 @@ static bool aes_xctr_arch(u8 *dst, const u8 *src, size_t len, u64 ctr,
if (!static_branch_likely(&have_aesni_avx) ||
unlikely(!irq_fpu_usable()))
return false;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (static_branch_likely(&have_vaes_avx512))
aes_xctr_crypt_vaes_avx512(dst, src, len, ctr, iv, key);
else if (static_branch_likely(&have_vaes_avx2))
@@ -344,7 +344,7 @@ aes_xts_crypt_x86(u8 *dst, const u8 *src, size_t len, u8 tweak[AES_BLOCK_SIZE],
if (!static_branch_likely(&have_aesni) || unlikely(!irq_fpu_usable()))
return false;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (IS_ENABLED(CONFIG_X86_64) &&
static_branch_likely(&have_vaes_avx512)) {
if (!cont)
diff --git a/lib/crypto/x86/blake2s.h b/lib/crypto/x86/blake2s.h
index 0f7c51f055c8..952db559eed3 100644
--- a/lib/crypto/x86/blake2s.h
+++ b/lib/crypto/x86/blake2s.h
@@ -34,7 +34,7 @@ static void blake2s_compress(struct blake2s_ctx *ctx,
const size_t blocks = min_t(size_t, nblocks,
SZ_4K / BLAKE2S_BLOCK_SIZE);
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (static_branch_likely(&blake2s_use_avx512))
blake2s_compress_avx512(ctx, data, blocks, inc);
else
diff --git a/lib/crypto/x86/chacha.h b/lib/crypto/x86/chacha.h
index fdf01b07742e..3bbfd21255de 100644
--- a/lib/crypto/x86/chacha.h
+++ b/lib/crypto/x86/chacha.h
@@ -152,7 +152,7 @@ static void hchacha_block_arch(const struct chacha_state *state,
if (!static_branch_likely(&chacha_use_simd)) {
hchacha_block_generic(state, out, nrounds);
} else {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
hchacha_block_ssse3(state, out, nrounds);
kernel_fpu_end();
}
@@ -168,7 +168,7 @@ static void chacha_crypt_arch(struct chacha_state *state, u8 *dst,
do {
size_t todo = min(bytes, SZ_4K);
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
chacha_dosimd(state, dst, src, todo, nrounds);
kernel_fpu_end();
diff --git a/lib/crypto/x86/gf128hash.h b/lib/crypto/x86/gf128hash.h
index 6b79b06caab0..5ef8f2f7b6f3 100644
--- a/lib/crypto/x86/gf128hash.h
+++ b/lib/crypto/x86/gf128hash.h
@@ -31,7 +31,7 @@ static void polyval_preparekey_arch(struct polyval_key *key,
static_assert(ARRAY_SIZE(key->h_powers) == NUM_H_POWERS);
memcpy(&key->h_powers[NUM_H_POWERS - 1], raw_key, POLYVAL_BLOCK_SIZE);
if (static_branch_likely(&have_pclmul_avx) && irq_fpu_usable()) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
for (int i = NUM_H_POWERS - 2; i >= 0; i--) {
key->h_powers[i] = key->h_powers[i + 1];
polyval_mul_pclmul_avx(
@@ -52,7 +52,7 @@ static void polyval_mul_x86(struct polyval_elem *a,
const struct polyval_elem *b)
{
if (static_branch_likely(&have_pclmul) && irq_fpu_usable()) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (static_branch_likely(&have_pclmul_avx))
polyval_mul_pclmul_avx(a, b);
else
@@ -88,7 +88,7 @@ static void ghash_blocks_arch(struct polyval_elem *acc,
size_t n = min_t(size_t, nblocks,
4096 / GHASH_BLOCK_SIZE);
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
ghash_blocks_pclmul(acc, &key->h, data, n);
kernel_fpu_end();
data += n * GHASH_BLOCK_SIZE;
@@ -110,7 +110,7 @@ static void polyval_blocks_arch(struct polyval_elem *acc,
size_t n = min_t(size_t, nblocks,
4096 / POLYVAL_BLOCK_SIZE);
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
polyval_blocks_pclmul_avx(acc, key, data, n);
kernel_fpu_end();
data += n * POLYVAL_BLOCK_SIZE;
diff --git a/lib/crypto/x86/nh.h b/lib/crypto/x86/nh.h
index 342636dcb750..e41512e5b4bb 100644
--- a/lib/crypto/x86/nh.h
+++ b/lib/crypto/x86/nh.h
@@ -21,7 +21,7 @@ static bool nh_arch(const u32 *key, const u8 *message, size_t message_len,
{
if (message_len >= 64 && static_branch_likely(&have_sse2) &&
irq_fpu_usable()) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (static_branch_likely(&have_avx2))
nh_avx2(key, message, message_len, hash);
else
diff --git a/lib/crypto/x86/poly1305.h b/lib/crypto/x86/poly1305.h
index a2034809b97b..3b92e20f4f9b 100644
--- a/lib/crypto/x86/poly1305.h
+++ b/lib/crypto/x86/poly1305.h
@@ -117,7 +117,7 @@ static void poly1305_blocks(struct poly1305_block_state *state, const u8 *inp,
do {
const size_t bytes = min(len, SZ_4K);
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
if (static_branch_likely(&poly1305_use_avx512))
poly1305_blocks_avx512(ctx, inp, bytes, padbit);
else if (static_branch_likely(&poly1305_use_avx2))
diff --git a/lib/crypto/x86/sha1.h b/lib/crypto/x86/sha1.h
index 6aff433466e7..9de5f388055e 100644
--- a/lib/crypto/x86/sha1.h
+++ b/lib/crypto/x86/sha1.h
@@ -16,7 +16,7 @@ DEFINE_STATIC_CALL(sha1_blocks_x86, sha1_blocks_generic);
const u8 *data, size_t nblocks) \
{ \
if (likely(irq_fpu_usable())) { \
- kernel_fpu_begin(); \
+ kernel_fpu_begin_mask(0); \
asm_fn(state, data, nblocks); \
kernel_fpu_end(); \
} else { \
@@ -36,7 +36,7 @@ static void sha1_blocks_avx2(struct sha1_block_state *state,
const u8 *data, size_t nblocks)
{
if (likely(irq_fpu_usable())) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
/* Select the optimal transform based on the number of blocks */
if (nblocks >= SHA1_AVX2_BLOCK_OPTSIZE)
sha1_transform_avx2(state, data, nblocks);
diff --git a/lib/crypto/x86/sha256.h b/lib/crypto/x86/sha256.h
index e98ffdaf4b14..72bce769aa92 100644
--- a/lib/crypto/x86/sha256.h
+++ b/lib/crypto/x86/sha256.h
@@ -18,7 +18,7 @@ DEFINE_STATIC_CALL(sha256_blocks_x86, sha256_blocks_generic);
size_t nblocks) \
{ \
if (likely(irq_fpu_usable())) { \
- kernel_fpu_begin(); \
+ kernel_fpu_begin_mask(0); \
asm_fn(state, data, nblocks); \
kernel_fpu_end(); \
} else { \
@@ -79,7 +79,7 @@ static bool sha256_finup_2x_arch(const struct __sha256_ctx *ctx,
*/
if (static_branch_likely(&have_sha_ni) && len >= SHA256_BLOCK_SIZE &&
len <= 65536 && likely(irq_fpu_usable())) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
sha256_ni_finup2x(ctx, data1, data2, len, out1, out2);
kernel_fpu_end();
kmsan_unpoison_memory(out1, SHA256_DIGEST_SIZE);
diff --git a/lib/crypto/x86/sha512.h b/lib/crypto/x86/sha512.h
index 4e177b4606bd..d16be7c03a41 100644
--- a/lib/crypto/x86/sha512.h
+++ b/lib/crypto/x86/sha512.h
@@ -16,7 +16,7 @@ DEFINE_STATIC_CALL(sha512_blocks_x86, sha512_blocks_generic);
size_t nblocks) \
{ \
if (likely(irq_fpu_usable())) { \
- kernel_fpu_begin(); \
+ kernel_fpu_begin_mask(0); \
asm_fn(state, data, nblocks); \
kernel_fpu_end(); \
} else { \
diff --git a/lib/crypto/x86/sm3.h b/lib/crypto/x86/sm3.h
index e06d4a22e4fa..493887838888 100644
--- a/lib/crypto/x86/sm3.h
+++ b/lib/crypto/x86/sm3.h
@@ -14,7 +14,7 @@ static void sm3_blocks_avx(struct sm3_block_state *state,
const u8 *data, size_t nblocks)
{
if (likely(irq_fpu_usable())) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
sm3_transform_avx(state, data, nblocks);
kernel_fpu_end();
} else {
--
2.56.0
next prev parent reply other threads:[~2026-11-05 0:56 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-05 0:55 [PATCH 0/3] crc, crypto: Opt out of initializing MXCSR and 387 state Eric Biggers
2026-11-05 0:55 ` [PATCH 1/3] lib/crc: x86: " Eric Biggers
2026-11-07 21:34 ` Eric Biggers
2026-11-05 0:55 ` Eric Biggers [this message]
2026-11-07 21:56 ` [PATCH 2/3] lib/crypto: " Eric Biggers
2026-11-05 0:55 ` [PATCH 3/3] crypto: x86 - " Eric Biggers
2026-11-07 21:57 ` Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261105005523.84925-3-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=ardb@kernel.org \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.