From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
x86@kernel.org, Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 3/3] crypto: x86 - Opt out of initializing MXCSR and 387 state
Date: Wed, 4 Nov 2026 16:55:23 -0800 [thread overview]
Message-ID: <20261105005523.84925-4-ebiggers@kernel.org> (raw)
In-Reply-To: <20261105005523.84925-1-ebiggers@kernel.org>
The crypto code using "kernel-mode FPU" is integer code that does not
use any MMX or x87 instructions. To the extent that float-domain
instructions are used at all, they are only moves and shuffles, neither
of which raises SIMD floating-point exceptions or depends on MXCSR.
Therefore, there's no need to initialize MXCSR or the 387 FPU state.
Opt out of this initialization by using kernel_fpu_begin_mask(0) instead
of kernel_fpu_begin(). This eliminates an 'ldmxcsr' instruction in
every kernel-mode FPU section, reducing overhead, which matters most for
short input lengths. On 32-bit kernels it also eliminates 'fninit'.
A precedent for this optimization can be found in
commit a58db7ad80e8 ("netfilter: nft_set_pipapo_avx2: Skip LDMXCSR, we
don't need a valid MXCSR state").
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/x86/crypto/aegis128-aesni-glue.c | 6 +++---
arch/x86/crypto/aesni-intel_glue.c | 10 +++++-----
arch/x86/crypto/aria_aesni_avx2_glue.c | 4 ++--
arch/x86/crypto/aria_aesni_avx_glue.c | 2 +-
arch/x86/crypto/aria_gfni_avx512_glue.c | 6 +++---
arch/x86/crypto/ecb_cbc_helpers.h | 2 +-
arch/x86/crypto/sm4_aesni_avx_glue.c | 6 +++---
7 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c
index 09fc0b15b0e9..5aa0adf9b285 100644
--- a/arch/x86/crypto/aegis128-aesni-glue.c
+++ b/arch/x86/crypto/aegis128-aesni-glue.c
@@ -124,7 +124,7 @@ crypto_aegis128_aesni_process_crypt(struct aegis_state *state,
kernel_fpu_end();
err = skcipher_walk_done(walk,
walk->nbytes % AEGIS128_BLOCK_SIZE);
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
}
if (walk->nbytes) {
@@ -138,7 +138,7 @@ crypto_aegis128_aesni_process_crypt(struct aegis_state *state,
walk->nbytes);
kernel_fpu_end();
err = skcipher_walk_done(walk, 0);
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
}
return err;
}
@@ -191,7 +191,7 @@ crypto_aegis128_aesni_crypt(struct aead_request *req,
if (err)
return err;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aegis128_aesni_init(&state, &ctx->key, req->iv);
crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen);
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index d2fb53e23eda..3b58d77d2f5d 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -458,7 +458,7 @@ static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
return err;
if (likely(crypto_simd_usable())) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aes_gcm_precompute(key, flags);
kernel_fpu_end();
} else {
@@ -527,7 +527,7 @@ static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
/*
* Initialize @ghash_acc, then pass all @assoclen bytes of associated data
* (a.k.a. additional authenticated data) from @sg_src through the GHASH update
- * assembly function. kernel_fpu_begin() must have already been called.
+ * assembly function. The caller must be in a kernel-mode FPU section.
*/
static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16],
struct scatterlist *sg_src, unsigned int assoclen,
@@ -578,7 +578,7 @@ static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16],
scatterwalk_done_src(&walk, orig_len_this_step);
if (need_resched()) {
kernel_fpu_end();
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
}
}
if (unlikely(pos))
@@ -631,7 +631,7 @@ gcm_crypt(struct aead_request *req, int flags)
* section and start a new one if there are multiple data segments or if
* rescheduling is needed while processing the associated data.
*/
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
/* Pass the associated data through GHASH. */
gcm_process_assoc(key, ghash_acc, req->src, assoclen, flags);
@@ -653,7 +653,7 @@ gcm_crypt(struct aead_request *req, int flags)
err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
if (err)
return err;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
}
/* Last segment: process all remaining data. */
aes_gcm_update(key, le_ctr, ghash_acc, walk.src.virt.addr,
diff --git a/arch/x86/crypto/aria_aesni_avx2_glue.c b/arch/x86/crypto/aria_aesni_avx2_glue.c
index 371be2fb6469..7eece77fde91 100644
--- a/arch/x86/crypto/aria_aesni_avx2_glue.c
+++ b/arch/x86/crypto/aria_aesni_avx2_glue.c
@@ -99,7 +99,7 @@ static int aria_avx2_ctr_encrypt(struct skcipher_request *req)
u8 *dst = walk.dst.virt.addr;
while (nbytes >= ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aria_ops.aria_ctr_crypt_32way(ctx, dst, src,
&req_ctx->keystream[0],
walk.iv);
@@ -110,7 +110,7 @@ static int aria_avx2_ctr_encrypt(struct skcipher_request *req)
}
while (nbytes >= ARIA_AESNI_PARALLEL_BLOCK_SIZE) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aria_ops.aria_ctr_crypt_16way(ctx, dst, src,
&req_ctx->keystream[0],
walk.iv);
diff --git a/arch/x86/crypto/aria_aesni_avx_glue.c b/arch/x86/crypto/aria_aesni_avx_glue.c
index d23fc91c0ebd..29f0aa96d8dc 100644
--- a/arch/x86/crypto/aria_aesni_avx_glue.c
+++ b/arch/x86/crypto/aria_aesni_avx_glue.c
@@ -97,7 +97,7 @@ static int aria_avx_ctr_encrypt(struct skcipher_request *req)
u8 *dst = walk.dst.virt.addr;
while (nbytes >= ARIA_AESNI_PARALLEL_BLOCK_SIZE) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aria_ops.aria_ctr_crypt_16way(ctx, dst, src,
&req_ctx->keystream[0],
walk.iv);
diff --git a/arch/x86/crypto/aria_gfni_avx512_glue.c b/arch/x86/crypto/aria_gfni_avx512_glue.c
index e05bbeb22d4a..d58a26cc2646 100644
--- a/arch/x86/crypto/aria_gfni_avx512_glue.c
+++ b/arch/x86/crypto/aria_gfni_avx512_glue.c
@@ -87,7 +87,7 @@ static int aria_avx512_ctr_encrypt(struct skcipher_request *req)
u8 *dst = walk.dst.virt.addr;
while (nbytes >= ARIA_GFNI_AVX512_PARALLEL_BLOCK_SIZE) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aria_ops.aria_ctr_crypt_64way(ctx, dst, src,
&req_ctx->keystream[0],
walk.iv);
@@ -98,7 +98,7 @@ static int aria_avx512_ctr_encrypt(struct skcipher_request *req)
}
while (nbytes >= ARIA_AESNI_AVX2_PARALLEL_BLOCK_SIZE) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aria_ops.aria_ctr_crypt_32way(ctx, dst, src,
&req_ctx->keystream[0],
walk.iv);
@@ -109,7 +109,7 @@ static int aria_avx512_ctr_encrypt(struct skcipher_request *req)
}
while (nbytes >= ARIA_AESNI_PARALLEL_BLOCK_SIZE) {
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
aria_ops.aria_ctr_crypt_16way(ctx, dst, src,
&req_ctx->keystream[0],
walk.iv);
diff --git a/arch/x86/crypto/ecb_cbc_helpers.h b/arch/x86/crypto/ecb_cbc_helpers.h
index 11955bd01af1..ef0bb85a2ce0 100644
--- a/arch/x86/crypto/ecb_cbc_helpers.h
+++ b/arch/x86/crypto/ecb_cbc_helpers.h
@@ -24,7 +24,7 @@
const u8 *src = walk.src.virt.addr; \
u8 *dst = walk.dst.virt.addr; \
u8 __maybe_unused buf[(bsize)]; \
- if (do_fpu) kernel_fpu_begin()
+ if (do_fpu) kernel_fpu_begin_mask(0)
#define CBC_WALK_START(req, bsize, fpu_blocks) \
ECB_WALK_START(req, bsize, fpu_blocks)
diff --git a/arch/x86/crypto/sm4_aesni_avx_glue.c b/arch/x86/crypto/sm4_aesni_avx_glue.c
index ed383da5ff46..1521587665e5 100644
--- a/arch/x86/crypto/sm4_aesni_avx_glue.c
+++ b/arch/x86/crypto/sm4_aesni_avx_glue.c
@@ -48,7 +48,7 @@ static int ecb_do_crypt(struct skcipher_request *req, const u32 *rkey)
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
while (nbytes >= SM4_CRYPT8_BLOCK_SIZE) {
sm4_aesni_avx_crypt8(rkey, dst, src, 8);
dst += SM4_CRYPT8_BLOCK_SIZE;
@@ -136,7 +136,7 @@ int sm4_avx_cbc_decrypt(struct skcipher_request *req,
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
while (nbytes >= bsize) {
func(ctx->rkey_dec, dst, src, walk.iv);
@@ -201,7 +201,7 @@ int sm4_avx_ctr_crypt(struct skcipher_request *req,
const u8 *src = walk.src.virt.addr;
u8 *dst = walk.dst.virt.addr;
- kernel_fpu_begin();
+ kernel_fpu_begin_mask(0);
while (nbytes >= bsize) {
func(ctx->rkey_enc, dst, src, walk.iv);
--
2.56.0
next prev parent reply other threads:[~2026-11-05 0:56 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-05 0:55 [PATCH 0/3] crc, crypto: Opt out of initializing MXCSR and 387 state Eric Biggers
2026-11-05 0:55 ` [PATCH 1/3] lib/crc: x86: " Eric Biggers
2026-11-07 21:34 ` Eric Biggers
2026-11-05 0:55 ` [PATCH 2/3] lib/crypto: " Eric Biggers
2026-11-07 21:56 ` Eric Biggers
2026-11-05 0:55 ` Eric Biggers [this message]
2026-11-07 21:57 ` [PATCH 3/3] crypto: x86 - " Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261105005523.84925-4-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=ardb@kernel.org \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.