From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
sparclinux@vger.kernel.org, Andreas Larsson <andreas@gaisler.com>,
"David S . Miller" <davem@davemloft.net>,
Stian Halseth <stian@itx.no>, Magnus Lindholm <linmag7@gmail.com>,
John Paul Adrian Glaubitz <glaubitz@physik.fu-berlin.de>,
Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 2/4] lib/crypto: sparc/aes-ctr: Migrate optimized code into library
Date: Tue, 17 Nov 2026 20:10:31 -0800 [thread overview]
Message-ID: <20261118041033.50014-3-ebiggers@kernel.org> (raw)
In-Reply-To: <20261118041033.50014-1-ebiggers@kernel.org>
Instead of exposing the sparc64-optimized AES-CTR code via a
sparc64-specific crypto_skcipher algorithm, implement the AES-CTR
library function. This is simpler, makes the AES-CTR library function
sparc64-optimized, and fixes the longstanding issue where the
sparc64-optimized AES-CTR code was disabled by default. AES-CTR support
remains available through crypto_skcipher via crypto/aes.c.
The library functions take size_t lengths, so update the CTR assembly
functions to match.
Handle the CTR assembly code's data alignment requirement by explicitly
using an on-stack bounce buffer when the data is misaligned. (Note that
bounce buffering was used in the old code too; it was just hidden in the
skcipher_walk_* functions via the configured cra_alignmask=7.) Also use
a bounce buffer to handle any partial block at the end of the message.
Now that ctr-aes-sparc64 is removed, also increase the priority of
ctr-aes-lib on SPARC, and re-enable ccm-aes-lib, gcm-aes-lib, and
rfc4106-gcm-aes-lib.
Tested in QEMU (commit 97c88c61d45d) with aes_ctr_kunit
(https://lore.kernel.org/r/20261116062849.1071038-6-ebiggers@kernel.org/)
and CONFIG_CRYPTO_SELFTESTS_FULL=y, using the patch that makes the SPARC
kernel detect QEMU's support for the SPARC crypto opcodes
(https://lore.kernel.org/r/20261117112914.2300417-1-stian@itx.no/).
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/sparc/crypto/Kconfig | 4 +-
arch/sparc/crypto/aes_glue.c | 63 ------------------------
crypto/aes.c | 7 ++-
include/crypto/aes.h | 6 ---
lib/crypto/sparc/aes.h | 92 ++++++++++++++++++++++++++++++++++--
lib/crypto/sparc/aes_asm.S | 8 ++--
6 files changed, 99 insertions(+), 81 deletions(-)
diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig
index 8db3f6eea5dc3..b7edee0da73e6 100644
--- a/arch/sparc/crypto/Kconfig
+++ b/arch/sparc/crypto/Kconfig
@@ -3,12 +3,12 @@
menu "Accelerated Cryptographic Algorithms for CPU (sparc64)"
config CRYPTO_AES_SPARC64
- tristate "Ciphers: AES, modes: ECB, CBC, CTR"
+ tristate "Ciphers: AES, modes: ECB, CBC"
depends on SPARC64
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preseving ciphers: AES with ECB, CBC, and CTR modes
+ Length-preserving ciphers: AES with ECB and CBC modes
Architecture: sparc64 using crypto instructions
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index 661561837415a..8fa231ab595c2 100644
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -42,8 +42,6 @@ struct aes_ops {
unsigned int len, u64 *iv);
void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
- void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
};
struct crypto_sparc64_aes_ctx {
@@ -60,7 +58,6 @@ static struct aes_ops aes128_ops = {
.ecb_decrypt = aes_sparc64_ecb_decrypt_128,
.cbc_encrypt = aes_sparc64_cbc_encrypt_128,
.cbc_decrypt = aes_sparc64_cbc_decrypt_128,
- .ctr_crypt = aes_sparc64_ctr_crypt_128,
};
static struct aes_ops aes192_ops = {
@@ -70,7 +67,6 @@ static struct aes_ops aes192_ops = {
.ecb_decrypt = aes_sparc64_ecb_decrypt_192,
.cbc_encrypt = aes_sparc64_cbc_encrypt_192,
.cbc_decrypt = aes_sparc64_cbc_decrypt_192,
- .ctr_crypt = aes_sparc64_ctr_crypt_192,
};
static struct aes_ops aes256_ops = {
@@ -80,7 +76,6 @@ static struct aes_ops aes256_ops = {
.ecb_decrypt = aes_sparc64_ecb_decrypt_256,
.cbc_encrypt = aes_sparc64_cbc_encrypt_256,
.cbc_decrypt = aes_sparc64_cbc_decrypt_256,
- .ctr_crypt = aes_sparc64_ctr_crypt_256,
};
static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
@@ -214,49 +209,6 @@ static int cbc_decrypt(struct skcipher_request *req)
return err;
}
-static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx,
- struct skcipher_walk *walk)
-{
- u8 *ctrblk = walk->iv;
- u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
- const u8 *src = walk->src.virt.addr;
- u8 *dst = walk->dst.virt.addr;
- unsigned int nbytes = walk->nbytes;
-
- ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
- keystream, AES_BLOCK_SIZE);
- crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes);
- crypto_inc(ctrblk, AES_BLOCK_SIZE);
-}
-
-static int ctr_crypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, true);
- if (err)
- return err;
-
- ctx->ops->load_encrypt_keys(&ctx->key[0]);
- while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
- ctx->ops->ctr_crypt(&ctx->key[0], walk.src.virt.addr,
- walk.dst.virt.addr,
- round_down(nbytes, AES_BLOCK_SIZE),
- walk.iv);
- err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
- }
- if (walk.nbytes) {
- ctr_crypt_final(ctx, &walk);
- err = skcipher_walk_done(&walk, 0);
- }
- fprs_write(0);
- return err;
-}
-
static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ecb(aes)",
@@ -285,21 +237,6 @@ static struct skcipher_alg skcipher_algs[] = {
.setkey = aes_set_key_skcipher,
.encrypt = cbc_encrypt,
.decrypt = cbc_decrypt,
- }, {
- .base.cra_name = "ctr(aes)",
- .base.cra_driver_name = "ctr-aes-sparc64",
- .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
- .base.cra_blocksize = 1,
- .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
- .base.cra_alignmask = 7,
- .base.cra_module = THIS_MODULE,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = aes_set_key_skcipher,
- .encrypt = ctr_crypt,
- .decrypt = ctr_crypt,
- .chunksize = AES_BLOCK_SIZE,
}
};
diff --git a/crypto/aes.c b/crypto/aes.c
index 9508a975d8eaf..96639c96b0497 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -672,6 +672,7 @@ static struct skcipher_alg skcipher_algs[] = {
.base.cra_name = "ctr(aes)",
.base.cra_driver_name = "ctr-aes-lib",
.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_SPARC) ||
IS_ENABLED(CONFIG_X86)) ? 500 : 110,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct aes_enckey),
@@ -1008,8 +1009,7 @@ static struct aead_alg aead_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_PPC) || \
- IS_ENABLED(CONFIG_S390) || \
- IS_ENABLED(CONFIG_SPARC))
+ IS_ENABLED(CONFIG_S390))
{
.base.cra_name = "gcm(aes)",
.base.cra_driver_name = "gcm-aes-lib",
@@ -1051,8 +1051,7 @@ static struct aead_alg aead_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_PPC) || \
- IS_ENABLED(CONFIG_S390) || \
- IS_ENABLED(CONFIG_SPARC))
+ IS_ENABLED(CONFIG_S390))
{
.base.cra_name = "ccm(aes)",
.base.cra_driver_name = "ccm-aes-lib",
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index fa5c3d363ac9b..6dd6d394235cc 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -332,12 +332,6 @@ void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
-void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
#endif
/**
diff --git a/lib/crypto/sparc/aes.h b/lib/crypto/sparc/aes.h
index 8b8ab0820bb3e..c9c21b6505d0a 100644
--- a/lib/crypto/sparc/aes.h
+++ b/lib/crypto/sparc/aes.h
@@ -33,9 +33,6 @@ EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_128);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_192);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_256);
void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, u32 *output);
@@ -156,6 +153,95 @@ static __always_inline void aes_sparc64_load_round_keys(aes_encrypt_arg key_arg,
}
}
+/*
+ * Size of the on-stack bounce buffer used when src or dst is not 8-byte aligned
+ * (required by the SPARC64 AES assembly). 8 AES blocks (128 bytes) keeps stack
+ * usage small while remaining below 192 bytes, the lowest threshold at which
+ * any SPARC64 memcpy() implementation (U3memcpy) unconditionally enters VIS/FPU
+ * code and would clobber the round keys loaded in %f8-%f62 and clear %fprs.
+ * (NG4memcpy and M7memcpy on real crypto-capable CPUs check FPRS_FEF and fall
+ * back to integer copies, but QEMU may emulate older CPUs with +crypto.)
+ */
+#define MISALIGNED_CHUNK_SIZE (8 * AES_BLOCK_SIZE)
+
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
+void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+
+static __always_inline void aes_sparc64_ctr_crypt(const struct aes_enckey *key,
+ const u64 *in, u64 *out,
+ size_t len, u64 ctr[2])
+{
+ const u64 *rk = key->k.sparc_rndkeys;
+
+ if (key->len == AES_KEYSIZE_128)
+ aes_sparc64_ctr_crypt_128(rk, in, out, len, ctr);
+ else if (key->len == AES_KEYSIZE_192)
+ aes_sparc64_ctr_crypt_192(rk, in, out, len, ctr);
+ else
+ aes_sparc64_ctr_crypt_256(rk, in, out, len, ctr);
+}
+
+/* len may be any value here. */
+#define aes_ctr_arch aes_ctr_arch
+static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
+ u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
+{
+ u64 aligned_ctr[2];
+
+ if (!static_branch_likely(&have_aes_opcodes))
+ return false;
+
+ if (unlikely(len == 0))
+ return true;
+
+ memcpy(aligned_ctr, ctr, sizeof(aligned_ctr));
+ aes_sparc64_load_round_keys(key, /* enc= */ true);
+ if (IS_ALIGNED((uintptr_t)src | (uintptr_t)dst, 8)) {
+ size_t blocks_len = round_down(len, AES_BLOCK_SIZE);
+
+ if (blocks_len)
+ aes_sparc64_ctr_crypt(key, (const u64 *)src, (u64 *)dst,
+ blocks_len, aligned_ctr);
+ len -= blocks_len;
+ if (len) {
+ u64 tmp[2] = {};
+
+ memcpy(tmp, src + blocks_len, len);
+ aes_sparc64_ctr_crypt(key, tmp, tmp, AES_BLOCK_SIZE,
+ aligned_ctr);
+ memcpy(dst + blocks_len, tmp, len);
+ memzero_explicit(tmp, sizeof(tmp));
+ }
+ } else {
+ u64 tmp[MISALIGNED_CHUNK_SIZE / sizeof(u64)];
+
+ do {
+ size_t n = min(len, sizeof(tmp));
+ size_t blocks_len = round_up(n, AES_BLOCK_SIZE);
+
+ if (unlikely(n < blocks_len))
+ memset((u8 *)tmp + n, 0, blocks_len - n);
+ memcpy(tmp, src, n);
+ aes_sparc64_ctr_crypt(key, tmp, tmp, blocks_len,
+ aligned_ctr);
+ memcpy(dst, tmp, n);
+ src += n;
+ dst += n;
+ len -= n;
+ } while (len);
+ memzero_explicit(tmp, sizeof(tmp));
+ }
+ fprs_write(0);
+ memcpy(ctr, aligned_ctr, sizeof(aligned_ctr));
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CTR */
+
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
void aes_sparc64_xts_encrypt_128(const u64 *key, const u64 *input, u64 *output,
size_t len, u64 tweak[2]);
diff --git a/lib/crypto/sparc/aes_asm.S b/lib/crypto/sparc/aes_asm.S
index fd90b129972fb..c95a86462b776 100644
--- a/lib/crypto/sparc/aes_asm.S
+++ b/lib/crypto/sparc/aes_asm.S
@@ -1356,6 +1356,7 @@ ENTRY(aes_sparc64_cbc_decrypt_256)
nop
ENDPROC(aes_sparc64_cbc_decrypt_256)
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
.align 32
ENTRY(aes_sparc64_ctr_crypt_128)
/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
@@ -1363,7 +1364,7 @@ ENTRY(aes_sparc64_ctr_crypt_128)
ldx [%o4 + 0x08], %g7
subcc %o3, 0x10, %o3
ldx [%o0 + 0x00], %g1
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: xor %g1, %g3, %o5
MOVXTOD_O5_F0
@@ -1424,7 +1425,7 @@ ENTRY(aes_sparc64_ctr_crypt_192)
ldx [%o4 + 0x08], %g7
subcc %o3, 0x10, %o3
ldx [%o0 + 0x00], %g1
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: xor %g1, %g3, %o5
MOVXTOD_O5_F0
@@ -1485,7 +1486,7 @@ ENTRY(aes_sparc64_ctr_crypt_256)
ldx [%o4 + 0x08], %g7
subcc %o3, 0x10, %o3
ldx [%o0 + 0x00], %g1
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: xor %g1, %g3, %o5
MOVXTOD_O5_F0
@@ -1542,6 +1543,7 @@ ENTRY(aes_sparc64_ctr_crypt_256)
retl
stx %g7, [%o4 + 0x08]
ENDPROC(aes_sparc64_ctr_crypt_256)
+#endif /* CONFIG_CRYPTO_LIB_AES_CTR */
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
/* XTS. The tweak is kept in %l0/%l1 as little-endian words, so that
--
2.56.1
next prev parent reply other threads:[~2026-11-18 4:10 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-18 4:10 [PATCH 0/4] Migrate SPARC64 accelerated AES modes into library Eric Biggers
2026-11-18 4:10 ` [PATCH 1/4] lib/crypto: sparc/aes: Add and use round key load helper Eric Biggers
2026-11-18 4:10 ` Eric Biggers [this message]
2026-11-18 4:10 ` [PATCH 3/4] lib/crypto: sparc/aes-ecb: Migrate optimized code into library Eric Biggers
2026-11-18 4:10 ` [PATCH 4/4] lib/crypto: sparc/aes-cbc: " Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261118041033.50014-3-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=andreas@gaisler.com \
--cc=ardb@kernel.org \
--cc=davem@davemloft.net \
--cc=glaubitz@physik.fu-berlin.de \
--cc=herbert@gondor.apana.org.au \
--cc=linmag7@gmail.com \
--cc=linux-crypto@vger.kernel.org \
--cc=sparclinux@vger.kernel.org \
--cc=stian@itx.no \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox