* [PATCH 1/4] lib/crypto: sparc/aes: Add and use round key load helper
2026-11-18 4:10 [PATCH 0/4] Migrate SPARC64 accelerated AES modes into library Eric Biggers
@ 2026-11-18 4:10 ` Eric Biggers
2026-11-18 4:10 ` [PATCH 2/4] lib/crypto: sparc/aes-ctr: Migrate optimized code into library Eric Biggers
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Eric Biggers @ 2026-11-18 4:10 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, sparclinux,
Andreas Larsson, David S . Miller, Stian Halseth, Magnus Lindholm,
John Paul Adrian Glaubitz, Eric Biggers
In preparation for migrating more SPARC-optimized AES modes into libaes,
add a helper function aes_sparc64_load_round_keys() that loads either
the encryption or decryption round keys.
Make aes_xts_crypt_sparc64() call it. (The AES-192 support is not used
yet, but the other modes will use it.)
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
lib/crypto/sparc/aes.h | 44 +++++++++++++++++++++++++++++-------------
1 file changed, 31 insertions(+), 13 deletions(-)
diff --git a/lib/crypto/sparc/aes.h b/lib/crypto/sparc/aes.h
index e1a060cf49539..8b8ab0820bb3e 100644
--- a/lib/crypto/sparc/aes.h
+++ b/lib/crypto/sparc/aes.h
@@ -133,6 +133,29 @@ static void aes_decrypt_arch(const struct aes_key *key,
}
}
+static __always_inline void aes_sparc64_load_round_keys(aes_encrypt_arg key_arg,
+ bool enc)
+{
+ const struct aes_enckey *key = key_arg.enc_key;
+ const u64 *rk = key->k.sparc_rndkeys;
+
+ if (enc) {
+ if (key->len == AES_KEYSIZE_128)
+ aes_sparc64_load_encrypt_keys_128(rk);
+ else if (key->len == AES_KEYSIZE_192)
+ aes_sparc64_load_encrypt_keys_192(rk);
+ else
+ aes_sparc64_load_encrypt_keys_256(rk);
+ } else {
+ if (key->len == AES_KEYSIZE_128)
+ aes_sparc64_load_decrypt_keys_128(rk);
+ else if (key->len == AES_KEYSIZE_192)
+ aes_sparc64_load_decrypt_keys_192(rk);
+ else
+ aes_sparc64_load_decrypt_keys_256(rk);
+ }
+}
+
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
void aes_sparc64_xts_encrypt_128(const u64 *key, const u64 *input, u64 *output,
size_t len, u64 tweak[2]);
@@ -167,22 +190,17 @@ aes_xts_crypt_sparc64(u8 *dst, const u8 *src, size_t len,
else
aes_encrypt_arch(&key->tweak_key, (u8 *)t, tweak);
- if (k->len == AES_KEYSIZE_128) {
- if (enc) {
- aes_sparc64_load_encrypt_keys_128(rk);
+ aes_sparc64_load_round_keys(k, enc);
+ if (enc) {
+ if (k->len == AES_KEYSIZE_128)
aes_sparc64_xts_encrypt_128(rk, in, out, len, t);
- } else {
- aes_sparc64_load_decrypt_keys_128(rk);
- aes_sparc64_xts_decrypt_128(rk_end, in, out, len, t);
- }
- } else {
- if (enc) {
- aes_sparc64_load_encrypt_keys_256(rk);
+ else
aes_sparc64_xts_encrypt_256(rk, in, out, len, t);
- } else {
- aes_sparc64_load_decrypt_keys_256(rk);
+ } else {
+ if (k->len == AES_KEYSIZE_128)
+ aes_sparc64_xts_decrypt_128(rk_end, in, out, len, t);
+ else
aes_sparc64_xts_decrypt_256(rk_end, in, out, len, t);
- }
}
fprs_write(0);
--
2.56.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 2/4] lib/crypto: sparc/aes-ctr: Migrate optimized code into library
2026-11-18 4:10 [PATCH 0/4] Migrate SPARC64 accelerated AES modes into library Eric Biggers
2026-11-18 4:10 ` [PATCH 1/4] lib/crypto: sparc/aes: Add and use round key load helper Eric Biggers
@ 2026-11-18 4:10 ` Eric Biggers
2026-11-18 4:10 ` [PATCH 3/4] lib/crypto: sparc/aes-ecb: " Eric Biggers
2026-11-18 4:10 ` [PATCH 4/4] lib/crypto: sparc/aes-cbc: " Eric Biggers
3 siblings, 0 replies; 5+ messages in thread
From: Eric Biggers @ 2026-11-18 4:10 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, sparclinux,
Andreas Larsson, David S . Miller, Stian Halseth, Magnus Lindholm,
John Paul Adrian Glaubitz, Eric Biggers
Instead of exposing the sparc64-optimized AES-CTR code via a
sparc64-specific crypto_skcipher algorithm, implement the AES-CTR
library function. This is simpler, makes the AES-CTR library function
sparc64-optimized, and fixes the longstanding issue where the
sparc64-optimized AES-CTR code was disabled by default. AES-CTR support
remains available through crypto_skcipher via crypto/aes.c.
The library functions take size_t lengths, so update the CTR assembly
functions to match.
Handle the CTR assembly code's data alignment requirement by explicitly
using an on-stack bounce buffer when the data is misaligned. (Note that
bounce buffering was used in the old code too; it was just hidden in the
skcipher_walk_* functions via the configured cra_alignmask=7.) Also use
a bounce buffer to handle any partial block at the end of the message.
Now that ctr-aes-sparc64 is removed, also increase the priority of
ctr-aes-lib on SPARC, and re-enable ccm-aes-lib, gcm-aes-lib, and
rfc4106-gcm-aes-lib.
Tested in QEMU (commit 97c88c61d45d) with aes_ctr_kunit
(https://lore.kernel.org/r/20261116062849.1071038-6-ebiggers@kernel.org/)
and CONFIG_CRYPTO_SELFTESTS_FULL=y, using the patch that makes the SPARC
kernel detect QEMU's support for the SPARC crypto opcodes
(https://lore.kernel.org/r/20261117112914.2300417-1-stian@itx.no/).
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/sparc/crypto/Kconfig | 4 +-
arch/sparc/crypto/aes_glue.c | 63 ------------------------
crypto/aes.c | 7 ++-
include/crypto/aes.h | 6 ---
lib/crypto/sparc/aes.h | 92 ++++++++++++++++++++++++++++++++++--
lib/crypto/sparc/aes_asm.S | 8 ++--
6 files changed, 99 insertions(+), 81 deletions(-)
diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig
index 8db3f6eea5dc3..b7edee0da73e6 100644
--- a/arch/sparc/crypto/Kconfig
+++ b/arch/sparc/crypto/Kconfig
@@ -3,12 +3,12 @@
menu "Accelerated Cryptographic Algorithms for CPU (sparc64)"
config CRYPTO_AES_SPARC64
- tristate "Ciphers: AES, modes: ECB, CBC, CTR"
+ tristate "Ciphers: AES, modes: ECB, CBC"
depends on SPARC64
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preseving ciphers: AES with ECB, CBC, and CTR modes
+ Length-preserving ciphers: AES with ECB and CBC modes
Architecture: sparc64 using crypto instructions
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index 661561837415a..8fa231ab595c2 100644
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -42,8 +42,6 @@ struct aes_ops {
unsigned int len, u64 *iv);
void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
- void (*ctr_crypt)(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
};
struct crypto_sparc64_aes_ctx {
@@ -60,7 +58,6 @@ static struct aes_ops aes128_ops = {
.ecb_decrypt = aes_sparc64_ecb_decrypt_128,
.cbc_encrypt = aes_sparc64_cbc_encrypt_128,
.cbc_decrypt = aes_sparc64_cbc_decrypt_128,
- .ctr_crypt = aes_sparc64_ctr_crypt_128,
};
static struct aes_ops aes192_ops = {
@@ -70,7 +67,6 @@ static struct aes_ops aes192_ops = {
.ecb_decrypt = aes_sparc64_ecb_decrypt_192,
.cbc_encrypt = aes_sparc64_cbc_encrypt_192,
.cbc_decrypt = aes_sparc64_cbc_decrypt_192,
- .ctr_crypt = aes_sparc64_ctr_crypt_192,
};
static struct aes_ops aes256_ops = {
@@ -80,7 +76,6 @@ static struct aes_ops aes256_ops = {
.ecb_decrypt = aes_sparc64_ecb_decrypt_256,
.cbc_encrypt = aes_sparc64_cbc_encrypt_256,
.cbc_decrypt = aes_sparc64_cbc_decrypt_256,
- .ctr_crypt = aes_sparc64_ctr_crypt_256,
};
static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
@@ -214,49 +209,6 @@ static int cbc_decrypt(struct skcipher_request *req)
return err;
}
-static void ctr_crypt_final(const struct crypto_sparc64_aes_ctx *ctx,
- struct skcipher_walk *walk)
-{
- u8 *ctrblk = walk->iv;
- u64 keystream[AES_BLOCK_SIZE / sizeof(u64)];
- const u8 *src = walk->src.virt.addr;
- u8 *dst = walk->dst.virt.addr;
- unsigned int nbytes = walk->nbytes;
-
- ctx->ops->ecb_encrypt(&ctx->key[0], (const u64 *)ctrblk,
- keystream, AES_BLOCK_SIZE);
- crypto_xor_cpy(dst, (u8 *) keystream, src, nbytes);
- crypto_inc(ctrblk, AES_BLOCK_SIZE);
-}
-
-static int ctr_crypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, true);
- if (err)
- return err;
-
- ctx->ops->load_encrypt_keys(&ctx->key[0]);
- while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) {
- ctx->ops->ctr_crypt(&ctx->key[0], walk.src.virt.addr,
- walk.dst.virt.addr,
- round_down(nbytes, AES_BLOCK_SIZE),
- walk.iv);
- err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
- }
- if (walk.nbytes) {
- ctr_crypt_final(ctx, &walk);
- err = skcipher_walk_done(&walk, 0);
- }
- fprs_write(0);
- return err;
-}
-
static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ecb(aes)",
@@ -285,21 +237,6 @@ static struct skcipher_alg skcipher_algs[] = {
.setkey = aes_set_key_skcipher,
.encrypt = cbc_encrypt,
.decrypt = cbc_decrypt,
- }, {
- .base.cra_name = "ctr(aes)",
- .base.cra_driver_name = "ctr-aes-sparc64",
- .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
- .base.cra_blocksize = 1,
- .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
- .base.cra_alignmask = 7,
- .base.cra_module = THIS_MODULE,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = aes_set_key_skcipher,
- .encrypt = ctr_crypt,
- .decrypt = ctr_crypt,
- .chunksize = AES_BLOCK_SIZE,
}
};
diff --git a/crypto/aes.c b/crypto/aes.c
index 9508a975d8eaf..96639c96b0497 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -672,6 +672,7 @@ static struct skcipher_alg skcipher_algs[] = {
.base.cra_name = "ctr(aes)",
.base.cra_driver_name = "ctr-aes-lib",
.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_SPARC) ||
IS_ENABLED(CONFIG_X86)) ? 500 : 110,
.base.cra_blocksize = 1,
.base.cra_ctxsize = sizeof(struct aes_enckey),
@@ -1008,8 +1009,7 @@ static struct aead_alg aead_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_PPC) || \
- IS_ENABLED(CONFIG_S390) || \
- IS_ENABLED(CONFIG_SPARC))
+ IS_ENABLED(CONFIG_S390))
{
.base.cra_name = "gcm(aes)",
.base.cra_driver_name = "gcm-aes-lib",
@@ -1051,8 +1051,7 @@ static struct aead_alg aead_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_PPC) || \
- IS_ENABLED(CONFIG_S390) || \
- IS_ENABLED(CONFIG_SPARC))
+ IS_ENABLED(CONFIG_S390))
{
.base.cra_name = "ccm(aes)",
.base.cra_driver_name = "ccm-aes-lib",
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index fa5c3d363ac9b..6dd6d394235cc 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -332,12 +332,6 @@ void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
-void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
#endif
/**
diff --git a/lib/crypto/sparc/aes.h b/lib/crypto/sparc/aes.h
index 8b8ab0820bb3e..c9c21b6505d0a 100644
--- a/lib/crypto/sparc/aes.h
+++ b/lib/crypto/sparc/aes.h
@@ -33,9 +33,6 @@ EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_128);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_192);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_ctr_crypt_256);
void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, u32 *output);
@@ -156,6 +153,95 @@ static __always_inline void aes_sparc64_load_round_keys(aes_encrypt_arg key_arg,
}
}
+/*
+ * Size of the on-stack bounce buffer used when src or dst is not 8-byte aligned
+ * (required by the SPARC64 AES assembly). 8 AES blocks (128 bytes) keeps stack
+ * usage small while remaining below 192 bytes, the lowest threshold at which
+ * any SPARC64 memcpy() implementation (U3memcpy) unconditionally enters VIS/FPU
+ * code and would clobber the round keys loaded in %f8-%f62 and clear %fprs.
+ * (NG4memcpy and M7memcpy on real crypto-capable CPUs check FPRS_FEF and fall
+ * back to integer copies, but QEMU may emulate older CPUs with +crypto.)
+ */
+#define MISALIGNED_CHUNK_SIZE (8 * AES_BLOCK_SIZE)
+
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
+void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+void aes_sparc64_ctr_crypt_192(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+void aes_sparc64_ctr_crypt_256(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+
+static __always_inline void aes_sparc64_ctr_crypt(const struct aes_enckey *key,
+ const u64 *in, u64 *out,
+ size_t len, u64 ctr[2])
+{
+ const u64 *rk = key->k.sparc_rndkeys;
+
+ if (key->len == AES_KEYSIZE_128)
+ aes_sparc64_ctr_crypt_128(rk, in, out, len, ctr);
+ else if (key->len == AES_KEYSIZE_192)
+ aes_sparc64_ctr_crypt_192(rk, in, out, len, ctr);
+ else
+ aes_sparc64_ctr_crypt_256(rk, in, out, len, ctr);
+}
+
+/* len may be any value here. */
+#define aes_ctr_arch aes_ctr_arch
+static bool aes_ctr_arch(u8 *dst, const u8 *src, size_t len,
+ u8 ctr[AES_BLOCK_SIZE], const struct aes_enckey *key)
+{
+ u64 aligned_ctr[2];
+
+ if (!static_branch_likely(&have_aes_opcodes))
+ return false;
+
+ if (unlikely(len == 0))
+ return true;
+
+ memcpy(aligned_ctr, ctr, sizeof(aligned_ctr));
+ aes_sparc64_load_round_keys(key, /* enc= */ true);
+ if (IS_ALIGNED((uintptr_t)src | (uintptr_t)dst, 8)) {
+ size_t blocks_len = round_down(len, AES_BLOCK_SIZE);
+
+ if (blocks_len)
+ aes_sparc64_ctr_crypt(key, (const u64 *)src, (u64 *)dst,
+ blocks_len, aligned_ctr);
+ len -= blocks_len;
+ if (len) {
+ u64 tmp[2] = {};
+
+ memcpy(tmp, src + blocks_len, len);
+ aes_sparc64_ctr_crypt(key, tmp, tmp, AES_BLOCK_SIZE,
+ aligned_ctr);
+ memcpy(dst + blocks_len, tmp, len);
+ memzero_explicit(tmp, sizeof(tmp));
+ }
+ } else {
+ u64 tmp[MISALIGNED_CHUNK_SIZE / sizeof(u64)];
+
+ do {
+ size_t n = min(len, sizeof(tmp));
+ size_t blocks_len = round_up(n, AES_BLOCK_SIZE);
+
+ if (unlikely(n < blocks_len))
+ memset((u8 *)tmp + n, 0, blocks_len - n);
+ memcpy(tmp, src, n);
+ aes_sparc64_ctr_crypt(key, tmp, tmp, blocks_len,
+ aligned_ctr);
+ memcpy(dst, tmp, n);
+ src += n;
+ dst += n;
+ len -= n;
+ } while (len);
+ memzero_explicit(tmp, sizeof(tmp));
+ }
+ fprs_write(0);
+ memcpy(ctr, aligned_ctr, sizeof(aligned_ctr));
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CTR */
+
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
void aes_sparc64_xts_encrypt_128(const u64 *key, const u64 *input, u64 *output,
size_t len, u64 tweak[2]);
diff --git a/lib/crypto/sparc/aes_asm.S b/lib/crypto/sparc/aes_asm.S
index fd90b129972fb..c95a86462b776 100644
--- a/lib/crypto/sparc/aes_asm.S
+++ b/lib/crypto/sparc/aes_asm.S
@@ -1356,6 +1356,7 @@ ENTRY(aes_sparc64_cbc_decrypt_256)
nop
ENDPROC(aes_sparc64_cbc_decrypt_256)
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
.align 32
ENTRY(aes_sparc64_ctr_crypt_128)
/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
@@ -1363,7 +1364,7 @@ ENTRY(aes_sparc64_ctr_crypt_128)
ldx [%o4 + 0x08], %g7
subcc %o3, 0x10, %o3
ldx [%o0 + 0x00], %g1
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: xor %g1, %g3, %o5
MOVXTOD_O5_F0
@@ -1424,7 +1425,7 @@ ENTRY(aes_sparc64_ctr_crypt_192)
ldx [%o4 + 0x08], %g7
subcc %o3, 0x10, %o3
ldx [%o0 + 0x00], %g1
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: xor %g1, %g3, %o5
MOVXTOD_O5_F0
@@ -1485,7 +1486,7 @@ ENTRY(aes_sparc64_ctr_crypt_256)
ldx [%o4 + 0x08], %g7
subcc %o3, 0x10, %o3
ldx [%o0 + 0x00], %g1
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: xor %g1, %g3, %o5
MOVXTOD_O5_F0
@@ -1542,6 +1543,7 @@ ENTRY(aes_sparc64_ctr_crypt_256)
retl
stx %g7, [%o4 + 0x08]
ENDPROC(aes_sparc64_ctr_crypt_256)
+#endif /* CONFIG_CRYPTO_LIB_AES_CTR */
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_XTS)
/* XTS. The tweak is kept in %l0/%l1 as little-endian words, so that
--
2.56.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 3/4] lib/crypto: sparc/aes-ecb: Migrate optimized code into library
2026-11-18 4:10 [PATCH 0/4] Migrate SPARC64 accelerated AES modes into library Eric Biggers
2026-11-18 4:10 ` [PATCH 1/4] lib/crypto: sparc/aes: Add and use round key load helper Eric Biggers
2026-11-18 4:10 ` [PATCH 2/4] lib/crypto: sparc/aes-ctr: Migrate optimized code into library Eric Biggers
@ 2026-11-18 4:10 ` Eric Biggers
2026-11-18 4:10 ` [PATCH 4/4] lib/crypto: sparc/aes-cbc: " Eric Biggers
3 siblings, 0 replies; 5+ messages in thread
From: Eric Biggers @ 2026-11-18 4:10 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, sparclinux,
Andreas Larsson, David S . Miller, Stian Halseth, Magnus Lindholm,
John Paul Adrian Glaubitz, Eric Biggers
Instead of exposing the sparc64-optimized AES-ECB code via a
sparc64-specific crypto_skcipher algorithm, implement the AES-ECB
library functions. This is simpler, makes the AES-ECB library
functions sparc64-optimized, and fixes the longstanding issue where
the sparc64-optimized AES-ECB code was disabled by default. AES-ECB
support remains available through crypto_skcipher via crypto/aes.c.
The library functions take size_t lengths, so update the ECB assembly
functions to match.
Handle the ECB assembly code's data alignment requirement by explicitly
using an on-stack bounce buffer when the data is misaligned. (Note that
bounce buffering was used in the old code too; it was just hidden in the
skcipher_walk_* functions via the configured cra_alignmask=7.)
Now that ecb-aes-sparc64 is removed, also increase the priority of
ecb-aes-lib on SPARC.
Tested in QEMU (commit 97c88c61d45d) with aes_ecb_kunit
(https://lore.kernel.org/r/20261116062849.1071038-3-ebiggers@kernel.org/)
and CONFIG_CRYPTO_SELFTESTS_FULL=y, using the patch that makes the SPARC
kernel detect QEMU's support for the SPARC crypto opcodes
(https://lore.kernel.org/r/20261117112914.2300417-1-stian@itx.no/).
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/sparc/crypto/Kconfig | 4 +-
arch/sparc/crypto/aes_glue.c | 72 ---------------------------
crypto/aes.c | 1 +
include/crypto/aes.h | 12 -----
lib/crypto/sparc/aes.h | 94 +++++++++++++++++++++++++++++++++---
lib/crypto/sparc/aes_asm.S | 14 +++---
6 files changed, 99 insertions(+), 98 deletions(-)
diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig
index b7edee0da73e6..783b057aeebfb 100644
--- a/arch/sparc/crypto/Kconfig
+++ b/arch/sparc/crypto/Kconfig
@@ -3,12 +3,12 @@
menu "Accelerated Cryptographic Algorithms for CPU (sparc64)"
config CRYPTO_AES_SPARC64
- tristate "Ciphers: AES, modes: ECB, CBC"
+ tristate "Ciphers: AES, modes: CBC"
depends on SPARC64
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preserving ciphers: AES with ECB and CBC modes
+ Length-preserving ciphers: AES with CBC mode
Architecture: sparc64 using crypto instructions
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index 8fa231ab595c2..cb36070ca39bf 100644
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -34,10 +34,6 @@
struct aes_ops {
void (*load_encrypt_keys)(const u64 *key);
void (*load_decrypt_keys)(const u64 *key);
- void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
- unsigned int len);
- void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
- unsigned int len);
void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
@@ -54,8 +50,6 @@ struct crypto_sparc64_aes_ctx {
static struct aes_ops aes128_ops = {
.load_encrypt_keys = aes_sparc64_load_encrypt_keys_128,
.load_decrypt_keys = aes_sparc64_load_decrypt_keys_128,
- .ecb_encrypt = aes_sparc64_ecb_encrypt_128,
- .ecb_decrypt = aes_sparc64_ecb_decrypt_128,
.cbc_encrypt = aes_sparc64_cbc_encrypt_128,
.cbc_decrypt = aes_sparc64_cbc_decrypt_128,
};
@@ -63,8 +57,6 @@ static struct aes_ops aes128_ops = {
static struct aes_ops aes192_ops = {
.load_encrypt_keys = aes_sparc64_load_encrypt_keys_192,
.load_decrypt_keys = aes_sparc64_load_decrypt_keys_192,
- .ecb_encrypt = aes_sparc64_ecb_encrypt_192,
- .ecb_decrypt = aes_sparc64_ecb_decrypt_192,
.cbc_encrypt = aes_sparc64_cbc_encrypt_192,
.cbc_decrypt = aes_sparc64_cbc_decrypt_192,
};
@@ -72,8 +64,6 @@ static struct aes_ops aes192_ops = {
static struct aes_ops aes256_ops = {
.load_encrypt_keys = aes_sparc64_load_encrypt_keys_256,
.load_decrypt_keys = aes_sparc64_load_decrypt_keys_256,
- .ecb_encrypt = aes_sparc64_ecb_encrypt_256,
- .ecb_decrypt = aes_sparc64_ecb_decrypt_256,
.cbc_encrypt = aes_sparc64_cbc_encrypt_256,
.cbc_decrypt = aes_sparc64_cbc_decrypt_256,
};
@@ -109,55 +99,6 @@ static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
return 0;
}
-static int ecb_encrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, true);
- if (err)
- return err;
-
- ctx->ops->load_encrypt_keys(&ctx->key[0]);
- while ((nbytes = walk.nbytes) != 0) {
- ctx->ops->ecb_encrypt(&ctx->key[0], walk.src.virt.addr,
- walk.dst.virt.addr,
- round_down(nbytes, AES_BLOCK_SIZE));
- err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
- }
- fprs_write(0);
- return err;
-}
-
-static int ecb_decrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- const u64 *key_end;
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, true);
- if (err)
- return err;
-
- ctx->ops->load_decrypt_keys(&ctx->key[0]);
- key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
- while ((nbytes = walk.nbytes) != 0) {
- ctx->ops->ecb_decrypt(key_end, walk.src.virt.addr,
- walk.dst.virt.addr,
- round_down(nbytes, AES_BLOCK_SIZE));
- err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
- }
- fprs_write(0);
-
- return err;
-}
-
static int cbc_encrypt(struct skcipher_request *req)
{
struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -211,19 +152,6 @@ static int cbc_decrypt(struct skcipher_request *req)
static struct skcipher_alg skcipher_algs[] = {
{
- .base.cra_name = "ecb(aes)",
- .base.cra_driver_name = "ecb-aes-sparc64",
- .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
- .base.cra_alignmask = 7,
- .base.cra_module = THIS_MODULE,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .setkey = aes_set_key_skcipher,
- .encrypt = ecb_encrypt,
- .decrypt = ecb_decrypt,
- }, {
.base.cra_name = "cbc(aes)",
.base.cra_driver_name = "cbc-aes-sparc64",
.base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
diff --git a/crypto/aes.c b/crypto/aes.c
index 96639c96b0497..2e4ea9d7a74d8 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -611,6 +611,7 @@ static struct skcipher_alg skcipher_algs[] = {
.base.cra_name = "ecb(aes)",
.base.cra_driver_name = "ecb-aes-lib",
.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_SPARC) ||
IS_ENABLED(CONFIG_X86)) ? 500 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index 6dd6d394235cc..e77e8660f347e 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -308,18 +308,6 @@ void aes_sparc64_load_encrypt_keys_256(const u64 *key);
void aes_sparc64_load_decrypt_keys_128(const u64 *key);
void aes_sparc64_load_decrypt_keys_192(const u64 *key);
void aes_sparc64_load_decrypt_keys_256(const u64 *key);
-void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, u64 *output,
- unsigned int len);
-void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, u64 *output,
- unsigned int len);
-void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, u64 *output,
- unsigned int len);
-void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, u64 *output,
- unsigned int len);
-void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, u64 *output,
- unsigned int len);
-void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, u64 *output,
- unsigned int len);
void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, u64 *output,
unsigned int len, u64 *iv);
void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, u64 *output,
diff --git a/lib/crypto/sparc/aes.h b/lib/crypto/sparc/aes.h
index c9c21b6505d0a..b13e8046467ea 100644
--- a/lib/crypto/sparc/aes.h
+++ b/lib/crypto/sparc/aes.h
@@ -21,12 +21,6 @@ EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_256);
EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_128);
EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_192);
EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_256);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_128);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_192);
EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256);
@@ -164,6 +158,94 @@ static __always_inline void aes_sparc64_load_round_keys(aes_encrypt_arg key_arg,
*/
#define MISALIGNED_CHUNK_SIZE (8 * AES_BLOCK_SIZE)
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
+void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, u64 *output,
+ size_t len);
+void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, u64 *output,
+ size_t len);
+void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, u64 *output,
+ size_t len);
+void aes_sparc64_ecb_decrypt_128(const u64 *key_end, const u64 *input,
+ u64 *output, size_t len);
+void aes_sparc64_ecb_decrypt_192(const u64 *key_end, const u64 *input,
+ u64 *output, size_t len);
+void aes_sparc64_ecb_decrypt_256(const u64 *key_end, const u64 *input,
+ u64 *output, size_t len);
+
+static __always_inline void aes_sparc64_ecb_crypt(const struct aes_enckey *key,
+ const u64 *in, u64 *out,
+ size_t len, bool enc)
+{
+ const u64 *rk = key->k.sparc_rndkeys;
+ const u64 *rk_end = &rk[2 * (key->nrounds + 1)];
+
+ if (enc) {
+ if (key->len == AES_KEYSIZE_128)
+ aes_sparc64_ecb_encrypt_128(rk, in, out, len);
+ else if (key->len == AES_KEYSIZE_192)
+ aes_sparc64_ecb_encrypt_192(rk, in, out, len);
+ else
+ aes_sparc64_ecb_encrypt_256(rk, in, out, len);
+ } else {
+ if (key->len == AES_KEYSIZE_128)
+ aes_sparc64_ecb_decrypt_128(rk_end, in, out, len);
+ else if (key->len == AES_KEYSIZE_192)
+ aes_sparc64_ecb_decrypt_192(rk_end, in, out, len);
+ else
+ aes_sparc64_ecb_decrypt_256(rk_end, in, out, len);
+ }
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+static __always_inline bool aes_ecb_crypt_sparc64(u8 *dst, const u8 *src,
+ size_t len,
+ aes_encrypt_arg key_arg,
+ bool enc)
+{
+ const struct aes_enckey *key = key_arg.enc_key;
+
+ if (!static_branch_likely(&have_aes_opcodes))
+ return false;
+
+ aes_sparc64_load_round_keys(key, enc);
+
+ if (IS_ALIGNED((uintptr_t)src | (uintptr_t)dst, 8)) {
+ aes_sparc64_ecb_crypt(key, (const u64 *)src, (u64 *)dst, len,
+ enc);
+ } else {
+ u64 tmp[MISALIGNED_CHUNK_SIZE / sizeof(u64)];
+
+ do {
+ size_t n = min(len, sizeof(tmp));
+
+ memcpy(tmp, src, n);
+ aes_sparc64_ecb_crypt(key, tmp, tmp, n, enc);
+ memcpy(dst, tmp, n);
+ src += n;
+ dst += n;
+ len -= n;
+ } while (len);
+ memzero_explicit(tmp, sizeof(tmp));
+ }
+ fprs_write(0);
+ return true;
+}
+
+#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
+static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_enckey *key)
+{
+ return aes_ecb_crypt_sparc64(dst, src, len, key, true);
+}
+
+#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
+static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_key *key)
+{
+ return aes_ecb_crypt_sparc64(dst, src, len, key, false);
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output,
size_t len, u64 iv[2]);
diff --git a/lib/crypto/sparc/aes_asm.S b/lib/crypto/sparc/aes_asm.S
index c95a86462b776..fc63f3233ca23 100644
--- a/lib/crypto/sparc/aes_asm.S
+++ b/lib/crypto/sparc/aes_asm.S
@@ -909,12 +909,13 @@ ENTRY(aes_sparc64_load_decrypt_keys_256)
ldd [%o0 + 0x00], %f62
ENDPROC(aes_sparc64_load_decrypt_keys_256)
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
.align 32
ENTRY(aes_sparc64_ecb_encrypt_128)
/* %o0=key, %o1=input, %o2=output, %o3=len */
ldx [%o0 + 0x00], %g1
subcc %o3, 0x10, %o3
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: ldx [%o1 + 0x00], %g3
ldx [%o1 + 0x08], %g7
@@ -957,7 +958,7 @@ ENTRY(aes_sparc64_ecb_encrypt_192)
/* %o0=key, %o1=input, %o2=output, %o3=len */
ldx [%o0 + 0x00], %g1
subcc %o3, 0x10, %o3
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: ldx [%o1 + 0x00], %g3
ldx [%o1 + 0x08], %g7
@@ -1000,7 +1001,7 @@ ENTRY(aes_sparc64_ecb_encrypt_256)
/* %o0=key, %o1=input, %o2=output, %o3=len */
ldx [%o0 + 0x00], %g1
subcc %o3, 0x10, %o3
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 + 0x08], %g2
1: ldx [%o1 + 0x00], %g3
ldx [%o1 + 0x08], %g7
@@ -1047,7 +1048,7 @@ ENTRY(aes_sparc64_ecb_decrypt_128)
/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
ldx [%o0 - 0x10], %g1
subcc %o3, 0x10, %o3
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 - 0x08], %g2
1: ldx [%o1 + 0x00], %g3
ldx [%o1 + 0x08], %g7
@@ -1090,7 +1091,7 @@ ENTRY(aes_sparc64_ecb_decrypt_192)
/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
ldx [%o0 - 0x10], %g1
subcc %o3, 0x10, %o3
- be 10f
+ be,pn %xcc, 10f
ldx [%o0 - 0x08], %g2
1: ldx [%o1 + 0x00], %g3
ldx [%o1 + 0x08], %g7
@@ -1134,7 +1135,7 @@ ENTRY(aes_sparc64_ecb_decrypt_256)
ldx [%o0 - 0x10], %g1
subcc %o3, 0x10, %o3
ldx [%o0 - 0x08], %g2
- be 10f
+ be,pn %xcc, 10f
sub %o0, 0xf0, %o0
1: ldx [%o1 + 0x00], %g3
ldx [%o1 + 0x08], %g7
@@ -1175,6 +1176,7 @@ ENTRY(aes_sparc64_ecb_decrypt_256)
11: retl
nop
ENDPROC(aes_sparc64_ecb_decrypt_256)
+#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
.align 32
ENTRY(aes_sparc64_cbc_encrypt_128)
--
2.56.1
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH 4/4] lib/crypto: sparc/aes-cbc: Migrate optimized code into library
2026-11-18 4:10 [PATCH 0/4] Migrate SPARC64 accelerated AES modes into library Eric Biggers
` (2 preceding siblings ...)
2026-11-18 4:10 ` [PATCH 3/4] lib/crypto: sparc/aes-ecb: " Eric Biggers
@ 2026-11-18 4:10 ` Eric Biggers
3 siblings, 0 replies; 5+ messages in thread
From: Eric Biggers @ 2026-11-18 4:10 UTC (permalink / raw)
To: linux-crypto
Cc: Ard Biesheuvel, Jason A . Donenfeld, Herbert Xu, sparclinux,
Andreas Larsson, David S . Miller, Stian Halseth, Magnus Lindholm,
John Paul Adrian Glaubitz, Eric Biggers
Instead of exposing the sparc64-optimized AES-CBC code via a
sparc64-specific crypto_skcipher algorithm, implement the AES-CBC
library functions. This is simpler, makes the AES-CBC and
AES-CBC-CTS library functions sparc64-optimized, and fixes the
longstanding issue where the sparc64-optimized AES-CBC code was
disabled by default. AES-CBC and AES-CBC-CTS support remains
available through crypto_skcipher via crypto/aes.c.
The library functions take size_t lengths, so update the prototypes of
the CBC assembly functions to match. The assembly code actually already
treated the length argument as 64-bit.
Handle the CBC assembly code's data alignment requirement by explicitly
using an on-stack bounce buffer when the data is misaligned. (Note that
bounce buffering was used in the old code too; it was just hidden in the
skcipher_walk_* functions via the configured cra_alignmask=7.)
Now that cbc-aes-sparc64 is removed, also increase the priority of
cbc-aes-lib on SPARC, and re-enable cts-cbc-aes-lib.
Tested in QEMU (commit 97c88c61d45d) with aes_cbc_kunit
(https://lore.kernel.org/r/20261116062849.1071038-4-ebiggers@kernel.org/)
and CONFIG_CRYPTO_SELFTESTS_FULL=y, using the patch that makes the SPARC
kernel detect QEMU's support for the SPARC crypto opcodes
(https://lore.kernel.org/r/20261117112914.2300417-1-stian@itx.no/).
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/sparc/crypto/Kconfig | 10 --
arch/sparc/crypto/Makefile | 2 -
arch/sparc/crypto/aes_glue.c | 209 -----------------------------------
crypto/aes.c | 5 +-
include/crypto/aes.h | 21 ----
lib/crypto/sparc/aes.h | 118 +++++++++++++++++---
lib/crypto/sparc/aes_asm.S | 2 +
7 files changed, 109 insertions(+), 258 deletions(-)
delete mode 100644 arch/sparc/crypto/aes_glue.c
diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig
index 783b057aeebfb..5cff41005241c 100644
--- a/arch/sparc/crypto/Kconfig
+++ b/arch/sparc/crypto/Kconfig
@@ -2,16 +2,6 @@
menu "Accelerated Cryptographic Algorithms for CPU (sparc64)"
-config CRYPTO_AES_SPARC64
- tristate "Ciphers: AES, modes: CBC"
- depends on SPARC64
- select CRYPTO_LIB_AES
- select CRYPTO_SKCIPHER
- help
- Length-preserving ciphers: AES with CBC mode
-
- Architecture: sparc64 using crypto instructions
-
config CRYPTO_CAMELLIA_SPARC64
tristate "Ciphers: Camellia, modes: ECB, CBC"
depends on SPARC64
diff --git a/arch/sparc/crypto/Makefile b/arch/sparc/crypto/Makefile
index ab4a7765babf5..b95c66c2f81cc 100644
--- a/arch/sparc/crypto/Makefile
+++ b/arch/sparc/crypto/Makefile
@@ -3,8 +3,6 @@
# Arch-specific CryptoAPI modules.
#
-obj-$(CONFIG_CRYPTO_AES_SPARC64) += aes-sparc64.o
obj-$(CONFIG_CRYPTO_CAMELLIA_SPARC64) += camellia-sparc64.o
-aes-sparc64-y := aes_glue.o
camellia-sparc64-y := camellia_asm.o camellia_glue.o
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
deleted file mode 100644
index cb36070ca39bf..0000000000000
--- a/arch/sparc/crypto/aes_glue.c
+++ /dev/null
@@ -1,209 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-only
-/* Glue code for AES encryption optimized for sparc64 crypto opcodes.
- *
- * This is based largely upon arch/x86/crypto/aesni-intel_glue.c
- *
- * Copyright (C) 2008, Intel Corp.
- * Author: Huang Ying <ying.huang@intel.com>
- *
- * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
- * interface for 64-bit kernels.
- * Authors: Adrian Hoban <adrian.hoban@intel.com>
- * Gabriele Paoloni <gabriele.paoloni@intel.com>
- * Tadeusz Struk (tadeusz.struk@intel.com)
- * Aidan O'Mahony (aidan.o.mahony@intel.com)
- * Copyright (c) 2010, Intel Corporation.
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <linux/crypto.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include <linux/mm.h>
-#include <linux/types.h>
-#include <crypto/algapi.h>
-#include <crypto/aes.h>
-#include <crypto/internal/skcipher.h>
-
-#include <asm/fpumacro.h>
-#include <asm/opcodes.h>
-#include <asm/pstate.h>
-#include <asm/elf.h>
-
-struct aes_ops {
- void (*load_encrypt_keys)(const u64 *key);
- void (*load_decrypt_keys)(const u64 *key);
- void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
- void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-};
-
-struct crypto_sparc64_aes_ctx {
- struct aes_ops *ops;
- u64 key[AES_MAX_KEYLENGTH / sizeof(u64)];
- u32 key_length;
- u32 expanded_key_length;
-};
-
-static struct aes_ops aes128_ops = {
- .load_encrypt_keys = aes_sparc64_load_encrypt_keys_128,
- .load_decrypt_keys = aes_sparc64_load_decrypt_keys_128,
- .cbc_encrypt = aes_sparc64_cbc_encrypt_128,
- .cbc_decrypt = aes_sparc64_cbc_decrypt_128,
-};
-
-static struct aes_ops aes192_ops = {
- .load_encrypt_keys = aes_sparc64_load_encrypt_keys_192,
- .load_decrypt_keys = aes_sparc64_load_decrypt_keys_192,
- .cbc_encrypt = aes_sparc64_cbc_encrypt_192,
- .cbc_decrypt = aes_sparc64_cbc_decrypt_192,
-};
-
-static struct aes_ops aes256_ops = {
- .load_encrypt_keys = aes_sparc64_load_encrypt_keys_256,
- .load_decrypt_keys = aes_sparc64_load_decrypt_keys_256,
- .cbc_encrypt = aes_sparc64_cbc_encrypt_256,
- .cbc_decrypt = aes_sparc64_cbc_decrypt_256,
-};
-
-static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
- unsigned int key_len)
-{
- struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
-
- switch (key_len) {
- case AES_KEYSIZE_128:
- ctx->expanded_key_length = 0xb0;
- ctx->ops = &aes128_ops;
- break;
-
- case AES_KEYSIZE_192:
- ctx->expanded_key_length = 0xd0;
- ctx->ops = &aes192_ops;
- break;
-
- case AES_KEYSIZE_256:
- ctx->expanded_key_length = 0xf0;
- ctx->ops = &aes256_ops;
- break;
-
- default:
- return -EINVAL;
- }
-
- aes_sparc64_key_expand((const u32 *)in_key, &ctx->key[0], key_len);
- ctx->key_length = key_len;
-
- return 0;
-}
-
-static int cbc_encrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, true);
- if (err)
- return err;
-
- ctx->ops->load_encrypt_keys(&ctx->key[0]);
- while ((nbytes = walk.nbytes) != 0) {
- ctx->ops->cbc_encrypt(&ctx->key[0], walk.src.virt.addr,
- walk.dst.virt.addr,
- round_down(nbytes, AES_BLOCK_SIZE),
- walk.iv);
- err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
- }
- fprs_write(0);
- return err;
-}
-
-static int cbc_decrypt(struct skcipher_request *req)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- const u64 *key_end;
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, true);
- if (err)
- return err;
-
- ctx->ops->load_decrypt_keys(&ctx->key[0]);
- key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
- while ((nbytes = walk.nbytes) != 0) {
- ctx->ops->cbc_decrypt(key_end, walk.src.virt.addr,
- walk.dst.virt.addr,
- round_down(nbytes, AES_BLOCK_SIZE),
- walk.iv);
- err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
- }
- fprs_write(0);
-
- return err;
-}
-
-static struct skcipher_alg skcipher_algs[] = {
- {
- .base.cra_name = "cbc(aes)",
- .base.cra_driver_name = "cbc-aes-sparc64",
- .base.cra_priority = SPARC_CR_OPCODE_PRIORITY,
- .base.cra_blocksize = AES_BLOCK_SIZE,
- .base.cra_ctxsize = sizeof(struct crypto_sparc64_aes_ctx),
- .base.cra_alignmask = 7,
- .base.cra_module = THIS_MODULE,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .ivsize = AES_BLOCK_SIZE,
- .setkey = aes_set_key_skcipher,
- .encrypt = cbc_encrypt,
- .decrypt = cbc_decrypt,
- }
-};
-
-static bool __init sparc64_has_aes_opcode(void)
-{
- unsigned long cfr;
-
- if (!(sparc64_elf_hwcap & HWCAP_SPARC_CRYPTO))
- return false;
-
- __asm__ __volatile__("rd %%asr26, %0" : "=r" (cfr));
- if (!(cfr & CFR_AES))
- return false;
-
- return true;
-}
-
-static int __init aes_sparc64_mod_init(void)
-{
- if (!sparc64_has_aes_opcode()) {
- pr_info("sparc64 aes opcodes not available.\n");
- return -ENODEV;
- }
- pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
- return crypto_register_skciphers(skcipher_algs,
- ARRAY_SIZE(skcipher_algs));
-}
-
-static void __exit aes_sparc64_mod_fini(void)
-{
- crypto_unregister_skciphers(skcipher_algs, ARRAY_SIZE(skcipher_algs));
-}
-
-module_init(aes_sparc64_mod_init);
-module_exit(aes_sparc64_mod_fini);
-
-MODULE_LICENSE("GPL");
-MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated");
-
-MODULE_ALIAS_CRYPTO("aes");
-
-#include "crop_devid.c"
diff --git a/crypto/aes.c b/crypto/aes.c
index 2e4ea9d7a74d8..a2db6351feefa 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -628,6 +628,7 @@ static struct skcipher_alg skcipher_algs[] = {
.base.cra_name = "cbc(aes)",
.base.cra_driver_name = "cbc-aes-lib",
.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_SPARC) ||
IS_ENABLED(CONFIG_X86)) ? 500 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
@@ -650,12 +651,12 @@ static struct skcipher_alg skcipher_algs[] = {
!(IS_ENABLED(CONFIG_ARM) || \
IS_ENABLED(CONFIG_ARM64) || \
IS_ENABLED(CONFIG_PPC) || \
- IS_ENABLED(CONFIG_S390) || \
- IS_ENABLED(CONFIG_SPARC))
+ IS_ENABLED(CONFIG_S390))
{
.base.cra_name = "cts(cbc(aes))",
.base.cra_driver_name = "cts-cbc-aes-lib",
.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_SPARC) ||
IS_ENABLED(CONFIG_X86)) ? 500 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index e77e8660f347e..a62cf6ef96c99 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -299,27 +299,6 @@ void aes_p8_xts_encrypt(const u8 *in, u8 *out, size_t len,
void aes_p8_xts_decrypt(const u8 *in, u8 *out, size_t len,
const struct p8_aes_key *key1,
const struct p8_aes_key *key2, u8 *iv);
-#elif defined(CONFIG_SPARC64)
-void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
- unsigned int key_len);
-void aes_sparc64_load_encrypt_keys_128(const u64 *key);
-void aes_sparc64_load_encrypt_keys_192(const u64 *key);
-void aes_sparc64_load_encrypt_keys_256(const u64 *key);
-void aes_sparc64_load_decrypt_keys_128(const u64 *key);
-void aes_sparc64_load_decrypt_keys_192(const u64 *key);
-void aes_sparc64_load_decrypt_keys_256(const u64 *key);
-void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_cbc_decrypt_128(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_cbc_decrypt_192(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
-void aes_sparc64_cbc_decrypt_256(const u64 *key, const u64 *input, u64 *output,
- unsigned int len, u64 *iv);
#endif
/**
diff --git a/lib/crypto/sparc/aes.h b/lib/crypto/sparc/aes.h
index b13e8046467ea..96251c22f4a31 100644
--- a/lib/crypto/sparc/aes.h
+++ b/lib/crypto/sparc/aes.h
@@ -14,26 +14,20 @@
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_aes_opcodes);
-EXPORT_SYMBOL_GPL(aes_sparc64_key_expand);
-EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_cbc_decrypt_256);
-
+void aes_sparc64_key_expand(const u32 *in_key, u64 *output_key,
+ unsigned int key_len);
void aes_sparc64_encrypt_128(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_encrypt_192(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_encrypt_256(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_decrypt_128(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_decrypt_192(const u64 *key, const u32 *input, u32 *output);
void aes_sparc64_decrypt_256(const u64 *key, const u32 *input, u32 *output);
+void aes_sparc64_load_encrypt_keys_128(const u64 *key);
+void aes_sparc64_load_encrypt_keys_192(const u64 *key);
+void aes_sparc64_load_encrypt_keys_256(const u64 *key);
+void aes_sparc64_load_decrypt_keys_128(const u64 *key);
+void aes_sparc64_load_decrypt_keys_192(const u64 *key);
+void aes_sparc64_load_decrypt_keys_256(const u64 *key);
static void aes_preparekey_arch(union aes_enckey_arch *k,
union aes_invkey_arch *inv_k,
@@ -246,6 +240,101 @@ static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
}
#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC)
+void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+void aes_sparc64_cbc_encrypt_256(const u64 *key, const u64 *input, u64 *output,
+ size_t len, u64 iv[2]);
+void aes_sparc64_cbc_decrypt_128(const u64 *key_end, const u64 *input,
+ u64 *output, size_t len, u64 iv[2]);
+void aes_sparc64_cbc_decrypt_192(const u64 *key_end, const u64 *input,
+ u64 *output, size_t len, u64 iv[2]);
+void aes_sparc64_cbc_decrypt_256(const u64 *key_end, const u64 *input,
+ u64 *output, size_t len, u64 iv[2]);
+
+static __always_inline void aes_sparc64_cbc_crypt(const struct aes_enckey *key,
+ const u64 *in, u64 *out,
+ size_t len, u64 iv[2],
+ bool enc)
+{
+ const u64 *rk = key->k.sparc_rndkeys;
+ const u64 *rk_end = &rk[2 * (key->nrounds + 1)];
+
+ if (enc) {
+ if (key->len == AES_KEYSIZE_128)
+ aes_sparc64_cbc_encrypt_128(rk, in, out, len, iv);
+ else if (key->len == AES_KEYSIZE_192)
+ aes_sparc64_cbc_encrypt_192(rk, in, out, len, iv);
+ else
+ aes_sparc64_cbc_encrypt_256(rk, in, out, len, iv);
+ } else {
+ if (key->len == AES_KEYSIZE_128)
+ aes_sparc64_cbc_decrypt_128(rk_end, in, out, len, iv);
+ else if (key->len == AES_KEYSIZE_192)
+ aes_sparc64_cbc_decrypt_192(rk_end, in, out, len, iv);
+ else
+ aes_sparc64_cbc_decrypt_256(rk_end, in, out, len, iv);
+ }
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+static __always_inline bool
+aes_cbc_crypt_sparc64(u8 *dst, const u8 *src, size_t len, u8 iv[AES_BLOCK_SIZE],
+ aes_encrypt_arg key_arg, bool enc)
+{
+ const struct aes_enckey *key = key_arg.enc_key;
+ u64 aligned_iv[2];
+
+ if (!static_branch_likely(&have_aes_opcodes))
+ return false;
+
+ memcpy(aligned_iv, iv, sizeof(aligned_iv));
+
+ aes_sparc64_load_round_keys(key, enc);
+
+ if (IS_ALIGNED((uintptr_t)src | (uintptr_t)dst, 8)) {
+ aes_sparc64_cbc_crypt(key, (const u64 *)src, (u64 *)dst, len,
+ aligned_iv, enc);
+ } else {
+ u64 tmp[MISALIGNED_CHUNK_SIZE / sizeof(u64)];
+
+ do {
+ size_t n = min(len, sizeof(tmp));
+
+ memcpy(tmp, src, n);
+ aes_sparc64_cbc_crypt(key, tmp, tmp, n, aligned_iv,
+ enc);
+ memcpy(dst, tmp, n);
+ src += n;
+ dst += n;
+ len -= n;
+ } while (len);
+ memzero_explicit(tmp, sizeof(tmp));
+ }
+ fprs_write(0);
+ memcpy(iv, aligned_iv, sizeof(aligned_iv));
+ return true;
+}
+
+#define aes_cbc_encrypt_arch aes_cbc_encrypt_arch
+static bool aes_cbc_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_enckey *key)
+{
+ return aes_cbc_crypt_sparc64(dst, src, len, iv, key, true);
+}
+
+#define aes_cbc_decrypt_arch aes_cbc_decrypt_arch
+static bool aes_cbc_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ u8 iv[AES_BLOCK_SIZE],
+ const struct aes_key *key)
+{
+ return aes_cbc_crypt_sparc64(dst, src, len, iv, key, false);
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
+
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output,
size_t len, u64 iv[2]);
@@ -407,4 +496,5 @@ static void aes_mod_init_arch(void)
return;
static_branch_enable(&have_aes_opcodes);
+ pr_info("Using sparc64 aes opcodes optimized AES implementation\n");
}
diff --git a/lib/crypto/sparc/aes_asm.S b/lib/crypto/sparc/aes_asm.S
index fc63f3233ca23..35c3c29769bad 100644
--- a/lib/crypto/sparc/aes_asm.S
+++ b/lib/crypto/sparc/aes_asm.S
@@ -1178,6 +1178,7 @@ ENTRY(aes_sparc64_ecb_decrypt_256)
ENDPROC(aes_sparc64_ecb_decrypt_256)
#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CBC)
.align 32
ENTRY(aes_sparc64_cbc_encrypt_128)
/* %o0=key, %o1=input, %o2=output, %o3=len, %o4=IV */
@@ -1357,6 +1358,7 @@ ENTRY(aes_sparc64_cbc_decrypt_256)
retl
nop
ENDPROC(aes_sparc64_cbc_decrypt_256)
+#endif /* CONFIG_CRYPTO_LIB_AES_CBC */
#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
.align 32
--
2.56.1
^ permalink raw reply related [flat|nested] 5+ messages in thread