Linux cryptographic layer development
 help / color / mirror / Atom feed
From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>,
	"Jason A . Donenfeld" <Jason@zx2c4.com>,
	Herbert Xu <herbert@gondor.apana.org.au>,
	sparclinux@vger.kernel.org, Andreas Larsson <andreas@gaisler.com>,
	"David S . Miller" <davem@davemloft.net>,
	Stian Halseth <stian@itx.no>, Magnus Lindholm <linmag7@gmail.com>,
	John Paul Adrian Glaubitz <glaubitz@physik.fu-berlin.de>,
	Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH 3/4] lib/crypto: sparc/aes-ecb: Migrate optimized code into library
Date: Tue, 17 Nov 2026 20:10:32 -0800	[thread overview]
Message-ID: <20261118041033.50014-4-ebiggers@kernel.org> (raw)
In-Reply-To: <20261118041033.50014-1-ebiggers@kernel.org>

Instead of exposing the sparc64-optimized AES-ECB code via a
sparc64-specific crypto_skcipher algorithm, implement the AES-ECB
library functions.  This is simpler, makes the AES-ECB library
functions sparc64-optimized, and fixes the longstanding issue where
the sparc64-optimized AES-ECB code was disabled by default.  AES-ECB
support remains available through crypto_skcipher via crypto/aes.c.

The library functions take size_t lengths, so update the ECB assembly
functions to match.

Handle the ECB assembly code's data alignment requirement by explicitly
using an on-stack bounce buffer when the data is misaligned.  (Note that
bounce buffering was used in the old code too; it was just hidden in the
skcipher_walk_* functions via the configured cra_alignmask=7.)

Now that ecb-aes-sparc64 is removed, also increase the priority of
ecb-aes-lib on SPARC.

Tested in QEMU (commit 97c88c61d45d) with aes_ecb_kunit
(https://lore.kernel.org/r/20261116062849.1071038-3-ebiggers@kernel.org/)
and CONFIG_CRYPTO_SELFTESTS_FULL=y, using the patch that makes the SPARC
kernel detect QEMU's support for the SPARC crypto opcodes
(https://lore.kernel.org/r/20261117112914.2300417-1-stian@itx.no/).

Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
 arch/sparc/crypto/Kconfig    |  4 +-
 arch/sparc/crypto/aes_glue.c | 72 ---------------------------
 crypto/aes.c                 |  1 +
 include/crypto/aes.h         | 12 -----
 lib/crypto/sparc/aes.h       | 94 +++++++++++++++++++++++++++++++++---
 lib/crypto/sparc/aes_asm.S   | 14 +++---
 6 files changed, 99 insertions(+), 98 deletions(-)

diff --git a/arch/sparc/crypto/Kconfig b/arch/sparc/crypto/Kconfig
index b7edee0da73e6..783b057aeebfb 100644
--- a/arch/sparc/crypto/Kconfig
+++ b/arch/sparc/crypto/Kconfig
@@ -3,12 +3,12 @@
 menu "Accelerated Cryptographic Algorithms for CPU (sparc64)"
 
 config CRYPTO_AES_SPARC64
-	tristate "Ciphers: AES, modes: ECB, CBC"
+	tristate "Ciphers: AES, modes: CBC"
 	depends on SPARC64
 	select CRYPTO_LIB_AES
 	select CRYPTO_SKCIPHER
 	help
-	  Length-preserving ciphers: AES with ECB and CBC modes
+	  Length-preserving ciphers: AES with CBC mode
 
 	  Architecture: sparc64 using crypto instructions
 
diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c
index 8fa231ab595c2..cb36070ca39bf 100644
--- a/arch/sparc/crypto/aes_glue.c
+++ b/arch/sparc/crypto/aes_glue.c
@@ -34,10 +34,6 @@
 struct aes_ops {
 	void (*load_encrypt_keys)(const u64 *key);
 	void (*load_decrypt_keys)(const u64 *key);
-	void (*ecb_encrypt)(const u64 *key, const u64 *input, u64 *output,
-			    unsigned int len);
-	void (*ecb_decrypt)(const u64 *key, const u64 *input, u64 *output,
-			    unsigned int len);
 	void (*cbc_encrypt)(const u64 *key, const u64 *input, u64 *output,
 			    unsigned int len, u64 *iv);
 	void (*cbc_decrypt)(const u64 *key, const u64 *input, u64 *output,
@@ -54,8 +50,6 @@ struct crypto_sparc64_aes_ctx {
 static struct aes_ops aes128_ops = {
 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_128,
 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_128,
-	.ecb_encrypt		= aes_sparc64_ecb_encrypt_128,
-	.ecb_decrypt		= aes_sparc64_ecb_decrypt_128,
 	.cbc_encrypt		= aes_sparc64_cbc_encrypt_128,
 	.cbc_decrypt		= aes_sparc64_cbc_decrypt_128,
 };
@@ -63,8 +57,6 @@ static struct aes_ops aes128_ops = {
 static struct aes_ops aes192_ops = {
 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_192,
 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_192,
-	.ecb_encrypt		= aes_sparc64_ecb_encrypt_192,
-	.ecb_decrypt		= aes_sparc64_ecb_decrypt_192,
 	.cbc_encrypt		= aes_sparc64_cbc_encrypt_192,
 	.cbc_decrypt		= aes_sparc64_cbc_decrypt_192,
 };
@@ -72,8 +64,6 @@ static struct aes_ops aes192_ops = {
 static struct aes_ops aes256_ops = {
 	.load_encrypt_keys	= aes_sparc64_load_encrypt_keys_256,
 	.load_decrypt_keys	= aes_sparc64_load_decrypt_keys_256,
-	.ecb_encrypt		= aes_sparc64_ecb_encrypt_256,
-	.ecb_decrypt		= aes_sparc64_ecb_decrypt_256,
 	.cbc_encrypt		= aes_sparc64_cbc_encrypt_256,
 	.cbc_decrypt		= aes_sparc64_cbc_decrypt_256,
 };
@@ -109,55 +99,6 @@ static int aes_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key,
 	return 0;
 }
 
-static int ecb_encrypt(struct skcipher_request *req)
-{
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
-	struct skcipher_walk walk;
-	unsigned int nbytes;
-	int err;
-
-	err = skcipher_walk_virt(&walk, req, true);
-	if (err)
-		return err;
-
-	ctx->ops->load_encrypt_keys(&ctx->key[0]);
-	while ((nbytes = walk.nbytes) != 0) {
-		ctx->ops->ecb_encrypt(&ctx->key[0], walk.src.virt.addr,
-				      walk.dst.virt.addr,
-				      round_down(nbytes, AES_BLOCK_SIZE));
-		err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
-	}
-	fprs_write(0);
-	return err;
-}
-
-static int ecb_decrypt(struct skcipher_request *req)
-{
-	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
-	const struct crypto_sparc64_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
-	const u64 *key_end;
-	struct skcipher_walk walk;
-	unsigned int nbytes;
-	int err;
-
-	err = skcipher_walk_virt(&walk, req, true);
-	if (err)
-		return err;
-
-	ctx->ops->load_decrypt_keys(&ctx->key[0]);
-	key_end = &ctx->key[ctx->expanded_key_length / sizeof(u64)];
-	while ((nbytes = walk.nbytes) != 0) {
-		ctx->ops->ecb_decrypt(key_end, walk.src.virt.addr,
-				      walk.dst.virt.addr,
-				      round_down(nbytes, AES_BLOCK_SIZE));
-		err = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE);
-	}
-	fprs_write(0);
-
-	return err;
-}
-
 static int cbc_encrypt(struct skcipher_request *req)
 {
 	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
@@ -211,19 +152,6 @@ static int cbc_decrypt(struct skcipher_request *req)
 
 static struct skcipher_alg skcipher_algs[] = {
 	{
-		.base.cra_name		= "ecb(aes)",
-		.base.cra_driver_name	= "ecb-aes-sparc64",
-		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
-		.base.cra_blocksize	= AES_BLOCK_SIZE,
-		.base.cra_ctxsize	= sizeof(struct crypto_sparc64_aes_ctx),
-		.base.cra_alignmask	= 7,
-		.base.cra_module	= THIS_MODULE,
-		.min_keysize		= AES_MIN_KEY_SIZE,
-		.max_keysize		= AES_MAX_KEY_SIZE,
-		.setkey			= aes_set_key_skcipher,
-		.encrypt		= ecb_encrypt,
-		.decrypt		= ecb_decrypt,
-	}, {
 		.base.cra_name		= "cbc(aes)",
 		.base.cra_driver_name	= "cbc-aes-sparc64",
 		.base.cra_priority	= SPARC_CR_OPCODE_PRIORITY,
diff --git a/crypto/aes.c b/crypto/aes.c
index 96639c96b0497..2e4ea9d7a74d8 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -611,6 +611,7 @@ static struct skcipher_alg skcipher_algs[] = {
 		.base.cra_name = "ecb(aes)",
 		.base.cra_driver_name = "ecb-aes-lib",
 		.base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+				      IS_ENABLED(CONFIG_SPARC) ||
 				      IS_ENABLED(CONFIG_X86)) ? 500 : 110,
 		.base.cra_blocksize = AES_BLOCK_SIZE,
 		.base.cra_ctxsize = sizeof(struct aes_key),
diff --git a/include/crypto/aes.h b/include/crypto/aes.h
index 6dd6d394235cc..e77e8660f347e 100644
--- a/include/crypto/aes.h
+++ b/include/crypto/aes.h
@@ -308,18 +308,6 @@ void aes_sparc64_load_encrypt_keys_256(const u64 *key);
 void aes_sparc64_load_decrypt_keys_128(const u64 *key);
 void aes_sparc64_load_decrypt_keys_192(const u64 *key);
 void aes_sparc64_load_decrypt_keys_256(const u64 *key);
-void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, u64 *output,
-				 unsigned int len);
-void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, u64 *output,
-				 unsigned int len);
-void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, u64 *output,
-				 unsigned int len);
-void aes_sparc64_ecb_decrypt_128(const u64 *key, const u64 *input, u64 *output,
-				 unsigned int len);
-void aes_sparc64_ecb_decrypt_192(const u64 *key, const u64 *input, u64 *output,
-				 unsigned int len);
-void aes_sparc64_ecb_decrypt_256(const u64 *key, const u64 *input, u64 *output,
-				 unsigned int len);
 void aes_sparc64_cbc_encrypt_128(const u64 *key, const u64 *input, u64 *output,
 				 unsigned int len, u64 *iv);
 void aes_sparc64_cbc_encrypt_192(const u64 *key, const u64 *input, u64 *output,
diff --git a/lib/crypto/sparc/aes.h b/lib/crypto/sparc/aes.h
index c9c21b6505d0a..b13e8046467ea 100644
--- a/lib/crypto/sparc/aes.h
+++ b/lib/crypto/sparc/aes.h
@@ -21,12 +21,6 @@ EXPORT_SYMBOL_GPL(aes_sparc64_load_encrypt_keys_256);
 EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_128);
 EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_192);
 EXPORT_SYMBOL_GPL(aes_sparc64_load_decrypt_keys_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_encrypt_256);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_128);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_192);
-EXPORT_SYMBOL_GPL(aes_sparc64_ecb_decrypt_256);
 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_128);
 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_192);
 EXPORT_SYMBOL_GPL(aes_sparc64_cbc_encrypt_256);
@@ -164,6 +158,94 @@ static __always_inline void aes_sparc64_load_round_keys(aes_encrypt_arg key_arg,
  */
 #define MISALIGNED_CHUNK_SIZE (8 * AES_BLOCK_SIZE)
 
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
+void aes_sparc64_ecb_encrypt_128(const u64 *key, const u64 *input, u64 *output,
+				 size_t len);
+void aes_sparc64_ecb_encrypt_192(const u64 *key, const u64 *input, u64 *output,
+				 size_t len);
+void aes_sparc64_ecb_encrypt_256(const u64 *key, const u64 *input, u64 *output,
+				 size_t len);
+void aes_sparc64_ecb_decrypt_128(const u64 *key_end, const u64 *input,
+				 u64 *output, size_t len);
+void aes_sparc64_ecb_decrypt_192(const u64 *key_end, const u64 *input,
+				 u64 *output, size_t len);
+void aes_sparc64_ecb_decrypt_256(const u64 *key_end, const u64 *input,
+				 u64 *output, size_t len);
+
+static __always_inline void aes_sparc64_ecb_crypt(const struct aes_enckey *key,
+						  const u64 *in, u64 *out,
+						  size_t len, bool enc)
+{
+	const u64 *rk = key->k.sparc_rndkeys;
+	const u64 *rk_end = &rk[2 * (key->nrounds + 1)];
+
+	if (enc) {
+		if (key->len == AES_KEYSIZE_128)
+			aes_sparc64_ecb_encrypt_128(rk, in, out, len);
+		else if (key->len == AES_KEYSIZE_192)
+			aes_sparc64_ecb_encrypt_192(rk, in, out, len);
+		else
+			aes_sparc64_ecb_encrypt_256(rk, in, out, len);
+	} else {
+		if (key->len == AES_KEYSIZE_128)
+			aes_sparc64_ecb_decrypt_128(rk_end, in, out, len);
+		else if (key->len == AES_KEYSIZE_192)
+			aes_sparc64_ecb_decrypt_192(rk_end, in, out, len);
+		else
+			aes_sparc64_ecb_decrypt_256(rk_end, in, out, len);
+	}
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+static __always_inline bool aes_ecb_crypt_sparc64(u8 *dst, const u8 *src,
+						  size_t len,
+						  aes_encrypt_arg key_arg,
+						  bool enc)
+{
+	const struct aes_enckey *key = key_arg.enc_key;
+
+	if (!static_branch_likely(&have_aes_opcodes))
+		return false;
+
+	aes_sparc64_load_round_keys(key, enc);
+
+	if (IS_ALIGNED((uintptr_t)src | (uintptr_t)dst, 8)) {
+		aes_sparc64_ecb_crypt(key, (const u64 *)src, (u64 *)dst, len,
+				      enc);
+	} else {
+		u64 tmp[MISALIGNED_CHUNK_SIZE / sizeof(u64)];
+
+		do {
+			size_t n = min(len, sizeof(tmp));
+
+			memcpy(tmp, src, n);
+			aes_sparc64_ecb_crypt(key, tmp, tmp, n, enc);
+			memcpy(dst, tmp, n);
+			src += n;
+			dst += n;
+			len -= n;
+		} while (len);
+		memzero_explicit(tmp, sizeof(tmp));
+	}
+	fprs_write(0);
+	return true;
+}
+
+#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
+static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+				 const struct aes_enckey *key)
+{
+	return aes_ecb_crypt_sparc64(dst, src, len, key, true);
+}
+
+#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
+static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+				 const struct aes_key *key)
+{
+	return aes_ecb_crypt_sparc64(dst, src, len, key, false);
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+
 #if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_CTR)
 void aes_sparc64_ctr_crypt_128(const u64 *key, const u64 *input, u64 *output,
 			       size_t len, u64 iv[2]);
diff --git a/lib/crypto/sparc/aes_asm.S b/lib/crypto/sparc/aes_asm.S
index c95a86462b776..fc63f3233ca23 100644
--- a/lib/crypto/sparc/aes_asm.S
+++ b/lib/crypto/sparc/aes_asm.S
@@ -909,12 +909,13 @@ ENTRY(aes_sparc64_load_decrypt_keys_256)
 	 ldd		[%o0 + 0x00], %f62
 ENDPROC(aes_sparc64_load_decrypt_keys_256)
 
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
 	.align		32
 ENTRY(aes_sparc64_ecb_encrypt_128)
 	/* %o0=key, %o1=input, %o2=output, %o3=len */
 	ldx		[%o0 + 0x00], %g1
 	subcc		%o3, 0x10, %o3
-	be		10f
+	be,pn		%xcc, 10f
 	 ldx		[%o0 + 0x08], %g2
 1:	ldx		[%o1 + 0x00], %g3
 	ldx		[%o1 + 0x08], %g7
@@ -957,7 +958,7 @@ ENTRY(aes_sparc64_ecb_encrypt_192)
 	/* %o0=key, %o1=input, %o2=output, %o3=len */
 	ldx		[%o0 + 0x00], %g1
 	subcc		%o3, 0x10, %o3
-	be		10f
+	be,pn		%xcc, 10f
 	 ldx		[%o0 + 0x08], %g2
 1:	ldx		[%o1 + 0x00], %g3
 	ldx		[%o1 + 0x08], %g7
@@ -1000,7 +1001,7 @@ ENTRY(aes_sparc64_ecb_encrypt_256)
 	/* %o0=key, %o1=input, %o2=output, %o3=len */
 	ldx		[%o0 + 0x00], %g1
 	subcc		%o3, 0x10, %o3
-	be		10f
+	be,pn		%xcc, 10f
 	 ldx		[%o0 + 0x08], %g2
 1:	ldx		[%o1 + 0x00], %g3
 	ldx		[%o1 + 0x08], %g7
@@ -1047,7 +1048,7 @@ ENTRY(aes_sparc64_ecb_decrypt_128)
 	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
 	ldx		[%o0 - 0x10], %g1
 	subcc		%o3, 0x10, %o3
-	be		10f
+	be,pn		%xcc, 10f
 	 ldx		[%o0 - 0x08], %g2
 1:	ldx		[%o1 + 0x00], %g3
 	ldx		[%o1 + 0x08], %g7
@@ -1090,7 +1091,7 @@ ENTRY(aes_sparc64_ecb_decrypt_192)
 	/* %o0=&key[key_len], %o1=input, %o2=output, %o3=len */
 	ldx		[%o0 - 0x10], %g1
 	subcc		%o3, 0x10, %o3
-	be		10f
+	be,pn		%xcc, 10f
 	 ldx		[%o0 - 0x08], %g2
 1:	ldx		[%o1 + 0x00], %g3
 	ldx		[%o1 + 0x08], %g7
@@ -1134,7 +1135,7 @@ ENTRY(aes_sparc64_ecb_decrypt_256)
 	ldx		[%o0 - 0x10], %g1
 	subcc		%o3, 0x10, %o3
 	ldx		[%o0 - 0x08], %g2
-	be		10f
+	be,pn		%xcc, 10f
 	 sub		%o0, 0xf0, %o0
 1:	ldx		[%o1 + 0x00], %g3
 	ldx		[%o1 + 0x08], %g7
@@ -1175,6 +1176,7 @@ ENTRY(aes_sparc64_ecb_decrypt_256)
 11:	retl
 	 nop
 ENDPROC(aes_sparc64_ecb_decrypt_256)
+#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
 
 	.align		32
 ENTRY(aes_sparc64_cbc_encrypt_128)
-- 
2.56.1


  parent reply	other threads:[~2026-11-18  4:10 UTC|newest]

Thread overview: 5+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-18  4:10 [PATCH 0/4] Migrate SPARC64 accelerated AES modes into library Eric Biggers
2026-11-18  4:10 ` [PATCH 1/4] lib/crypto: sparc/aes: Add and use round key load helper Eric Biggers
2026-11-18  4:10 ` [PATCH 2/4] lib/crypto: sparc/aes-ctr: Migrate optimized code into library Eric Biggers
2026-11-18  4:10 ` Eric Biggers [this message]
2026-11-18  4:10 ` [PATCH 4/4] lib/crypto: sparc/aes-cbc: " Eric Biggers

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261118041033.50014-4-ebiggers@kernel.org \
    --to=ebiggers@kernel.org \
    --cc=Jason@zx2c4.com \
    --cc=andreas@gaisler.com \
    --cc=ardb@kernel.org \
    --cc=davem@davemloft.net \
    --cc=glaubitz@physik.fu-berlin.de \
    --cc=herbert@gondor.apana.org.au \
    --cc=linmag7@gmail.com \
    --cc=linux-crypto@vger.kernel.org \
    --cc=sparclinux@vger.kernel.org \
    --cc=stian@itx.no \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox