From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
x86@kernel.org, linux-riscv@lists.infradead.org,
Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH v2 17/20] lib/crypto: riscv/aes-ecb: Migrate optimized code into library
Date: Sun, 27 Sep 2026 15:43:08 -0700 [thread overview]
Message-ID: <20260927224418.109759-18-ebiggers@kernel.org> (raw)
In-Reply-To: <20260927224418.109759-1-ebiggers@kernel.org>
Instead of exposing the riscv-optimized AES-ECB code via a
riscv-specific crypto_skcipher algorithm, just implement the AES-ECB
library functions. This is simpler, it makes the AES-ECB library
functions be riscv-optimized, and it also fixes the longstanding issue
where the riscv-optimized AES-ECB code was disabled by default. AES-ECB
support still remains available through crypto_skcipher via
crypto/aes.c, but individual architectures no longer need to handle it.
To match what the library expects, update the assembly functions to
operate on struct aes_enckey or struct aes_key rather than struct
crypto_aes_ctx, and adjust the argument order.
Bump up the priority of the corresponding library-based algorithm on
riscv now that it no longer has to be lower than arch/riscv/crypto/.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/riscv/crypto/Kconfig | 4 +-
arch/riscv/crypto/aes-riscv64-glue.c | 61 +-------------------------
arch/riscv/crypto/aes-riscv64-zvkned.S | 39 ----------------
crypto/aes.c | 3 +-
lib/crypto/riscv/aes-riscv64-zvkned.S | 51 +++++++++++++++++++++
lib/crypto/riscv/aes.h | 33 ++++++++++++++
6 files changed, 89 insertions(+), 102 deletions(-)
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 6905232ddb03..84c41824b433 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -3,13 +3,13 @@
menu "Accelerated Cryptographic Algorithms for CPU (riscv)"
config CRYPTO_AES_RISCV64
- tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTS"
+ tristate "Ciphers: AES, modes: CBC, CTS, CTR, XTS"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XTS
+ Length-preserving ciphers: AES with CBC, CTS, CTR, XTS
Architecture: riscv64 using:
- Zvkned vector crypto extension
diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c
index bbd920c9e29d..f7c492dcfd57 100644
--- a/arch/riscv/crypto/aes-riscv64-glue.c
+++ b/arch/riscv/crypto/aes-riscv64-glue.c
@@ -22,11 +22,6 @@
#include <linux/minmax.h>
#include <linux/module.h>
-asmlinkage void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len);
-asmlinkage void aes_ecb_decrypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len);
-
asmlinkage void aes_cbc_encrypt_zvkned(const struct crypto_aes_ctx *key,
const u8 *in, u8 *out, size_t len,
u8 iv[AES_BLOCK_SIZE]);
@@ -86,44 +81,6 @@ static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
return riscv64_aes_setkey(ctx, key, keylen);
}
-/* AES-ECB */
-
-static inline int riscv64_aes_ecb_crypt(struct skcipher_request *req, bool enc)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes) != 0) {
- kernel_vector_begin();
- if (enc)
- aes_ecb_encrypt_zvkned(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & ~(AES_BLOCK_SIZE - 1));
- else
- aes_ecb_decrypt_zvkned(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & ~(AES_BLOCK_SIZE - 1));
- kernel_vector_end();
- err = skcipher_walk_done(&walk, nbytes & (AES_BLOCK_SIZE - 1));
- }
-
- return err;
-}
-
-static int riscv64_aes_ecb_encrypt(struct skcipher_request *req)
-{
- return riscv64_aes_ecb_crypt(req, true);
-}
-
-static int riscv64_aes_ecb_decrypt(struct skcipher_request *req)
-{
- return riscv64_aes_ecb_crypt(req, false);
-}
-
/* AES-CBC */
static int riscv64_aes_cbc_crypt(struct skcipher_request *req, bool enc)
@@ -411,21 +368,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
static struct skcipher_alg riscv64_zvkned_aes_skcipher_algs[] = {
{
- .setkey = riscv64_aes_setkey_skcipher,
- .encrypt = riscv64_aes_ecb_encrypt,
- .decrypt = riscv64_aes_ecb_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .walksize = 8 * AES_BLOCK_SIZE, /* matches LMUL=8 */
- .base = {
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_priority = 300,
- .cra_name = "ecb(aes)",
- .cra_driver_name = "ecb-aes-riscv64-zvkned",
- .cra_module = THIS_MODULE,
- },
- }, {
.setkey = riscv64_aes_setkey_skcipher,
.encrypt = riscv64_aes_cbc_encrypt,
.decrypt = riscv64_aes_cbc_decrypt,
@@ -555,11 +497,10 @@ static void __exit riscv64_aes_mod_exit(void)
module_init(riscv64_aes_mod_init);
module_exit(riscv64_aes_mod_exit);
-MODULE_DESCRIPTION("AES-ECB/CBC/CTS/CTR/XTS (RISC-V accelerated)");
+MODULE_DESCRIPTION("AES-CBC/CTS/CTR/XTS (RISC-V accelerated)");
MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("ecb(aes)");
MODULE_ALIAS_CRYPTO("cbc(aes)");
MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
MODULE_ALIAS_CRYPTO("ctr(aes)");
diff --git a/arch/riscv/crypto/aes-riscv64-zvkned.S b/arch/riscv/crypto/aes-riscv64-zvkned.S
index d0fc4581a380..00f8a06596d3 100644
--- a/arch/riscv/crypto/aes-riscv64-zvkned.S
+++ b/arch/riscv/crypto/aes-riscv64-zvkned.S
@@ -56,45 +56,6 @@
#define LEN a3
#define IVP a4
-.macro __aes_ecb_crypt enc, keylen
- srli t0, LEN, 2
- // t0 is the remaining length in 32-bit words. It's a multiple of 4.
-1:
- vsetvli t1, t0, e32, m8, ta, ma
- sub t0, t0, t1 // Subtract number of words processed
- slli t1, t1, 2 // Words to bytes
- vle32.v v16, (INP)
- aes_crypt v16, \enc, \keylen
- vse32.v v16, (OUTP)
- add INP, INP, t1
- add OUTP, OUTP, t1
- bnez t0, 1b
-
- ret
-.endm
-
-.macro aes_ecb_crypt enc
- aes_begin KEYP, 128f, 192f
- __aes_ecb_crypt \enc, 256
-128:
- __aes_ecb_crypt \enc, 128
-192:
- __aes_ecb_crypt \enc, 192
-.endm
-
-// void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key,
-// const u8 *in, u8 *out, size_t len);
-//
-// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
-SYM_FUNC_START(aes_ecb_encrypt_zvkned)
- aes_ecb_crypt 1
-SYM_FUNC_END(aes_ecb_encrypt_zvkned)
-
-// Same prototype and calling convention as the encryption function
-SYM_FUNC_START(aes_ecb_decrypt_zvkned)
- aes_ecb_crypt 0
-SYM_FUNC_END(aes_ecb_decrypt_zvkned)
-
.macro aes_cbc_encrypt keylen
vle32.v v16, (IVP) // Load IV
1:
diff --git a/crypto/aes.c b/crypto/aes.c
index 756fce2157be..9b9e5b413fc4 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -610,7 +610,8 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ecb(aes)",
.base.cra_driver_name = "ecb-aes-lib",
- .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+ .base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_X86)) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,
diff --git a/lib/crypto/riscv/aes-riscv64-zvkned.S b/lib/crypto/riscv/aes-riscv64-zvkned.S
index fb35f694b5ac..3700028e62ea 100644
--- a/lib/crypto/riscv/aes-riscv64-zvkned.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned.S
@@ -81,3 +81,54 @@ SYM_FUNC_END(aes_encrypt_zvkned)
SYM_FUNC_START(aes_decrypt_zvkned)
aes_crypt_zvkned 0
SYM_FUNC_END(aes_decrypt_zvkned)
+
+#undef KEYP
+#undef OUTP
+#undef INP
+
+#define DST a0
+#define SRC a1
+#define LEN a2
+#define KEYP a3
+
+.macro __aes_ecb_crypt enc, keylen
+ srli t0, LEN, 2
+ // t0 is the remaining length in 32-bit words. It's a multiple of 4.
+1:
+ vsetvli t1, t0, e32, m8, ta, ma
+ sub t0, t0, t1 // Subtract number of words processed
+ slli t1, t1, 2 // Words to bytes
+ vle32.v v16, (SRC)
+ aes_crypt v16, \enc, \keylen
+ vse32.v v16, (DST)
+ add SRC, SRC, t1
+ add DST, DST, t1
+ bnez t0, 1b
+
+ ret
+.endm
+
+.macro aes_ecb_crypt enc
+ aes_begin KEYP, 128f, 192f
+ __aes_ecb_crypt \enc, 256
+128:
+ __aes_ecb_crypt \enc, 128
+192:
+ __aes_ecb_crypt \enc, 192
+.endm
+
+// void aes_ecb_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// const struct aes_enckey *key);
+//
+// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
+SYM_FUNC_START(aes_ecb_encrypt_zvkned)
+ aes_ecb_crypt 1
+SYM_FUNC_END(aes_ecb_encrypt_zvkned)
+
+// void aes_ecb_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// const struct aes_key *key);
+//
+// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
+SYM_FUNC_START(aes_ecb_decrypt_zvkned)
+ aes_ecb_crypt 0
+SYM_FUNC_END(aes_ecb_decrypt_zvkned)
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
index 9de9dbd1e887..5b36d0147d93 100644
--- a/lib/crypto/riscv/aes.h
+++ b/lib/crypto/riscv/aes.h
@@ -60,6 +60,39 @@ static void aes_decrypt_arch(const struct aes_key *key,
}
}
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
+void aes_ecb_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ const struct aes_enckey *key);
+void aes_ecb_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ const struct aes_key *key);
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
+static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ aes_ecb_encrypt_zvkned(dst, src, len, key);
+ kernel_vector_end();
+ return true;
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
+static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_key *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ aes_ecb_decrypt_zvkned(dst, src, len, key);
+ kernel_vector_end();
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
--
2.55.0
_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv
WARNING: multiple messages have this Message-ID (diff)
From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Herbert Xu <herbert@gondor.apana.org.au>,
x86@kernel.org, linux-riscv@lists.infradead.org,
Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH v2 17/20] lib/crypto: riscv/aes-ecb: Migrate optimized code into library
Date: Sun, 27 Sep 2026 15:43:08 -0700 [thread overview]
Message-ID: <20260927224418.109759-18-ebiggers@kernel.org> (raw)
In-Reply-To: <20260927224418.109759-1-ebiggers@kernel.org>
Instead of exposing the riscv-optimized AES-ECB code via a
riscv-specific crypto_skcipher algorithm, just implement the AES-ECB
library functions. This is simpler, it makes the AES-ECB library
functions be riscv-optimized, and it also fixes the longstanding issue
where the riscv-optimized AES-ECB code was disabled by default. AES-ECB
support still remains available through crypto_skcipher via
crypto/aes.c, but individual architectures no longer need to handle it.
To match what the library expects, update the assembly functions to
operate on struct aes_enckey or struct aes_key rather than struct
crypto_aes_ctx, and adjust the argument order.
Bump up the priority of the corresponding library-based algorithm on
riscv now that it no longer has to be lower than arch/riscv/crypto/.
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/riscv/crypto/Kconfig | 4 +-
arch/riscv/crypto/aes-riscv64-glue.c | 61 +-------------------------
arch/riscv/crypto/aes-riscv64-zvkned.S | 39 ----------------
crypto/aes.c | 3 +-
lib/crypto/riscv/aes-riscv64-zvkned.S | 51 +++++++++++++++++++++
lib/crypto/riscv/aes.h | 33 ++++++++++++++
6 files changed, 89 insertions(+), 102 deletions(-)
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 6905232ddb03..84c41824b433 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -3,13 +3,13 @@
menu "Accelerated Cryptographic Algorithms for CPU (riscv)"
config CRYPTO_AES_RISCV64
- tristate "Ciphers: AES, modes: ECB, CBC, CTS, CTR, XTS"
+ tristate "Ciphers: AES, modes: CBC, CTS, CTR, XTS"
depends on 64BIT && TOOLCHAIN_HAS_VECTOR_CRYPTO && \
RISCV_EFFICIENT_VECTOR_UNALIGNED_ACCESS
select CRYPTO_LIB_AES
select CRYPTO_SKCIPHER
help
- Length-preserving ciphers: AES with ECB, CBC, CTS, CTR, XTS
+ Length-preserving ciphers: AES with CBC, CTS, CTR, XTS
Architecture: riscv64 using:
- Zvkned vector crypto extension
diff --git a/arch/riscv/crypto/aes-riscv64-glue.c b/arch/riscv/crypto/aes-riscv64-glue.c
index bbd920c9e29d..f7c492dcfd57 100644
--- a/arch/riscv/crypto/aes-riscv64-glue.c
+++ b/arch/riscv/crypto/aes-riscv64-glue.c
@@ -22,11 +22,6 @@
#include <linux/minmax.h>
#include <linux/module.h>
-asmlinkage void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len);
-asmlinkage void aes_ecb_decrypt_zvkned(const struct crypto_aes_ctx *key,
- const u8 *in, u8 *out, size_t len);
-
asmlinkage void aes_cbc_encrypt_zvkned(const struct crypto_aes_ctx *key,
const u8 *in, u8 *out, size_t len,
u8 iv[AES_BLOCK_SIZE]);
@@ -86,44 +81,6 @@ static int riscv64_aes_setkey_skcipher(struct crypto_skcipher *tfm,
return riscv64_aes_setkey(ctx, key, keylen);
}
-/* AES-ECB */
-
-static inline int riscv64_aes_ecb_crypt(struct skcipher_request *req, bool enc)
-{
- struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
- const struct crypto_aes_ctx *ctx = crypto_skcipher_ctx(tfm);
- struct skcipher_walk walk;
- unsigned int nbytes;
- int err;
-
- err = skcipher_walk_virt(&walk, req, false);
- while ((nbytes = walk.nbytes) != 0) {
- kernel_vector_begin();
- if (enc)
- aes_ecb_encrypt_zvkned(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & ~(AES_BLOCK_SIZE - 1));
- else
- aes_ecb_decrypt_zvkned(ctx, walk.src.virt.addr,
- walk.dst.virt.addr,
- nbytes & ~(AES_BLOCK_SIZE - 1));
- kernel_vector_end();
- err = skcipher_walk_done(&walk, nbytes & (AES_BLOCK_SIZE - 1));
- }
-
- return err;
-}
-
-static int riscv64_aes_ecb_encrypt(struct skcipher_request *req)
-{
- return riscv64_aes_ecb_crypt(req, true);
-}
-
-static int riscv64_aes_ecb_decrypt(struct skcipher_request *req)
-{
- return riscv64_aes_ecb_crypt(req, false);
-}
-
/* AES-CBC */
static int riscv64_aes_cbc_crypt(struct skcipher_request *req, bool enc)
@@ -411,21 +368,6 @@ static int riscv64_aes_xts_decrypt(struct skcipher_request *req)
static struct skcipher_alg riscv64_zvkned_aes_skcipher_algs[] = {
{
- .setkey = riscv64_aes_setkey_skcipher,
- .encrypt = riscv64_aes_ecb_encrypt,
- .decrypt = riscv64_aes_ecb_decrypt,
- .min_keysize = AES_MIN_KEY_SIZE,
- .max_keysize = AES_MAX_KEY_SIZE,
- .walksize = 8 * AES_BLOCK_SIZE, /* matches LMUL=8 */
- .base = {
- .cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct crypto_aes_ctx),
- .cra_priority = 300,
- .cra_name = "ecb(aes)",
- .cra_driver_name = "ecb-aes-riscv64-zvkned",
- .cra_module = THIS_MODULE,
- },
- }, {
.setkey = riscv64_aes_setkey_skcipher,
.encrypt = riscv64_aes_cbc_encrypt,
.decrypt = riscv64_aes_cbc_decrypt,
@@ -555,11 +497,10 @@ static void __exit riscv64_aes_mod_exit(void)
module_init(riscv64_aes_mod_init);
module_exit(riscv64_aes_mod_exit);
-MODULE_DESCRIPTION("AES-ECB/CBC/CTS/CTR/XTS (RISC-V accelerated)");
+MODULE_DESCRIPTION("AES-CBC/CTS/CTR/XTS (RISC-V accelerated)");
MODULE_AUTHOR("Jerry Shih <jerry.shih@sifive.com>");
MODULE_LICENSE("GPL");
MODULE_ALIAS_CRYPTO("aes");
-MODULE_ALIAS_CRYPTO("ecb(aes)");
MODULE_ALIAS_CRYPTO("cbc(aes)");
MODULE_ALIAS_CRYPTO("cts(cbc(aes))");
MODULE_ALIAS_CRYPTO("ctr(aes)");
diff --git a/arch/riscv/crypto/aes-riscv64-zvkned.S b/arch/riscv/crypto/aes-riscv64-zvkned.S
index d0fc4581a380..00f8a06596d3 100644
--- a/arch/riscv/crypto/aes-riscv64-zvkned.S
+++ b/arch/riscv/crypto/aes-riscv64-zvkned.S
@@ -56,45 +56,6 @@
#define LEN a3
#define IVP a4
-.macro __aes_ecb_crypt enc, keylen
- srli t0, LEN, 2
- // t0 is the remaining length in 32-bit words. It's a multiple of 4.
-1:
- vsetvli t1, t0, e32, m8, ta, ma
- sub t0, t0, t1 // Subtract number of words processed
- slli t1, t1, 2 // Words to bytes
- vle32.v v16, (INP)
- aes_crypt v16, \enc, \keylen
- vse32.v v16, (OUTP)
- add INP, INP, t1
- add OUTP, OUTP, t1
- bnez t0, 1b
-
- ret
-.endm
-
-.macro aes_ecb_crypt enc
- aes_begin KEYP, 128f, 192f
- __aes_ecb_crypt \enc, 256
-128:
- __aes_ecb_crypt \enc, 128
-192:
- __aes_ecb_crypt \enc, 192
-.endm
-
-// void aes_ecb_encrypt_zvkned(const struct crypto_aes_ctx *key,
-// const u8 *in, u8 *out, size_t len);
-//
-// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
-SYM_FUNC_START(aes_ecb_encrypt_zvkned)
- aes_ecb_crypt 1
-SYM_FUNC_END(aes_ecb_encrypt_zvkned)
-
-// Same prototype and calling convention as the encryption function
-SYM_FUNC_START(aes_ecb_decrypt_zvkned)
- aes_ecb_crypt 0
-SYM_FUNC_END(aes_ecb_decrypt_zvkned)
-
.macro aes_cbc_encrypt keylen
vle32.v v16, (IVP) // Load IV
1:
diff --git a/crypto/aes.c b/crypto/aes.c
index 756fce2157be..9b9e5b413fc4 100644
--- a/crypto/aes.c
+++ b/crypto/aes.c
@@ -610,7 +610,8 @@ static struct skcipher_alg skcipher_algs[] = {
{
.base.cra_name = "ecb(aes)",
.base.cra_driver_name = "ecb-aes-lib",
- .base.cra_priority = IS_ENABLED(CONFIG_X86) ? 300 : 110,
+ .base.cra_priority = (IS_ENABLED(CONFIG_RISCV) ||
+ IS_ENABLED(CONFIG_X86)) ? 300 : 110,
.base.cra_blocksize = AES_BLOCK_SIZE,
.base.cra_ctxsize = sizeof(struct aes_key),
.base.cra_module = THIS_MODULE,
diff --git a/lib/crypto/riscv/aes-riscv64-zvkned.S b/lib/crypto/riscv/aes-riscv64-zvkned.S
index fb35f694b5ac..3700028e62ea 100644
--- a/lib/crypto/riscv/aes-riscv64-zvkned.S
+++ b/lib/crypto/riscv/aes-riscv64-zvkned.S
@@ -81,3 +81,54 @@ SYM_FUNC_END(aes_encrypt_zvkned)
SYM_FUNC_START(aes_decrypt_zvkned)
aes_crypt_zvkned 0
SYM_FUNC_END(aes_decrypt_zvkned)
+
+#undef KEYP
+#undef OUTP
+#undef INP
+
+#define DST a0
+#define SRC a1
+#define LEN a2
+#define KEYP a3
+
+.macro __aes_ecb_crypt enc, keylen
+ srli t0, LEN, 2
+ // t0 is the remaining length in 32-bit words. It's a multiple of 4.
+1:
+ vsetvli t1, t0, e32, m8, ta, ma
+ sub t0, t0, t1 // Subtract number of words processed
+ slli t1, t1, 2 // Words to bytes
+ vle32.v v16, (SRC)
+ aes_crypt v16, \enc, \keylen
+ vse32.v v16, (DST)
+ add SRC, SRC, t1
+ add DST, DST, t1
+ bnez t0, 1b
+
+ ret
+.endm
+
+.macro aes_ecb_crypt enc
+ aes_begin KEYP, 128f, 192f
+ __aes_ecb_crypt \enc, 256
+128:
+ __aes_ecb_crypt \enc, 128
+192:
+ __aes_ecb_crypt \enc, 192
+.endm
+
+// void aes_ecb_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// const struct aes_enckey *key);
+//
+// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
+SYM_FUNC_START(aes_ecb_encrypt_zvkned)
+ aes_ecb_crypt 1
+SYM_FUNC_END(aes_ecb_encrypt_zvkned)
+
+// void aes_ecb_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+// const struct aes_key *key);
+//
+// |len| must be nonzero and a multiple of 16 (AES_BLOCK_SIZE).
+SYM_FUNC_START(aes_ecb_decrypt_zvkned)
+ aes_ecb_crypt 0
+SYM_FUNC_END(aes_ecb_decrypt_zvkned)
diff --git a/lib/crypto/riscv/aes.h b/lib/crypto/riscv/aes.h
index 9de9dbd1e887..5b36d0147d93 100644
--- a/lib/crypto/riscv/aes.h
+++ b/lib/crypto/riscv/aes.h
@@ -60,6 +60,39 @@ static void aes_decrypt_arch(const struct aes_key *key,
}
}
+#if IS_ENABLED(CONFIG_CRYPTO_LIB_AES_ECB)
+void aes_ecb_encrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ const struct aes_enckey *key);
+void aes_ecb_decrypt_zvkned(u8 *dst, const u8 *src, size_t len,
+ const struct aes_key *key);
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_ecb_encrypt_arch aes_ecb_encrypt_arch
+static bool aes_ecb_encrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_enckey *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ aes_ecb_encrypt_zvkned(dst, src, len, key);
+ kernel_vector_end();
+ return true;
+}
+
+/* len is always a positive multiple of AES_BLOCK_SIZE here. */
+#define aes_ecb_decrypt_arch aes_ecb_decrypt_arch
+static bool aes_ecb_decrypt_arch(u8 *dst, const u8 *src, size_t len,
+ const struct aes_key *key)
+{
+ if (!static_branch_likely(&have_zvkned) || unlikely(!may_use_simd()))
+ return false;
+ kernel_vector_begin();
+ aes_ecb_decrypt_zvkned(dst, src, len, key);
+ kernel_vector_end();
+ return true;
+}
+#endif /* CONFIG_CRYPTO_LIB_AES_ECB */
+
#define aes_mod_init_arch aes_mod_init_arch
static void aes_mod_init_arch(void)
{
--
2.55.0
next prev parent reply other threads:[~2026-09-27 22:45 UTC|newest]
Thread overview: 46+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-27 22:42 [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes into library Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 01/20] crypto: aes - Fix undesired override of some optimized AES modes Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 02/20] lib/crypto: aes-xctr: Pass counter by value to aes_xctr_arch() Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 03/20] lib/crypto: x86/aes: Clean up aes-aesni.S in preparation for AES modes Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 04/20] lib/crypto: x86/aes-ecb: Add AES-NI optimization Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 05/20] lib/crypto: x86/aes-cbc: " Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 06/20] lib/crypto: x86/aes-ctr: " Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 07/20] lib/crypto: x86/aes-xts: " Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:42 ` [PATCH v2 08/20] crypto: x86/aes - Drop superseded 32-bit build support Eric Biggers
2026-09-27 22:42 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 09/20] crypto: x86/aes-ecb - Remove superseded ECB skcipher Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 10/20] crypto: x86/aes-cbc - Remove superseded CBC skciphers Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 11/20] crypto: x86/aes-ctr - Remove superseded CTR skcipher Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 12/20] crypto: x86/aes-xts - Remove superseded XTS skcipher Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 13/20] lib/crypto: x86/aes-ctr: Migrate AVX-optimized code into library Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 14/20] lib/crypto: x86/aes-xts: " Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 15/20] lib/crypto: riscv/aes: Copy aes-macros.S to library Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 16/20] lib/crypto: riscv/aes: Pass key struct to assembly code Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` Eric Biggers [this message]
2026-09-27 22:43 ` [PATCH v2 17/20] lib/crypto: riscv/aes-ecb: Migrate optimized code into library Eric Biggers
2026-09-27 22:43 ` [PATCH v2 18/20] lib/crypto: riscv/aes-cbc: " Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 19/20] lib/crypto: riscv/aes-ctr: " Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-27 22:43 ` [PATCH v2 20/20] lib/crypto: riscv/aes-xts: " Eric Biggers
2026-09-27 22:43 ` Eric Biggers
2026-09-30 13:15 ` [PATCH v2 00/20] Migrate x86 and RISC-V accelerated AES modes " Ard Biesheuvel
2026-09-30 13:15 ` Ard Biesheuvel
2026-09-30 16:56 ` Eric Biggers
2026-09-30 16:56 ` Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260927224418.109759-18-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=ardb@kernel.org \
--cc=herbert@gondor.apana.org.au \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-riscv@lists.infradead.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.