From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: linux-kernel@vger.kernel.org,
linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org,
linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org,
sparclinux@vger.kernel.org, x86@kernel.org,
Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH v3 11/16] lib/crypto: riscv/sha512: Migrate optimized SHA-512 code to library
Date: Mon, 30 Jun 2025 09:03:15 -0700 [thread overview]
Message-ID: <20250630160320.2888-12-ebiggers@kernel.org> (raw)
In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org>
Instead of exposing the riscv-optimized SHA-512 code via riscv-specific
crypto_shash algorithms, instead just implement the sha512_blocks()
library function. This is much simpler, it makes the SHA-512 (and
SHA-384) library functions be riscv-optimized, and it fixes the
longstanding issue where the riscv-optimized SHA-512 code was disabled
by default. SHA-512 still remains available through crypto_shash, but
individual architectures no longer need to handle it.
To match sha512_blocks(), change the type of the nblocks parameter of
the assembly function from int to size_t. The assembly function
actually already treated it as size_t.
Note: to see the diff from arch/riscv/crypto/sha512-riscv64-glue.c to
lib/crypto/riscv/sha512.h, view this commit with 'git show -M10'.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/riscv/crypto/Kconfig | 12 --
arch/riscv/crypto/Makefile | 3 -
arch/riscv/crypto/sha512-riscv64-glue.c | 130 ------------------
lib/crypto/Kconfig | 1 +
lib/crypto/Makefile | 2 +
.../riscv}/sha512-riscv64-zvknhb-zvkb.S | 4 +-
lib/crypto/riscv/sha512.h | 41 ++++++
7 files changed, 46 insertions(+), 147 deletions(-)
delete mode 100644 arch/riscv/crypto/sha512-riscv64-glue.c
rename {arch/riscv/crypto => lib/crypto/riscv}/sha512-riscv64-zvknhb-zvkb.S (98%)
create mode 100644 lib/crypto/riscv/sha512.h
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 53e4e1eacf554..a75d6325607b4 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -26,22 +26,10 @@ config CRYPTO_GHASH_RISCV64
GCM GHASH function (NIST SP 800-38D)
Architecture: riscv64 using:
- Zvkg vector crypto extension
-config CRYPTO_SHA512_RISCV64
- tristate "Hash functions: SHA-384 and SHA-512"
- depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
- select CRYPTO_LIB_SHA512
- select CRYPTO_SHA512
- help
- SHA-384 and SHA-512 secure hash algorithm (FIPS 180)
-
- Architecture: riscv64 using:
- - Zvknhb vector crypto extension
- - Zvkb vector crypto extension
-
config CRYPTO_SM3_RISCV64
tristate "Hash functions: SM3 (ShangMi 3)"
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
select CRYPTO_HASH
select CRYPTO_LIB_SM3
diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index e10e8257734e3..183495a95cc0e 100644
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -5,13 +5,10 @@ aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
-obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o
-sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o
-
obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o
sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o
diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sha512-riscv64-glue.c
deleted file mode 100644
index b3dbc71de07b0..0000000000000
--- a/arch/riscv/crypto/sha512-riscv64-glue.c
+++ /dev/null
@@ -1,130 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * SHA-512 and SHA-384 using the RISC-V vector crypto extensions
- *
- * Copyright (C) 2023 VRULL GmbH
- * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
- *
- * Copyright (C) 2023 SiFive, Inc.
- * Author: Jerry Shih <jerry.shih@sifive.com>
- */
-
-#include <asm/simd.h>
-#include <asm/vector.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/sha512_base.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-/*
- * Note: the asm function only uses the 'state' field of struct sha512_state.
- * It is assumed to be the first field.
- */
-asmlinkage void sha512_transform_zvknhb_zvkb(
- struct sha512_state *state, const u8 *data, int num_blocks);
-
-static void sha512_block(struct sha512_state *state, const u8 *data,
- int num_blocks)
-{
- /*
- * Ensure struct sha512_state begins directly with the SHA-512
- * 512-bit internal state, as this is what the asm function expects.
- */
- BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
-
- if (crypto_simd_usable()) {
- kernel_vector_begin();
- sha512_transform_zvknhb_zvkb(state, data, num_blocks);
- kernel_vector_end();
- } else {
- struct __sha512_ctx ctx = {};
-
- static_assert(sizeof(ctx.state) == sizeof(state->state));
- memcpy(&ctx.state, state->state, sizeof(ctx.state));
- __sha512_update(&ctx, data,
- (size_t)num_blocks * SHA512_BLOCK_SIZE);
- memcpy(state->state, &ctx.state, sizeof(state->state));
- }
-}
-
-static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- return sha512_base_do_update_blocks(desc, data, len, sha512_block);
-}
-
-static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- sha512_base_do_finup(desc, data, len, sha512_block);
- return sha512_base_finish(desc, out);
-}
-
-static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- return sha512_base_init(desc) ?:
- riscv64_sha512_finup(desc, data, len, out);
-}
-
-static struct shash_alg riscv64_sha512_algs[] = {
- {
- .init = sha512_base_init,
- .update = riscv64_sha512_update,
- .finup = riscv64_sha512_finup,
- .digest = riscv64_sha512_digest,
- .descsize = SHA512_STATE_SIZE,
- .digestsize = SHA512_DIGEST_SIZE,
- .base = {
- .cra_blocksize = SHA512_BLOCK_SIZE,
- .cra_priority = 300,
- .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .cra_name = "sha512",
- .cra_driver_name = "sha512-riscv64-zvknhb-zvkb",
- .cra_module = THIS_MODULE,
- },
- }, {
- .init = sha384_base_init,
- .update = riscv64_sha512_update,
- .finup = riscv64_sha512_finup,
- .descsize = SHA512_STATE_SIZE,
- .digestsize = SHA384_DIGEST_SIZE,
- .base = {
- .cra_blocksize = SHA384_BLOCK_SIZE,
- .cra_priority = 300,
- .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .cra_name = "sha384",
- .cra_driver_name = "sha384-riscv64-zvknhb-zvkb",
- .cra_module = THIS_MODULE,
- },
- },
-};
-
-static int __init riscv64_sha512_mod_init(void)
-{
- if (riscv_isa_extension_available(NULL, ZVKNHB) &&
- riscv_isa_extension_available(NULL, ZVKB) &&
- riscv_vector_vlen() >= 128)
- return crypto_register_shashes(riscv64_sha512_algs,
- ARRAY_SIZE(riscv64_sha512_algs));
-
- return -ENODEV;
-}
-
-static void __exit riscv64_sha512_mod_exit(void)
-{
- crypto_unregister_shashes(riscv64_sha512_algs,
- ARRAY_SIZE(riscv64_sha512_algs));
-}
-
-module_init(riscv64_sha512_mod_init);
-module_exit(riscv64_sha512_mod_exit);
-
-MODULE_DESCRIPTION("SHA-512 (RISC-V accelerated)");
-MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("sha512");
-MODULE_ALIAS_CRYPTO("sha384");
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 303ea15e3e900..656f2c08c46cd 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -178,10 +178,11 @@ config CRYPTO_LIB_SHA512_ARCH
bool
depends on CRYPTO_LIB_SHA512 && !UML
default y if ARM && !CPU_V7M
default y if ARM64
default y if MIPS && CPU_CAVIUM_OCTEON
+ default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
config CRYPTO_LIB_SM3
tristate
if !KMSAN # avoid false positives from assembly
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 22269ab06d700..bc42464a279b8 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -90,10 +90,12 @@ libsha512-y += arm64/sha512-core.o
$(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.pl
$(call cmd,perlasm_with_args)
clean-files += arm64/sha512-core.S
libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o
endif
+
+libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o
endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
obj-$(CONFIG_MPILIB) += mpi/
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
diff --git a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
similarity index 98%
rename from arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S
rename to lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
index 89f4a10d12dd6..b41eebf605462 100644
--- a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S
+++ b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
@@ -91,12 +91,12 @@
sha512_4rounds \last, W1, W2, W3, W0
sha512_4rounds \last, W2, W3, W0, W1
sha512_4rounds \last, W3, W0, W1, W2
.endm
-// void sha512_transform_zvknhb_zvkb(u64 state[8], const u8 *data,
-// int num_blocks);
+// void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state,
+// const u8 *data, size_t nblocks);
SYM_FUNC_START(sha512_transform_zvknhb_zvkb)
// Setup mask for the vmerge to replace the first word (idx==0) in
// message scheduling. There are 4 words, so an 8-bit mask suffices.
vsetivli zero, 1, e8, m1, ta, ma
diff --git a/lib/crypto/riscv/sha512.h b/lib/crypto/riscv/sha512.h
new file mode 100644
index 0000000000000..9d0abede322f7
--- /dev/null
+++ b/lib/crypto/riscv/sha512.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SHA-512 and SHA-384 using the RISC-V vector crypto extensions
+ *
+ * Copyright (C) 2023 VRULL GmbH
+ * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
+ *
+ * Copyright (C) 2023 SiFive, Inc.
+ * Author: Jerry Shih <jerry.shih@sifive.com>
+ */
+
+#include <asm/simd.h>
+#include <asm/vector.h>
+#include <crypto/internal/simd.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
+
+asmlinkage void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state,
+ const u8 *data, size_t nblocks);
+
+static void sha512_blocks(struct sha512_block_state *state,
+ const u8 *data, size_t nblocks)
+{
+ if (static_branch_likely(&have_extensions) &&
+ likely(crypto_simd_usable())) {
+ kernel_vector_begin();
+ sha512_transform_zvknhb_zvkb(state, data, nblocks);
+ kernel_vector_end();
+ } else {
+ sha512_blocks_generic(state, data, nblocks);
+ }
+}
+
+#define sha512_mod_init_arch sha512_mod_init_arch
+static inline void sha512_mod_init_arch(void)
+{
+ if (riscv_isa_extension_available(NULL, ZVKNHB) &&
+ riscv_isa_extension_available(NULL, ZVKB) &&
+ riscv_vector_vlen() >= 128)
+ static_branch_enable(&have_extensions);
+}
--
2.50.0
WARNING: multiple messages have this Message-ID (diff)
From: Eric Biggers <ebiggers@kernel.org>
To: linux-crypto@vger.kernel.org
Cc: linux-kernel@vger.kernel.org,
linux-arm-kernel@lists.infradead.org, linux-mips@vger.kernel.org,
linux-riscv@lists.infradead.org, linux-s390@vger.kernel.org,
sparclinux@vger.kernel.org, x86@kernel.org,
Ard Biesheuvel <ardb@kernel.org>,
"Jason A . Donenfeld" <Jason@zx2c4.com>,
Eric Biggers <ebiggers@kernel.org>
Subject: [PATCH v3 11/16] lib/crypto: riscv/sha512: Migrate optimized SHA-512 code to library
Date: Mon, 30 Jun 2025 09:03:15 -0700 [thread overview]
Message-ID: <20250630160320.2888-12-ebiggers@kernel.org> (raw)
In-Reply-To: <20250630160320.2888-1-ebiggers@kernel.org>
Instead of exposing the riscv-optimized SHA-512 code via riscv-specific
crypto_shash algorithms, instead just implement the sha512_blocks()
library function. This is much simpler, it makes the SHA-512 (and
SHA-384) library functions be riscv-optimized, and it fixes the
longstanding issue where the riscv-optimized SHA-512 code was disabled
by default. SHA-512 still remains available through crypto_shash, but
individual architectures no longer need to handle it.
To match sha512_blocks(), change the type of the nblocks parameter of
the assembly function from int to size_t. The assembly function
actually already treated it as size_t.
Note: to see the diff from arch/riscv/crypto/sha512-riscv64-glue.c to
lib/crypto/riscv/sha512.h, view this commit with 'git show -M10'.
Acked-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
---
arch/riscv/crypto/Kconfig | 12 --
arch/riscv/crypto/Makefile | 3 -
arch/riscv/crypto/sha512-riscv64-glue.c | 130 ------------------
lib/crypto/Kconfig | 1 +
lib/crypto/Makefile | 2 +
.../riscv}/sha512-riscv64-zvknhb-zvkb.S | 4 +-
lib/crypto/riscv/sha512.h | 41 ++++++
7 files changed, 46 insertions(+), 147 deletions(-)
delete mode 100644 arch/riscv/crypto/sha512-riscv64-glue.c
rename {arch/riscv/crypto => lib/crypto/riscv}/sha512-riscv64-zvknhb-zvkb.S (98%)
create mode 100644 lib/crypto/riscv/sha512.h
diff --git a/arch/riscv/crypto/Kconfig b/arch/riscv/crypto/Kconfig
index 53e4e1eacf554..a75d6325607b4 100644
--- a/arch/riscv/crypto/Kconfig
+++ b/arch/riscv/crypto/Kconfig
@@ -26,22 +26,10 @@ config CRYPTO_GHASH_RISCV64
GCM GHASH function (NIST SP 800-38D)
Architecture: riscv64 using:
- Zvkg vector crypto extension
-config CRYPTO_SHA512_RISCV64
- tristate "Hash functions: SHA-384 and SHA-512"
- depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
- select CRYPTO_LIB_SHA512
- select CRYPTO_SHA512
- help
- SHA-384 and SHA-512 secure hash algorithm (FIPS 180)
-
- Architecture: riscv64 using:
- - Zvknhb vector crypto extension
- - Zvkb vector crypto extension
-
config CRYPTO_SM3_RISCV64
tristate "Hash functions: SM3 (ShangMi 3)"
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
select CRYPTO_HASH
select CRYPTO_LIB_SM3
diff --git a/arch/riscv/crypto/Makefile b/arch/riscv/crypto/Makefile
index e10e8257734e3..183495a95cc0e 100644
--- a/arch/riscv/crypto/Makefile
+++ b/arch/riscv/crypto/Makefile
@@ -5,13 +5,10 @@ aes-riscv64-y := aes-riscv64-glue.o aes-riscv64-zvkned.o \
aes-riscv64-zvkned-zvbb-zvkg.o aes-riscv64-zvkned-zvkb.o
obj-$(CONFIG_CRYPTO_GHASH_RISCV64) += ghash-riscv64.o
ghash-riscv64-y := ghash-riscv64-glue.o ghash-riscv64-zvkg.o
-obj-$(CONFIG_CRYPTO_SHA512_RISCV64) += sha512-riscv64.o
-sha512-riscv64-y := sha512-riscv64-glue.o sha512-riscv64-zvknhb-zvkb.o
-
obj-$(CONFIG_CRYPTO_SM3_RISCV64) += sm3-riscv64.o
sm3-riscv64-y := sm3-riscv64-glue.o sm3-riscv64-zvksh-zvkb.o
obj-$(CONFIG_CRYPTO_SM4_RISCV64) += sm4-riscv64.o
sm4-riscv64-y := sm4-riscv64-glue.o sm4-riscv64-zvksed-zvkb.o
diff --git a/arch/riscv/crypto/sha512-riscv64-glue.c b/arch/riscv/crypto/sha512-riscv64-glue.c
deleted file mode 100644
index b3dbc71de07b0..0000000000000
--- a/arch/riscv/crypto/sha512-riscv64-glue.c
+++ /dev/null
@@ -1,130 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * SHA-512 and SHA-384 using the RISC-V vector crypto extensions
- *
- * Copyright (C) 2023 VRULL GmbH
- * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
- *
- * Copyright (C) 2023 SiFive, Inc.
- * Author: Jerry Shih <jerry.shih@sifive.com>
- */
-
-#include <asm/simd.h>
-#include <asm/vector.h>
-#include <crypto/internal/hash.h>
-#include <crypto/internal/simd.h>
-#include <crypto/sha512_base.h>
-#include <linux/kernel.h>
-#include <linux/module.h>
-
-/*
- * Note: the asm function only uses the 'state' field of struct sha512_state.
- * It is assumed to be the first field.
- */
-asmlinkage void sha512_transform_zvknhb_zvkb(
- struct sha512_state *state, const u8 *data, int num_blocks);
-
-static void sha512_block(struct sha512_state *state, const u8 *data,
- int num_blocks)
-{
- /*
- * Ensure struct sha512_state begins directly with the SHA-512
- * 512-bit internal state, as this is what the asm function expects.
- */
- BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0);
-
- if (crypto_simd_usable()) {
- kernel_vector_begin();
- sha512_transform_zvknhb_zvkb(state, data, num_blocks);
- kernel_vector_end();
- } else {
- struct __sha512_ctx ctx = {};
-
- static_assert(sizeof(ctx.state) == sizeof(state->state));
- memcpy(&ctx.state, state->state, sizeof(ctx.state));
- __sha512_update(&ctx, data,
- (size_t)num_blocks * SHA512_BLOCK_SIZE);
- memcpy(state->state, &ctx.state, sizeof(state->state));
- }
-}
-
-static int riscv64_sha512_update(struct shash_desc *desc, const u8 *data,
- unsigned int len)
-{
- return sha512_base_do_update_blocks(desc, data, len, sha512_block);
-}
-
-static int riscv64_sha512_finup(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- sha512_base_do_finup(desc, data, len, sha512_block);
- return sha512_base_finish(desc, out);
-}
-
-static int riscv64_sha512_digest(struct shash_desc *desc, const u8 *data,
- unsigned int len, u8 *out)
-{
- return sha512_base_init(desc) ?:
- riscv64_sha512_finup(desc, data, len, out);
-}
-
-static struct shash_alg riscv64_sha512_algs[] = {
- {
- .init = sha512_base_init,
- .update = riscv64_sha512_update,
- .finup = riscv64_sha512_finup,
- .digest = riscv64_sha512_digest,
- .descsize = SHA512_STATE_SIZE,
- .digestsize = SHA512_DIGEST_SIZE,
- .base = {
- .cra_blocksize = SHA512_BLOCK_SIZE,
- .cra_priority = 300,
- .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .cra_name = "sha512",
- .cra_driver_name = "sha512-riscv64-zvknhb-zvkb",
- .cra_module = THIS_MODULE,
- },
- }, {
- .init = sha384_base_init,
- .update = riscv64_sha512_update,
- .finup = riscv64_sha512_finup,
- .descsize = SHA512_STATE_SIZE,
- .digestsize = SHA384_DIGEST_SIZE,
- .base = {
- .cra_blocksize = SHA384_BLOCK_SIZE,
- .cra_priority = 300,
- .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY |
- CRYPTO_AHASH_ALG_FINUP_MAX,
- .cra_name = "sha384",
- .cra_driver_name = "sha384-riscv64-zvknhb-zvkb",
- .cra_module = THIS_MODULE,
- },
- },
-};
-
-static int __init riscv64_sha512_mod_init(void)
-{
- if (riscv_isa_extension_available(NULL, ZVKNHB) &&
- riscv_isa_extension_available(NULL, ZVKB) &&
- riscv_vector_vlen() >= 128)
- return crypto_register_shashes(riscv64_sha512_algs,
- ARRAY_SIZE(riscv64_sha512_algs));
-
- return -ENODEV;
-}
-
-static void __exit riscv64_sha512_mod_exit(void)
-{
- crypto_unregister_shashes(riscv64_sha512_algs,
- ARRAY_SIZE(riscv64_sha512_algs));
-}
-
-module_init(riscv64_sha512_mod_init);
-module_exit(riscv64_sha512_mod_exit);
-
-MODULE_DESCRIPTION("SHA-512 (RISC-V accelerated)");
-MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
-MODULE_LICENSE("GPL");
-MODULE_ALIAS_CRYPTO("sha512");
-MODULE_ALIAS_CRYPTO("sha384");
diff --git a/lib/crypto/Kconfig b/lib/crypto/Kconfig
index 303ea15e3e900..656f2c08c46cd 100644
--- a/lib/crypto/Kconfig
+++ b/lib/crypto/Kconfig
@@ -178,10 +178,11 @@ config CRYPTO_LIB_SHA512_ARCH
bool
depends on CRYPTO_LIB_SHA512 && !UML
default y if ARM && !CPU_V7M
default y if ARM64
default y if MIPS && CPU_CAVIUM_OCTEON
+ default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
config CRYPTO_LIB_SM3
tristate
if !KMSAN # avoid false positives from assembly
diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile
index 22269ab06d700..bc42464a279b8 100644
--- a/lib/crypto/Makefile
+++ b/lib/crypto/Makefile
@@ -90,10 +90,12 @@ libsha512-y += arm64/sha512-core.o
$(obj)/arm64/sha512-core.S: $(src)/../../arch/arm64/lib/crypto/sha2-armv8.pl
$(call cmd,perlasm_with_args)
clean-files += arm64/sha512-core.S
libsha512-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha512-ce-core.o
endif
+
+libsha512-$(CONFIG_RISCV) += riscv/sha512-riscv64-zvknhb-zvkb.o
endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
obj-$(CONFIG_MPILIB) += mpi/
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
diff --git a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
similarity index 98%
rename from arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S
rename to lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
index 89f4a10d12dd6..b41eebf605462 100644
--- a/arch/riscv/crypto/sha512-riscv64-zvknhb-zvkb.S
+++ b/lib/crypto/riscv/sha512-riscv64-zvknhb-zvkb.S
@@ -91,12 +91,12 @@
sha512_4rounds \last, W1, W2, W3, W0
sha512_4rounds \last, W2, W3, W0, W1
sha512_4rounds \last, W3, W0, W1, W2
.endm
-// void sha512_transform_zvknhb_zvkb(u64 state[8], const u8 *data,
-// int num_blocks);
+// void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state,
+// const u8 *data, size_t nblocks);
SYM_FUNC_START(sha512_transform_zvknhb_zvkb)
// Setup mask for the vmerge to replace the first word (idx==0) in
// message scheduling. There are 4 words, so an 8-bit mask suffices.
vsetivli zero, 1, e8, m1, ta, ma
diff --git a/lib/crypto/riscv/sha512.h b/lib/crypto/riscv/sha512.h
new file mode 100644
index 0000000000000..9d0abede322f7
--- /dev/null
+++ b/lib/crypto/riscv/sha512.h
@@ -0,0 +1,41 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * SHA-512 and SHA-384 using the RISC-V vector crypto extensions
+ *
+ * Copyright (C) 2023 VRULL GmbH
+ * Author: Heiko Stuebner <heiko.stuebner@vrull.eu>
+ *
+ * Copyright (C) 2023 SiFive, Inc.
+ * Author: Jerry Shih <jerry.shih@sifive.com>
+ */
+
+#include <asm/simd.h>
+#include <asm/vector.h>
+#include <crypto/internal/simd.h>
+
+static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
+
+asmlinkage void sha512_transform_zvknhb_zvkb(struct sha512_block_state *state,
+ const u8 *data, size_t nblocks);
+
+static void sha512_blocks(struct sha512_block_state *state,
+ const u8 *data, size_t nblocks)
+{
+ if (static_branch_likely(&have_extensions) &&
+ likely(crypto_simd_usable())) {
+ kernel_vector_begin();
+ sha512_transform_zvknhb_zvkb(state, data, nblocks);
+ kernel_vector_end();
+ } else {
+ sha512_blocks_generic(state, data, nblocks);
+ }
+}
+
+#define sha512_mod_init_arch sha512_mod_init_arch
+static inline void sha512_mod_init_arch(void)
+{
+ if (riscv_isa_extension_available(NULL, ZVKNHB) &&
+ riscv_isa_extension_available(NULL, ZVKB) &&
+ riscv_vector_vlen() >= 128)
+ static_branch_enable(&have_extensions);
+}
--
2.50.0
_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv
next prev parent reply other threads:[~2025-06-30 17:50 UTC|newest]
Thread overview: 34+ messages / expand[flat|nested] mbox.gz Atom feed top
2025-06-30 16:03 [PATCH v3 00/16] SHA-512 library functions Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 01/16] crypto: sha512 - Rename conflicting symbols Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 02/16] lib/crypto: sha512: Add support for SHA-384 and SHA-512 Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 03/16] lib/crypto: sha512: Add HMAC-SHA384 and HMAC-SHA512 support Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 04/16] crypto: riscv/sha512 - Stop depending on sha512_generic_block_fn Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 05/16] crypto: sha512 - Replace sha512_generic with wrapper around SHA-512 library Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 06/16] crypto: sha512 - Use same state format as legacy drivers Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 07/16] lib/crypto: arm/sha512: Migrate optimized SHA-512 code to library Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 08/16] lib/crypto: arm64/sha512: " Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 09/16] mips: cavium-octeon: Move octeon-crypto.h into asm directory Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 10/16] lib/crypto: mips/sha512: Migrate optimized SHA-512 code to library Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` Eric Biggers [this message]
2025-06-30 16:03 ` [PATCH v3 11/16] lib/crypto: riscv/sha512: " Eric Biggers
2025-06-30 16:03 ` [PATCH v3 12/16] lib/crypto: s390/sha512: " Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 13/16] lib/crypto: sparc/sha512: " Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 14/16] lib/crypto: x86/sha512: " Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 15/16] lib/crypto: x86/sha512: Remove unnecessary checks for nblocks==0 Eric Biggers
2025-06-30 16:03 ` Eric Biggers
2025-06-30 16:03 ` [PATCH v3 16/16] crypto: sha512 - Remove sha512_base.h Eric Biggers
2025-06-30 16:03 ` Eric Biggers
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20250630160320.2888-12-ebiggers@kernel.org \
--to=ebiggers@kernel.org \
--cc=Jason@zx2c4.com \
--cc=ardb@kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mips@vger.kernel.org \
--cc=linux-riscv@lists.infradead.org \
--cc=linux-s390@vger.kernel.org \
--cc=sparclinux@vger.kernel.org \
--cc=x86@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.