From: David Laight <david.laight.linux@gmail.com>
To: Shreesh Adiga <16567adigashreesh@gmail.com>
Cc: Eric Biggers <ebiggers@kernel.org>,
Ard Biesheuvel <ardb@kernel.org>,
Demian Shulhan <demyansh@gmail.com>,
linux-crypto@vger.kernel.org, linux-kernel@vger.kernel.org
Subject: Re: [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation
Date: Tue, 17 Nov 2026 09:00:32 +0000 [thread overview]
Message-ID: <20261117090032.0abf7198@pumpkin> (raw)
In-Reply-To: <20261114172955.1598929-1-16567adigashreesh@gmail.com>
On Sat, 14 Nov 2026 22:59:51 +0530
Shreesh Adiga <16567adigashreesh@gmail.com> wrote:
> Add a 4x-folding implementation for CPUs with FEAT_SHA3 which benefits
> newer ARMv9 CPUs like Cortex-X925.
> Usage of EOR3 from SHA3 extension improves performance by 20% compared
> to the regular PMULL without EOR3 on Cortex-X925 measure in userspace
> application via libbenchmark for 512 byte input size.
> Measured about 3x speedup for 512 byte input compared with the existing
> NEON single fold implementation on Cortex-X925 in userspace.
>
> Signed-off-by: Shreesh Adiga <16567adigashreesh@gmail.com>
> ---
> Changes in v2:
> Improved the 4x to 1x reduction by using x^255 and x^319 x2 reduction.
> Added static keys for the CPU feature checks instead of regular
> branches.
>
> lib/crc/arm64/crc64.h | 24 +++++++++++--
> lib/crc/crc64-neon.c | 83 +++++++++++++++++++++++++++++++++++++++++++
> 2 files changed, 104 insertions(+), 3 deletions(-)
>
> diff --git a/lib/crc/arm64/crc64.h b/lib/crc/arm64/crc64.h
> index c7a69e1f3d8f..e5be0ef21140 100644
> --- a/lib/crc/arm64/crc64.h
> +++ b/lib/crc/arm64/crc64.h
> @@ -9,20 +9,38 @@
> #include <linux/sizes.h>
>
> u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len);
> +
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_eor3);
> +static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_pmull);
>
> #define crc64_be_arch crc64_be_generic
>
> static inline u64 crc64_nvme_arch(u64 crc, const u8 *p, size_t len)
> {
> - if (len >= 128 && cpu_have_named_feature(PMULL) &&
> + if (len >= 128 && static_branch_likely(&have_pmull) &&
Would it be better to swap the order of the checks?
Otherwise I think you get the 'len >= 128' conditional branch
even when the static branch is false.
David
> likely(may_use_simd())) {
> size_t chunk = len & ~15;
>
> - scoped_ksimd()
> - crc = crc64_nvme_neon(crc, p, chunk);
> + scoped_ksimd() {
> + if (static_branch_likely(&have_eor3))
> + crc = crc64_nvme_neon_pmull_eor3(crc, p, chunk);
> + else
> + crc = crc64_nvme_neon(crc, p, chunk);
> + }
>
> p += chunk;
> len &= 15;
> }
> return crc64_nvme_generic(crc, p, len);
> }
> +
> +#define crc64_mod_init_arch crc64_mod_init_arch
> +static void crc64_mod_init_arch(void)
> +{
> + if (cpu_have_named_feature(PMULL)) {
> + static_branch_enable(&have_pmull);
> + if (cpu_have_named_feature(SHA3))
> + static_branch_enable(&have_eor3);
> + }
> +}
> diff --git a/lib/crc/crc64-neon.c b/lib/crc/crc64-neon.c
> index 4753fb94a4be..767e55edc0e0 100644
> --- a/lib/crc/crc64-neon.c
> +++ b/lib/crc/crc64-neon.c
> @@ -10,6 +10,17 @@
>
> u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len);
>
> +#ifdef CONFIG_ARM64
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len);
> +
> +/* x^575 mod G, x^511 mod G */
> +static const u64 fold_consts_4x_val[2] = { 0x0c32cdb31e18a84aULL,
> + 0x62242240ace5045aULL };
> +/* x^319 mod G, x^255 mod G */
> +static const u64 fold_consts_2x_val[2] = { 0xb0bc2e589204f500ULL,
> + 0xe1e0bb9d45d7a44cULL };
> +#endif
> +
> /* x^191 mod G, x^127 mod G */
> static const u64 fold_consts_val[2] = { 0xeadc41fd2ba3d420ULL,
> 0x21e9761e252621acULL };
> @@ -45,3 +56,75 @@ u64 crc64_nvme_neon(u64 crc, const u8 *p, size_t len)
>
> return vgetq_lane_u64(v0, 1);
> }
> +
> +#ifdef CONFIG_ARM64
> +#define SHA3_TARGET __attribute__((target("arch=armv8.2-a+crypto+sha3")))
> +
> +SHA3_TARGET
> +static inline uint64x2_t fold_single_round(uint64x2_t fold,
> + uint64x2_t temp,
> + uint64x2_t consts)
> +{
> + uint64x2_t temp1 = pmull64(consts, fold);
> + uint64x2_t temp2 = pmull64_high(consts, fold);
> +
> + return veor3q_u64(temp, temp1, temp2);
> +}
> +
> +SHA3_TARGET
> +u64 crc64_nvme_neon_pmull_eor3(u64 crc, const u8 *p, size_t len)
> +{
> + /* Assumes len >= 64. */
> + uint64x2_t fold_consts = vld1q_u64(fold_consts_val);
> + uint64x2_t fold_consts_4x = vld1q_u64(fold_consts_4x_val);
> + uint64x2_t fold_consts_2x = vld1q_u64(fold_consts_2x_val);
> + uint64x2_t v0 = { crc, 0 };
> + uint64x2_t zero = { };
> + uint64x2_t v1, v2, v3;
> +
> + v0 ^= vreinterpretq_u64_u8(vld1q_u8(p));
> + v1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
> + v2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
> + v3 = vreinterpretq_u64_u8(vld1q_u8(p + 48));
> +
> + p += 64;
> + len -= 64;
> + while (len >= 64) {
> + uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p + 0));
> + uint64x2_t t1 = vreinterpretq_u64_u8(vld1q_u8(p + 16));
> + uint64x2_t t2 = vreinterpretq_u64_u8(vld1q_u8(p + 32));
> + uint64x2_t t3 = vreinterpretq_u64_u8(vld1q_u8(p + 48));
> +
> + v0 = fold_single_round(v0, t0, fold_consts_4x);
> + v1 = fold_single_round(v1, t1, fold_consts_4x);
> + v2 = fold_single_round(v2, t2, fold_consts_4x);
> + v3 = fold_single_round(v3, t3, fold_consts_4x);
> + p += 64;
> + len -= 64;
> + }
> +
> + v0 = fold_single_round(v0, v2, fold_consts_2x);
> + v1 = fold_single_round(v1, v3, fold_consts_2x);
> + v0 = fold_single_round(v0, v1, fold_consts);
> +
> + while (len >= 16) {
> + uint64x2_t t0 = vreinterpretq_u64_u8(vld1q_u8(p));
> +
> + v0 = fold_single_round(v0, t0, fold_consts);
> + p += 16;
> + len -= 16;
> + }
> +
> + /* Multiply the 128-bit value by x^64 and reduce it back to 128 bits. */
> + v0 = vextq_u64(v0, zero, 1) ^ pmull64_hi_lo(fold_consts, v0);
> +
> + /* Final Barrett reduction */
> + uint64x2_t bconsts = vld1q_u64(bconsts_val);
> + uint64x2_t final = pmull64(bconsts, v0);
> +
> + v0 = veor3q_u64(v0, vextq_u64(zero, final, 1),
> + pmull64_hi_lo(bconsts, final));
> +
> + return vgetq_lane_u64(v0, 1);
> +}
> +#endif
prev parent reply other threads:[~2026-11-17 9:00 UTC|newest]
Thread overview: 8+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-14 17:29 [PATCH v2] lib/crc: arm64: Add NEON PMULL+EOR3 crc64_nvme implementation Shreesh Adiga
2026-11-17 0:01 ` Eric Biggers
2026-11-17 6:50 ` Shreesh Adiga
2026-11-17 7:11 ` Eric Biggers
2026-11-17 14:53 ` Shreesh Adiga
2026-11-17 0:45 ` Demian Shulhan
2026-11-17 7:06 ` Shreesh Adiga
2026-11-17 9:00 ` David Laight [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261117090032.0abf7198@pumpkin \
--to=david.laight.linux@gmail.com \
--cc=16567adigashreesh@gmail.com \
--cc=ardb@kernel.org \
--cc=demyansh@gmail.com \
--cc=ebiggers@kernel.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kernel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.