From: Demian Shulhan <demyansh@gmail.com>
To: Catalin Marinas <catalin.marinas@arm.com>, Will Deacon <will@kernel.org>
Cc: Robin Murphy <robin.murphy@arm.com>,
David Laight <david.laight.linux@gmail.com>,
Nathan Chancellor <nathan@kernel.org>,
Jakub Kicinski <kuba@kernel.org>,
Eric Dumazet <edumazet@google.com>,
Andrew Morton <akpm@linux-foundation.org>,
David Gow <david@davidgow.net>,
Brendan Higgins <brendan.higgins@linux.dev>,
linux-arm-kernel@lists.infradead.org, netdev@vger.kernel.org,
kunit-dev@googlegroups.com, linux-kselftest@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: [PATCH v3 1/2] arm64: csum: Add fused copy and Internet checksum
Date: Fri, 20 Nov 2026 23:03:23 +0100 [thread overview]
Message-ID: <20261120220330.1067165-2-demyansh@gmail.com> (raw)
In-Reply-To: <20261120220330.1067165-1-demyansh@gmail.com>
arm64 falls back to the generic csum_partial_copy_nocheck(), which does
two passes (memcpy() + csum_partial()) and hits software GSO paths
(e.g. TCP over WireGuard) and UDP/ICMP replies.
Implement a single-pass csum_partial_copy_nocheck() in
arch/arm64/lib/csum.c using a 64-byte carry-tree primitive unrolled 4x
to 256 bytes (~21 instructions per 64B, pure leaf function with zero
spills on GCC 15 and Clang 21) and 16-byte @src alignment for
len >= 272. Each summed word is read once into GPRs before storing and
summing, so the checksum always matches @dst under concurrent @src
modification.
In-kernel KUnit benchmark on ARM Neoverse-V2 (3.0 GHz,
CONFIG_CHECKSUM_KUNIT_BENCHMARK=y, GCC 15.2 / Clang 21.1, median of 3):
case (src/dst) memcpy+csum_partial fused (GCC / Clang) speedup
---------------- ------------------- ------------------- -----------
16B (+0 / +0) 4.0 / 3.7 GB/s 6.9 / 5.9 GB/s 1.73x/1.61x
64B (+0 / +0) 9.4 / 10.5 GB/s 19.1 / 22.0 GB/s 2.04x/2.08x
256B (+0 / +0) 16.3 / 18.4 GB/s 33.2 / 35.2 GB/s 2.04x/1.92x
512B (+0 / +0) 19.4 / 21.0 GB/s 36.5 / 39.3 GB/s 1.88x/1.87x
1368B (+0 / +0) 21.7 / 22.2 GB/s 39.5 / 41.2 GB/s 1.82x/1.85x
1448B (+0 / +0) 21.8 / 22.2 GB/s 39.2 / 41.1 GB/s 1.79x/1.86x
4096B (+0 / +0) 23.4 / 23.5 GB/s 42.5 / 43.9 GB/s 1.82x/1.87x
16384B (+0 / +0) 23.8 / 24.0 GB/s 43.3 / 45.1 GB/s 1.82x/1.87x
16B (+1 / +3) 1.5 / 2.6 GB/s 6.9 / 5.9 GB/s 4.44x/2.25x
128B (+1 / +3) 6.4 / 10.5 GB/s 24.3 / 27.2 GB/s 3.78x/2.60x
1448B (+1 / +3) 17.8 / 18.0 GB/s 38.0 / 39.0 GB/s 2.14x/2.16x
1500B (+1 / +3) 17.9 / 18.2 GB/s 36.5 / 39.6 GB/s 2.04x/2.18x
4096B (+1 / +3) 19.1 / 19.5 GB/s 40.6 / 42.0 GB/s 2.12x/2.15x
16384B (+1 / +3) 19.4 / 19.6 GB/s 42.3 / 43.8 GB/s 2.18x/2.24x
Suggested-by: David Laight <david.laight.linux@gmail.com>
Signed-off-by: Demian Shulhan <demyansh@gmail.com>
---
arch/arm64/include/asm/checksum.h | 3 +
arch/arm64/lib/csum.c | 189 ++++++++++++++++++++++++++++++
2 files changed, 192 insertions(+)
diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h
index dc52b733675d..9f766db01152 100644
--- a/arch/arm64/include/asm/checksum.h
+++ b/arch/arm64/include/asm/checksum.h
@@ -44,6 +44,9 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
extern unsigned int do_csum(const unsigned char *buff, int len);
#define do_csum do_csum
+#define _HAVE_ARCH_CSUM_AND_COPY
+__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
+
#include <asm-generic/checksum.h>
#endif /* __ASM_CHECKSUM_H */
diff --git a/arch/arm64/lib/csum.c b/arch/arm64/lib/csum.c
index 75db718e5b6a..2bf2d63cfd42 100644
--- a/arch/arm64/lib/csum.c
+++ b/arch/arm64/lib/csum.c
@@ -1,9 +1,12 @@
// SPDX-License-Identifier: GPL-2.0-only
// Copyright (C) 2019-2020 Arm Ltd.
+#include <linux/bitops.h>
#include <linux/compiler.h>
+#include <linux/export.h>
#include <linux/kasan-checks.h>
#include <linux/kernel.h>
+#include <linux/unaligned.h>
#include <net/checksum.h>
@@ -116,6 +119,192 @@ unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
return sum >> 16;
}
+static __always_inline __uint128_t get_unaligned128(const u8 *p)
+{
+ u64 lo = get_unaligned((const u64 *)p);
+ u64 hi = get_unaligned((const u64 *)(p + 8));
+
+ return ((__uint128_t)hi << 64) | lo;
+}
+
+static __always_inline void put_unaligned128(__uint128_t val, u8 *p)
+{
+ put_unaligned((u64)val, (u64 *)p);
+ put_unaligned((u64)(val >> 64), (u64 *)(p + 8));
+}
+
+static __always_inline __wsum csum_fold_acc_cy(__uint128_t acc, u64 cy)
+{
+ __uint128_t t;
+ u64 s;
+
+ acc += (acc >> 64) | (acc << 64);
+ t = (__uint128_t)(u64)(acc >> 64) + cy;
+ s = (u64)t + (u64)(t >> 64);
+ s += (s >> 32) | (s << 32);
+ return (__force __wsum)(u32)(s >> 32);
+}
+
+static __always_inline void csum_tree64_ovf(const u8 *s, u8 *d,
+ __uint128_t *acc, s64 *cy)
+{
+ __uint128_t v0 = get_unaligned128(s);
+ __uint128_t v1 = get_unaligned128(s + 16);
+ __uint128_t v2, v3;
+ s64 o;
+
+ put_unaligned128(v0, d);
+ put_unaligned128(v1, d + 16);
+ o = __builtin_add_overflow(v0, v1, &v0);
+
+ v2 = get_unaligned128(s + 32);
+ v3 = get_unaligned128(s + 48);
+ put_unaligned128(v2, d + 32);
+ put_unaligned128(v3, d + 48);
+ o += __builtin_add_overflow(v2, v3, &v2);
+
+ o += __builtin_add_overflow(v0, v2, &v0);
+ *cy += o + __builtin_add_overflow(*acc, v0, acc);
+}
+
+static __always_inline void csum_align_src16(const u8 **ps, u8 **pd,
+ unsigned int *pulen,
+ __uint128_t *pacc, s64 *pc0)
+{
+ uintptr_t off = (uintptr_t)(*ps) & 15;
+
+ if (unlikely(off) && *pulen >= 272U) {
+ unsigned int adv = 16U - (unsigned int)off;
+ u64 lo = get_unaligned((const u64 *)(*ps));
+ u64 hi = get_unaligned((const u64 *)(*ps + 8));
+ u64 odd = (u64)off & 1ULL;
+ u64 m = ~0ULL >> ((off & 7) * 8);
+ u64 h;
+
+ put_unaligned(lo, (u64 *)(*pd));
+ put_unaligned(hi, (u64 *)(*pd + 8));
+ lo = (off & 8) ? (lo & m) : lo;
+ hi = (off & 8) ? 0ULL : (hi & m);
+ h = lo + hi;
+ h += h < lo;
+ *pacc = ror64(h, (unsigned int)(odd * 56ULL));
+ /*
+ * At len <= INT_MAX, the 256B loop adds <= 16 carries per
+ * iteration (< 2^27 total before bit 62 is cleared, and c0
+ * stays < 2^33 overall), so bit 62 of @c0 cannot overflow.
+ */
+ *pc0 = (s64)(odd << 62);
+ *ps += adv;
+ *pd += adv;
+ *pulen -= adv;
+ }
+}
+
+static __always_inline __wsum csum_fold_final(__uint128_t acc, u64 cy, u64 odd)
+{
+ __wsum res = csum_fold_acc_cy(acc, cy);
+
+ if (unlikely(odd))
+ return (__force __wsum)ror32((__force u32)res, 8);
+ return res;
+}
+
+/*
+ * Single-pass copy and ones' complement sum. Each summed word is read once
+ * into a GPR before storing and summing, so the returned sum always matches
+ * @dst even if @src is concurrently modified.
+ */
+__wsum __no_sanitize_address csum_partial_copy_nocheck(const void *src, void *dst, int len)
+{
+ const u8 *s = src;
+ u8 *d = dst;
+ __uint128_t acc = 0;
+ s64 c0 = 0;
+ u64 odd = 0;
+ unsigned int ulen = (unsigned int)len;
+
+ if (unlikely(len <= 0))
+ return 0;
+
+ kasan_check_read(src, len);
+ kasan_check_write(dst, len);
+
+ if (ulen >= 64U) {
+ if (ulen >= 256U) {
+ csum_align_src16(&s, &d, &ulen, &acc, &c0);
+ ulen -= 256U;
+ do {
+ csum_tree64_ovf(s, d, &acc, &c0);
+ csum_tree64_ovf(s + 64, d + 64, &acc, &c0);
+ csum_tree64_ovf(s + 128, d + 128, &acc, &c0);
+ csum_tree64_ovf(s + 192, d + 192, &acc, &c0);
+ s += 256;
+ d += 256;
+ } while (!__builtin_sub_overflow(ulen, 256U, &ulen));
+ odd = (u64)c0 >> 62;
+ c0 &= (1LL << 62) - 1;
+ ulen &= 255U;
+ if (!ulen)
+ goto fold;
+ }
+ if (ulen >= 64U) {
+ ulen -= 64U;
+ do {
+ csum_tree64_ovf(s, d, &acc, &c0);
+ s += 64;
+ d += 64;
+ } while (!__builtin_sub_overflow(ulen, 64U, &ulen));
+ ulen &= 63U;
+ if (!ulen)
+ goto fold;
+ }
+ }
+ if (likely(ulen >= 16U)) {
+ ulen -= 16U;
+ do {
+ __uint128_t v = get_unaligned128(s);
+
+ put_unaligned128(v, d);
+ s += 16;
+ d += 16;
+ c0 += __builtin_add_overflow(acc, v, &acc);
+ } while (!__builtin_sub_overflow(ulen, 16U, &ulen));
+ }
+ if (likely(ulen & 8U)) {
+ u64 v = get_unaligned((const u64 *)s);
+
+ put_unaligned(v, (u64 *)d);
+ s += 8;
+ d += 8;
+ c0 += __builtin_add_overflow(acc, v, &acc);
+ }
+ if (likely(ulen & 4U)) {
+ u32 v = get_unaligned((const u32 *)s);
+
+ put_unaligned(v, (u32 *)d);
+ s += 4;
+ d += 4;
+ c0 += v;
+ }
+ if (likely(ulen & 2U)) {
+ u16 v = get_unaligned((const u16 *)s);
+
+ put_unaligned(v, (u16 *)d);
+ s += 2;
+ d += 2;
+ c0 += v;
+ }
+ if (ulen & 1U) {
+ u8 v = *s;
+
+ *d = v;
+ c0 += v;
+ }
+fold:
+ return csum_fold_final(acc, (u64)c0, odd);
+}
+EXPORT_SYMBOL(csum_partial_copy_nocheck);
+
__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
const struct in6_addr *daddr,
__u32 len, __u8 proto, __wsum csum)
--
2.53.0
next prev parent reply other threads:[~2026-11-20 22:03 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-20 22:03 [PATCH v3 0/2] arm64: csum: Add fused copy and Internet checksum Demian Shulhan
2026-11-20 22:03 ` Demian Shulhan [this message]
2026-11-20 22:03 ` [PATCH v3 2/2] lib/tests: checksum: Add KUnit tests for csum_partial_copy_nocheck() Demian Shulhan
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120220330.1067165-2-demyansh@gmail.com \
--to=demyansh@gmail.com \
--cc=akpm@linux-foundation.org \
--cc=brendan.higgins@linux.dev \
--cc=catalin.marinas@arm.com \
--cc=david.laight.linux@gmail.com \
--cc=david@davidgow.net \
--cc=edumazet@google.com \
--cc=kuba@kernel.org \
--cc=kunit-dev@googlegroups.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=nathan@kernel.org \
--cc=netdev@vger.kernel.org \
--cc=robin.murphy@arm.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.