From: Demian Shulhan <demyansh@gmail.com>
To: Catalin Marinas <catalin.marinas@arm.com>, Will Deacon <will@kernel.org>
Cc: Robin Murphy <robin.murphy@arm.com>,
David Laight <david.laight.linux@gmail.com>,
Nathan Chancellor <nathan@kernel.org>,
Jakub Kicinski <kuba@kernel.org>,
Eric Dumazet <edumazet@google.com>,
Andrew Morton <akpm@linux-foundation.org>,
David Gow <david@davidgow.net>,
Brendan Higgins <brendan.higgins@linux.dev>,
linux-arm-kernel@lists.infradead.org, netdev@vger.kernel.org,
kunit-dev@googlegroups.com, linux-kselftest@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: [PATCH v3 1/2] arm64: csum: Add fused copy and Internet checksum
Date: Fri, 20 Nov 2026 23:03:23 +0100 [thread overview]
Message-ID: <20261120220330.1067165-2-demyansh@gmail.com> (raw)
In-Reply-To: <20261120220330.1067165-1-demyansh@gmail.com>
arm64 falls back to the generic csum_partial_copy_nocheck(), which does
two passes (memcpy() + csum_partial()) and hits software GSO paths
(e.g. TCP over WireGuard) and UDP/ICMP replies.
Implement a single-pass csum_partial_copy_nocheck() in
arch/arm64/lib/csum.c using a 64-byte carry-tree primitive unrolled 4x
to 256 bytes (~21 instructions per 64B, pure leaf function with zero
spills on GCC 15 and Clang 21) and 16-byte @src alignment for
len >= 272. Each summed word is read once into GPRs before storing and
summing, so the checksum always matches @dst under concurrent @src
modification.
In-kernel KUnit benchmark on ARM Neoverse-V2 (3.0 GHz,
CONFIG_CHECKSUM_KUNIT_BENCHMARK=y, GCC 15.2 / Clang 21.1, median of 3):
case (src/dst) memcpy+csum_partial fused (GCC / Clang) speedup
---------------- ------------------- ------------------- -----------
16B (+0 / +0) 4.0 / 3.7 GB/s 6.9 / 5.9 GB/s 1.73x/1.61x
64B (+0 / +0) 9.4 / 10.5 GB/s 19.1 / 22.0 GB/s 2.04x/2.08x
256B (+0 / +0) 16.3 / 18.4 GB/s 33.2 / 35.2 GB/s 2.04x/1.92x
512B (+0 / +0) 19.4 / 21.0 GB/s 36.5 / 39.3 GB/s 1.88x/1.87x
1368B (+0 / +0) 21.7 / 22.2 GB/s 39.5 / 41.2 GB/s 1.82x/1.85x
1448B (+0 / +0) 21.8 / 22.2 GB/s 39.2 / 41.1 GB/s 1.79x/1.86x
4096B (+0 / +0) 23.4 / 23.5 GB/s 42.5 / 43.9 GB/s 1.82x/1.87x
16384B (+0 / +0) 23.8 / 24.0 GB/s 43.3 / 45.1 GB/s 1.82x/1.87x
16B (+1 / +3) 1.5 / 2.6 GB/s 6.9 / 5.9 GB/s 4.44x/2.25x
128B (+1 / +3) 6.4 / 10.5 GB/s 24.3 / 27.2 GB/s 3.78x/2.60x
1448B (+1 / +3) 17.8 / 18.0 GB/s 38.0 / 39.0 GB/s 2.14x/2.16x
1500B (+1 / +3) 17.9 / 18.2 GB/s 36.5 / 39.6 GB/s 2.04x/2.18x
4096B (+1 / +3) 19.1 / 19.5 GB/s 40.6 / 42.0 GB/s 2.12x/2.15x
16384B (+1 / +3) 19.4 / 19.6 GB/s 42.3 / 43.8 GB/s 2.18x/2.24x
Suggested-by: David Laight <david.laight.linux@gmail.com>
Signed-off-by: Demian Shulhan <demyansh@gmail.com>
---
arch/arm64/include/asm/checksum.h | 3 +
arch/arm64/lib/csum.c | 189 ++++++++++++++++++++++++++++++
2 files changed, 192 insertions(+)
diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h
index dc52b733675d..9f766db01152 100644
--- a/arch/arm64/include/asm/checksum.h
+++ b/arch/arm64/include/asm/checksum.h
@@ -44,6 +44,9 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
extern unsigned int do_csum(const unsigned char *buff, int len);
#define do_csum do_csum
+#define _HAVE_ARCH_CSUM_AND_COPY
+__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
+
#include <asm-generic/checksum.h>
#endif /* __ASM_CHECKSUM_H */
diff --git a/arch/arm64/lib/csum.c b/arch/arm64/lib/csum.c
index 75db718e5b6a..2bf2d63cfd42 100644
--- a/arch/arm64/lib/csum.c
+++ b/arch/arm64/lib/csum.c
@@ -1,9 +1,12 @@
// SPDX-License-Identifier: GPL-2.0-only
// Copyright (C) 2019-2020 Arm Ltd.
+#include <linux/bitops.h>
#include <linux/compiler.h>
+#include <linux/export.h>
#include <linux/kasan-checks.h>
#include <linux/kernel.h>
+#include <linux/unaligned.h>
#include <net/checksum.h>
@@ -116,6 +119,192 @@ unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
return sum >> 16;
}
+static __always_inline __uint128_t get_unaligned128(const u8 *p)
+{
+ u64 lo = get_unaligned((const u64 *)p);
+ u64 hi = get_unaligned((const u64 *)(p + 8));
+
+ return ((__uint128_t)hi << 64) | lo;
+}
+
+static __always_inline void put_unaligned128(__uint128_t val, u8 *p)
+{
+ put_unaligned((u64)val, (u64 *)p);
+ put_unaligned((u64)(val >> 64), (u64 *)(p + 8));
+}
+
+static __always_inline __wsum csum_fold_acc_cy(__uint128_t acc, u64 cy)
+{
+ __uint128_t t;
+ u64 s;
+
+ acc += (acc >> 64) | (acc << 64);
+ t = (__uint128_t)(u64)(acc >> 64) + cy;
+ s = (u64)t + (u64)(t >> 64);
+ s += (s >> 32) | (s << 32);
+ return (__force __wsum)(u32)(s >> 32);
+}
+
+static __always_inline void csum_tree64_ovf(const u8 *s, u8 *d,
+ __uint128_t *acc, s64 *cy)
+{
+ __uint128_t v0 = get_unaligned128(s);
+ __uint128_t v1 = get_unaligned128(s + 16);
+ __uint128_t v2, v3;
+ s64 o;
+
+ put_unaligned128(v0, d);
+ put_unaligned128(v1, d + 16);
+ o = __builtin_add_overflow(v0, v1, &v0);
+
+ v2 = get_unaligned128(s + 32);
+ v3 = get_unaligned128(s + 48);
+ put_unaligned128(v2, d + 32);
+ put_unaligned128(v3, d + 48);
+ o += __builtin_add_overflow(v2, v3, &v2);
+
+ o += __builtin_add_overflow(v0, v2, &v0);
+ *cy += o + __builtin_add_overflow(*acc, v0, acc);
+}
+
+static __always_inline void csum_align_src16(const u8 **ps, u8 **pd,
+ unsigned int *pulen,
+ __uint128_t *pacc, s64 *pc0)
+{
+ uintptr_t off = (uintptr_t)(*ps) & 15;
+
+ if (unlikely(off) && *pulen >= 272U) {
+ unsigned int adv = 16U - (unsigned int)off;
+ u64 lo = get_unaligned((const u64 *)(*ps));
+ u64 hi = get_unaligned((const u64 *)(*ps + 8));
+ u64 odd = (u64)off & 1ULL;
+ u64 m = ~0ULL >> ((off & 7) * 8);
+ u64 h;
+
+ put_unaligned(lo, (u64 *)(*pd));
+ put_unaligned(hi, (u64 *)(*pd + 8));
+ lo = (off & 8) ? (lo & m) : lo;
+ hi = (off & 8) ? 0ULL : (hi & m);
+ h = lo + hi;
+ h += h < lo;
+ *pacc = ror64(h, (unsigned int)(odd * 56ULL));
+ /*
+ * At len <= INT_MAX, the 256B loop adds <= 16 carries per
+ * iteration (< 2^27 total before bit 62 is cleared, and c0
+ * stays < 2^33 overall), so bit 62 of @c0 cannot overflow.
+ */
+ *pc0 = (s64)(odd << 62);
+ *ps += adv;
+ *pd += adv;
+ *pulen -= adv;
+ }
+}
+
+static __always_inline __wsum csum_fold_final(__uint128_t acc, u64 cy, u64 odd)
+{
+ __wsum res = csum_fold_acc_cy(acc, cy);
+
+ if (unlikely(odd))
+ return (__force __wsum)ror32((__force u32)res, 8);
+ return res;
+}
+
+/*
+ * Single-pass copy and ones' complement sum. Each summed word is read once
+ * into a GPR before storing and summing, so the returned sum always matches
+ * @dst even if @src is concurrently modified.
+ */
+__wsum __no_sanitize_address csum_partial_copy_nocheck(const void *src, void *dst, int len)
+{
+ const u8 *s = src;
+ u8 *d = dst;
+ __uint128_t acc = 0;
+ s64 c0 = 0;
+ u64 odd = 0;
+ unsigned int ulen = (unsigned int)len;
+
+ if (unlikely(len <= 0))
+ return 0;
+
+ kasan_check_read(src, len);
+ kasan_check_write(dst, len);
+
+ if (ulen >= 64U) {
+ if (ulen >= 256U) {
+ csum_align_src16(&s, &d, &ulen, &acc, &c0);
+ ulen -= 256U;
+ do {
+ csum_tree64_ovf(s, d, &acc, &c0);
+ csum_tree64_ovf(s + 64, d + 64, &acc, &c0);
+ csum_tree64_ovf(s + 128, d + 128, &acc, &c0);
+ csum_tree64_ovf(s + 192, d + 192, &acc, &c0);
+ s += 256;
+ d += 256;
+ } while (!__builtin_sub_overflow(ulen, 256U, &ulen));
+ odd = (u64)c0 >> 62;
+ c0 &= (1LL << 62) - 1;
+ ulen &= 255U;
+ if (!ulen)
+ goto fold;
+ }
+ if (ulen >= 64U) {
+ ulen -= 64U;
+ do {
+ csum_tree64_ovf(s, d, &acc, &c0);
+ s += 64;
+ d += 64;
+ } while (!__builtin_sub_overflow(ulen, 64U, &ulen));
+ ulen &= 63U;
+ if (!ulen)
+ goto fold;
+ }
+ }
+ if (likely(ulen >= 16U)) {
+ ulen -= 16U;
+ do {
+ __uint128_t v = get_unaligned128(s);
+
+ put_unaligned128(v, d);
+ s += 16;
+ d += 16;
+ c0 += __builtin_add_overflow(acc, v, &acc);
+ } while (!__builtin_sub_overflow(ulen, 16U, &ulen));
+ }
+ if (likely(ulen & 8U)) {
+ u64 v = get_unaligned((const u64 *)s);
+
+ put_unaligned(v, (u64 *)d);
+ s += 8;
+ d += 8;
+ c0 += __builtin_add_overflow(acc, v, &acc);
+ }
+ if (likely(ulen & 4U)) {
+ u32 v = get_unaligned((const u32 *)s);
+
+ put_unaligned(v, (u32 *)d);
+ s += 4;
+ d += 4;
+ c0 += v;
+ }
+ if (likely(ulen & 2U)) {
+ u16 v = get_unaligned((const u16 *)s);
+
+ put_unaligned(v, (u16 *)d);
+ s += 2;
+ d += 2;
+ c0 += v;
+ }
+ if (ulen & 1U) {
+ u8 v = *s;
+
+ *d = v;
+ c0 += v;
+ }
+fold:
+ return csum_fold_final(acc, (u64)c0, odd);
+}
+EXPORT_SYMBOL(csum_partial_copy_nocheck);
+
__sum16 csum_ipv6_magic(const struct in6_addr *saddr,
const struct in6_addr *daddr,
__u32 len, __u8 proto, __wsum csum)
--
2.53.0
next prev parent reply other threads:[~2026-11-20 22:03 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-20 22:03 [PATCH v3 0/2] arm64: csum: Add fused copy and Internet checksum Demian Shulhan
2026-11-20 22:03 ` Demian Shulhan [this message]
2026-11-20 22:03 ` [PATCH v3 2/2] lib/tests: checksum: Add KUnit tests for csum_partial_copy_nocheck() Demian Shulhan
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261120220330.1067165-2-demyansh@gmail.com \
--to=demyansh@gmail.com \
--cc=akpm@linux-foundation.org \
--cc=brendan.higgins@linux.dev \
--cc=catalin.marinas@arm.com \
--cc=david.laight.linux@gmail.com \
--cc=david@davidgow.net \
--cc=edumazet@google.com \
--cc=kuba@kernel.org \
--cc=kunit-dev@googlegroups.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=nathan@kernel.org \
--cc=netdev@vger.kernel.org \
--cc=robin.murphy@arm.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox