[PATCH v2 0/2] arm64: csum: Add fused copy and Internet checksum

[PATCH v2 0/2] arm64: csum: Add fused copy and Internet checksum

[Demian Shulhan]

Add a single-pass csum_partial_copy_nocheck() implementation for arm64
and extend the checksum KUnit suite to cover copy-and-checksum across
alignments, lengths, guard pages, and interrupt contexts.

Changes in v2:
- Drop the NEON path and separate csum-copy* files, folding the scalar
  implementation directly into arch/arm64/lib/csum.c (resolves the
  <asm/neon-intrinsics.h> include issue noted by Nathan Chancellor).
- Switch the scalar loop to four __uint128_t accumulators (David Laight).
- Canonicalize 0xffff (-0) to 0x0000 (+0) after csum_fold() in KUnit to
  fix x86_64 test failures on zero/all-zero inputs (Jakub Kicinski).

Link: https://lore.kernel.org/r/20260927131838.6774-1-demyansh@gmail.com

Demian Shulhan (2):
  arm64: csum: Add fused copy and Internet checksum
  lib/tests: checksum: Add KUnit tests for csum_partial_copy_nocheck()

 arch/arm64/include/asm/checksum.h |   3 +
 arch/arm64/lib/csum.c             | 100 ++++++++
 lib/Kconfig.debug                 |  10 +
 lib/tests/checksum_kunit.c        | 410 ++++++++++++++++++++++++++++++
 4 files changed, 523 insertions(+)


base-commit: 97d73c8251226cf27cc6656b4c542869bb8c69fa
-- 
2.53.0

[PATCH v2 1/2] arm64: csum: Add fused copy and Internet checksum

[Demian Shulhan]

arm64 currently falls back to the generic csum_partial_copy_nocheck() in
lib/checksum.c, which performs memcpy() followed by a second pass in
csum_partial(). This double pass is hit on software GSO paths
(skb_segment() without SG or TX checksum offload, e.g. TCP over
WireGuard) and UDP/ICMP replies.

Implement a single-pass csum_partial_copy_nocheck() in
arch/arm64/lib/csum.c using four independent __uint128_t accumulators
over a 32-byte stride, followed by 16/8/4/2/1-byte tail handling. Each
word is read once into a GPR before storing and summing, emitting paired
ldp/stp and adds/cinc carry propagation with no stack frame.

Throughput on ARM Neoverse-V2 (Google Axion, 3.0 GHz, GCC 15.2 -O2,
warm L1 cache):

   len   memcpy + do_csum   csum_partial_copy_nocheck   speedup
  ----   ----------------   -------------------------   -------
    64         10245 MB/s                  15902 MB/s    +55.2%
   256         18851 MB/s                  26026 MB/s    +38.1%
   512         22465 MB/s                  28421 MB/s    +26.5%
  1024         24330 MB/s                  29900 MB/s    +22.9%
  1368         24192 MB/s                  30221 MB/s    +24.9%
  1448         24188 MB/s                  30279 MB/s    +25.2%
  1500         24165 MB/s                  30096 MB/s    +24.5%
  4096         25299 MB/s                  29946 MB/s    +18.4%

Suggested-by: David Laight <david.laight.linux@gmail.com>
Signed-off-by: Demian Shulhan <demyansh@gmail.com>
---
 arch/arm64/include/asm/checksum.h |   3 +
 arch/arm64/lib/csum.c             | 100 ++++++++++++++++++++++++++++++
 2 files changed, 103 insertions(+)

diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h
index dc52b733675d..9f766db01152 100644
--- a/arch/arm64/include/asm/checksum.h
+++ b/arch/arm64/include/asm/checksum.h
@@ -44,6 +44,9 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
 extern unsigned int do_csum(const unsigned char *buff, int len);
 #define do_csum do_csum
 
+#define _HAVE_ARCH_CSUM_AND_COPY
+__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
+
 #include <asm-generic/checksum.h>
 
 #endif	/* __ASM_CHECKSUM_H */
diff --git a/arch/arm64/lib/csum.c b/arch/arm64/lib/csum.c
index 75db718e5b6a..a9617cde28c1 100644
--- a/arch/arm64/lib/csum.c
+++ b/arch/arm64/lib/csum.c
@@ -2,8 +2,10 @@
 // Copyright (C) 2019-2020 Arm Ltd.
 
 #include <linux/compiler.h>
+#include <linux/export.h>
 #include <linux/kasan-checks.h>
 #include <linux/kernel.h>
+#include <linux/unaligned.h>
 
 #include <net/checksum.h>
 
@@ -116,6 +118,104 @@ unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
 	return sum >> 16;
 }
 
+static __always_inline __wsum csum_fold128(__uint128_t sum)
+{
+	u64 s;
+
+	sum += (sum >> 64) | (sum << 64);
+	s = sum >> 64;
+	s += (s >> 32) | (s << 32);
+	return (__force __wsum)(u32)(s >> 32);
+}
+
+/*
+ * Single-pass copy and ones' complement sum. Each word is read once into a
+ * GPR before storing and summing, so the returned sum always matches @dst
+ * even if @src is concurrently modified.
+ */
+__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len)
+{
+	const u8 *s = src;
+	u8 *d = dst;
+	__uint128_t sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
+	unsigned int n;
+
+	if (unlikely(len <= 0))
+		return 0;
+
+	n = (unsigned int)len >> 5;
+	if (n) {
+		do {
+			u64 v0, v1;
+
+			v0 = get_unaligned((const u64 *)s);
+			v1 = get_unaligned((const u64 *)(s + 8));
+			put_unaligned(v0, (u64 *)d);
+			put_unaligned(v1, (u64 *)(d + 8));
+			sum0 += v0;
+			sum1 += v1;
+
+			v0 = get_unaligned((const u64 *)(s + 16));
+			v1 = get_unaligned((const u64 *)(s + 24));
+			put_unaligned(v0, (u64 *)(d + 16));
+			put_unaligned(v1, (u64 *)(d + 24));
+			sum2 += v0;
+			sum3 += v1;
+
+			s += 32;
+			d += 32;
+		} while (--n);
+
+		sum0 += sum2;
+		sum1 += sum3;
+	}
+
+	if (len & 16) {
+		u64 v0 = get_unaligned((const u64 *)s);
+		u64 v1 = get_unaligned((const u64 *)(s + 8));
+
+		put_unaligned(v0, (u64 *)d);
+		put_unaligned(v1, (u64 *)(d + 8));
+		sum0 += v0;
+		sum1 += v1;
+		s += 16;
+		d += 16;
+	}
+	if (len & 8) {
+		u64 v = get_unaligned((const u64 *)s);
+
+		put_unaligned(v, (u64 *)d);
+		sum0 += v;
+		s += 8;
+		d += 8;
+	}
+	if (len & 4) {
+		u32 v = get_unaligned((const u32 *)s);
+
+		put_unaligned(v, (u32 *)d);
+		sum1 += v;
+		s += 4;
+		d += 4;
+	}
+	if (len & 2) {
+		u16 v = get_unaligned((const u16 *)s);
+
+		put_unaligned(v, (u16 *)d);
+		sum0 += v;
+		s += 2;
+		d += 2;
+	}
+	if (len & 1) {
+		u8 v = *s;
+
+		*d = v;
+		sum1 += v;
+	}
+
+	return csum_fold128(sum0 + sum1);
+}
+EXPORT_SYMBOL(csum_partial_copy_nocheck);
+
 __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
 			const struct in6_addr *daddr,
 			__u32 len, __u8 proto, __wsum csum)
-- 
2.53.0

[PATCH v2 2/2] lib/tests: checksum: Add KUnit tests for csum_partial_copy_nocheck()

[Demian Shulhan]

Extend the checksum KUnit test suite to cover csum_partial_copy_nocheck()
across alignments, tail lengths, bit patterns, vmalloc() guard pages with
canary checks, and concurrent task/softirq/hardirq execution.

Compare folded checksums against both a standalone C reference and
csum_partial(), canonicalizing 0xffff (-0) to 0x0000 (+0) after
csum_fold() so zero and all-zero inputs match across architectures such
as x86_64.

Also add an optional throughput benchmark gated behind
CONFIG_CHECKSUM_KUNIT_BENCHMARK (default n).

Signed-off-by: Demian Shulhan <demyansh@gmail.com>
---
 lib/Kconfig.debug          |  10 +
 lib/tests/checksum_kunit.c | 410 +++++++++++++++++++++++++++++++++++++
 2 files changed, 420 insertions(+)

diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 2d22638ec13a..588343ced260 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -2770,6 +2770,16 @@ config CHECKSUM_KUNIT
 
 	  If unsure, say N.
 
+config CHECKSUM_KUNIT_BENCHMARK
+	bool "Benchmark for the checksum functions"
+	depends on CHECKSUM_KUNIT
+	help
+	  Include a benchmark of csum_partial_copy_nocheck() against
+	  memcpy() followed by csum_partial() in the checksum KUnit test
+	  suite, for a range of buffer lengths and misalignments. The
+	  throughput of both is logged as test output. Only meaningful on
+	  real hardware and is not intended for production builds.
+
 config UTIL_MACROS_KUNIT
 	tristate "KUnit test util_macros.h functions at runtime" if !KUNIT_ALL_TESTS
 	depends on KUNIT
diff --git a/lib/tests/checksum_kunit.c b/lib/tests/checksum_kunit.c
index be04aa42125c..69611daa7c0c 100644
--- a/lib/tests/checksum_kunit.c
+++ b/lib/tests/checksum_kunit.c
@@ -3,8 +3,15 @@
  * Test cases csum_partial, csum_fold, ip_fast_csum, csum_ipv6_magic
  */
 
+#include <kunit/run-in-irq-context.h>
 #include <kunit/test.h>
+#include <linux/math64.h>
+#include <linux/prandom.h>
+#include <linux/preempt.h>
+#include <linux/string.h>
+#include <linux/vmalloc.h>
 #include <asm/checksum.h>
+#include <net/checksum.h>
 #include <net/ip6_checksum.h>
 
 #define MAX_LEN 512
@@ -619,18 +626,421 @@ static void test_csum_ipv6_magic(struct kunit *test)
 	}
 }
 
+/*
+ * csum_partial_copy_nocheck() tests: verify copied bytes, folded checksum
+ * against a standalone reference and csum_partial(), canary/guard-page
+ * bounds, and concurrent task/softirq/hardirq execution.
+ */
+#define COPY_MAX_OFF 16
+#define COPY_MAX_LEN 4352
+#define COPY_BENCH_MAX_LEN 16384
+#define COPY_GUARD 128 /* canary span on each side of dst */
+#define COPY_CANARY 0xa5
+#define COPY_SEED 0x5eed
+
+static struct rnd_state copy_rng;
+/* vmalloc()ed with a page-aligned length, so each is followed by a guard page */
+static u8 *copy_src_area;
+static u8 *copy_dst_area;
+static u8 *copy_shadow; /* expected contents of copy_src_area */
+static size_t copy_area_len;
+
+static int checksum_suite_init(struct kunit_suite *suite)
+{
+	copy_area_len = round_up(COPY_BENCH_MAX_LEN + COPY_MAX_OFF, PAGE_SIZE);
+	copy_src_area = vmalloc(copy_area_len);
+	copy_dst_area = vmalloc(copy_area_len);
+	copy_shadow = vmalloc(copy_area_len);
+	if (!copy_src_area || !copy_dst_area || !copy_shadow) {
+		vfree(copy_src_area);
+		vfree(copy_dst_area);
+		vfree(copy_shadow);
+		return -ENOMEM;
+	}
+	prandom_seed_state(&copy_rng, COPY_SEED);
+	return 0;
+}
+
+static void checksum_suite_exit(struct kunit_suite *suite)
+{
+	vfree(copy_src_area);
+	vfree(copy_dst_area);
+	vfree(copy_shadow);
+}
+
+static void copy_fill_random(void)
+{
+	prandom_bytes_state(&copy_rng, copy_src_area, copy_area_len);
+	memcpy(copy_shadow, copy_src_area, copy_area_len);
+}
+
+static void copy_fill_bytes(const u8 *pattern, int pattern_len)
+{
+	size_t i;
+
+	for (i = 0; i < copy_area_len; i++)
+		copy_src_area[i] = pattern[i % pattern_len];
+	memcpy(copy_shadow, copy_src_area, copy_area_len);
+}
+
+/* Place @len bytes at misalignment @off as close as possible to the guard page. */
+static u8 *copy_place_at_end(u8 *area, int off, int len)
+{
+	u8 *end = area + copy_area_len;
+	int pad = ((unsigned long)(end - len) - off) & (COPY_MAX_OFF - 1);
+
+	return end - len - pad;
+}
+
+/* Naive reference: folded ones' complement sum, same semantics as csum_fold(). */
+static u16 ref_csum_fold(const u8 *buf, int len)
+{
+	u64 sum = 0;
+	u16 word;
+	int i;
+
+	for (i = 0; i + 1 < len; i += 2) {
+		memcpy(&word, buf + i, sizeof(word)); /* native endian */
+		sum += word;
+	}
+	if (len & 1) {
+		u8 pad[2] = { buf[len - 1], 0 };
+
+		memcpy(&word, pad, sizeof(word)); /* LE: low byte, BE: high byte */
+		sum += word;
+	}
+	while (sum >> 16)
+		sum = (sum & 0xffff) + (sum >> 16);
+
+	sum = ~sum & 0xffff;
+	return sum == 0xffff ? 0 : sum;
+}
+
+/*
+ * RFC 1071: 0xffff (-0) and 0x0000 (+0) are equivalent. Canonicalize to 0 so
+ * ~0U-seeded archs (e.g. x86_64) match 0-seeded refs on zero/all-zero inputs.
+ */
+static u16 fold_csum(__wsum csum)
+{
+	u16 sum = (__force u16)csum_fold(csum);
+
+	return sum == 0xffff ? 0 : sum;
+}
+
+static int first_non_canary(const u8 *p, int n)
+{
+	int i;
+
+	for (i = 0; i < n; i++)
+		if (p[i] != COPY_CANARY)
+			return i;
+	return -1;
+}
+
+/* Verify one csum_partial_copy_nocheck() call with @before/@after canaries. */
+static void check_csum_copy_at(struct kunit *test, const u8 *src, u8 *dst,
+			       int len, int before, int after)
+{
+	unsigned long src_off = (unsigned long)src & (COPY_MAX_OFF - 1);
+	unsigned long dst_off = (unsigned long)dst & (COPY_MAX_OFF - 1);
+	const u8 *lo, *hi;
+	u16 expect, got;
+	int bad;
+
+	memset(dst - before, COPY_CANARY, before + len + after);
+	expect = ref_csum_fold(src, len);
+
+	got = fold_csum(csum_partial_copy_nocheck(src, dst, len));
+
+	KUNIT_ASSERT_EQ_MSG(test, got, expect,
+			    "checksum mismatch src_off=%lu dst_off=%lu len=%d",
+			    src_off, dst_off, len);
+	KUNIT_ASSERT_EQ_MSG(test, memcmp(src, dst, len), 0,
+			    "copied data mismatch src_off=%lu dst_off=%lu len=%d",
+			    src_off, dst_off, len);
+
+	bad = first_non_canary(dst - before, before);
+	KUNIT_ASSERT_EQ_MSG(test, bad, -1,
+			    "wrote %d bytes before dst (src_off=%lu dst_off=%lu len=%d)",
+			    before - bad, src_off, dst_off, len);
+
+	bad = first_non_canary(dst + len, after);
+	KUNIT_ASSERT_EQ_MSG(test, bad, -1,
+			    "wrote at dst+len+%d (src_off=%lu dst_off=%lu len=%d)",
+			    bad, src_off, dst_off, len);
+
+	/* Source untouched, including a margin on either side. */
+	lo = max(src - COPY_GUARD, (const u8 *)copy_src_area);
+	hi = min(src + len + COPY_GUARD, (const u8 *)copy_src_area + copy_area_len);
+	KUNIT_ASSERT_EQ_MSG(test,
+			    memcmp(lo, copy_shadow + (lo - copy_src_area), hi - lo), 0,
+			    "source buffer modified (src_off=%lu dst_off=%lu len=%d)",
+			    src_off, dst_off, len);
+
+	/* Must be interchangeable with the arch csum_partial() semantics. */
+	KUNIT_EXPECT_EQ_MSG(test, got,
+			    fold_csum(csum_partial(src, len, 0)),
+			    "disagrees with csum_partial() src_off=%lu dst_off=%lu len=%d",
+			    src_off, dst_off, len);
+}
+
+/* Source against its guard page, destination between canaries. */
+static void check_csum_copy(struct kunit *test, int src_off, int dst_off,
+			    int len)
+{
+	KUNIT_ASSERT_TRUE(test, src_off >= 0 && src_off < COPY_MAX_OFF);
+	KUNIT_ASSERT_TRUE(test, dst_off >= 0 && dst_off < COPY_MAX_OFF);
+	KUNIT_ASSERT_TRUE(test, len >= 0 && len <= COPY_MAX_LEN);
+
+	check_csum_copy_at(test, copy_place_at_end(copy_src_area, src_off, len),
+			   copy_dst_area + COPY_GUARD + dst_off, len,
+			   COPY_GUARD, COPY_GUARD);
+}
+
+/* Representative source offsets used with all 16 destination offsets. */
+static const int copy_src_offs[] = { 0, 1, 3, 8, 15 };
+
+/* Every len 0..64 with every src/dst misalignment: the scalar tail ladder. */
+static void test_csum_copy_small_all_alignments(struct kunit *test)
+{
+	int len, src_off, dst_off;
+
+	copy_fill_random();
+	for (len = 0; len <= 64; len++)
+		for (src_off = 0; src_off < COPY_MAX_OFF; src_off++)
+			for (dst_off = 0; dst_off < COPY_MAX_OFF; dst_off++)
+				check_csum_copy(test, src_off, dst_off, len);
+}
+
+/* Large unaligned buffers exercising multi-block strides and GSO/MTU lengths. */
+static void test_csum_copy_large_unaligned(struct kunit *test)
+{
+	static const int extra_lens[] = {
+		1368, 1448, 1500, 2033, 2034, 2035, 2039, 2040, 2041, 2047,
+		2048, 2049, 2055, 2056, 2057, 2062, 2063, 3072, 4095, 4096,
+		4097, 4352,
+	};
+	int len, i, j, dst_off;
+
+	copy_fill_random();
+	for (len = 1024; len < 1024 + 80; len++)
+		for (i = 0; i < ARRAY_SIZE(copy_src_offs); i++)
+			for (dst_off = 0; dst_off < COPY_MAX_OFF; dst_off++)
+				check_csum_copy(test, copy_src_offs[i], dst_off,
+						len);
+
+	for (j = 0; j < ARRAY_SIZE(extra_lens); j++)
+		for (i = 0; i < ARRAY_SIZE(copy_src_offs); i++)
+			for (dst_off = 0; dst_off < COPY_MAX_OFF; dst_off++)
+				check_csum_copy(test, copy_src_offs[i], dst_off,
+						extra_lens[j]);
+}
+
+/* Patterns stressing carries, zero/MSB bytes, and odd-offset byte order. */
+static void test_csum_copy_patterns(struct kunit *test)
+{
+	static const u8 pat_ff[] = { 0xff };
+	static const u8 pat_00[] = { 0x00 };
+	static const u8 pat_80[] = { 0x80 };
+	static const u8 pat_ff00[] = { 0xff, 0x00 };
+	static const u8 pat_ramp[] = { 0x01, 0x02, 0x04, 0x08,
+				       0x10, 0x20, 0x40, 0x80 };
+	static const struct {
+		const u8 *pat;
+		int len;
+	} patterns[] = {
+		{ pat_ff, ARRAY_SIZE(pat_ff) },
+		{ pat_00, ARRAY_SIZE(pat_00) },
+		{ pat_80, ARRAY_SIZE(pat_80) },
+		{ pat_ff00, ARRAY_SIZE(pat_ff00) },
+		{ pat_ramp, ARRAY_SIZE(pat_ramp) },
+		{ NULL, 0 }, /* random */
+	};
+	static const int lens[] = {
+		1, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 63, 64, 65, 127, 128,
+		255, 256, 511, 512, 1023, 1024, 1025, 1368, 1448, 1500, 2048,
+		4095, 4096, 4352,
+	};
+	static const int offs[][2] = {
+		{ 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 }, { 3, 7 }, { 7, 3 },
+		{ 9, 5 }, { 15, 15 },
+	};
+	int p, l, o;
+
+	for (p = 0; p < ARRAY_SIZE(patterns); p++) {
+		if (patterns[p].pat)
+			copy_fill_bytes(patterns[p].pat, patterns[p].len);
+		else
+			copy_fill_random();
+
+		for (l = 0; l < ARRAY_SIZE(lens); l++)
+			for (o = 0; o < ARRAY_SIZE(offs); o++)
+				check_csum_copy(test, offs[o][0], offs[o][1],
+						lens[l]);
+	}
+}
+
+/* Place dst against its trailing guard page so any over-write faults. */
+static void test_csum_copy_dst_guard_page(struct kunit *test)
+{
+	static const int lens[] = {
+		1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17,
+		31, 32, 33, 63, 64, 65, 79, 80, 81, 127, 128, 129, 1023, 1024,
+		1025, 1368, 1448, 1500, 2048, 4095, 4096, 4097,
+	};
+	int l, src_off, dst_off;
+
+	copy_fill_random();
+	for (l = 0; l < ARRAY_SIZE(lens); l++)
+		for (dst_off = 0; dst_off < COPY_MAX_OFF; dst_off++) {
+			u8 *dst = copy_place_at_end(copy_dst_area, dst_off, lens[l]);
+			int after = copy_dst_area + copy_area_len - (dst + lens[l]);
+
+			src_off = (dst_off * 7 + lens[l]) & (COPY_MAX_OFF - 1);
+			check_csum_copy_at(test,
+					   copy_place_at_end(copy_src_area,
+							     src_off, lens[l]),
+					   dst, lens[l], COPY_GUARD, after);
+		}
+}
+
+/* len == 0: folded checksum must be zero and dst must not be touched. */
+static void test_csum_copy_zero_len(struct kunit *test)
+{
+	u8 *dst = copy_dst_area + COPY_GUARD;
+	__wsum sum;
+
+	copy_fill_random();
+	memset(copy_dst_area, COPY_CANARY, 2 * COPY_GUARD);
+	sum = csum_partial_copy_nocheck(copy_src_area, dst, 0);
+	KUNIT_EXPECT_EQ(test, fold_csum(sum), 0);
+	KUNIT_EXPECT_EQ(test, first_non_canary(copy_dst_area, 2 * COPY_GUARD), -1);
+}
+
+/* Concurrent execution across task, softirq, and hardirq contexts. */
+#define COPY_IRQ_NUM_MSGS 3
+#define COPY_IRQ_NUM_CTX 3 /* task, softirq, hardirq */
+
+static const int copy_irq_lens[COPY_IRQ_NUM_MSGS] = { 100, 1500, 4097 };
+
+struct copy_irq_test_state {
+	const u8 *src[COPY_IRQ_NUM_MSGS];
+	u16 expected[COPY_IRQ_NUM_MSGS];
+	u8 *dst[COPY_IRQ_NUM_CTX];
+	atomic_t seqno;
+};
+
+static bool copy_irq_test_func(void *state_)
+{
+	struct copy_irq_test_state *state = state_;
+	u32 i = (u32)atomic_inc_return(&state->seqno) % COPY_IRQ_NUM_MSGS;
+	int ctx = in_hardirq() ? 2 : in_serving_softirq() ? 1 : 0;
+	const u8 *src = state->src[i];
+	u8 *dst = state->dst[ctx];
+	int len = copy_irq_lens[i];
+	u16 got;
+
+	got = fold_csum(csum_partial_copy_nocheck(src, dst, len));
+	return got == state->expected[i] && memcmp(src, dst, len) == 0;
+}
+
+static void test_csum_copy_interrupt_context(struct kunit *test)
+{
+	struct copy_irq_test_state state = { };
+	int i;
+
+	copy_fill_random();
+	for (i = 0; i < COPY_IRQ_NUM_MSGS; i++) {
+		/* Distinct, misaligned messages. */
+		state.src[i] = copy_src_area + i * (COPY_MAX_LEN + 64) + 2 * i + 1;
+		state.expected[i] = ref_csum_fold(state.src[i], copy_irq_lens[i]);
+	}
+	for (i = 0; i < COPY_IRQ_NUM_CTX; i++)
+		state.dst[i] = copy_dst_area + i * (COPY_MAX_LEN + 64) + 2 * i + 1;
+
+	kunit_run_irq_test(test, copy_irq_test_func, 100000, &state);
+}
+
+/* Optional throughput benchmark vs memcpy() + csum_partial(). */
+static void test_csum_copy_benchmark(struct kunit *test)
+{
+	static const int lens[] = {
+		16, 40, 64, 128, 256, 512, 1000, 1024, 1368, 1448, 1500,
+		2048, 4096, 16384,
+	};
+	static const int offs[][2] = { { 0, 0 }, { 1, 3 } };
+	int l, o, i, len, num_iters;
+	u64 t, t_fused, t_split;
+	u32 acc = 0; /* keeps every call alive via OPTIMIZER_HIDE_VAR() */
+
+	if (!IS_ENABLED(CONFIG_CHECKSUM_KUNIT_BENCHMARK))
+		kunit_skip(test, "not enabled");
+
+	copy_fill_random();
+
+	/* warm-up */
+	for (i = 0; i < 10000000; i += COPY_BENCH_MAX_LEN)
+		acc ^= (__force u32)csum_partial_copy_nocheck(copy_src_area,
+							      copy_dst_area,
+							      COPY_BENCH_MAX_LEN);
+	OPTIMIZER_HIDE_VAR(acc);
+
+	for (o = 0; o < ARRAY_SIZE(offs); o++) {
+		const u8 *src = copy_src_area + offs[o][0];
+		u8 *dst = copy_dst_area + offs[o][1];
+
+		for (l = 0; l < ARRAY_SIZE(lens); l++) {
+			len = lens[l];
+			num_iters = 10000000 / (len + 128);
+
+			preempt_disable();
+			t = ktime_get_ns();
+			for (i = 0; i < num_iters; i++)
+				acc ^= (__force u32)csum_partial_copy_nocheck(src, dst, len);
+			t_fused = ktime_get_ns() - t;
+			OPTIMIZER_HIDE_VAR(acc);
+
+			t = ktime_get_ns();
+			for (i = 0; i < num_iters; i++) {
+				memcpy(dst, src, len);
+				acc ^= (__force u32)csum_partial(dst, len, 0);
+			}
+			t_split = ktime_get_ns() - t;
+			OPTIMIZER_HIDE_VAR(acc);
+			preempt_enable();
+
+			kunit_info(test,
+				   "src+%d dst+%d len=%5d: csum_partial_copy_nocheck %6llu MB/s, memcpy+csum_partial %6llu MB/s\n",
+				   offs[o][0], offs[o][1], len,
+				   div64_u64((u64)len * num_iters * 1000,
+					     max_t(u64, t_fused, 1)),
+				   div64_u64((u64)len * num_iters * 1000,
+					     max_t(u64, t_split, 1)));
+		}
+	}
+}
+
 static struct kunit_case __refdata checksum_test_cases[] = {
 	KUNIT_CASE(test_csum_fixed_random_inputs),
 	KUNIT_CASE(test_csum_all_carry_inputs),
 	KUNIT_CASE(test_csum_no_carry_inputs),
 	KUNIT_CASE(test_ip_fast_csum),
 	KUNIT_CASE(test_csum_ipv6_magic),
+	KUNIT_CASE(test_csum_copy_small_all_alignments),
+	KUNIT_CASE(test_csum_copy_large_unaligned),
+	KUNIT_CASE(test_csum_copy_patterns),
+	KUNIT_CASE(test_csum_copy_dst_guard_page),
+	KUNIT_CASE(test_csum_copy_zero_len),
+	KUNIT_CASE(test_csum_copy_interrupt_context),
+	KUNIT_CASE(test_csum_copy_benchmark),
 	{}
 };
 
 static struct kunit_suite checksum_test_suite = {
 	.name = "checksum",
 	.test_cases = checksum_test_cases,
+	.suite_init = checksum_suite_init,
+	.suite_exit = checksum_suite_exit,
 };
 
 kunit_test_suites(&checksum_test_suite);
-- 
2.53.0

Re: [PATCH v2 1/2] arm64: csum: Add fused copy and Internet checksum

[David Laight]

On Sat, 14 Nov 2026 03:57:00 +0000
Demian Shulhan <demyansh@gmail.com> wrote:

> arm64 currently falls back to the generic csum_partial_copy_nocheck() in
> lib/checksum.c, which performs memcpy() followed by a second pass in
> csum_partial(). This double pass is hit on software GSO paths
> (skb_segment() without SG or TX checksum offload, e.g. TCP over
> WireGuard) and UDP/ICMP replies.
> 
> Implement a single-pass csum_partial_copy_nocheck() in
> arch/arm64/lib/csum.c using four independent __uint128_t accumulators
> over a 32-byte stride, followed by 16/8/4/2/1-byte tail handling. Each
> word is read once into a GPR before storing and summing, emitting paired
> ldp/stp and adds/cinc carry propagation with no stack frame.

Is that right?
I thought ldp/stp required aligned addresses.
The code below is using get_unaligned().
I think the timings below are for ldp/stp?

> 
> Throughput on ARM Neoverse-V2 (Google Axion, 3.0 GHz, GCC 15.2 -O2,
> warm L1 cache):
> 
>    len   memcpy + do_csum   csum_partial_copy_nocheck   speedup
>   ----   ----------------   -------------------------   -------
>     64         10245 MB/s                  15902 MB/s    +55.2%
>    256         18851 MB/s                  26026 MB/s    +38.1%
>    512         22465 MB/s                  28421 MB/s    +26.5%
>   1024         24330 MB/s                  29900 MB/s    +22.9%
>   1368         24192 MB/s                  30221 MB/s    +24.9%
>   1448         24188 MB/s                  30279 MB/s    +25.2%
>   1500         24165 MB/s                  30096 MB/s    +24.5%
>   4096         25299 MB/s                  29946 MB/s    +18.4%

That is topping out at 10 bytes/clock.
Each loop iteration copies 32 bytes - so must take 3 clocks.
Which means it must be doing 16 byte read/write in a single clock.
So rather begs the question of why it is limited by the loop control
and checksum instructions rather than the memory accesses.
The checksum code needs 4 ALU instructions (for 16 bytes), if the CPU
can execute more than that then there is half a chance of getting the
loop down to 2 clocks.
Just moving one (or both) the pointer updates into the middle may help.
Otherwise you try using negative offsets from the end of the buffer
and 'reg + reg' addressing (will need more pointers since I don't think
arm has 'reg + reg + offset' addressing.

> 
> Suggested-by: David Laight <david.laight.linux@gmail.com>
> Signed-off-by: Demian Shulhan <demyansh@gmail.com>
> ---
>  arch/arm64/include/asm/checksum.h |   3 +
>  arch/arm64/lib/csum.c             | 100 ++++++++++++++++++++++++++++++
>  2 files changed, 103 insertions(+)
> 
> diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h
> index dc52b733675d..9f766db01152 100644
> --- a/arch/arm64/include/asm/checksum.h
> +++ b/arch/arm64/include/asm/checksum.h
> @@ -44,6 +44,9 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
>  extern unsigned int do_csum(const unsigned char *buff, int len);
>  #define do_csum do_csum
>  
> +#define _HAVE_ARCH_CSUM_AND_COPY
> +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
> +
>  #include <asm-generic/checksum.h>
>  
>  #endif	/* __ASM_CHECKSUM_H */
> diff --git a/arch/arm64/lib/csum.c b/arch/arm64/lib/csum.c
> index 75db718e5b6a..a9617cde28c1 100644
> --- a/arch/arm64/lib/csum.c
> +++ b/arch/arm64/lib/csum.c
> @@ -2,8 +2,10 @@
>  // Copyright (C) 2019-2020 Arm Ltd.
>  
>  #include <linux/compiler.h>
> +#include <linux/export.h>
>  #include <linux/kasan-checks.h>
>  #include <linux/kernel.h>
> +#include <linux/unaligned.h>
>  
>  #include <net/checksum.h>
>  
> @@ -116,6 +118,104 @@ unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
>  	return sum >> 16;
>  }
>  
> +static __always_inline __wsum csum_fold128(__uint128_t sum)
> +{
> +	u64 s;
> +
> +	sum += (sum >> 64) | (sum << 64);
> +	s = sum >> 64;
> +	s += (s >> 32) | (s << 32);
> +	return (__force __wsum)(u32)(s >> 32);
> +}
> +
> +/*
> + * Single-pass copy and ones' complement sum. Each word is read once into a
> + * GPR before storing and summing, so the returned sum always matches @dst
> + * even if @src is concurrently modified.
> + */
> +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len)
> +{
> +	const u8 *s = src;
> +	u8 *d = dst;
> +	__uint128_t sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
> +	unsigned int n;
> +
> +	if (unlikely(len <= 0))
> +		return 0;

Hmmm...
Does zero actually happen often enough for that to be a gain?
Even with the unlikely() it is likely to slow down the normal path.
And that assumes that the branch isn't mispredicted and that gcc
doesn't ignore the unlikely(). 

> +
> +	n = (unsigned int)len >> 5;
> +	if (n) {
> +		do {
> +			u64 v0, v1;
> +
> +			v0 = get_unaligned((const u64 *)s);
> +			v1 = get_unaligned((const u64 *)(s + 8));
> +			put_unaligned(v0, (u64 *)d);
> +			put_unaligned(v1, (u64 *)(d + 8));
> +			sum0 += v0;
> +			sum1 += v1;
> +
> +			v0 = get_unaligned((const u64 *)(s + 16));
> +			v1 = get_unaligned((const u64 *)(s + 24));
> +			put_unaligned(v0, (u64 *)(d + 16));
> +			put_unaligned(v1, (u64 *)(d + 24));
> +			sum2 += v0;
> +			sum3 += v1;

Is using sum0/1 here actually slower?
My 'gut feel' is that it won't be - and saves the adds below.

It is also worth checking that gcc doesn't zero extend v0 into a
register and then do a full 128bit add.
I'm sure I looked at the code using only 64bit variables for:
			sum0 += v0;
			carry0 += sum0 < v0;
both gcc and clang used 'add 0 with carry' for the second line.

> +
> +			s += 32;
> +			d += 32;
> +		} while (--n);

It is a shame to have to update 3 variables here.
They may be taking the 3rd clock.

David


> +
> +		sum0 += sum2;
> +		sum1 += sum3;
> +	}
> +
> +	if (len & 16) {
> +		u64 v0 = get_unaligned((const u64 *)s);
> +		u64 v1 = get_unaligned((const u64 *)(s + 8));
> +
> +		put_unaligned(v0, (u64 *)d);
> +		put_unaligned(v1, (u64 *)(d + 8));
> +		sum0 += v0;
> +		sum1 += v1;
> +		s += 16;
> +		d += 16;
> +	}
> +	if (len & 8) {
> +		u64 v = get_unaligned((const u64 *)s);
> +
> +		put_unaligned(v, (u64 *)d);
> +		sum0 += v;
> +		s += 8;
> +		d += 8;
> +	}
> +	if (len & 4) {
> +		u32 v = get_unaligned((const u32 *)s);
> +
> +		put_unaligned(v, (u32 *)d);
> +		sum1 += v;
> +		s += 4;
> +		d += 4;
> +	}
> +	if (len & 2) {
> +		u16 v = get_unaligned((const u16 *)s);
> +
> +		put_unaligned(v, (u16 *)d);
> +		sum0 += v;
> +		s += 2;
> +		d += 2;
> +	}
> +	if (len & 1) {
> +		u8 v = *s;
> +
> +		*d = v;
> +		sum1 += v;
> +	}
> +
> +	return csum_fold128(sum0 + sum1);
> +}
> +EXPORT_SYMBOL(csum_partial_copy_nocheck);
> +
>  __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
>  			const struct in6_addr *daddr,
>  			__u32 len, __u8 proto, __wsum csum)

Re: [PATCH v2 1/2] arm64: csum: Add fused copy and Internet checksum

[Demian Shulhan]

Hi David,

> Is that right?
> I thought ldp/stp required aligned addresses.
> The code below is using get_unaligned().
> I think the timings below are for ldp/stp?

Unlike 32-bit ARM (ldrd/strd), AArch64 GPR ldp/stp support unaligned
addresses in hardware on Normal memory. Both GCC 15 and Clang 21 merge
the adjacent get_unaligned()/put_unaligned() pairs into ldp/stp, and
the timings are from that code. Unaligned (src+1, dst+3) is only ~5%
slower due to cache-line splits (~28.8-29.1 GB/s vs ~30.3-31.1 GB/s).


> That is topping out at 10 bytes/clock.
> Each loop iteration copies 32 bytes - so must take 3 clocks.
> Which means it must be doing 16 byte read/write in a single clock.
> So rather begs the question of why it is limited by the loop control
> and checksum instructions rather than the memory accesses.
> The checksum code needs 4 ALU instructions (for 16 bytes), if the CPU
> can execute more than that then there is half a chance of getting the
> loop down to 2 clocks.

For 32 bytes, the loop is 16 instructions (4x ldp/stp, 8x adds/cinc,
2x add, cmp, b.ne). On Neoverse-V2 (6 ALU pipes, 8-wide dispatch) 12
ALU/branch ops plus 4 paired memory ops put the floor right around ~3
clocks/iteration (~5.2 IPC).


> Just moving one (or both) the pointer updates into the middle may help.
> Otherwise you try using negative offsets from the end of the buffer
> and 'reg + reg' addressing (will need more pointers since I don't think
> arm has 'reg + reg + offset' addressing.

Moving the pointer updates into the middle does help, it shaves up to
~5% off across sizes.

For 'reg + reg', ldp/stp only support immediate offsets, so indexed
addressing would force single-register ldr/str and double the memory
instructions.


> > + if (unlikely(len <= 0))
> > + return 0;
>
> Hmmm...
> Does zero actually happen often enough for that to be a gain?
> Even with the unlikely() it is likely to slow down the normal path.
> And that assumes that the branch isn't mispredicted and that gcc
> doesn't ignore the unlikely().

len == 0 would fall through fine, but since len is a signed int in the
csum_partial_copy_nocheck() prototype, this guards against negative len
wrapping (unsigned int)len >> 5 (matching do_csum() right above it).
Both compilers place the return 0 block out-of-line at the end of the
function.


> > + sum2 += v0;
> > + sum3 += v1;
>
> Is using sum0/1 here actually slower?
> My 'gut feel' is that it won't be - and saves the adds below.

For 64-128B your gut feel is right - 2 accumulators is 2-10% faster from
saving the post-loop fold, and on GCC 15 it matches 4 accumulators on
larger buffers too (GCC folds v0 + v2 first, keeping a 1-cycle chain in
18 insns). The catch is Clang 21: with 2 accumulators it chains both
adds/cinc updates onto the same accumulator and drops ~12% for >= 512B
(27.0 vs 30.5-31.2 GB/s).


> It is also worth checking that gcc doesn't zero extend v0 into a
> register and then do a full 128bit add.
> I'm sure I looked at the code using only 64bit variables for:
> sum0 += v0;
> carry0 += sum0 < v0;
> both gcc and clang used 'add 0 with carry' for the second line.

Checked - neither compiler zero-extends v0, both emit adds + cinc
directly on the 64-bit registers.


> > + s += 32;
> > + d += 32;
> > + } while (--n);
>
> It is a shame to have to update 3 variables here.
> They may be taking the 3rd clock.

Switching --n to an end pointer (s != end) and moving s += 32 / d += 32
between the loads and stores drops the 3rd update on Clang (80 -> 76
insns total) and cleans up GCC's loop setup (88 -> 81 insns), gaining
+0.4% to +4.9% across sizes.

Here is the resulting 16-instruction loop with an end pointer and 4
accumulators:

  GCC 15.2:                        Clang 21.1:
    ldp  x10, x9, [x0]               ldp  x17, x18, [x0]
    add  x11, x11, #0x20             ldp  x3, x4, [x0, #0x10]
    ldp  x8, x7, [x0, #16]           add  x0, x0, #0x20
    add  x0, x0, #0x20               adds x15, x15, x17
    stp  x10, x9, [x11, #-32]        stp  x17, x18, [x1]
    adds x3, x10, x3                 cinc x16, x16, hs
    stp  x8, x7, [x11, #-16]         adds x13, x13, x18
    cinc x12, x12, cs                stp  x3, x4, [x1, #0x10]
    adds x4, x9, x4                  cinc x14, x14, hs
    cinc x13, x13, cs                adds x8, x8, x3
    adds x6, x8, x6                  add  x1, x1, #0x20
    cinc x16, x16, cs                cinc x9, x9, hs
    adds x5, x7, x5                  adds x10, x10, x4
    cinc x15, x15, cs                cinc x11, x11, hs
    cmp  x14, x0                     cmp  x0, x12
    b.ne .Loop                       b.ne .Loop

I'll switch to the end-pointer loop with interleaved pointer updates in
v3.

Thanks!
Demian

сб, 14 лист. 2026 р. о 16:26 David Laight <david.laight.linux@gmail.com> пише:
>
> On Sat, 14 Nov 2026 03:57:00 +0000
> Demian Shulhan <demyansh@gmail.com> wrote:
>
> > arm64 currently falls back to the generic csum_partial_copy_nocheck() in
> > lib/checksum.c, which performs memcpy() followed by a second pass in
> > csum_partial(). This double pass is hit on software GSO paths
> > (skb_segment() without SG or TX checksum offload, e.g. TCP over
> > WireGuard) and UDP/ICMP replies.
> >
> > Implement a single-pass csum_partial_copy_nocheck() in
> > arch/arm64/lib/csum.c using four independent __uint128_t accumulators
> > over a 32-byte stride, followed by 16/8/4/2/1-byte tail handling. Each
> > word is read once into a GPR before storing and summing, emitting paired
> > ldp/stp and adds/cinc carry propagation with no stack frame.
>
> Is that right?
> I thought ldp/stp required aligned addresses.
> The code below is using get_unaligned().
> I think the timings below are for ldp/stp?
>
> >
> > Throughput on ARM Neoverse-V2 (Google Axion, 3.0 GHz, GCC 15.2 -O2,
> > warm L1 cache):
> >
> >    len   memcpy + do_csum   csum_partial_copy_nocheck   speedup
> >   ----   ----------------   -------------------------   -------
> >     64         10245 MB/s                  15902 MB/s    +55.2%
> >    256         18851 MB/s                  26026 MB/s    +38.1%
> >    512         22465 MB/s                  28421 MB/s    +26.5%
> >   1024         24330 MB/s                  29900 MB/s    +22.9%
> >   1368         24192 MB/s                  30221 MB/s    +24.9%
> >   1448         24188 MB/s                  30279 MB/s    +25.2%
> >   1500         24165 MB/s                  30096 MB/s    +24.5%
> >   4096         25299 MB/s                  29946 MB/s    +18.4%
>
> That is topping out at 10 bytes/clock.
> Each loop iteration copies 32 bytes - so must take 3 clocks.
> Which means it must be doing 16 byte read/write in a single clock.
> So rather begs the question of why it is limited by the loop control
> and checksum instructions rather than the memory accesses.
> The checksum code needs 4 ALU instructions (for 16 bytes), if the CPU
> can execute more than that then there is half a chance of getting the
> loop down to 2 clocks.
> Just moving one (or both) the pointer updates into the middle may help.
> Otherwise you try using negative offsets from the end of the buffer
> and 'reg + reg' addressing (will need more pointers since I don't think
> arm has 'reg + reg + offset' addressing.
>
> >
> > Suggested-by: David Laight <david.laight.linux@gmail.com>
> > Signed-off-by: Demian Shulhan <demyansh@gmail.com>
> > ---
> >  arch/arm64/include/asm/checksum.h |   3 +
> >  arch/arm64/lib/csum.c             | 100 ++++++++++++++++++++++++++++++
> >  2 files changed, 103 insertions(+)
> >
> > diff --git a/arch/arm64/include/asm/checksum.h b/arch/arm64/include/asm/checksum.h
> > index dc52b733675d..9f766db01152 100644
> > --- a/arch/arm64/include/asm/checksum.h
> > +++ b/arch/arm64/include/asm/checksum.h
> > @@ -44,6 +44,9 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl)
> >  extern unsigned int do_csum(const unsigned char *buff, int len);
> >  #define do_csum do_csum
> >
> > +#define _HAVE_ARCH_CSUM_AND_COPY
> > +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len);
> > +
> >  #include <asm-generic/checksum.h>
> >
> >  #endif       /* __ASM_CHECKSUM_H */
> > diff --git a/arch/arm64/lib/csum.c b/arch/arm64/lib/csum.c
> > index 75db718e5b6a..a9617cde28c1 100644
> > --- a/arch/arm64/lib/csum.c
> > +++ b/arch/arm64/lib/csum.c
> > @@ -2,8 +2,10 @@
> >  // Copyright (C) 2019-2020 Arm Ltd.
> >
> >  #include <linux/compiler.h>
> > +#include <linux/export.h>
> >  #include <linux/kasan-checks.h>
> >  #include <linux/kernel.h>
> > +#include <linux/unaligned.h>
> >
> >  #include <net/checksum.h>
> >
> > @@ -116,6 +118,104 @@ unsigned int __no_sanitize_address do_csum(const unsigned char *buff, int len)
> >       return sum >> 16;
> >  }
> >
> > +static __always_inline __wsum csum_fold128(__uint128_t sum)
> > +{
> > +     u64 s;
> > +
> > +     sum += (sum >> 64) | (sum << 64);
> > +     s = sum >> 64;
> > +     s += (s >> 32) | (s << 32);
> > +     return (__force __wsum)(u32)(s >> 32);
> > +}
> > +
> > +/*
> > + * Single-pass copy and ones' complement sum. Each word is read once into a
> > + * GPR before storing and summing, so the returned sum always matches @dst
> > + * even if @src is concurrently modified.
> > + */
> > +__wsum csum_partial_copy_nocheck(const void *src, void *dst, int len)
> > +{
> > +     const u8 *s = src;
> > +     u8 *d = dst;
> > +     __uint128_t sum0 = 0, sum1 = 0, sum2 = 0, sum3 = 0;
> > +     unsigned int n;
> > +
> > +     if (unlikely(len <= 0))
> > +             return 0;
>
> Hmmm...
> Does zero actually happen often enough for that to be a gain?
> Even with the unlikely() it is likely to slow down the normal path.
> And that assumes that the branch isn't mispredicted and that gcc
> doesn't ignore the unlikely().
>
> > +
> > +     n = (unsigned int)len >> 5;
> > +     if (n) {
> > +             do {
> > +                     u64 v0, v1;
> > +
> > +                     v0 = get_unaligned((const u64 *)s);
> > +                     v1 = get_unaligned((const u64 *)(s + 8));
> > +                     put_unaligned(v0, (u64 *)d);
> > +                     put_unaligned(v1, (u64 *)(d + 8));
> > +                     sum0 += v0;
> > +                     sum1 += v1;
> > +
> > +                     v0 = get_unaligned((const u64 *)(s + 16));
> > +                     v1 = get_unaligned((const u64 *)(s + 24));
> > +                     put_unaligned(v0, (u64 *)(d + 16));
> > +                     put_unaligned(v1, (u64 *)(d + 24));
> > +                     sum2 += v0;
> > +                     sum3 += v1;
>
> Is using sum0/1 here actually slower?
> My 'gut feel' is that it won't be - and saves the adds below.
>
> It is also worth checking that gcc doesn't zero extend v0 into a
> register and then do a full 128bit add.
> I'm sure I looked at the code using only 64bit variables for:
>                         sum0 += v0;
>                         carry0 += sum0 < v0;
> both gcc and clang used 'add 0 with carry' for the second line.
>
> > +
> > +                     s += 32;
> > +                     d += 32;
> > +             } while (--n);
>
> It is a shame to have to update 3 variables here.
> They may be taking the 3rd clock.
>
> David
>
>
> > +
> > +             sum0 += sum2;
> > +             sum1 += sum3;
> > +     }
> > +
> > +     if (len & 16) {
> > +             u64 v0 = get_unaligned((const u64 *)s);
> > +             u64 v1 = get_unaligned((const u64 *)(s + 8));
> > +
> > +             put_unaligned(v0, (u64 *)d);
> > +             put_unaligned(v1, (u64 *)(d + 8));
> > +             sum0 += v0;
> > +             sum1 += v1;
> > +             s += 16;
> > +             d += 16;
> > +     }
> > +     if (len & 8) {
> > +             u64 v = get_unaligned((const u64 *)s);
> > +
> > +             put_unaligned(v, (u64 *)d);
> > +             sum0 += v;
> > +             s += 8;
> > +             d += 8;
> > +     }
> > +     if (len & 4) {
> > +             u32 v = get_unaligned((const u32 *)s);
> > +
> > +             put_unaligned(v, (u32 *)d);
> > +             sum1 += v;
> > +             s += 4;
> > +             d += 4;
> > +     }
> > +     if (len & 2) {
> > +             u16 v = get_unaligned((const u16 *)s);
> > +
> > +             put_unaligned(v, (u16 *)d);
> > +             sum0 += v;
> > +             s += 2;
> > +             d += 2;
> > +     }
> > +     if (len & 1) {
> > +             u8 v = *s;
> > +
> > +             *d = v;
> > +             sum1 += v;
> > +     }
> > +
> > +     return csum_fold128(sum0 + sum1);
> > +}
> > +EXPORT_SYMBOL(csum_partial_copy_nocheck);
> > +
> >  __sum16 csum_ipv6_magic(const struct in6_addr *saddr,
> >                       const struct in6_addr *daddr,
> >                       __u32 len, __u8 proto, __wsum csum)
>

Re: [PATCH v2 1/2] arm64: csum: Add fused copy and Internet checksum

[David Laight]

On Sun, 15 Nov 2026 19:25:24 +0200
Demian Shulhan <demyansh@gmail.com> wrote:

> Hi David,
> 
> > Is that right?
> > I thought ldp/stp required aligned addresses.
> > The code below is using get_unaligned().
> > I think the timings below are for ldp/stp?  
> 
> Unlike 32-bit ARM (ldrd/strd), AArch64 GPR ldp/stp support unaligned
> addresses in hardware on Normal memory. Both GCC 15 and Clang 21 merge
> the adjacent get_unaligned()/put_unaligned() pairs into ldp/stp, and
> the timings are from that code. Unaligned (src+1, dst+3) is only ~5%
> slower due to cache-line splits (~28.8-29.1 GB/s vs ~30.3-31.1 GB/s).

Ok, I knew there was a problem - but it is 32-bit only.

> > That is topping out at 10 bytes/clock.
> > Each loop iteration copies 32 bytes - so must take 3 clocks.
> > Which means it must be doing 16 byte read/write in a single clock.
> > So rather begs the question of why it is limited by the loop control
> > and checksum instructions rather than the memory accesses.
> > The checksum code needs 4 ALU instructions (for 16 bytes), if the CPU
> > can execute more than that then there is half a chance of getting the
> > loop down to 2 clocks.  
> 
> For 32 bytes, the loop is 16 instructions (4x ldp/stp, 8x adds/cinc,
> 2x add, cmp, b.ne). On Neoverse-V2 (6 ALU pipes, 8-wide dispatch) 12
> ALU/branch ops plus 4 paired memory ops put the floor right around ~3
> clocks/iteration (~5.2 IPC).

The floor would be nearer 2.
There are 11 ALU and a branch to fit in two clocks.

> > Just moving one (or both) the pointer updates into the middle may help.
> > Otherwise you try using negative offsets from the end of the buffer
> 
> Moving the pointer updates into the middle does help, it shaves up to
> ~5% off across sizes.

Thought it might - you were trying to do to many alu ops at the end.

> > and 'reg + reg' addressing (will need more pointers since I don't think
> > arm has 'reg + reg + offset' addressing. 
> For 'reg + reg', ldp/stp only support immediate offsets, so indexed
> addressing would force single-register ldr/str and double the memory
> instructions.

I didn't think that was possible.
x86 can do 'reg + reg + offset' but I have a feeling it ends up using
an alu u-op for one of the adds (on intel and/or amd). Nothing else
explains why we can't get move that 12 bytes/clock for ip cksum.

> 
> 
> > > + if (unlikely(len <= 0))
> > > + return 0;  
> >
> > Hmmm...
> > Does zero actually happen often enough for that to be a gain?
> > Even with the unlikely() it is likely to slow down the normal path.
> > And that assumes that the branch isn't mispredicted and that gcc
> > doesn't ignore the unlikely().  
> 
> len == 0 would fall through fine, but since len is a signed int in the
> csum_partial_copy_nocheck() prototype, this guards against negative len
> wrapping (unsigned int)len >> 5 (matching do_csum() right above it).
> Both compilers place the return 0 block out-of-line at the end of the
> function.

You've still got a branch that can (plausibly) get predicted taken.
 
> > > + sum2 += v0;
> > > + sum3 += v1;  
> >
> > Is using sum0/1 here actually slower?
> > My 'gut feel' is that it won't be - and saves the adds below.  
> 
> For 64-128B your gut feel is right - 2 accumulators is 2-10% faster from
> saving the post-loop fold, and on GCC 15 it matches 4 accumulators on
> larger buffers too (GCC folds v0 + v2 first, keeping a 1-cycle chain in
> 18 insns). The catch is Clang 21: with 2 accumulators it chains both
> adds/cinc updates onto the same accumulator and drops ~12% for >= 512B
> (27.0 vs 30.5-31.2 GB/s).

I hate that - why can't the compilers do what they are told :-)
gcc always compiles (a + b) + (c + d) as (((a + b) + c) + d) making
the register dependency chain (which matters far more then the number
of instructions) one clock longer.

Sometimes an OPTIMIZER_HIDE_VAR() can help - but it tends to add
a register-register move.

> > It is also worth checking that gcc doesn't zero extend v0 into a
> > register and then do a full 128bit add.
> > I'm sure I looked at the code using only 64bit variables for:
> > sum0 += v0;
> > carry0 += sum0 < v0;
> > both gcc and clang used 'add 0 with carry' for the second line.  
> 
> Checked - neither compiler zero-extends v0, both emit adds + cinc
> directly on the 64-bit registers.

This week :-)
clang tends to get it right, gcc has a habit of allocating a register
for the high part.
For u64 on 32bit (effectively the same) I've seen it spill constant
zeros to stack and do multiplies by registers that are zero. 

> 
> 
> > > + s += 32;
> > > + d += 32;
> > > + } while (--n);  
> >
> > It is a shame to have to update 3 variables here.
> > They may be taking the 3rd clock.  
> 
> Switching --n to an end pointer (s != end) and moving s += 32 / d += 32
> between the loads and stores drops the 3rd update on Clang (80 -> 76
> insns total) and cleans up GCC's loop setup (88 -> 81 insns), gaining
> +0.4% to +4.9% across sizes.
> 
> Here is the resulting 16-instruction loop with an end pointer and 4
> accumulators:
> 
>   GCC 15.2:                        Clang 21.1:
>     ldp  x10, x9, [x0]               ldp  x17, x18, [x0]
>     add  x11, x11, #0x20             ldp  x3, x4, [x0, #0x10]
>     ldp  x8, x7, [x0, #16]           add  x0, x0, #0x20
>     add  x0, x0, #0x20               adds x15, x15, x17
>     stp  x10, x9, [x11, #-32]        stp  x17, x18, [x1]
>     adds x3, x10, x3                 cinc x16, x16, hs
>     stp  x8, x7, [x11, #-16]         adds x13, x13, x18
>     cinc x12, x12, cs                stp  x3, x4, [x1, #0x10]
>     adds x4, x9, x4                  cinc x14, x14, hs
>     cinc x13, x13, cs                adds x8, x8, x3
>     adds x6, x8, x6                  add  x1, x1, #0x20
>     cinc x16, x16, cs                cinc x9, x9, hs
>     adds x5, x7, x5                  adds x10, x10, x4
>     cinc x15, x15, cs                cinc x11, x11, hs
>     cmp  x14, x0                     cmp  x0, x12
>     b.ne .Loop                       b.ne .Loop

Both those manage to get the ldp early (probably good)
but there are a lot of alu instructions at the end.
If moving the 's/d += 32' moved those instructions to before
the stp, then trying to persuade the compiler to move the
sum0/1/2/3 earlier might help as well.
I was thinking on the lines of:
	v0 = get_unaligned((const u64 *)s);
	v1 = get_unaligned((const u64 *)s + 8);
	sum0 += v0;
	sum1 += v1;
	put_unaligned(v0, (u64 *)d);
	put_unaligned(v1, (u64 *)d + 8);
	s += 32;
	d += 32;
	v0 = get_unaligned((const u64 *)s + 16 - 32);
	v1 = get_unaligned((const u64 *)s + 24 - 32);
	sum2 += v0;
	sum3 += v1;
	put_unaligned(v0, (u64 *)d + 16 - 32);
	put_unaligned(v1, (u64 *)d + 24 - 32);

but it might make things worse
(and the compiler might just ignore you).

I always try to do tests in userspace.
Makes it much quicker to see the effect of changes and you can
test just the inner part of the loop separately from the tail code.

As it stands this code it reasonably generic.
Although cpu without ldp/stp probably don't need the extra unroll.

David

> 
> I'll switch to the end-pointer loop with interleaved pointer updates in
> v3.
> 
> Thanks!
> Demian