* [PATCH] raid6: arm64: add SVE optimized implementation for syndrome generation
From: Demian Shulhan @ 2026-03-17 11:17 UTC (permalink / raw)
To: Song Liu, Yu Kuai; +Cc: Li Nan, linux-raid, linux-kernel, Demian Shulhan
Implement Scalable Vector Extension (SVE) optimized routines for RAID6
syndrome generation and recovery on ARM64.
The SVE instruction set allows for variable vector lengths (from 128 to
2048 bits), scaling automatically with the hardware capabilities. This
implementation handles arbitrary SVE vector lengths using the `cntb`
instruction to determine the runtime vector length.
The implementation introduces `svex1`, `svex2`, and `svex4` algorithms.
The `svex4` algorithm utilizes loop unrolling by 4 blocks per iteration
and manual software pipelining (interleaving memory loads with XORs)
to minimize instruction dependency stalls and maximize CPU pipeline
utilization and memory bandwidth.
Performance was tested on an AWS Graviton3 (Neoverse-V1) instance which
features 256-bit SVE vector length. The `svex4` implementation outperforms
the existing 128-bit `neonx4` baseline for syndrome generation:
raid6: svex4 gen() 19688 MB/s
raid6: svex2 gen() 18610 MB/s
raid6: svex1 gen() 19254 MB/s
raid6: neonx8 gen() 18554 MB/s
raid6: neonx4 gen() 19612 MB/s
raid6: neonx2 gen() 16248 MB/s
raid6: neonx1 gen() 13591 MB/s
raid6: using algorithm svex4 gen() 19688 MB/s
raid6: .... xor() 11212 MB/s, rmw enabled
raid6: using neon recovery algorithm
Note that for the recovery path (`xor_syndrome`), NEON may still be
selected dynamically by the algorithm benchmark, as the recovery
workload is heavily memory-bound.
Signed-off-by: Demian Shulhan <demyansh@gmail.com>
---
include/linux/raid/pq.h | 3 +
lib/raid6/Makefile | 5 +
lib/raid6/algos.c | 5 +
lib/raid6/sve.c | 675 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 688 insertions(+)
create mode 100644 lib/raid6/sve.c
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 2467b3be15c9..787cc57aea9d 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -140,6 +140,9 @@ extern const struct raid6_calls raid6_neonx1;
extern const struct raid6_calls raid6_neonx2;
extern const struct raid6_calls raid6_neonx4;
extern const struct raid6_calls raid6_neonx8;
+extern const struct raid6_calls raid6_svex1;
+extern const struct raid6_calls raid6_svex2;
+extern const struct raid6_calls raid6_svex4;
/* Algorithm list */
extern const struct raid6_calls * const raid6_algos[];
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
index 5be0a4e60ab1..6cdaa6f206fb 100644
--- a/lib/raid6/Makefile
+++ b/lib/raid6/Makefile
@@ -8,6 +8,7 @@ raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o
raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \
vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
+raid6_pq-$(CONFIG_ARM64_SVE) += sve.o
raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o
raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o
raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o
@@ -67,6 +68,10 @@ CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU)
+
+CFLAGS_sve.o += $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_sve.o += $(CC_FLAGS_NO_FPU)
+
targets += neon1.c neon2.c neon4.c neon8.c
$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE
$(call if_changed,unroll)
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 799e0e5eac26..0ae73c3a4be3 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -66,6 +66,11 @@ const struct raid6_calls * const raid6_algos[] = {
&raid6_neonx2,
&raid6_neonx1,
#endif
+#ifdef CONFIG_ARM64_SVE
+ &raid6_svex4,
+ &raid6_svex2,
+ &raid6_svex1,
+#endif
#ifdef CONFIG_LOONGARCH
#ifdef CONFIG_CPU_HAS_LASX
&raid6_lasx,
diff --git a/lib/raid6/sve.c b/lib/raid6/sve.c
new file mode 100644
index 000000000000..afcf46b89a3d
--- /dev/null
+++ b/lib/raid6/sve.c
@@ -0,0 +1,675 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RAID-6 syndrome calculation using ARM SVE instructions
+ */
+
+#include <linux/raid/pq.h>
+
+#ifdef __KERNEL__
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+#else
+#define scoped_ksimd()
+#define system_supports_sve() (1)
+#endif
+
+static void raid6_sve1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "mov z1.d, z0.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4"
+ );
+}
+
+static void raid6_sve1_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "eor z0.d, z0.d, z1.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "eor z1.d, z1.d, z2.d\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4"
+ );
+}
+
+static void raid6_sve2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [x6, x8]\n"
+ "mov z1.d, z0.d\n"
+ "mov z6.d, z5.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+
+ "add x5, x5, x3\n"
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8"
+ );
+}
+
+static void raid6_sve2_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z6.b, p0/z, [x6, x8]\n"
+
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "ld1b z5.b, p0/z, [%[p], x8]\n"
+
+ "eor z0.d, z0.d, z1.d\n"
+ "eor z5.d, z5.d, z6.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [%[q], x8]\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+
+ "add x5, x5, x3\n"
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8"
+ );
+}
+
+static void raid6_sve4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z10.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z15.b, p0/z, [x6, x8]\n"
+
+ "mov z1.d, z0.d\n"
+ "mov z6.d, z5.d\n"
+ "mov z11.d, z10.d\n"
+ "mov z16.d, z15.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ // software pipelining: load data early
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [x6, x8]\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z10.d, z10.d, z12.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+ "eor z15.d, z15.d, z17.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z10.b, p0, [%[p], x8]\n"
+ "st1b z11.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z15.b, p0, [%[p], x8]\n"
+ "st1b z16.b, p0, [%[q], x8]\n"
+
+ "add x8, x3, x3\n"
+ "add x5, x5, x8, lsl #1\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8",
+ "z10", "z11", "z12", "z13",
+ "z15", "z16", "z17", "z18"
+ );
+}
+
+static void raid6_sve4_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ int z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z6.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z11.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z16.b, p0/z, [x6, x8]\n"
+
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [%[p], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z10.b, p0/z, [%[p], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z15.b, p0/z, [%[p], x8]\n"
+
+ "eor z0.d, z0.d, z1.d\n"
+ "eor z5.d, z5.d, z6.d\n"
+ "eor z10.d, z10.d, z11.d\n"
+ "eor z15.d, z15.d, z16.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ // software pipelining: load data early
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [x6, x8]\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z10.d, z10.d, z12.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+ "eor z15.d, z15.d, z17.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ // Load q and XOR
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [%[q], x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+
+ // Store results
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z10.b, p0, [%[p], x8]\n"
+ "st1b z11.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z15.b, p0, [%[p], x8]\n"
+ "st1b z16.b, p0, [%[q], x8]\n"
+
+ "add x8, x3, x3\n"
+ "add x5, x5, x8, lsl #1\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8",
+ "z10", "z11", "z12", "z13",
+ "z15", "z16", "z17", "z18"
+ );
+}
+
+#define RAID6_SVE_WRAPPER(_n) \
+ static void raid6_sve ## _n ## _gen_syndrome(int disks, \
+ size_t bytes, void **ptrs) \
+ { \
+ scoped_ksimd() \
+ raid6_sve ## _n ## _gen_syndrome_real(disks, \
+ (unsigned long)bytes, ptrs); \
+ } \
+ static void raid6_sve ## _n ## _xor_syndrome(int disks, \
+ int start, int stop, \
+ size_t bytes, void **ptrs) \
+ { \
+ scoped_ksimd() \
+ raid6_sve ## _n ## _xor_syndrome_real(disks, \
+ start, stop, (unsigned long)bytes, ptrs);\
+ } \
+ struct raid6_calls const raid6_svex ## _n = { \
+ raid6_sve ## _n ## _gen_syndrome, \
+ raid6_sve ## _n ## _xor_syndrome, \
+ raid6_have_sve, \
+ "svex" #_n, \
+ 0 \
+ }
+
+static int raid6_have_sve(void)
+{
+ return system_supports_sve();
+}
+
+RAID6_SVE_WRAPPER(1);
+RAID6_SVE_WRAPPER(2);
+RAID6_SVE_WRAPPER(4);
--
2.43.0
^ permalink raw reply related
* Re: Release mdadm-4.6
From: Xose Vazquez Perez @ 2026-03-18 9:35 UTC (permalink / raw)
To: Xiao Ni, linux-raid
Cc: Song Liu, Yu Kuai, Nigel Croxon, Mariusz Tkaczyk, Martin Wilck
In-Reply-To: <CALTww2_2pjjOSsVj-WT++m35555CjnPWokU7fut+HwUJVV-=ZQ@mail.gmail.com>
On 3/16/26 4:45 PM, Xiao Ni wrote:
> I'm glad to announce mdadm-4.5. It is published to both remotes
> (github and kernel.org). Here are some highlights from CHANGELOG.md.
>
> # Release [mdadm-4.6](https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git/log/?h=mdadm-4.6)
Could you please upload the corresponding tarball to the official
kernel.org dir ( https://kernel.org/pub/linux/utils/raid/mdadm/ ) ?
Thanks.
^ permalink raw reply
* Re: Release mdadm-4.6
From: Xiao Ni @ 2026-03-18 10:30 UTC (permalink / raw)
To: Xose Vazquez Perez
Cc: linux-raid, Song Liu, Yu Kuai, Nigel Croxon, Mariusz Tkaczyk,
Martin Wilck
In-Reply-To: <ebafe7e7-4166-4c1b-92b0-378db8f79ebf@gmail.com>
On Wed, Mar 18, 2026 at 5:35 PM Xose Vazquez Perez
<xose.vazquez@gmail.com> wrote:
>
> On 3/16/26 4:45 PM, Xiao Ni wrote:
>
> > I'm glad to announce mdadm-4.5. It is published to both remotes
> > (github and kernel.org). Here are some highlights from CHANGELOG.md.
> >
> > # Release [mdadm-4.6](https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git/log/?h=mdadm-4.6)
> Could you please upload the corresponding tarball to the official
> kernel.org dir ( https://kernel.org/pub/linux/utils/raid/mdadm/ ) ?
Hi, I'll try to do this. I took the maintainer job, but the job
doesn't contain this part. I'll figure it out. But it needs some time.
If you want to download the package in spec file, you can do like this:
1 Name: mdadm
2 Version: 4.4
3 # extraversion is used to define rhel internal version
4 %define extraversion 3
5 Release: %{extraversion}%{?dist}
6 Summary: The mdadm program controls Linux md devices
(software RAID arrays)
7 URL: https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git
8 License: GPLv2+
9
10 Source:
https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git/snapshot/%{name}-%{version}%{?subversion:-%{subversion}}.tar.gz
This is my way. Hope this helps you.
Best Regards
Xiao
>
> Thanks.
>
^ permalink raw reply
* Re: [PATCH] raid6: arm64: add SVE optimized implementation for syndrome generation
From: kernel test robot @ 2026-03-18 11:24 UTC (permalink / raw)
To: Demian Shulhan, Song Liu, Yu Kuai
Cc: llvm, oe-kbuild-all, Li Nan, linux-raid, linux-kernel,
Demian Shulhan
In-Reply-To: <20260317111706.2756977-1-demyansh@gmail.com>
Hi Demian,
kernel test robot noticed the following build warnings:
[auto build test WARNING on akpm-mm/mm-nonmm-unstable]
[also build test WARNING on linus/master v7.0-rc4 next-20260317]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Demian-Shulhan/raid6-arm64-add-SVE-optimized-implementation-for-syndrome-generation/20260317-224300
base: https://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm.git mm-nonmm-unstable
patch link: https://lore.kernel.org/r/20260317111706.2756977-1-demyansh%40gmail.com
patch subject: [PATCH] raid6: arm64: add SVE optimized implementation for syndrome generation
config: arm64-allmodconfig (https://download.01.org/0day-ci/archive/20260318/202603181940.cFwYmYoi-lkp@intel.com/config)
compiler: clang version 19.1.7 (https://github.com/llvm/llvm-project cd708029e0b2869e80abe31ddb175f7c35361f90)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20260318/202603181940.cFwYmYoi-lkp@intel.com/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202603181940.cFwYmYoi-lkp@intel.com/
All warnings (new ones prefixed by >>):
>> lib/raid6/sve.c:70:34: warning: value size does not match register size specified by the constraint and modifier [-Wasm-operand-widths]
70 | : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
| ^
lib/raid6/sve.c:34:22: note: use constraint modifier "w"
34 | "ldr x6, [%[dptr], %[z0], lsl #3]\n"
| ^~~~~
| %w[z0]
lib/raid6/sve.c:151:34: warning: value size does not match register size specified by the constraint and modifier [-Wasm-operand-widths]
151 | : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
| ^
lib/raid6/sve.c:96:22: note: use constraint modifier "w"
96 | "ldr x6, [%[dptr], %[z0], lsl #3]\n"
| ^~~~~
| %w[z0]
lib/raid6/sve.c:229:34: warning: value size does not match register size specified by the constraint and modifier [-Wasm-operand-widths]
229 | : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
| ^
lib/raid6/sve.c:176:22: note: use constraint modifier "w"
176 | "ldr x6, [%[dptr], %[z0], lsl #3]\n"
| ^~~~~
| %w[z0]
lib/raid6/sve.c:340:34: warning: value size does not match register size specified by the constraint and modifier [-Wasm-operand-widths]
340 | : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
| ^
lib/raid6/sve.c:256:22: note: use constraint modifier "w"
256 | "ldr x6, [%[dptr], %[z0], lsl #3]\n"
| ^~~~~
| %w[z0]
lib/raid6/sve.c:455:34: warning: value size does not match register size specified by the constraint and modifier [-Wasm-operand-widths]
455 | : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
| ^
lib/raid6/sve.c:366:22: note: use constraint modifier "w"
366 | "ldr x6, [%[dptr], %[z0], lsl #3]\n"
| ^~~~~
| %w[z0]
lib/raid6/sve.c:634:34: warning: value size does not match register size specified by the constraint and modifier [-Wasm-operand-widths]
634 | : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
| ^
lib/raid6/sve.c:484:22: note: use constraint modifier "w"
484 | "ldr x6, [%[dptr], %[z0], lsl #3]\n"
| ^~~~~
| %w[z0]
6 warnings generated.
vim +70 lib/raid6/sve.c
15
16 static void raid6_sve1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
17 {
18 u8 **dptr = (u8 **)ptrs;
19 u8 *p, *q;
20 int z0 = disks - 3;
21
22 p = dptr[z0 + 1];
23 q = dptr[z0 + 2];
24
25 asm volatile(
26 ".arch armv8.2-a+sve\n"
27 "ptrue p0.b\n"
28 "cntb x3\n"
29 "mov w4, #0x1d\n"
30 "dup z4.b, w4\n"
31 "mov x5, #0\n"
32
33 "0:\n"
34 "ldr x6, [%[dptr], %[z0], lsl #3]\n"
35 "ld1b z0.b, p0/z, [x6, x5]\n"
36 "mov z1.d, z0.d\n"
37
38 "mov w7, %w[z0]\n"
39 "sub w7, w7, #1\n"
40
41 "1:\n"
42 "cmp w7, #0\n"
43 "blt 2f\n"
44
45 "mov z3.d, z1.d\n"
46 "asr z3.b, p0/m, z3.b, #7\n"
47 "lsl z1.b, p0/m, z1.b, #1\n"
48
49 "and z3.d, z3.d, z4.d\n"
50 "eor z1.d, z1.d, z3.d\n"
51
52 "sxtw x8, w7\n"
53 "ldr x6, [%[dptr], x8, lsl #3]\n"
54 "ld1b z2.b, p0/z, [x6, x5]\n"
55
56 "eor z1.d, z1.d, z2.d\n"
57 "eor z0.d, z0.d, z2.d\n"
58
59 "sub w7, w7, #1\n"
60 "b 1b\n"
61 "2:\n"
62
63 "st1b z0.b, p0, [%[p], x5]\n"
64 "st1b z1.b, p0, [%[q], x5]\n"
65
66 "add x5, x5, x3\n"
67 "cmp x5, %[bytes]\n"
68 "blt 0b\n"
69 :
> 70 : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
71 [p] "r" (p), [q] "r" (q)
72 : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
73 "z0", "z1", "z2", "z3", "z4"
74 );
75 }
76
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply
* Re: Release mdadm-4.6
From: Xose Vazquez Perez @ 2026-03-18 12:06 UTC (permalink / raw)
To: Xiao Ni
Cc: linux-raid, Song Liu, Yu Kuai, Nigel Croxon, Mariusz Tkaczyk,
Martin Wilck
In-Reply-To: <CALTww28+ZdE4VzP1DqU=nFRibSPtu8L1qSOXKabv0ShdsAOO7A@mail.gmail.com>
On 3/18/26 11:30 AM, Xiao Ni wrote:
> Hi, I'll try to do this. I took the maintainer job, but the job
> doesn't contain this part. I'll figure it out. But it needs some time.
>
> If you want to download the package in spec file, you can do like this:
>
> 1 Name: mdadm
> 2 Version: 4.4
> 3 # extraversion is used to define rhel internal version
> 4 %define extraversion 3
> 5 Release: %{extraversion}%{?dist}
> 6 Summary: The mdadm program controls Linux md devices
> (software RAID arrays)
> 7 URL: https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git
> 8 License: GPLv2+
> 9
> 10 Source:
> https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git/snapshot/%{name}-%{version}%{?subversion:-%{subversion}}.tar.gz
All versions are listed at https://git.kernel.org/pub/scm/utils/mdadm/mdadm.git/refs/tags
But package maintainers, or automated build systems, often rely on the kernel.org url
( https://www.kernel.org/pub/linux/utils/raid/mdadm ) for stable source downloads.
Thanks.
^ permalink raw reply
* [PATCH v2] raid6: arm64: add SVE optimized implementation for syndrome generation
From: Demian Shulhan @ 2026-03-18 15:01 UTC (permalink / raw)
To: Song Liu, Yu Kuai
Cc: Li Nan, linux-raid, linux-kernel, Demian Shulhan,
kernel test robot
Implement Scalable Vector Extension (SVE) optimized routines for RAID6
syndrome generation and recovery on ARM64.
The SVE instruction set allows for variable vector lengths (from 128 to
2048 bits), scaling automatically with the hardware capabilities. This
implementation handles arbitrary SVE vector lengths using the `cntb`
instruction to determine the runtime vector length.
The implementation introduces `svex1`, `svex2`, and `svex4` algorithms.
The `svex4` algorithm utilizes loop unrolling by 4 blocks per iteration
and manual software pipelining (interleaving memory loads with XORs)
to minimize instruction dependency stalls and maximize CPU pipeline
utilization and memory bandwidth.
Performance was tested on an AWS Graviton3 (Neoverse-V1) instance which
features 256-bit SVE vector length. The `svex4` implementation outperforms
the existing 128-bit `neonx4` baseline for syndrome generation:
raid6: svex4 gen() 19688 MB/s
raid6: svex2 gen() 18610 MB/s
raid6: svex1 gen() 19254 MB/s
raid6: neonx8 gen() 18554 MB/s
raid6: neonx4 gen() 19612 MB/s
raid6: neonx2 gen() 16248 MB/s
raid6: neonx1 gen() 13591 MB/s
raid6: using algorithm svex4 gen() 19688 MB/s
raid6: .... xor() 11212 MB/s, rmw enabled
raid6: using neon recovery algorithm
Note that for the recovery path (`xor_syndrome`), NEON may still be
selected dynamically by the algorithm benchmark, as the recovery
workload is heavily memory-bound.
Signed-off-by: Demian Shulhan <demyansh@gmail.com>
Reported-by: kernel test robot <lkp@intel.com>
Closes: https://lore.kernel.org/oe-kbuild-all/202603181940.cFwYmYoi-lkp@intel.com/
---
include/linux/raid/pq.h | 3 +
lib/raid6/Makefile | 5 +
lib/raid6/algos.c | 5 +
lib/raid6/sve.c | 675 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 688 insertions(+)
create mode 100644 lib/raid6/sve.c
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 2467b3be15c9..787cc57aea9d 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -140,6 +140,9 @@ extern const struct raid6_calls raid6_neonx1;
extern const struct raid6_calls raid6_neonx2;
extern const struct raid6_calls raid6_neonx4;
extern const struct raid6_calls raid6_neonx8;
+extern const struct raid6_calls raid6_svex1;
+extern const struct raid6_calls raid6_svex2;
+extern const struct raid6_calls raid6_svex4;
/* Algorithm list */
extern const struct raid6_calls * const raid6_algos[];
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
index 5be0a4e60ab1..6cdaa6f206fb 100644
--- a/lib/raid6/Makefile
+++ b/lib/raid6/Makefile
@@ -8,6 +8,7 @@ raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o
raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \
vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
+raid6_pq-$(CONFIG_ARM64_SVE) += sve.o
raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o
raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o
raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o
@@ -67,6 +68,10 @@ CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU)
+
+CFLAGS_sve.o += $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_sve.o += $(CC_FLAGS_NO_FPU)
+
targets += neon1.c neon2.c neon4.c neon8.c
$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE
$(call if_changed,unroll)
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 799e0e5eac26..0ae73c3a4be3 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -66,6 +66,11 @@ const struct raid6_calls * const raid6_algos[] = {
&raid6_neonx2,
&raid6_neonx1,
#endif
+#ifdef CONFIG_ARM64_SVE
+ &raid6_svex4,
+ &raid6_svex2,
+ &raid6_svex1,
+#endif
#ifdef CONFIG_LOONGARCH
#ifdef CONFIG_CPU_HAS_LASX
&raid6_lasx,
diff --git a/lib/raid6/sve.c b/lib/raid6/sve.c
new file mode 100644
index 000000000000..d52937f806d4
--- /dev/null
+++ b/lib/raid6/sve.c
@@ -0,0 +1,675 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RAID-6 syndrome calculation using ARM SVE instructions
+ */
+
+#include <linux/raid/pq.h>
+
+#ifdef __KERNEL__
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+#else
+#define scoped_ksimd()
+#define system_supports_sve() (1)
+#endif
+
+static void raid6_sve1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "mov z1.d, z0.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4"
+ );
+}
+
+static void raid6_sve1_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "eor z0.d, z0.d, z1.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "eor z1.d, z1.d, z2.d\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4"
+ );
+}
+
+static void raid6_sve2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [x6, x8]\n"
+ "mov z1.d, z0.d\n"
+ "mov z6.d, z5.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+
+ "add x5, x5, x3\n"
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8"
+ );
+}
+
+static void raid6_sve2_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z6.b, p0/z, [x6, x8]\n"
+
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "ld1b z5.b, p0/z, [%[p], x8]\n"
+
+ "eor z0.d, z0.d, z1.d\n"
+ "eor z5.d, z5.d, z6.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [%[q], x8]\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+
+ "add x5, x5, x3\n"
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8"
+ );
+}
+
+static void raid6_sve4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z10.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z15.b, p0/z, [x6, x8]\n"
+
+ "mov z1.d, z0.d\n"
+ "mov z6.d, z5.d\n"
+ "mov z11.d, z10.d\n"
+ "mov z16.d, z15.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ // software pipelining: load data early
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [x6, x8]\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z10.d, z10.d, z12.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+ "eor z15.d, z15.d, z17.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z10.b, p0, [%[p], x8]\n"
+ "st1b z11.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z15.b, p0, [%[p], x8]\n"
+ "st1b z16.b, p0, [%[q], x8]\n"
+
+ "add x8, x3, x3\n"
+ "add x5, x5, x8, lsl #1\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8",
+ "z10", "z11", "z12", "z13",
+ "z15", "z16", "z17", "z18"
+ );
+}
+
+static void raid6_sve4_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z6.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z11.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z16.b, p0/z, [x6, x8]\n"
+
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [%[p], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z10.b, p0/z, [%[p], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z15.b, p0/z, [%[p], x8]\n"
+
+ "eor z0.d, z0.d, z1.d\n"
+ "eor z5.d, z5.d, z6.d\n"
+ "eor z10.d, z10.d, z11.d\n"
+ "eor z15.d, z15.d, z16.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ // software pipelining: load data early
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [x6, x8]\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z10.d, z10.d, z12.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+ "eor z15.d, z15.d, z17.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ // Load q and XOR
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [%[q], x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+
+ // Store results
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z10.b, p0, [%[p], x8]\n"
+ "st1b z11.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z15.b, p0, [%[p], x8]\n"
+ "st1b z16.b, p0, [%[q], x8]\n"
+
+ "add x8, x3, x3\n"
+ "add x5, x5, x8, lsl #1\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8",
+ "z10", "z11", "z12", "z13",
+ "z15", "z16", "z17", "z18"
+ );
+}
+
+#define RAID6_SVE_WRAPPER(_n) \
+ static void raid6_sve ## _n ## _gen_syndrome(int disks, \
+ size_t bytes, void **ptrs) \
+ { \
+ scoped_ksimd() \
+ raid6_sve ## _n ## _gen_syndrome_real(disks, \
+ (unsigned long)bytes, ptrs); \
+ } \
+ static void raid6_sve ## _n ## _xor_syndrome(int disks, \
+ int start, int stop, \
+ size_t bytes, void **ptrs) \
+ { \
+ scoped_ksimd() \
+ raid6_sve ## _n ## _xor_syndrome_real(disks, \
+ start, stop, (unsigned long)bytes, ptrs);\
+ } \
+ struct raid6_calls const raid6_svex ## _n = { \
+ raid6_sve ## _n ## _gen_syndrome, \
+ raid6_sve ## _n ## _xor_syndrome, \
+ raid6_have_sve, \
+ "svex" #_n, \
+ 0 \
+ }
+
+static int raid6_have_sve(void)
+{
+ return system_supports_sve();
+}
+
+RAID6_SVE_WRAPPER(1);
+RAID6_SVE_WRAPPER(2);
+RAID6_SVE_WRAPPER(4);
--
2.43.0
^ permalink raw reply related
* [PATCH v2] raid6: arm64: add SVE optimized implementation for syndrome generation
From: Demian Shulhan @ 2026-03-18 15:02 UTC (permalink / raw)
To: Song Liu, Yu Kuai
Cc: Li Nan, linux-raid, linux-kernel, Demian Shulhan,
kernel test robot
Implement Scalable Vector Extension (SVE) optimized routines for RAID6
syndrome generation and recovery on ARM64.
The SVE instruction set allows for variable vector lengths (from 128 to
2048 bits), scaling automatically with the hardware capabilities. This
implementation handles arbitrary SVE vector lengths using the `cntb`
instruction to determine the runtime vector length.
The implementation introduces `svex1`, `svex2`, and `svex4` algorithms.
The `svex4` algorithm utilizes loop unrolling by 4 blocks per iteration
and manual software pipelining (interleaving memory loads with XORs)
to minimize instruction dependency stalls and maximize CPU pipeline
utilization and memory bandwidth.
Performance was tested on an AWS Graviton3 (Neoverse-V1) instance which
features 256-bit SVE vector length. The `svex4` implementation outperforms
the existing 128-bit `neonx4` baseline for syndrome generation:
raid6: svex4 gen() 19688 MB/s
raid6: svex2 gen() 18610 MB/s
raid6: svex1 gen() 19254 MB/s
raid6: neonx8 gen() 18554 MB/s
raid6: neonx4 gen() 19612 MB/s
raid6: neonx2 gen() 16248 MB/s
raid6: neonx1 gen() 13591 MB/s
raid6: using algorithm svex4 gen() 19688 MB/s
raid6: .... xor() 11212 MB/s, rmw enabled
raid6: using neon recovery algorithm
Note that for the recovery path (`xor_syndrome`), NEON may still be
selected dynamically by the algorithm benchmark, as the recovery
workload is heavily memory-bound.
Signed-off-by: Demian Shulhan <demyansh@gmail.com>
Reported-by: kernel test robot <lkp@intel.com>
Closes: https://lore.kernel.org/oe-kbuild-all/202603181940.cFwYmYoi-lkp@intel.com/
---
include/linux/raid/pq.h | 3 +
lib/raid6/Makefile | 5 +
lib/raid6/algos.c | 5 +
lib/raid6/sve.c | 675 ++++++++++++++++++++++++++++++++++++++++
4 files changed, 688 insertions(+)
create mode 100644 lib/raid6/sve.c
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 2467b3be15c9..787cc57aea9d 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -140,6 +140,9 @@ extern const struct raid6_calls raid6_neonx1;
extern const struct raid6_calls raid6_neonx2;
extern const struct raid6_calls raid6_neonx4;
extern const struct raid6_calls raid6_neonx8;
+extern const struct raid6_calls raid6_svex1;
+extern const struct raid6_calls raid6_svex2;
+extern const struct raid6_calls raid6_svex4;
/* Algorithm list */
extern const struct raid6_calls * const raid6_algos[];
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
index 5be0a4e60ab1..6cdaa6f206fb 100644
--- a/lib/raid6/Makefile
+++ b/lib/raid6/Makefile
@@ -8,6 +8,7 @@ raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o
raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \
vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
+raid6_pq-$(CONFIG_ARM64_SVE) += sve.o
raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o
raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o
raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o
@@ -67,6 +68,10 @@ CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU)
CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU)
+
+CFLAGS_sve.o += $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_sve.o += $(CC_FLAGS_NO_FPU)
+
targets += neon1.c neon2.c neon4.c neon8.c
$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE
$(call if_changed,unroll)
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 799e0e5eac26..0ae73c3a4be3 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -66,6 +66,11 @@ const struct raid6_calls * const raid6_algos[] = {
&raid6_neonx2,
&raid6_neonx1,
#endif
+#ifdef CONFIG_ARM64_SVE
+ &raid6_svex4,
+ &raid6_svex2,
+ &raid6_svex1,
+#endif
#ifdef CONFIG_LOONGARCH
#ifdef CONFIG_CPU_HAS_LASX
&raid6_lasx,
diff --git a/lib/raid6/sve.c b/lib/raid6/sve.c
new file mode 100644
index 000000000000..d52937f806d4
--- /dev/null
+++ b/lib/raid6/sve.c
@@ -0,0 +1,675 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * RAID-6 syndrome calculation using ARM SVE instructions
+ */
+
+#include <linux/raid/pq.h>
+
+#ifdef __KERNEL__
+#include <asm/simd.h>
+#include <linux/cpufeature.h>
+#else
+#define scoped_ksimd()
+#define system_supports_sve() (1)
+#endif
+
+static void raid6_sve1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "mov z1.d, z0.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4"
+ );
+}
+
+static void raid6_sve1_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "eor z0.d, z0.d, z1.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "eor z1.d, z1.d, z2.d\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4"
+ );
+}
+
+static void raid6_sve2_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [x6, x8]\n"
+ "mov z1.d, z0.d\n"
+ "mov z6.d, z5.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+
+ "add x5, x5, x3\n"
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8"
+ );
+}
+
+static void raid6_sve2_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z6.b, p0/z, [x6, x8]\n"
+
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "ld1b z5.b, p0/z, [%[p], x8]\n"
+
+ "eor z0.d, z0.d, z1.d\n"
+ "eor z5.d, z5.d, z6.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [%[q], x8]\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+
+ "add x5, x5, x3\n"
+ "add x5, x5, x3\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8"
+ );
+}
+
+static void raid6_sve4_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = disks - 3;
+
+ p = dptr[z0 + 1];
+ q = dptr[z0 + 2];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z0.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z10.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z15.b, p0/z, [x6, x8]\n"
+
+ "mov z1.d, z0.d\n"
+ "mov z6.d, z5.d\n"
+ "mov z11.d, z10.d\n"
+ "mov z16.d, z15.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, #0\n"
+ "blt 2f\n"
+
+ // software pipelining: load data early
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [x6, x8]\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z10.d, z10.d, z12.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+ "eor z15.d, z15.d, z17.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z10.b, p0, [%[p], x8]\n"
+ "st1b z11.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z15.b, p0, [%[p], x8]\n"
+ "st1b z16.b, p0, [%[q], x8]\n"
+
+ "add x8, x3, x3\n"
+ "add x5, x5, x8, lsl #1\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8",
+ "z10", "z11", "z12", "z13",
+ "z15", "z16", "z17", "z18"
+ );
+}
+
+static void raid6_sve4_xor_syndrome_real(int disks, int start, int stop,
+ unsigned long bytes, void **ptrs)
+{
+ u8 **dptr = (u8 **)ptrs;
+ u8 *p, *q;
+ long z0 = stop;
+
+ p = dptr[disks - 2];
+ q = dptr[disks - 1];
+
+ asm volatile(
+ ".arch armv8.2-a+sve\n"
+ "ptrue p0.b\n"
+ "cntb x3\n"
+ "mov w4, #0x1d\n"
+ "dup z4.b, w4\n"
+ "mov x5, #0\n"
+
+ "0:\n"
+ "ldr x6, [%[dptr], %[z0], lsl #3]\n"
+ "ld1b z1.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z6.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z11.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z16.b, p0/z, [x6, x8]\n"
+
+ "ld1b z0.b, p0/z, [%[p], x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z5.b, p0/z, [%[p], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z10.b, p0/z, [%[p], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z15.b, p0/z, [%[p], x8]\n"
+
+ "eor z0.d, z0.d, z1.d\n"
+ "eor z5.d, z5.d, z6.d\n"
+ "eor z10.d, z10.d, z11.d\n"
+ "eor z15.d, z15.d, z16.d\n"
+
+ "mov w7, %w[z0]\n"
+ "sub w7, w7, #1\n"
+
+ "1:\n"
+ "cmp w7, %w[start]\n"
+ "blt 2f\n"
+
+ // software pipelining: load data early
+ "sxtw x8, w7\n"
+ "ldr x6, [%[dptr], x8, lsl #3]\n"
+ "ld1b z2.b, p0/z, [x6, x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [x6, x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [x6, x8]\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z0.d, z0.d, z2.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z5.d, z5.d, z7.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z10.d, z10.d, z12.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+ "eor z15.d, z15.d, z17.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 1b\n"
+ "2:\n"
+
+ "mov w7, %w[start]\n"
+ "sub w7, w7, #1\n"
+ "3:\n"
+ "cmp w7, #0\n"
+ "blt 4f\n"
+
+ // math block 1
+ "mov z3.d, z1.d\n"
+ "asr z3.b, p0/m, z3.b, #7\n"
+ "lsl z1.b, p0/m, z1.b, #1\n"
+ "and z3.d, z3.d, z4.d\n"
+ "eor z1.d, z1.d, z3.d\n"
+
+ // math block 2
+ "mov z8.d, z6.d\n"
+ "asr z8.b, p0/m, z8.b, #7\n"
+ "lsl z6.b, p0/m, z6.b, #1\n"
+ "and z8.d, z8.d, z4.d\n"
+ "eor z6.d, z6.d, z8.d\n"
+
+ // math block 3
+ "mov z13.d, z11.d\n"
+ "asr z13.b, p0/m, z13.b, #7\n"
+ "lsl z11.b, p0/m, z11.b, #1\n"
+ "and z13.d, z13.d, z4.d\n"
+ "eor z11.d, z11.d, z13.d\n"
+
+ // math block 4
+ "mov z18.d, z16.d\n"
+ "asr z18.b, p0/m, z18.b, #7\n"
+ "lsl z16.b, p0/m, z16.b, #1\n"
+ "and z18.d, z18.d, z4.d\n"
+ "eor z16.d, z16.d, z18.d\n"
+
+ "sub w7, w7, #1\n"
+ "b 3b\n"
+ "4:\n"
+
+ // Load q and XOR
+ "ld1b z2.b, p0/z, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "ld1b z7.b, p0/z, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z12.b, p0/z, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "ld1b z17.b, p0/z, [%[q], x8]\n"
+
+ "eor z1.d, z1.d, z2.d\n"
+ "eor z6.d, z6.d, z7.d\n"
+ "eor z11.d, z11.d, z12.d\n"
+ "eor z16.d, z16.d, z17.d\n"
+
+ // Store results
+ "st1b z0.b, p0, [%[p], x5]\n"
+ "st1b z1.b, p0, [%[q], x5]\n"
+ "add x8, x5, x3\n"
+ "st1b z5.b, p0, [%[p], x8]\n"
+ "st1b z6.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z10.b, p0, [%[p], x8]\n"
+ "st1b z11.b, p0, [%[q], x8]\n"
+ "add x8, x8, x3\n"
+ "st1b z15.b, p0, [%[p], x8]\n"
+ "st1b z16.b, p0, [%[q], x8]\n"
+
+ "add x8, x3, x3\n"
+ "add x5, x5, x8, lsl #1\n"
+ "cmp x5, %[bytes]\n"
+ "blt 0b\n"
+ :
+ : [dptr] "r" (dptr), [z0] "r" (z0), [bytes] "r" (bytes),
+ [p] "r" (p), [q] "r" (q), [start] "r" (start)
+ : "memory", "p0", "x3", "x4", "x5", "x6", "x7", "x8",
+ "z0", "z1", "z2", "z3", "z4",
+ "z5", "z6", "z7", "z8",
+ "z10", "z11", "z12", "z13",
+ "z15", "z16", "z17", "z18"
+ );
+}
+
+#define RAID6_SVE_WRAPPER(_n) \
+ static void raid6_sve ## _n ## _gen_syndrome(int disks, \
+ size_t bytes, void **ptrs) \
+ { \
+ scoped_ksimd() \
+ raid6_sve ## _n ## _gen_syndrome_real(disks, \
+ (unsigned long)bytes, ptrs); \
+ } \
+ static void raid6_sve ## _n ## _xor_syndrome(int disks, \
+ int start, int stop, \
+ size_t bytes, void **ptrs) \
+ { \
+ scoped_ksimd() \
+ raid6_sve ## _n ## _xor_syndrome_real(disks, \
+ start, stop, (unsigned long)bytes, ptrs);\
+ } \
+ struct raid6_calls const raid6_svex ## _n = { \
+ raid6_sve ## _n ## _gen_syndrome, \
+ raid6_sve ## _n ## _xor_syndrome, \
+ raid6_have_sve, \
+ "svex" #_n, \
+ 0 \
+ }
+
+static int raid6_have_sve(void)
+{
+ return system_supports_sve();
+}
+
+RAID6_SVE_WRAPPER(1);
+RAID6_SVE_WRAPPER(2);
+RAID6_SVE_WRAPPER(4);
--
2.43.0
^ permalink raw reply related
* [PATCH] md/raid5: skip 2-failure compute when other disk is R5_LOCKED
From: FengWei Shih @ 2026-03-19 5:33 UTC (permalink / raw)
To: song, yukuai; +Cc: linan122, linux-raid, linux-kernel, FengWei Shih
When skip_copy is enabled on a doubly-degraded RAID6, a device that is
being written to will be in R5_LOCKED state with R5_UPTODATE cleared.
If a new read triggers fetch_block() while the write is still in
flight, the 2-failure compute path may select this locked device as a
compute target because it is not R5_UPTODATE.
Because skip_copy makes the device page point directly to the bio page,
reconstructing data into it might be risky. Also, since the compute
marks the device R5_UPTODATE, it triggers WARN_ON in ops_run_io()
which checks that R5_SkipCopy and R5_UPTODATE are not both set.
This can be reproduced by running small-range concurrent read/write on
a doubly-degraded RAID6 with skip_copy enabled, for example:
mdadm -C /dev/md0 -l6 -n6 -R -f /dev/loop[0-3] missing missing
echo 1 > /sys/block/md0/md/skip_copy
fio --filename=/dev/md0 --rw=randrw --bs=4k --numjobs=8 \
--iodepth=32 --size=4M --runtime=30 --time_based --direct=1
Fix by checking R5_LOCKED before proceeding with the compute. The
compute will be retried once the lock is cleared on IO completion.
Signed-off-by: FengWei Shih <dannyshih@synology.com>
---
drivers/md/raid5.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index bded2b86f0ef..bb493acfec29 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -3916,6 +3916,8 @@ static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
break;
}
BUG_ON(other < 0);
+ if (test_bit(R5_LOCKED, &sh->dev[other].flags))
+ return 0;
pr_debug("Computing stripe %llu blocks %d,%d\n",
(unsigned long long)sh->sector,
disk_idx, other);
--
2.25.1
Disclaimer: The contents of this e-mail message and any attachments are confidential and are intended solely for addressee. The information may also be legally privileged. This transmission is sent in trust, for the sole purpose of delivery to the intended recipient. If you have received this transmission in error, any use, reproduction or dissemination of this transmission is strictly prohibited. If you are not the intended recipient, please immediately notify the sender by reply e-mail or phone and delete this message and its attachments, if any.
^ permalink raw reply related
* Re: [PATCH] md/raid5: skip 2-failure compute when other disk is R5_LOCKED
From: Yu Kuai @ 2026-03-20 4:04 UTC (permalink / raw)
To: FengWei Shih, song, yukuai; +Cc: linan122, linux-raid, linux-kernel
In-Reply-To: <20260319053351.3676794-1-dannyshih@synology.com>
在 2026/3/19 13:33, FengWei Shih 写道:
> When skip_copy is enabled on a doubly-degraded RAID6, a device that is
> being written to will be in R5_LOCKED state with R5_UPTODATE cleared.
> If a new read triggers fetch_block() while the write is still in
> flight, the 2-failure compute path may select this locked device as a
> compute target because it is not R5_UPTODATE.
>
> Because skip_copy makes the device page point directly to the bio page,
> reconstructing data into it might be risky. Also, since the compute
> marks the device R5_UPTODATE, it triggers WARN_ON in ops_run_io()
> which checks that R5_SkipCopy and R5_UPTODATE are not both set.
>
> This can be reproduced by running small-range concurrent read/write on
> a doubly-degraded RAID6 with skip_copy enabled, for example:
>
> mdadm -C /dev/md0 -l6 -n6 -R -f /dev/loop[0-3] missing missing
> echo 1 > /sys/block/md0/md/skip_copy
> fio --filename=/dev/md0 --rw=randrw --bs=4k --numjobs=8 \
> --iodepth=32 --size=4M --runtime=30 --time_based --direct=1
>
> Fix by checking R5_LOCKED before proceeding with the compute. The
> compute will be retried once the lock is cleared on IO completion.
>
> Signed-off-by: FengWei Shih<dannyshih@synology.com>
> ---
> drivers/md/raid5.c | 2 ++
> 1 file changed, 2 insertions(+)
Reviewed-by: Yu Kuai <yukuai@fnnas.com>
--
Thansk,
Kuai
^ permalink raw reply
* Re: [PATCH] md/raid0: use kvzalloc/kvfree for strip_zone and devlist allocations
From: Yu Kuai @ 2026-03-20 4:08 UTC (permalink / raw)
To: Gregory Price, song, linan122, yukuai
Cc: linux-raid, linux-kernel, linux-mm, syzbot+924649752adf0d3ac9dd,
akpm
In-Reply-To: <20260308234202.3118119-1-gourry@gourry.net>
在 2026/3/9 7:42, Gregory Price 写道:
> syzbot reported a WARNING at mm/page_alloc.c:__alloc_frozen_pages_noprof()
> triggered by create_strip_zones() in the RAID0 driver.
>
> When raid_disks is large, the allocation size exceeds MAX_PAGE_ORDER (4MB
> on x86), causing WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER).
>
> Convert the strip_zone and devlist allocations from kzalloc/kzalloc_objs to
> kvzalloc/kvzalloc_objs, which first attempts a contiguous allocation with
> __GFP_NOWARN and then falls back to vmalloc for large sizes. Convert the
> corresponding kfree calls to kvfree.
>
> Both arrays are pure metadata lookup tables (arrays of pointers and zone
> descriptors) accessed only via indexing, so they do not require physically
> contiguous memory.
>
> Reported-by:syzbot+924649752adf0d3ac9dd@syzkaller.appspotmail.com
> Signed-off-by: Gregory Price<gourry@gourry.net>
> ---
> drivers/md/raid0.c | 18 +++++++++---------
> 1 file changed, 9 insertions(+), 9 deletions(-)
Reviewed-by: Yu Kuai <yukuai@fnnas.com>
--
Thansk,
Kuai
^ permalink raw reply
* [PATCH RESEND v3 0/3] lib/raid6: Optimize raid6_select_algo to ensure
From: sunliming @ 2026-03-20 6:00 UTC (permalink / raw)
To: song, yukuai, akpm; +Cc: linux-raid, linux-kernel, sunliming
From: sunliming <sunliming@kylinos.cn>
The selection of RAID6 PQ functions involves a dual-strategy approach:
prioritizing startup speed leads to quickly choosing a usable algorithm,
while prioritizing performance leads to selecting an optimal algorithm
via benchmarking. This choice is determined by the RAID6_PQ_BENCHMARK
configuration. This patch series achieves both fast startup and optimal
algorithm selection by initially choosing an algorithm quickly at startup,
then asynchronously determining the optimal algorithm through benchmarking.
Since all RAID6 PQ function algorithms are functionally equivalent despite
performance differences, this approach should be effective.
---
Changes in v3:
Remove the __init annotation from raid6_benchmark_work and benchmark_work_func,
as they are still referenced during initialization.
Changes in v2:
Select the highest-priority algorithm instead of the first one.
Add the cancel_work_sync function in the exit function to handle the
work queue cleanup.
---
sunliming (3):
ib/raid6: Divide the raid6 algorithm selection process into two parts
lib/raid6: Optimizing the raid6_select_algo time through asynchronous
processing
lib/raid6: Delete the RAID6_PQ_BENCHMARK config
include/linux/raid/pq.h | 3 --
lib/Kconfig | 8 ----
lib/raid6/algos.c | 88 ++++++++++++++++++++++++++++++-----------
3 files changed, 65 insertions(+), 34 deletions(-)
--
2.25.1
^ permalink raw reply
* [PATCH RESEND v3 1/3] lib/raid6: Divide the raid6 algorithm selection process into two parts
From: sunliming @ 2026-03-20 6:00 UTC (permalink / raw)
To: song, yukuai, akpm; +Cc: linux-raid, linux-kernel, sunliming
In-Reply-To: <20260320060021.30762-1-sunliming@linux.dev>
From: sunliming <sunliming@kylinos.cn>
Divide the RAID6 algorithm selection process into two parts: fast selection
and benchmark selection. To prepare for the asynchronous processing of
the benchmark phase.
Signed-off-by: sunliming <sunliming@kylinos.cn>
---
lib/raid6/algos.c | 76 +++++++++++++++++++++++++++++++++--------------
1 file changed, 54 insertions(+), 22 deletions(-)
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 799e0e5eac26..c21e3ad99d97 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -152,8 +152,32 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void)
return best;
}
-static inline const struct raid6_calls *raid6_choose_gen(
- void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
+/* Quick selection: select the highest priority valid algorithm. */
+static inline int raid6_choose_gen_fast(void)
+{
+ int ret = 0;
+ const struct raid6_calls *const *algo;
+ const struct raid6_calls *best = NULL;
+
+ for (best = NULL, algo = raid6_algos; *algo; algo++)
+ if (!best || (*algo)->priority > best->priority)
+ if (!(*algo)->valid || (*algo)->valid())
+ best = *algo;
+
+ if (best) {
+ raid6_call = *best;
+ pr_info("raid6: skipped pq benchmark and selected %s\n",
+ best->name);
+ } else {
+ pr_err("raid6: No valid algorithm found even for fast selection!\n");
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
+static inline const struct raid6_calls *raid6_gen_benchmark(
+ void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
{
unsigned long perf, bestgenperf, j0, j1;
int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
@@ -165,11 +189,6 @@ static inline const struct raid6_calls *raid6_choose_gen(
if ((*algo)->valid && !(*algo)->valid())
continue;
- if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
- best = *algo;
- break;
- }
-
perf = 0;
preempt_disable();
@@ -200,12 +219,6 @@ static inline const struct raid6_calls *raid6_choose_gen(
raid6_call = *best;
- if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
- pr_info("raid6: skipped pq benchmark and selected %s\n",
- best->name);
- goto out;
- }
-
pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
best->name,
(bestgenperf * HZ * (disks - 2)) >>
@@ -239,15 +252,13 @@ static inline const struct raid6_calls *raid6_choose_gen(
/* Try to pick the best algorithm */
/* This code uses the gfmul table as convenient data set to abuse */
-int __init raid6_select_algo(void)
+static int raid6_choose_gen_benmark(void)
{
const int disks = RAID6_TEST_DISKS;
-
const struct raid6_calls *gen_best;
- const struct raid6_recov_calls *rec_best;
char *disk_ptr, *p;
void *dptrs[RAID6_TEST_DISKS];
- int i, cycle;
+ int i, cycle, ret = 0;
/* prepare the buffer and fill it circularly with gfmul table */
disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
@@ -269,15 +280,36 @@ int __init raid6_select_algo(void)
if ((disks - 2) * PAGE_SIZE % 65536)
memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
- /* select raid gen_syndrome function */
- gen_best = raid6_choose_gen(&dptrs, disks);
+ gen_best = raid6_gen_benchmark(&dptrs, disks);
+ if (!gen_best)
+ ret = -EINVAL;
+
+ free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
+
+ return ret;
+}
+
+int __init raid6_select_algo(void)
+{
+ int ret = 0;
+ const struct raid6_recov_calls *rec_best = NULL;
+
+ /* select raid gen_syndrome functions */
+ if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK))
+ ret = raid6_choose_gen_fast();
+ else
+ ret = raid6_choose_gen_benmark();
+
+ if (ret < 0)
+ goto out;
/* select raid recover functions */
rec_best = raid6_choose_recov();
+ if (!rec_best)
+ ret = -EINVAL;
- free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
-
- return gen_best && rec_best ? 0 : -EINVAL;
+out:
+ return ret;
}
static void raid6_exit(void)
--
2.25.1
^ permalink raw reply related
* [PATCH RESEND v3 2/3] lib/raid6: Optimizing the raid6_select_algo time through asynchronous processing
From: sunliming @ 2026-03-20 6:00 UTC (permalink / raw)
To: song, yukuai, akpm; +Cc: linux-raid, linux-kernel, sunliming
In-Reply-To: <20260320060021.30762-1-sunliming@linux.dev>
From: sunliming <sunliming@kylinos.cn>
Optimizing the raid6_select_algo time. In raid6_select_algo(), an raid6 gen
algorithm is first selected quickly through synchronous processing, while
the time-consuming process of selecting the optimal algorithm via benchmarking
is handled asynchronously. This approach speeds up the overall startup time
and ultimately ensures the selection of an optimal algorithm.
Signed-off-by: sunliming <sunliming@kylinos.cn>
---
lib/raid6/algos.c | 30 ++++++++++++++++++++----------
1 file changed, 20 insertions(+), 10 deletions(-)
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index c21e3ad99d97..b8b5515ac7a6 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -12,6 +12,7 @@
*/
#include <linux/raid/pq.h>
+#include <linux/workqueue.h>
#ifndef __KERNEL__
#include <sys/mman.h>
#include <stdio.h>
@@ -166,7 +167,7 @@ static inline int raid6_choose_gen_fast(void)
if (best) {
raid6_call = *best;
- pr_info("raid6: skipped pq benchmark and selected %s\n",
+ pr_info("raid6: raid6: fast selected %s, async benchmark pending\n",
best->name);
} else {
pr_err("raid6: No valid algorithm found even for fast selection!\n");
@@ -213,7 +214,7 @@ static inline const struct raid6_calls *raid6_gen_benchmark(
}
if (!best) {
- pr_err("raid6: Yikes! No algorithm found!\n");
+ pr_warn("raid6: async benchmark failed to find any algorithm\n");
goto out;
}
@@ -289,24 +290,33 @@ static int raid6_choose_gen_benmark(void)
return ret;
}
+static struct work_struct raid6_benchmark_work;
+
+static void benchmark_work_func(struct work_struct *work)
+{
+ raid6_choose_gen_benmark();
+}
+
int __init raid6_select_algo(void)
{
int ret = 0;
const struct raid6_recov_calls *rec_best = NULL;
- /* select raid gen_syndrome functions */
- if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK))
- ret = raid6_choose_gen_fast();
- else
- ret = raid6_choose_gen_benmark();
-
+ /* phase 1: synchronous fast selection generation algorithm */
+ ret = raid6_choose_gen_fast();
if (ret < 0)
goto out;
/* select raid recover functions */
rec_best = raid6_choose_recov();
- if (!rec_best)
+ if (!rec_best) {
ret = -EINVAL;
+ goto out;
+ }
+
+ /* phase 2: asynchronous performance benchmarking */
+ INIT_WORK(&raid6_benchmark_work, benchmark_work_func);
+ schedule_work(&raid6_benchmark_work);
out:
return ret;
@@ -314,7 +324,7 @@ int __init raid6_select_algo(void)
static void raid6_exit(void)
{
- do { } while (0);
+ cancel_work_sync(&raid6_benchmark_work);
}
subsys_initcall(raid6_select_algo);
--
2.25.1
^ permalink raw reply related
* [PATCH RESEND v3 0/3] lib/raid6: Optimize raid6_select_algo to ensure
From: sunliming @ 2026-03-20 6:07 UTC (permalink / raw)
To: song, yukuai, akpm; +Cc: linux-raid, linux-kernel, sunliming
From: sunliming <sunliming@kylinos.cn>
The selection of RAID6 PQ functions involves a dual-strategy approach:
prioritizing startup speed leads to quickly choosing a usable algorithm,
while prioritizing performance leads to selecting an optimal algorithm
via benchmarking. This choice is determined by the RAID6_PQ_BENCHMARK
configuration. This patch series achieves both fast startup and optimal
algorithm selection by initially choosing an algorithm quickly at startup,
then asynchronously determining the optimal algorithm through benchmarking.
Since all RAID6 PQ function algorithms are functionally equivalent despite
performance differences, this approach should be effective.
---
Changes in v3:
Remove the __init annotation from raid6_benchmark_work and benchmark_work_func,
as they are still referenced during initialization.
Changes in v2:
Select the highest-priority algorithm instead of the first one.
Add the cancel_work_sync function in the exit function to handle the
work queue cleanup.
---
sunliming (3):
ib/raid6: Divide the raid6 algorithm selection process into two parts
lib/raid6: Optimizing the raid6_select_algo time through asynchronous
processing
lib/raid6: Delete the RAID6_PQ_BENCHMARK config
include/linux/raid/pq.h | 3 --
lib/Kconfig | 8 ----
lib/raid6/algos.c | 88 ++++++++++++++++++++++++++++++-----------
3 files changed, 65 insertions(+), 34 deletions(-)
--
2.25.1
^ permalink raw reply
* [PATCH RESEND v3 1/3] lib/raid6: Divide the raid6 algorithm selection process into two parts
From: sunliming @ 2026-03-20 6:07 UTC (permalink / raw)
To: song, yukuai, akpm; +Cc: linux-raid, linux-kernel, sunliming
In-Reply-To: <20260320060750.31334-1-sunliming@linux.dev>
From: sunliming <sunliming@kylinos.cn>
Divide the RAID6 algorithm selection process into two parts: fast selection
and benchmark selection. To prepare for the asynchronous processing of
the benchmark phase.
Signed-off-by: sunliming <sunliming@kylinos.cn>
---
lib/raid6/algos.c | 76 +++++++++++++++++++++++++++++++++--------------
1 file changed, 54 insertions(+), 22 deletions(-)
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 799e0e5eac26..c21e3ad99d97 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -152,8 +152,32 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void)
return best;
}
-static inline const struct raid6_calls *raid6_choose_gen(
- void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
+/* Quick selection: select the highest priority valid algorithm. */
+static inline int raid6_choose_gen_fast(void)
+{
+ int ret = 0;
+ const struct raid6_calls *const *algo;
+ const struct raid6_calls *best = NULL;
+
+ for (best = NULL, algo = raid6_algos; *algo; algo++)
+ if (!best || (*algo)->priority > best->priority)
+ if (!(*algo)->valid || (*algo)->valid())
+ best = *algo;
+
+ if (best) {
+ raid6_call = *best;
+ pr_info("raid6: skipped pq benchmark and selected %s\n",
+ best->name);
+ } else {
+ pr_err("raid6: No valid algorithm found even for fast selection!\n");
+ ret = -EINVAL;
+ }
+
+ return ret;
+}
+
+static inline const struct raid6_calls *raid6_gen_benchmark(
+ void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
{
unsigned long perf, bestgenperf, j0, j1;
int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
@@ -165,11 +189,6 @@ static inline const struct raid6_calls *raid6_choose_gen(
if ((*algo)->valid && !(*algo)->valid())
continue;
- if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
- best = *algo;
- break;
- }
-
perf = 0;
preempt_disable();
@@ -200,12 +219,6 @@ static inline const struct raid6_calls *raid6_choose_gen(
raid6_call = *best;
- if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
- pr_info("raid6: skipped pq benchmark and selected %s\n",
- best->name);
- goto out;
- }
-
pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
best->name,
(bestgenperf * HZ * (disks - 2)) >>
@@ -239,15 +252,13 @@ static inline const struct raid6_calls *raid6_choose_gen(
/* Try to pick the best algorithm */
/* This code uses the gfmul table as convenient data set to abuse */
-int __init raid6_select_algo(void)
+static int raid6_choose_gen_benmark(void)
{
const int disks = RAID6_TEST_DISKS;
-
const struct raid6_calls *gen_best;
- const struct raid6_recov_calls *rec_best;
char *disk_ptr, *p;
void *dptrs[RAID6_TEST_DISKS];
- int i, cycle;
+ int i, cycle, ret = 0;
/* prepare the buffer and fill it circularly with gfmul table */
disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
@@ -269,15 +280,36 @@ int __init raid6_select_algo(void)
if ((disks - 2) * PAGE_SIZE % 65536)
memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
- /* select raid gen_syndrome function */
- gen_best = raid6_choose_gen(&dptrs, disks);
+ gen_best = raid6_gen_benchmark(&dptrs, disks);
+ if (!gen_best)
+ ret = -EINVAL;
+
+ free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
+
+ return ret;
+}
+
+int __init raid6_select_algo(void)
+{
+ int ret = 0;
+ const struct raid6_recov_calls *rec_best = NULL;
+
+ /* select raid gen_syndrome functions */
+ if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK))
+ ret = raid6_choose_gen_fast();
+ else
+ ret = raid6_choose_gen_benmark();
+
+ if (ret < 0)
+ goto out;
/* select raid recover functions */
rec_best = raid6_choose_recov();
+ if (!rec_best)
+ ret = -EINVAL;
- free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
-
- return gen_best && rec_best ? 0 : -EINVAL;
+out:
+ return ret;
}
static void raid6_exit(void)
--
2.25.1
^ permalink raw reply related
* [PATCH RESEND v3 2/3] lib/raid6: Optimizing the raid6_select_algo time through asynchronous processing
From: sunliming @ 2026-03-20 6:07 UTC (permalink / raw)
To: song, yukuai, akpm; +Cc: linux-raid, linux-kernel, sunliming
In-Reply-To: <20260320060750.31334-1-sunliming@linux.dev>
From: sunliming <sunliming@kylinos.cn>
Optimizing the raid6_select_algo time. In raid6_select_algo(), an raid6 gen
algorithm is first selected quickly through synchronous processing, while
the time-consuming process of selecting the optimal algorithm via benchmarking
is handled asynchronously. This approach speeds up the overall startup time
and ultimately ensures the selection of an optimal algorithm.
Signed-off-by: sunliming <sunliming@kylinos.cn>
---
lib/raid6/algos.c | 30 ++++++++++++++++++++----------
1 file changed, 20 insertions(+), 10 deletions(-)
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index c21e3ad99d97..b8b5515ac7a6 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -12,6 +12,7 @@
*/
#include <linux/raid/pq.h>
+#include <linux/workqueue.h>
#ifndef __KERNEL__
#include <sys/mman.h>
#include <stdio.h>
@@ -166,7 +167,7 @@ static inline int raid6_choose_gen_fast(void)
if (best) {
raid6_call = *best;
- pr_info("raid6: skipped pq benchmark and selected %s\n",
+ pr_info("raid6: raid6: fast selected %s, async benchmark pending\n",
best->name);
} else {
pr_err("raid6: No valid algorithm found even for fast selection!\n");
@@ -213,7 +214,7 @@ static inline const struct raid6_calls *raid6_gen_benchmark(
}
if (!best) {
- pr_err("raid6: Yikes! No algorithm found!\n");
+ pr_warn("raid6: async benchmark failed to find any algorithm\n");
goto out;
}
@@ -289,24 +290,33 @@ static int raid6_choose_gen_benmark(void)
return ret;
}
+static struct work_struct raid6_benchmark_work;
+
+static void benchmark_work_func(struct work_struct *work)
+{
+ raid6_choose_gen_benmark();
+}
+
int __init raid6_select_algo(void)
{
int ret = 0;
const struct raid6_recov_calls *rec_best = NULL;
- /* select raid gen_syndrome functions */
- if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK))
- ret = raid6_choose_gen_fast();
- else
- ret = raid6_choose_gen_benmark();
-
+ /* phase 1: synchronous fast selection generation algorithm */
+ ret = raid6_choose_gen_fast();
if (ret < 0)
goto out;
/* select raid recover functions */
rec_best = raid6_choose_recov();
- if (!rec_best)
+ if (!rec_best) {
ret = -EINVAL;
+ goto out;
+ }
+
+ /* phase 2: asynchronous performance benchmarking */
+ INIT_WORK(&raid6_benchmark_work, benchmark_work_func);
+ schedule_work(&raid6_benchmark_work);
out:
return ret;
@@ -314,7 +324,7 @@ int __init raid6_select_algo(void)
static void raid6_exit(void)
{
- do { } while (0);
+ cancel_work_sync(&raid6_benchmark_work);
}
subsys_initcall(raid6_select_algo);
--
2.25.1
^ permalink raw reply related
* [PATCH RESEND v3 3/3] lib/raid6: Delete the RAID6_PQ_BENCHMARK config
From: sunliming @ 2026-03-20 6:07 UTC (permalink / raw)
To: song, yukuai, akpm; +Cc: linux-raid, linux-kernel, sunliming
In-Reply-To: <20260320060750.31334-1-sunliming@linux.dev>
From: sunliming <sunliming@kylinos.cn>
Now RAID6 PQ functions is automatically choosed and the
RAID6_PQ_BENCHMARK is not needed.
Signed-off-by: sunliming <sunliming@kylinos.cn>
---
include/linux/raid/pq.h | 3 ---
lib/Kconfig | 8 --------
2 files changed, 11 deletions(-)
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 2467b3be15c9..6378ec4ae4ba 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -67,9 +67,6 @@ extern const char raid6_empty_zero_page[PAGE_SIZE];
#define MODULE_DESCRIPTION(desc)
#define subsys_initcall(x)
#define module_exit(x)
-
-#define IS_ENABLED(x) (x)
-#define CONFIG_RAID6_PQ_BENCHMARK 1
#endif /* __KERNEL__ */
/* Routine choices */
diff --git a/lib/Kconfig b/lib/Kconfig
index 2923924bea78..841a0245a2c4 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -11,14 +11,6 @@ menu "Library routines"
config RAID6_PQ
tristate
-config RAID6_PQ_BENCHMARK
- bool "Automatically choose fastest RAID6 PQ functions"
- depends on RAID6_PQ
- default y
- help
- Benchmark all available RAID6 PQ functions on init and choose the
- fastest one.
-
config LINEAR_RANGES
tristate
--
2.25.1
^ permalink raw reply related
* Re: [PATCH RESEND v3 2/3] lib/raid6: Optimizing the raid6_select_algo time through asynchronous processing
From: Paul Menzel @ 2026-03-20 11:24 UTC (permalink / raw)
To: sunliming; +Cc: song, yukuai, akpm, linux-raid, linux-kernel, sunliming
In-Reply-To: <20260320060750.31334-3-sunliming@linux.dev>
Dear sunliming,
Thank you for your patch. Two formal things, and one “real” question.
Am 20.03.26 um 07:07 schrieb sunliming@linux.dev:
> From: sunliming <sunliming@kylinos.cn>
It’d be great if you could spell your name with spaces. Sun Li Ming or
Sunli Ming? (`git config --global user.name "Sun Li Ming"` would
configure it.)
> Optimizing the raid6_select_algo time. In raid6_select_algo(), an raid6 gen
> algorithm is first selected quickly through synchronous processing, while
> the time-consuming process of selecting the optimal algorithm via benchmarking
Please adhere to the text width of 75 characters.
(`scripts/checkpatch.pl` should have found this.)
> is handled asynchronously. This approach speeds up the overall startup time
> and ultimately ensures the selection of an optimal algorithm.
It’d be great if you shared the test system and exact numbers, and new
log messages.
Is there any problem with the benchmark, if it is run asynchronously,
because other things are run on the CPU?
If you are working on this, I was wondering if the current approach of
selecting the algorithm during each boot is the best choice, and if it
shouldn’t be dynamically configurable like schedulers and other things
over sysfs and the Linux kernel command line, and also if the chosen
algorithm can’t be cached as long as – I guess – the processor/CPU stays
the same.
> Signed-off-by: sunliming <sunliming@kylinos.cn>
(Ditto regarding the name.)
Kind regards,
Paul
> ---
> lib/raid6/algos.c | 30 ++++++++++++++++++++----------
> 1 file changed, 20 insertions(+), 10 deletions(-)
>
> diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
> index c21e3ad99d97..b8b5515ac7a6 100644
> --- a/lib/raid6/algos.c
> +++ b/lib/raid6/algos.c
> @@ -12,6 +12,7 @@
> */
>
> #include <linux/raid/pq.h>
> +#include <linux/workqueue.h>
> #ifndef __KERNEL__
> #include <sys/mman.h>
> #include <stdio.h>
> @@ -166,7 +167,7 @@ static inline int raid6_choose_gen_fast(void)
>
> if (best) {
> raid6_call = *best;
> - pr_info("raid6: skipped pq benchmark and selected %s\n",
> + pr_info("raid6: raid6: fast selected %s, async benchmark pending\n",
> best->name);
> } else {
> pr_err("raid6: No valid algorithm found even for fast selection!\n");
> @@ -213,7 +214,7 @@ static inline const struct raid6_calls *raid6_gen_benchmark(
> }
>
> if (!best) {
> - pr_err("raid6: Yikes! No algorithm found!\n");
> + pr_warn("raid6: async benchmark failed to find any algorithm\n");
> goto out;
> }
>
> @@ -289,24 +290,33 @@ static int raid6_choose_gen_benmark(void)
> return ret;
> }
>
> +static struct work_struct raid6_benchmark_work;
> +
> +static void benchmark_work_func(struct work_struct *work)
> +{
> + raid6_choose_gen_benmark();
> +}
> +
> int __init raid6_select_algo(void)
> {
> int ret = 0;
> const struct raid6_recov_calls *rec_best = NULL;
>
> - /* select raid gen_syndrome functions */
> - if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK))
> - ret = raid6_choose_gen_fast();
> - else
> - ret = raid6_choose_gen_benmark();
> -
> + /* phase 1: synchronous fast selection generation algorithm */
> + ret = raid6_choose_gen_fast();
> if (ret < 0)
> goto out;
>
> /* select raid recover functions */
> rec_best = raid6_choose_recov();
> - if (!rec_best)
> + if (!rec_best) {
> ret = -EINVAL;
> + goto out;
> + }
> +
> + /* phase 2: asynchronous performance benchmarking */
> + INIT_WORK(&raid6_benchmark_work, benchmark_work_func);
> + schedule_work(&raid6_benchmark_work);
>
> out:
> return ret;
> @@ -314,7 +324,7 @@ int __init raid6_select_algo(void)
>
> static void raid6_exit(void)
> {
> - do { } while (0);
> + cancel_work_sync(&raid6_benchmark_work);
> }
>
> subsys_initcall(raid6_select_algo);
^ permalink raw reply
* Re: [PATCH] md/raid0: use kvzalloc/kvfree for strip_zone and devlist allocations
From: Li Nan @ 2026-03-21 6:29 UTC (permalink / raw)
To: Gregory Price, song, yukuai
Cc: linux-raid, linux-kernel, linux-mm, syzbot+924649752adf0d3ac9dd,
akpm
In-Reply-To: <20260308234202.3118119-1-gourry@gourry.net>
在 2026/3/9 7:42, Gregory Price 写道:
> syzbot reported a WARNING at mm/page_alloc.c:__alloc_frozen_pages_noprof()
> triggered by create_strip_zones() in the RAID0 driver.
>
> When raid_disks is large, the allocation size exceeds MAX_PAGE_ORDER (4MB
> on x86), causing WARN_ON_ONCE_GFP(order > MAX_PAGE_ORDER).
>
> Convert the strip_zone and devlist allocations from kzalloc/kzalloc_objs to
> kvzalloc/kvzalloc_objs, which first attempts a contiguous allocation with
> __GFP_NOWARN and then falls back to vmalloc for large sizes. Convert the
> corresponding kfree calls to kvfree.
>
> Both arrays are pure metadata lookup tables (arrays of pointers and zone
> descriptors) accessed only via indexing, so they do not require physically
> contiguous memory.
>
> Reported-by: syzbot+924649752adf0d3ac9dd@syzkaller.appspotmail.com
> Signed-off-by: Gregory Price <gourry@gourry.net>
> ---
> drivers/md/raid0.c | 18 +++++++++---------
> 1 file changed, 9 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c
> index ef0045db409f..5e38a51e349a 100644
> --- a/drivers/md/raid0.c
> +++ b/drivers/md/raid0.c
> @@ -143,13 +143,13 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
> }
>
> err = -ENOMEM;
> - conf->strip_zone = kzalloc_objs(struct strip_zone, conf->nr_strip_zones);
> + conf->strip_zone = kvzalloc_objs(struct strip_zone, conf->nr_strip_zones);
> if (!conf->strip_zone)
> goto abort;
> - conf->devlist = kzalloc(array3_size(sizeof(struct md_rdev *),
> - conf->nr_strip_zones,
> - mddev->raid_disks),
> - GFP_KERNEL);
> + conf->devlist = kvzalloc(array3_size(sizeof(struct md_rdev *),
> + conf->nr_strip_zones,
> + mddev->raid_disks),
> + GFP_KERNEL);
> if (!conf->devlist)
> goto abort;
>
> @@ -291,8 +291,8 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf)
>
> return 0;
> abort:
> - kfree(conf->strip_zone);
> - kfree(conf->devlist);
> + kvfree(conf->strip_zone);
> + kvfree(conf->devlist);
> kfree(conf);
> *private_conf = ERR_PTR(err);
> return err;
> @@ -373,8 +373,8 @@ static void raid0_free(struct mddev *mddev, void *priv)
> {
> struct r0conf *conf = priv;
>
> - kfree(conf->strip_zone);
> - kfree(conf->devlist);
> + kvfree(conf->strip_zone);
> + kvfree(conf->devlist);
> kfree(conf);
> }
>
LGTM
Reviewed-by: Li Nan <linan122@huawei.com>
--
Thanks,
Nan
^ permalink raw reply
* Re: [PATCH 1/2] md/raid1: fix the comparing region of interval tree
From: Yu Kuai @ 2026-03-22 18:28 UTC (permalink / raw)
To: Xiao Ni, yukuai; +Cc: linux-raid, ncroxon
In-Reply-To: <20260305011839.5118-2-xni@redhat.com>
在 2026/3/5 9:18, Xiao Ni 写道:
> Interval tree uses [start, end] as a region which stores in the tree.
> In raid1, it uses the wrong end value. For example:
> bio(A,B) is too big and needs to be split to bio1(A,C-1), bio2(C,B).
> The region of bio1 is [A,C] and the region of bio2 is [C,B]. So bio1 and
> bio2 overlap which is not right.
>
> Fix this problem by using right end value of the region.
>
> Fixes: d0d2d8ba0494 ("md/raid1: introduce wait_for_serialization")
> Signed-off-by: Xiao Ni<xni@redhat.com>
> ---
> drivers/md/raid1.c | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
patch 1 applied.
--
Thansk,
Kuai
^ permalink raw reply
* Re: [PATCH v2 0/5] md/md-llbitmap: fixes and proactive parity building support
From: Yu Kuai @ 2026-03-22 18:30 UTC (permalink / raw)
To: song; +Cc: linan122, xni, colyli, linux-raid, linux-kernel, yukuai
In-Reply-To: <20260223024038.3084853-1-yukuai@fnnas.com>
在 2026/2/23 10:40, Yu Kuai 写道:
> Changes in v2:
> - add error message in error path in patch 3;
>
> This series contains fixes and enhancements for the md-llbitmap (lockless
> bitmap) implementation.
>
> Patches 1-2 are bug fixes:
> - Patch 1 fixes bitmap data being read from spare disks that are not yet
> in sync, which could lead to incorrect dirty bit tracking.
> - Patch 2 fixes a race condition where the state machine could transition
> before the barrier is properly raised.
>
> Patch 3 improves compatibility with older mdadm versions by detecting
> on-disk bitmap version and falling back to the correct bitmap_ops when
> there's a version mismatch.
>
> Patch 4 adds support for proactive XOR parity building in RAID-456 arrays.
> This allows users to pre-build parity for unwritten regions via sysfs
> before any user data is written, which can improve write performance for
> workloads that will eventually use all storage. New states (CleanUnwritten,
> NeedSyncUnwritten, SyncingUnwritten) are added to track these regions
> separately from normal dirty/syncing states.
>
> Patch 5 optimizes initial array sync for RAID-456 arrays on devices that
> support write_zeroes with unmap. By zeroing all disks upfront, parity is
> automatically consistent (0 XOR 0 = 0), allowing the bitmap to be
> initialized to BitCleanUnwritten and skipping the initial sync entirely.
> This significantly reduces array initialization time on modern NVMe SSDs.
>
> Yu Kuai (5):
> md/md-llbitmap: skip reading rdevs that are not in_sync
> md/md-llbitmap: raise barrier before state machine transition
> md: add fallback to correct bitmap_ops on version mismatch
> md/md-llbitmap: add CleanUnwritten state for RAID-5 proactive parity
> building
> md/md-llbitmap: optimize initial sync with write_zeroes_unmap support
>
> drivers/md/md-llbitmap.c | 213 +++++++++++++++++++++++++++++++++++----
> drivers/md/md.c | 117 ++++++++++++++++++++-
> 2 files changed, 309 insertions(+), 21 deletions(-)
Apply patch 1 and 2 to md-7.1, will send a new version for others soon.
--
Thansk,
Kuai
^ permalink raw reply
* Re: [PATCH] md/raid5: skip 2-failure compute when other disk is R5_LOCKED
From: Yu Kuai @ 2026-03-22 18:31 UTC (permalink / raw)
To: FengWei Shih, song, yukuai; +Cc: linan122, linux-raid, linux-kernel
In-Reply-To: <20260319053351.3676794-1-dannyshih@synology.com>
在 2026/3/19 13:33, FengWei Shih 写道:
> When skip_copy is enabled on a doubly-degraded RAID6, a device that is
> being written to will be in R5_LOCKED state with R5_UPTODATE cleared.
> If a new read triggers fetch_block() while the write is still in
> flight, the 2-failure compute path may select this locked device as a
> compute target because it is not R5_UPTODATE.
>
> Because skip_copy makes the device page point directly to the bio page,
> reconstructing data into it might be risky. Also, since the compute
> marks the device R5_UPTODATE, it triggers WARN_ON in ops_run_io()
> which checks that R5_SkipCopy and R5_UPTODATE are not both set.
>
> This can be reproduced by running small-range concurrent read/write on
> a doubly-degraded RAID6 with skip_copy enabled, for example:
>
> mdadm -C /dev/md0 -l6 -n6 -R -f /dev/loop[0-3] missing missing
> echo 1 > /sys/block/md0/md/skip_copy
> fio --filename=/dev/md0 --rw=randrw --bs=4k --numjobs=8 \
> --iodepth=32 --size=4M --runtime=30 --time_based --direct=1
>
> Fix by checking R5_LOCKED before proceeding with the compute. The
> compute will be retried once the lock is cleared on IO completion.
>
> Signed-off-by: FengWei Shih<dannyshih@synology.com>
> ---
> drivers/md/raid5.c | 2 ++
> 1 file changed, 2 insertions(+)
Applied
--
Thansk,
Kuai
^ permalink raw reply
* Re: [PATCH] md: remove unused mddev argument from export_rdev
From: Yu Kuai @ 2026-03-22 18:32 UTC (permalink / raw)
To: Chen Cheng, Song Liu, yukuai; +Cc: Li Nan, linux-raid
In-Reply-To: <20260304111417.20777-1-chencheng@fnnas.com>
在 2026/3/4 19:14, Chen Cheng 写道:
> The mddev argument in export_rdev() is never used. Remove it to
> simplify callers.
>
> Signed-off-by: Chen Cheng<chencheng@fnnas.com>
> ---
> drivers/md/md.c | 28 ++++++++++++++--------------
> 1 file changed, 14 insertions(+), 14 deletions(-)
Applied
--
Thansk,
Kuai
^ permalink raw reply
* Re: [PATCH 1/2] md/raid5: remove stale md_raid5_kick_device() declaration
From: Yu Kuai @ 2026-03-22 18:32 UTC (permalink / raw)
To: Chen Cheng, Song Liu, yukuai; +Cc: Li Nan, linux-raid
In-Reply-To: <20260304110919.15071-1-chencheng@fnnas.com>
在 2026/3/4 19:09, Chen Cheng 写道:
> Remove the unused md_raid5_kick_device() declaration from raid5.h -
> no definition exists for this function.
>
> Signed-off-by: Chen Cheng<chencheng@fnnas.com>
> ---
> drivers/md/raid5.h | 1 -
> 1 file changed, 1 deletion(-)
Applied
--
Thansk,
Kuai
^ permalink raw reply
* Re: [PATCH 2/2] md/raid5: move handle_stripe() comment to correct location
From: Yu Kuai @ 2026-03-22 18:33 UTC (permalink / raw)
To: Chen Cheng, Song Liu, yukuai; +Cc: Li Nan, linux-raid
In-Reply-To: <20260304111001.15767-1-chencheng@fnnas.com>
在 2026/3/4 19:10, Chen Cheng 写道:
> Move the handle_stripe() documentation comment from above
> analyse_stripe() to directly above handle_stripe() where it belongs.
>
> Signed-off-by: Chen Cheng<chencheng@fnnas.com>
> ---
> drivers/md/raid5.c | 26 ++++++++++++--------------
> 1 file changed, 12 insertions(+), 14 deletions(-)
Applied
--
Thansk,
Kuai
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox