* [PATCH 13/26] loongarch: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR into lib/raid and include it it in xor.ko
instead of always building it into the main kernel image.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/loongarch/include/asm/xor.h | 24 ++----------
arch/loongarch/include/asm/xor_simd.h | 34 ----------------
arch/loongarch/lib/Makefile | 2 -
lib/raid/xor/Makefile | 2 +
.../lib => lib/raid/xor/loongarch}/xor_simd.c | 0
.../lib => lib/raid/xor/loongarch}/xor_simd.h | 0
.../raid/xor/loongarch}/xor_simd_glue.c | 39 +++++++++++--------
.../raid/xor/loongarch}/xor_template.c | 0
8 files changed, 27 insertions(+), 74 deletions(-)
delete mode 100644 arch/loongarch/include/asm/xor_simd.h
rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_simd.c (100%)
rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_simd.h (100%)
rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_simd_glue.c (64%)
rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_template.c (100%)
diff --git a/arch/loongarch/include/asm/xor.h b/arch/loongarch/include/asm/xor.h
index d17c0e3b047f..7e32f72f8b03 100644
--- a/arch/loongarch/include/asm/xor.h
+++ b/arch/loongarch/include/asm/xor.h
@@ -6,27 +6,6 @@
#define _ASM_LOONGARCH_XOR_H
#include <asm/cpu-features.h>
-#include <asm/xor_simd.h>
-
-#ifdef CONFIG_CPU_HAS_LSX
-static struct xor_block_template xor_block_lsx = {
- .name = "lsx",
- .do_2 = xor_lsx_2,
- .do_3 = xor_lsx_3,
- .do_4 = xor_lsx_4,
- .do_5 = xor_lsx_5,
-};
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-static struct xor_block_template xor_block_lasx = {
- .name = "lasx",
- .do_2 = xor_lasx_2,
- .do_3 = xor_lasx_3,
- .do_4 = xor_lasx_4,
- .do_5 = xor_lasx_5,
-};
-#endif /* CONFIG_CPU_HAS_LASX */
/*
* For grins, also test the generic routines.
@@ -38,6 +17,9 @@ static struct xor_block_template xor_block_lasx = {
*/
#include <asm-generic/xor.h>
+extern struct xor_block_template xor_block_lsx;
+extern struct xor_block_template xor_block_lasx;
+
#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
diff --git a/arch/loongarch/include/asm/xor_simd.h b/arch/loongarch/include/asm/xor_simd.h
deleted file mode 100644
index 471b96332f38..000000000000
--- a/arch/loongarch/include/asm/xor_simd.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-#ifndef _ASM_LOONGARCH_XOR_SIMD_H
-#define _ASM_LOONGARCH_XOR_SIMD_H
-
-#ifdef CONFIG_CPU_HAS_LSX
-void xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-void xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LASX */
-
-#endif /* _ASM_LOONGARCH_XOR_SIMD_H */
diff --git a/arch/loongarch/lib/Makefile b/arch/loongarch/lib/Makefile
index ccea3bbd4353..827a88529a42 100644
--- a/arch/loongarch/lib/Makefile
+++ b/arch/loongarch/lib/Makefile
@@ -8,6 +8,4 @@ lib-y += delay.o memset.o memcpy.o memmove.o \
obj-$(CONFIG_ARCH_SUPPORTS_INT128) += tishift.o
-obj-$(CONFIG_CPU_HAS_LSX) += xor_simd.o xor_simd_glue.o
-
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 4ab0e7411ff7..e8868f5fc396 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -14,6 +14,8 @@ ifeq ($(CONFIG_ARM),y)
xor-$(CONFIG_KERNEL_MODE_NEON) += arm/xor-neon.o arm/xor-neon-glue.o
endif
xor-$(CONFIG_ARM64) += arm64/xor-neon.o arm64/xor-neon-glue.o
+xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o
+xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
diff --git a/arch/loongarch/lib/xor_simd.c b/lib/raid/xor/loongarch/xor_simd.c
similarity index 100%
rename from arch/loongarch/lib/xor_simd.c
rename to lib/raid/xor/loongarch/xor_simd.c
diff --git a/arch/loongarch/lib/xor_simd.h b/lib/raid/xor/loongarch/xor_simd.h
similarity index 100%
rename from arch/loongarch/lib/xor_simd.h
rename to lib/raid/xor/loongarch/xor_simd.h
diff --git a/arch/loongarch/lib/xor_simd_glue.c b/lib/raid/xor/loongarch/xor_simd_glue.c
similarity index 64%
rename from arch/loongarch/lib/xor_simd_glue.c
rename to lib/raid/xor/loongarch/xor_simd_glue.c
index 393f689dbcf6..11fa3b47ba83 100644
--- a/arch/loongarch/lib/xor_simd_glue.c
+++ b/lib/raid/xor/loongarch/xor_simd_glue.c
@@ -5,24 +5,23 @@
* Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
*/
-#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/raid/xor_impl.h>
#include <asm/fpu.h>
-#include <asm/xor_simd.h>
+#include <asm/xor.h>
#include "xor_simd.h"
#define MAKE_XOR_GLUE_2(flavor) \
-void xor_##flavor##_2(unsigned long bytes, unsigned long * __restrict p1, \
+static void xor_##flavor##_2(unsigned long bytes, unsigned long * __restrict p1,\
const unsigned long * __restrict p2) \
{ \
kernel_fpu_begin(); \
__xor_##flavor##_2(bytes, p1, p2); \
kernel_fpu_end(); \
} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_2)
#define MAKE_XOR_GLUE_3(flavor) \
-void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1, \
+static void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1,\
const unsigned long * __restrict p2, \
const unsigned long * __restrict p3) \
{ \
@@ -30,10 +29,9 @@ void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1, \
__xor_##flavor##_3(bytes, p1, p2, p3); \
kernel_fpu_end(); \
} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_3)
#define MAKE_XOR_GLUE_4(flavor) \
-void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1, \
+static void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1,\
const unsigned long * __restrict p2, \
const unsigned long * __restrict p3, \
const unsigned long * __restrict p4) \
@@ -42,10 +40,9 @@ void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1, \
__xor_##flavor##_4(bytes, p1, p2, p3, p4); \
kernel_fpu_end(); \
} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_4)
#define MAKE_XOR_GLUE_5(flavor) \
-void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1, \
+static void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1,\
const unsigned long * __restrict p2, \
const unsigned long * __restrict p3, \
const unsigned long * __restrict p4, \
@@ -55,18 +52,26 @@ void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1, \
__xor_##flavor##_5(bytes, p1, p2, p3, p4, p5); \
kernel_fpu_end(); \
} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_5)
-#define MAKE_XOR_GLUES(flavor) \
- MAKE_XOR_GLUE_2(flavor); \
- MAKE_XOR_GLUE_3(flavor); \
- MAKE_XOR_GLUE_4(flavor); \
- MAKE_XOR_GLUE_5(flavor)
+#define MAKE_XOR_GLUES(flavor) \
+ MAKE_XOR_GLUE_2(flavor); \
+ MAKE_XOR_GLUE_3(flavor); \
+ MAKE_XOR_GLUE_4(flavor); \
+ MAKE_XOR_GLUE_5(flavor); \
+ \
+struct xor_block_template xor_block_##flavor = { \
+ .name = __stringify(flavor), \
+ .do_2 = xor_##flavor##_2, \
+ .do_3 = xor_##flavor##_3, \
+ .do_4 = xor_##flavor##_4, \
+ .do_5 = xor_##flavor##_5, \
+}
+
#ifdef CONFIG_CPU_HAS_LSX
MAKE_XOR_GLUES(lsx);
-#endif
+#endif /* CONFIG_CPU_HAS_LSX */
#ifdef CONFIG_CPU_HAS_LASX
MAKE_XOR_GLUES(lasx);
-#endif
+#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/arch/loongarch/lib/xor_template.c b/lib/raid/xor/loongarch/xor_template.c
similarity index 100%
rename from arch/loongarch/lib/xor_template.c
rename to lib/raid/xor/loongarch/xor_template.c
--
2.47.3
^ permalink raw reply related
* [PATCH 12/26] arm64: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR into lib/raid and include it it in the main
xor.ko instead of building a separate module for it.
Note that this drops the CONFIG_KERNEL_MODE_NEON dependency, as that is
always set for arm64.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/arm64/include/asm/xor.h | 58 +------------------
arch/arm64/lib/Makefile | 6 --
lib/raid/xor/Makefile | 4 ++
lib/raid/xor/arm64/xor-neon-glue.c | 57 ++++++++++++++++++
.../lib => lib/raid/xor/arm64}/xor-neon.c | 20 +------
5 files changed, 67 insertions(+), 78 deletions(-)
create mode 100644 lib/raid/xor/arm64/xor-neon-glue.c
rename {arch/arm64/lib => lib/raid/xor/arm64}/xor-neon.c (95%)
diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h
index bfa6122f55ce..81718f010761 100644
--- a/arch/arm64/include/asm/xor.h
+++ b/arch/arm64/include/asm/xor.h
@@ -1,73 +1,21 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
- * arch/arm64/include/asm/xor.h
- *
* Authors: Jackie Liu <liuyun01@kylinos.cn>
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
-#include <linux/hardirq.h>
#include <asm-generic/xor.h>
-#include <asm/hwcap.h>
#include <asm/simd.h>
-#ifdef CONFIG_KERNEL_MODE_NEON
-
-extern struct xor_block_template const xor_block_inner_neon;
-
-static void
-xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_2(bytes, p1, p2);
-}
-
-static void
-xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_3(bytes, p1, p2, p3);
-}
-
-static void
-xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
-}
-
-static void
-xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);
-}
-
-static struct xor_block_template xor_block_arm64 = {
- .name = "arm64_neon",
- .do_2 = xor_neon_2,
- .do_3 = xor_neon_3,
- .do_4 = xor_neon_4,
- .do_5 = xor_neon_5
-};
+extern struct xor_block_template xor_block_arm64;
+void __init xor_neon_init(void);
#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
+ xor_neon_init();
xor_register(&xor_block_8regs);
xor_register(&xor_block_32regs);
if (cpu_has_neon())
xor_register(&xor_block_arm64);
}
-
-#endif /* ! CONFIG_KERNEL_MODE_NEON */
diff --git a/arch/arm64/lib/Makefile b/arch/arm64/lib/Makefile
index 633e5223d944..448c917494f3 100644
--- a/arch/arm64/lib/Makefile
+++ b/arch/arm64/lib/Makefile
@@ -5,12 +5,6 @@ lib-y := clear_user.o delay.o copy_from_user.o \
memset.o memcmp.o strcmp.o strncmp.o strlen.o \
strnlen.o strchr.o strrchr.o tishift.o
-ifeq ($(CONFIG_KERNEL_MODE_NEON), y)
-obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o
-CFLAGS_xor-neon.o += $(CC_FLAGS_FPU)
-CFLAGS_REMOVE_xor-neon.o += $(CC_FLAGS_NO_FPU)
-endif
-
lib-$(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) += uaccess_flushcache.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index fb760edae54b..4ab0e7411ff7 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -13,7 +13,11 @@ xor-$(CONFIG_ARM) += arm/xor.o
ifeq ($(CONFIG_ARM),y)
xor-$(CONFIG_KERNEL_MODE_NEON) += arm/xor-neon.o arm/xor-neon-glue.o
endif
+xor-$(CONFIG_ARM64) += arm64/xor-neon.o arm64/xor-neon-glue.o
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
CFLAGS_REMOVE_arm/xor-neon.o += $(CC_FLAGS_NO_FPU)
+
+CFLAGS_arm64/xor-neon.o += $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_arm64/xor-neon.o += $(CC_FLAGS_NO_FPU)
diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c
new file mode 100644
index 000000000000..067a2095659a
--- /dev/null
+++ b/lib/raid/xor/arm64/xor-neon-glue.c
@@ -0,0 +1,57 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Authors: Jackie Liu <liuyun01@kylinos.cn>
+ * Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
+ */
+
+#include <linux/raid/xor_impl.h>
+#include <asm/simd.h>
+#include <asm/xor.h>
+
+extern struct xor_block_template const xor_block_inner_neon;
+
+static void
+xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ scoped_ksimd()
+ xor_block_inner_neon.do_2(bytes, p1, p2);
+}
+
+static void
+xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ scoped_ksimd()
+ xor_block_inner_neon.do_3(bytes, p1, p2, p3);
+}
+
+static void
+xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ scoped_ksimd()
+ xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
+}
+
+static void
+xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ scoped_ksimd()
+ xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);
+}
+
+struct xor_block_template xor_block_arm64 = {
+ .name = "arm64_neon",
+ .do_2 = xor_neon_2,
+ .do_3 = xor_neon_3,
+ .do_4 = xor_neon_4,
+ .do_5 = xor_neon_5
+};
diff --git a/arch/arm64/lib/xor-neon.c b/lib/raid/xor/arm64/xor-neon.c
similarity index 95%
rename from arch/arm64/lib/xor-neon.c
rename to lib/raid/xor/arm64/xor-neon.c
index 351aba92d932..8d2d185090db 100644
--- a/arch/arm64/lib/xor-neon.c
+++ b/lib/raid/xor/arm64/xor-neon.c
@@ -1,15 +1,13 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
- * arch/arm64/lib/xor-neon.c
- *
* Authors: Jackie Liu <liuyun01@kylinos.cn>
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
-#include <linux/raid/xor.h>
#include <linux/raid/xor_impl.h>
-#include <linux/module.h>
+#include <linux/cache.h>
#include <asm/neon-intrinsics.h>
+#include <asm/xor.h>
static void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
@@ -180,7 +178,6 @@ struct xor_block_template xor_block_inner_neon __ro_after_init = {
.do_4 = xor_arm64_neon_4,
.do_5 = xor_arm64_neon_5,
};
-EXPORT_SYMBOL(xor_block_inner_neon);
static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
{
@@ -318,22 +315,11 @@ static void xor_arm64_eor3_5(unsigned long bytes,
} while (--lines > 0);
}
-static int __init xor_neon_init(void)
+void __init xor_neon_init(void)
{
if (cpu_have_named_feature(SHA3)) {
xor_block_inner_neon.do_3 = xor_arm64_eor3_3;
xor_block_inner_neon.do_4 = xor_arm64_eor3_4;
xor_block_inner_neon.do_5 = xor_arm64_eor3_5;
}
- return 0;
}
-module_init(xor_neon_init);
-
-static void __exit xor_neon_exit(void)
-{
-}
-module_exit(xor_neon_exit);
-
-MODULE_AUTHOR("Jackie Liu <liuyun01@kylinos.cn>");
-MODULE_DESCRIPTION("ARMv8 XOR Extensions");
-MODULE_LICENSE("GPL");
--
2.47.3
^ permalink raw reply related
* [PATCH 11/26] arm: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR into lib/raid and include it it in the main
xor.ko instead of building a separate module for it.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/arm/include/asm/xor.h | 190 +-----------------
arch/arm/lib/Makefile | 5 -
lib/raid/xor/Makefile | 8 +
lib/raid/xor/arm/xor-neon-glue.c | 58 ++++++
{arch/arm/lib => lib/raid/xor/arm}/xor-neon.c | 10 +-
lib/raid/xor/arm/xor.c | 136 +++++++++++++
6 files changed, 205 insertions(+), 202 deletions(-)
create mode 100644 lib/raid/xor/arm/xor-neon-glue.c
rename {arch/arm/lib => lib/raid/xor/arm}/xor-neon.c (74%)
create mode 100644 lib/raid/xor/arm/xor.c
diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h
index b2dcd49186e2..989c55872ef6 100644
--- a/arch/arm/include/asm/xor.h
+++ b/arch/arm/include/asm/xor.h
@@ -1,198 +1,12 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
- * arch/arm/include/asm/xor.h
- *
* Copyright (C) 2001 Russell King
*/
#include <asm-generic/xor.h>
-#include <asm/hwcap.h>
#include <asm/neon.h>
-#define __XOR(a1, a2) a1 ^= a2
-
-#define GET_BLOCK_2(dst) \
- __asm__("ldmia %0, {%1, %2}" \
- : "=r" (dst), "=r" (a1), "=r" (a2) \
- : "0" (dst))
-
-#define GET_BLOCK_4(dst) \
- __asm__("ldmia %0, {%1, %2, %3, %4}" \
- : "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \
- : "0" (dst))
-
-#define XOR_BLOCK_2(src) \
- __asm__("ldmia %0!, {%1, %2}" \
- : "=r" (src), "=r" (b1), "=r" (b2) \
- : "0" (src)); \
- __XOR(a1, b1); __XOR(a2, b2);
-
-#define XOR_BLOCK_4(src) \
- __asm__("ldmia %0!, {%1, %2, %3, %4}" \
- : "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \
- : "0" (src)); \
- __XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4)
-
-#define PUT_BLOCK_2(dst) \
- __asm__ __volatile__("stmia %0!, {%2, %3}" \
- : "=r" (dst) \
- : "0" (dst), "r" (a1), "r" (a2))
-
-#define PUT_BLOCK_4(dst) \
- __asm__ __volatile__("stmia %0!, {%2, %3, %4, %5}" \
- : "=r" (dst) \
- : "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))
-
-static void
-xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned int lines = bytes / sizeof(unsigned long) / 4;
- register unsigned int a1 __asm__("r4");
- register unsigned int a2 __asm__("r5");
- register unsigned int a3 __asm__("r6");
- register unsigned int a4 __asm__("r10");
- register unsigned int b1 __asm__("r8");
- register unsigned int b2 __asm__("r9");
- register unsigned int b3 __asm__("ip");
- register unsigned int b4 __asm__("lr");
-
- do {
- GET_BLOCK_4(p1);
- XOR_BLOCK_4(p2);
- PUT_BLOCK_4(p1);
- } while (--lines);
-}
-
-static void
-xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned int lines = bytes / sizeof(unsigned long) / 4;
- register unsigned int a1 __asm__("r4");
- register unsigned int a2 __asm__("r5");
- register unsigned int a3 __asm__("r6");
- register unsigned int a4 __asm__("r10");
- register unsigned int b1 __asm__("r8");
- register unsigned int b2 __asm__("r9");
- register unsigned int b3 __asm__("ip");
- register unsigned int b4 __asm__("lr");
-
- do {
- GET_BLOCK_4(p1);
- XOR_BLOCK_4(p2);
- XOR_BLOCK_4(p3);
- PUT_BLOCK_4(p1);
- } while (--lines);
-}
-
-static void
-xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned int lines = bytes / sizeof(unsigned long) / 2;
- register unsigned int a1 __asm__("r8");
- register unsigned int a2 __asm__("r9");
- register unsigned int b1 __asm__("ip");
- register unsigned int b2 __asm__("lr");
-
- do {
- GET_BLOCK_2(p1);
- XOR_BLOCK_2(p2);
- XOR_BLOCK_2(p3);
- XOR_BLOCK_2(p4);
- PUT_BLOCK_2(p1);
- } while (--lines);
-}
-
-static void
-xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- unsigned int lines = bytes / sizeof(unsigned long) / 2;
- register unsigned int a1 __asm__("r8");
- register unsigned int a2 __asm__("r9");
- register unsigned int b1 __asm__("ip");
- register unsigned int b2 __asm__("lr");
-
- do {
- GET_BLOCK_2(p1);
- XOR_BLOCK_2(p2);
- XOR_BLOCK_2(p3);
- XOR_BLOCK_2(p4);
- XOR_BLOCK_2(p5);
- PUT_BLOCK_2(p1);
- } while (--lines);
-}
-
-static struct xor_block_template xor_block_arm4regs = {
- .name = "arm4regs",
- .do_2 = xor_arm4regs_2,
- .do_3 = xor_arm4regs_3,
- .do_4 = xor_arm4regs_4,
- .do_5 = xor_arm4regs_5,
-};
-
-#ifdef CONFIG_KERNEL_MODE_NEON
-
-extern struct xor_block_template const xor_block_neon_inner;
-
-static void
-xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- kernel_neon_begin();
- xor_block_neon_inner.do_2(bytes, p1, p2);
- kernel_neon_end();
-}
-
-static void
-xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- kernel_neon_begin();
- xor_block_neon_inner.do_3(bytes, p1, p2, p3);
- kernel_neon_end();
-}
-
-static void
-xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- kernel_neon_begin();
- xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
- kernel_neon_end();
-}
-
-static void
-xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- kernel_neon_begin();
- xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
- kernel_neon_end();
-}
-
-static struct xor_block_template xor_block_neon = {
- .name = "neon",
- .do_2 = xor_neon_2,
- .do_3 = xor_neon_3,
- .do_4 = xor_neon_4,
- .do_5 = xor_neon_5
-};
-
-#endif /* CONFIG_KERNEL_MODE_NEON */
+extern struct xor_block_template xor_block_arm4regs;
+extern struct xor_block_template xor_block_neon;
#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
diff --git a/arch/arm/lib/Makefile b/arch/arm/lib/Makefile
index 0ca5aae1bcc3..9295055cdfc9 100644
--- a/arch/arm/lib/Makefile
+++ b/arch/arm/lib/Makefile
@@ -39,9 +39,4 @@ endif
$(obj)/csumpartialcopy.o: $(obj)/csumpartialcopygeneric.S
$(obj)/csumpartialcopyuser.o: $(obj)/csumpartialcopygeneric.S
-ifeq ($(CONFIG_KERNEL_MODE_NEON),y)
- CFLAGS_xor-neon.o += $(CC_FLAGS_FPU)
- obj-$(CONFIG_XOR_BLOCKS) += xor-neon.o
-endif
-
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 6d03c27c37c7..fb760edae54b 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -9,3 +9,11 @@ xor-y += xor-8regs-prefetch.o
xor-y += xor-32regs-prefetch.o
xor-$(CONFIG_ALPHA) += alpha/xor.o
+xor-$(CONFIG_ARM) += arm/xor.o
+ifeq ($(CONFIG_ARM),y)
+xor-$(CONFIG_KERNEL_MODE_NEON) += arm/xor-neon.o arm/xor-neon-glue.o
+endif
+
+
+CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_arm/xor-neon.o += $(CC_FLAGS_NO_FPU)
diff --git a/lib/raid/xor/arm/xor-neon-glue.c b/lib/raid/xor/arm/xor-neon-glue.c
new file mode 100644
index 000000000000..c7b162b383a2
--- /dev/null
+++ b/lib/raid/xor/arm/xor-neon-glue.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2001 Russell King
+ */
+#include <linux/raid/xor_impl.h>
+#include <asm/xor.h>
+
+extern struct xor_block_template const xor_block_neon_inner;
+
+static void
+xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ kernel_neon_begin();
+ xor_block_neon_inner.do_2(bytes, p1, p2);
+ kernel_neon_end();
+}
+
+static void
+xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ kernel_neon_begin();
+ xor_block_neon_inner.do_3(bytes, p1, p2, p3);
+ kernel_neon_end();
+}
+
+static void
+xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ kernel_neon_begin();
+ xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
+ kernel_neon_end();
+}
+
+static void
+xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ kernel_neon_begin();
+ xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
+ kernel_neon_end();
+}
+
+struct xor_block_template xor_block_neon = {
+ .name = "neon",
+ .do_2 = xor_neon_2,
+ .do_3 = xor_neon_3,
+ .do_4 = xor_neon_4,
+ .do_5 = xor_neon_5
+};
diff --git a/arch/arm/lib/xor-neon.c b/lib/raid/xor/arm/xor-neon.c
similarity index 74%
rename from arch/arm/lib/xor-neon.c
rename to lib/raid/xor/arm/xor-neon.c
index b5be50567991..c9d4378b0f0e 100644
--- a/arch/arm/lib/xor-neon.c
+++ b/lib/raid/xor/arm/xor-neon.c
@@ -1,16 +1,9 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
- * linux/arch/arm/lib/xor-neon.c
- *
* Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
-#include <linux/raid/xor.h>
#include <linux/raid/xor_impl.h>
-#include <linux/module.h>
-
-MODULE_DESCRIPTION("NEON accelerated XOR implementation");
-MODULE_LICENSE("GPL");
#ifndef __ARM_NEON__
#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon'
@@ -27,7 +20,7 @@ MODULE_LICENSE("GPL");
#endif
#define NO_TEMPLATE
-#include "../../../lib/raid/xor/xor-8regs.c"
+#include "../xor-8regs.c"
struct xor_block_template const xor_block_neon_inner = {
.name = "__inner_neon__",
@@ -36,4 +29,3 @@ struct xor_block_template const xor_block_neon_inner = {
.do_4 = xor_8regs_4,
.do_5 = xor_8regs_5,
};
-EXPORT_SYMBOL(xor_block_neon_inner);
diff --git a/lib/raid/xor/arm/xor.c b/lib/raid/xor/arm/xor.c
new file mode 100644
index 000000000000..2263341dbbcd
--- /dev/null
+++ b/lib/raid/xor/arm/xor.c
@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2001 Russell King
+ */
+#include <linux/raid/xor_impl.h>
+#include <asm/xor.h>
+
+#define __XOR(a1, a2) a1 ^= a2
+
+#define GET_BLOCK_2(dst) \
+ __asm__("ldmia %0, {%1, %2}" \
+ : "=r" (dst), "=r" (a1), "=r" (a2) \
+ : "0" (dst))
+
+#define GET_BLOCK_4(dst) \
+ __asm__("ldmia %0, {%1, %2, %3, %4}" \
+ : "=r" (dst), "=r" (a1), "=r" (a2), "=r" (a3), "=r" (a4) \
+ : "0" (dst))
+
+#define XOR_BLOCK_2(src) \
+ __asm__("ldmia %0!, {%1, %2}" \
+ : "=r" (src), "=r" (b1), "=r" (b2) \
+ : "0" (src)); \
+ __XOR(a1, b1); __XOR(a2, b2);
+
+#define XOR_BLOCK_4(src) \
+ __asm__("ldmia %0!, {%1, %2, %3, %4}" \
+ : "=r" (src), "=r" (b1), "=r" (b2), "=r" (b3), "=r" (b4) \
+ : "0" (src)); \
+ __XOR(a1, b1); __XOR(a2, b2); __XOR(a3, b3); __XOR(a4, b4)
+
+#define PUT_BLOCK_2(dst) \
+ __asm__ __volatile__("stmia %0!, {%2, %3}" \
+ : "=r" (dst) \
+ : "0" (dst), "r" (a1), "r" (a2))
+
+#define PUT_BLOCK_4(dst) \
+ __asm__ __volatile__("stmia %0!, {%2, %3, %4, %5}" \
+ : "=r" (dst) \
+ : "0" (dst), "r" (a1), "r" (a2), "r" (a3), "r" (a4))
+
+static void
+xor_arm4regs_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ unsigned int lines = bytes / sizeof(unsigned long) / 4;
+ register unsigned int a1 __asm__("r4");
+ register unsigned int a2 __asm__("r5");
+ register unsigned int a3 __asm__("r6");
+ register unsigned int a4 __asm__("r10");
+ register unsigned int b1 __asm__("r8");
+ register unsigned int b2 __asm__("r9");
+ register unsigned int b3 __asm__("ip");
+ register unsigned int b4 __asm__("lr");
+
+ do {
+ GET_BLOCK_4(p1);
+ XOR_BLOCK_4(p2);
+ PUT_BLOCK_4(p1);
+ } while (--lines);
+}
+
+static void
+xor_arm4regs_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ unsigned int lines = bytes / sizeof(unsigned long) / 4;
+ register unsigned int a1 __asm__("r4");
+ register unsigned int a2 __asm__("r5");
+ register unsigned int a3 __asm__("r6");
+ register unsigned int a4 __asm__("r10");
+ register unsigned int b1 __asm__("r8");
+ register unsigned int b2 __asm__("r9");
+ register unsigned int b3 __asm__("ip");
+ register unsigned int b4 __asm__("lr");
+
+ do {
+ GET_BLOCK_4(p1);
+ XOR_BLOCK_4(p2);
+ XOR_BLOCK_4(p3);
+ PUT_BLOCK_4(p1);
+ } while (--lines);
+}
+
+static void
+xor_arm4regs_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ unsigned int lines = bytes / sizeof(unsigned long) / 2;
+ register unsigned int a1 __asm__("r8");
+ register unsigned int a2 __asm__("r9");
+ register unsigned int b1 __asm__("ip");
+ register unsigned int b2 __asm__("lr");
+
+ do {
+ GET_BLOCK_2(p1);
+ XOR_BLOCK_2(p2);
+ XOR_BLOCK_2(p3);
+ XOR_BLOCK_2(p4);
+ PUT_BLOCK_2(p1);
+ } while (--lines);
+}
+
+static void
+xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ unsigned int lines = bytes / sizeof(unsigned long) / 2;
+ register unsigned int a1 __asm__("r8");
+ register unsigned int a2 __asm__("r9");
+ register unsigned int b1 __asm__("ip");
+ register unsigned int b2 __asm__("lr");
+
+ do {
+ GET_BLOCK_2(p1);
+ XOR_BLOCK_2(p2);
+ XOR_BLOCK_2(p3);
+ XOR_BLOCK_2(p4);
+ XOR_BLOCK_2(p5);
+ PUT_BLOCK_2(p1);
+ } while (--lines);
+}
+
+struct xor_block_template xor_block_arm4regs = {
+ .name = "arm4regs",
+ .do_2 = xor_arm4regs_2,
+ .do_3 = xor_arm4regs_3,
+ .do_4 = xor_arm4regs_4,
+ .do_5 = xor_arm4regs_5,
+};
--
2.47.3
^ permalink raw reply related
* [PATCH 10/26] alpha: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR code out of line into lib/raid.
Note that the giant inline assembly block might be better off as a
separate assembly source file now, but I'll leave that to the alpha
maintainers.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Magnus Lindholm <linmag7@gmail.com>
Tested-by: Magnus Lindholm <linmag7@gmail.com>
---
arch/alpha/include/asm/xor.h | 853 +----------------------------------
lib/raid/xor/Makefile | 2 +
lib/raid/xor/alpha/xor.c | 849 ++++++++++++++++++++++++++++++++++
3 files changed, 855 insertions(+), 849 deletions(-)
create mode 100644 lib/raid/xor/alpha/xor.c
diff --git a/arch/alpha/include/asm/xor.h b/arch/alpha/include/asm/xor.h
index 4c8085711df1..e517be577a09 100644
--- a/arch/alpha/include/asm/xor.h
+++ b/arch/alpha/include/asm/xor.h
@@ -1,856 +1,11 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm-alpha/xor.h
- *
- * Optimized RAID-5 checksumming functions for alpha EV5 and EV6
- */
-
-extern void
-xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-extern void
-xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-extern void
-xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-extern void
-xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-
-extern void
-xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-extern void
-xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-extern void
-xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-extern void
-xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-asm(" \n\
- .text \n\
- .align 3 \n\
- .ent xor_alpha_2 \n\
-xor_alpha_2: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- .align 4 \n\
-2: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,8($17) \n\
- ldq $3,8($18) \n\
- \n\
- ldq $4,16($17) \n\
- ldq $5,16($18) \n\
- ldq $6,24($17) \n\
- ldq $7,24($18) \n\
- \n\
- ldq $19,32($17) \n\
- ldq $20,32($18) \n\
- ldq $21,40($17) \n\
- ldq $22,40($18) \n\
- \n\
- ldq $23,48($17) \n\
- ldq $24,48($18) \n\
- ldq $25,56($17) \n\
- xor $0,$1,$0 # 7 cycles from $1 load \n\
- \n\
- ldq $27,56($18) \n\
- xor $2,$3,$2 \n\
- stq $0,0($17) \n\
- xor $4,$5,$4 \n\
- \n\
- stq $2,8($17) \n\
- xor $6,$7,$6 \n\
- stq $4,16($17) \n\
- xor $19,$20,$19 \n\
- \n\
- stq $6,24($17) \n\
- xor $21,$22,$21 \n\
- stq $19,32($17) \n\
- xor $23,$24,$23 \n\
- \n\
- stq $21,40($17) \n\
- xor $25,$27,$25 \n\
- stq $23,48($17) \n\
- subq $16,1,$16 \n\
- \n\
- stq $25,56($17) \n\
- addq $17,64,$17 \n\
- addq $18,64,$18 \n\
- bgt $16,2b \n\
- \n\
- ret \n\
- .end xor_alpha_2 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_3 \n\
-xor_alpha_3: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- .align 4 \n\
-3: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,8($17) \n\
- \n\
- ldq $4,8($18) \n\
- ldq $6,16($17) \n\
- ldq $7,16($18) \n\
- ldq $21,24($17) \n\
- \n\
- ldq $22,24($18) \n\
- ldq $24,32($17) \n\
- ldq $25,32($18) \n\
- ldq $5,8($19) \n\
- \n\
- ldq $20,16($19) \n\
- ldq $23,24($19) \n\
- ldq $27,32($19) \n\
- nop \n\
- \n\
- xor $0,$1,$1 # 8 cycles from $0 load \n\
- xor $3,$4,$4 # 6 cycles from $4 load \n\
- xor $6,$7,$7 # 6 cycles from $7 load \n\
- xor $21,$22,$22 # 5 cycles from $22 load \n\
- \n\
- xor $1,$2,$2 # 9 cycles from $2 load \n\
- xor $24,$25,$25 # 5 cycles from $25 load \n\
- stq $2,0($17) \n\
- xor $4,$5,$5 # 6 cycles from $5 load \n\
- \n\
- stq $5,8($17) \n\
- xor $7,$20,$20 # 7 cycles from $20 load \n\
- stq $20,16($17) \n\
- xor $22,$23,$23 # 7 cycles from $23 load \n\
- \n\
- stq $23,24($17) \n\
- xor $25,$27,$27 # 7 cycles from $27 load \n\
- stq $27,32($17) \n\
- nop \n\
- \n\
- ldq $0,40($17) \n\
- ldq $1,40($18) \n\
- ldq $3,48($17) \n\
- ldq $4,48($18) \n\
- \n\
- ldq $6,56($17) \n\
- ldq $7,56($18) \n\
- ldq $2,40($19) \n\
- ldq $5,48($19) \n\
- \n\
- ldq $20,56($19) \n\
- xor $0,$1,$1 # 4 cycles from $1 load \n\
- xor $3,$4,$4 # 5 cycles from $4 load \n\
- xor $6,$7,$7 # 5 cycles from $7 load \n\
- \n\
- xor $1,$2,$2 # 4 cycles from $2 load \n\
- xor $4,$5,$5 # 5 cycles from $5 load \n\
- stq $2,40($17) \n\
- xor $7,$20,$20 # 4 cycles from $20 load \n\
- \n\
- stq $5,48($17) \n\
- subq $16,1,$16 \n\
- stq $20,56($17) \n\
- addq $19,64,$19 \n\
- \n\
- addq $18,64,$18 \n\
- addq $17,64,$17 \n\
- bgt $16,3b \n\
- ret \n\
- .end xor_alpha_3 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_4 \n\
-xor_alpha_4: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- .align 4 \n\
-4: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,0($20) \n\
- \n\
- ldq $4,8($17) \n\
- ldq $5,8($18) \n\
- ldq $6,8($19) \n\
- ldq $7,8($20) \n\
- \n\
- ldq $21,16($17) \n\
- ldq $22,16($18) \n\
- ldq $23,16($19) \n\
- ldq $24,16($20) \n\
- \n\
- ldq $25,24($17) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- ldq $27,24($18) \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- ldq $0,24($19) \n\
- xor $1,$3,$3 \n\
- ldq $1,24($20) \n\
- xor $4,$5,$5 # 7 cycles from $5 load \n\
- \n\
- stq $3,0($17) \n\
- xor $6,$7,$7 \n\
- xor $21,$22,$22 # 7 cycles from $22 load \n\
- xor $5,$7,$7 \n\
- \n\
- stq $7,8($17) \n\
- xor $23,$24,$24 # 7 cycles from $24 load \n\
- ldq $2,32($17) \n\
- xor $22,$24,$24 \n\
- \n\
- ldq $3,32($18) \n\
- ldq $4,32($19) \n\
- ldq $5,32($20) \n\
- xor $25,$27,$27 # 8 cycles from $27 load \n\
- \n\
- ldq $6,40($17) \n\
- ldq $7,40($18) \n\
- ldq $21,40($19) \n\
- ldq $22,40($20) \n\
- \n\
- stq $24,16($17) \n\
- xor $0,$1,$1 # 9 cycles from $1 load \n\
- xor $2,$3,$3 # 5 cycles from $3 load \n\
- xor $27,$1,$1 \n\
- \n\
- stq $1,24($17) \n\
- xor $4,$5,$5 # 5 cycles from $5 load \n\
- ldq $23,48($17) \n\
- ldq $24,48($18) \n\
- \n\
- ldq $25,48($19) \n\
- xor $3,$5,$5 \n\
- ldq $27,48($20) \n\
- ldq $0,56($17) \n\
- \n\
- ldq $1,56($18) \n\
- ldq $2,56($19) \n\
- xor $6,$7,$7 # 8 cycles from $6 load \n\
- ldq $3,56($20) \n\
- \n\
- stq $5,32($17) \n\
- xor $21,$22,$22 # 8 cycles from $22 load \n\
- xor $7,$22,$22 \n\
- xor $23,$24,$24 # 5 cycles from $24 load \n\
- \n\
- stq $22,40($17) \n\
- xor $25,$27,$27 # 5 cycles from $27 load \n\
- xor $24,$27,$27 \n\
- xor $0,$1,$1 # 5 cycles from $1 load \n\
- \n\
- stq $27,48($17) \n\
- xor $2,$3,$3 # 4 cycles from $3 load \n\
- xor $1,$3,$3 \n\
- subq $16,1,$16 \n\
- \n\
- stq $3,56($17) \n\
- addq $20,64,$20 \n\
- addq $19,64,$19 \n\
- addq $18,64,$18 \n\
- \n\
- addq $17,64,$17 \n\
- bgt $16,4b \n\
- ret \n\
- .end xor_alpha_4 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_5 \n\
-xor_alpha_5: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- .align 4 \n\
-5: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,0($20) \n\
- \n\
- ldq $4,0($21) \n\
- ldq $5,8($17) \n\
- ldq $6,8($18) \n\
- ldq $7,8($19) \n\
- \n\
- ldq $22,8($20) \n\
- ldq $23,8($21) \n\
- ldq $24,16($17) \n\
- ldq $25,16($18) \n\
- \n\
- ldq $27,16($19) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- ldq $28,16($20) \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- ldq $0,16($21) \n\
- xor $1,$3,$3 \n\
- ldq $1,24($17) \n\
- xor $3,$4,$4 # 7 cycles from $4 load \n\
- \n\
- stq $4,0($17) \n\
- xor $5,$6,$6 # 7 cycles from $6 load \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- xor $6,$23,$23 # 7 cycles from $23 load \n\
- \n\
- ldq $2,24($18) \n\
- xor $22,$23,$23 \n\
- ldq $3,24($19) \n\
- xor $24,$25,$25 # 8 cycles from $25 load \n\
- \n\
- stq $23,8($17) \n\
- xor $25,$27,$27 # 8 cycles from $27 load \n\
- ldq $4,24($20) \n\
- xor $28,$0,$0 # 7 cycles from $0 load \n\
- \n\
- ldq $5,24($21) \n\
- xor $27,$0,$0 \n\
- ldq $6,32($17) \n\
- ldq $7,32($18) \n\
- \n\
- stq $0,16($17) \n\
- xor $1,$2,$2 # 6 cycles from $2 load \n\
- ldq $22,32($19) \n\
- xor $3,$4,$4 # 4 cycles from $4 load \n\
- \n\
- ldq $23,32($20) \n\
- xor $2,$4,$4 \n\
- ldq $24,32($21) \n\
- ldq $25,40($17) \n\
- \n\
- ldq $27,40($18) \n\
- ldq $28,40($19) \n\
- ldq $0,40($20) \n\
- xor $4,$5,$5 # 7 cycles from $5 load \n\
- \n\
- stq $5,24($17) \n\
- xor $6,$7,$7 # 7 cycles from $7 load \n\
- ldq $1,40($21) \n\
- ldq $2,48($17) \n\
- \n\
- ldq $3,48($18) \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- ldq $4,48($19) \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- \n\
- ldq $5,48($20) \n\
- xor $22,$24,$24 \n\
- ldq $6,48($21) \n\
- xor $25,$27,$27 # 7 cycles from $27 load \n\
- \n\
- stq $24,32($17) \n\
- xor $27,$28,$28 # 8 cycles from $28 load \n\
- ldq $7,56($17) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- \n\
- ldq $22,56($18) \n\
- ldq $23,56($19) \n\
- ldq $24,56($20) \n\
- ldq $25,56($21) \n\
- \n\
- xor $28,$1,$1 \n\
- xor $2,$3,$3 # 9 cycles from $3 load \n\
- xor $3,$4,$4 # 9 cycles from $4 load \n\
- xor $5,$6,$6 # 8 cycles from $6 load \n\
- \n\
- stq $1,40($17) \n\
- xor $4,$6,$6 \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- \n\
- stq $6,48($17) \n\
- xor $22,$24,$24 \n\
- subq $16,1,$16 \n\
- xor $24,$25,$25 # 8 cycles from $25 load \n\
- \n\
- stq $25,56($17) \n\
- addq $21,64,$21 \n\
- addq $20,64,$20 \n\
- addq $19,64,$19 \n\
- \n\
- addq $18,64,$18 \n\
- addq $17,64,$17 \n\
- bgt $16,5b \n\
- ret \n\
- .end xor_alpha_5 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_prefetch_2 \n\
-xor_alpha_prefetch_2: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- \n\
- ldq $31, 0($17) \n\
- ldq $31, 0($18) \n\
- \n\
- ldq $31, 64($17) \n\
- ldq $31, 64($18) \n\
- \n\
- ldq $31, 128($17) \n\
- ldq $31, 128($18) \n\
- \n\
- ldq $31, 192($17) \n\
- ldq $31, 192($18) \n\
- .align 4 \n\
-2: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,8($17) \n\
- ldq $3,8($18) \n\
- \n\
- ldq $4,16($17) \n\
- ldq $5,16($18) \n\
- ldq $6,24($17) \n\
- ldq $7,24($18) \n\
- \n\
- ldq $19,32($17) \n\
- ldq $20,32($18) \n\
- ldq $21,40($17) \n\
- ldq $22,40($18) \n\
- \n\
- ldq $23,48($17) \n\
- ldq $24,48($18) \n\
- ldq $25,56($17) \n\
- ldq $27,56($18) \n\
- \n\
- ldq $31,256($17) \n\
- xor $0,$1,$0 # 8 cycles from $1 load \n\
- ldq $31,256($18) \n\
- xor $2,$3,$2 \n\
- \n\
- stq $0,0($17) \n\
- xor $4,$5,$4 \n\
- stq $2,8($17) \n\
- xor $6,$7,$6 \n\
- \n\
- stq $4,16($17) \n\
- xor $19,$20,$19 \n\
- stq $6,24($17) \n\
- xor $21,$22,$21 \n\
- \n\
- stq $19,32($17) \n\
- xor $23,$24,$23 \n\
- stq $21,40($17) \n\
- xor $25,$27,$25 \n\
- \n\
- stq $23,48($17) \n\
- subq $16,1,$16 \n\
- stq $25,56($17) \n\
- addq $17,64,$17 \n\
- \n\
- addq $18,64,$18 \n\
- bgt $16,2b \n\
- ret \n\
- .end xor_alpha_prefetch_2 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_prefetch_3 \n\
-xor_alpha_prefetch_3: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- \n\
- ldq $31, 0($17) \n\
- ldq $31, 0($18) \n\
- ldq $31, 0($19) \n\
- \n\
- ldq $31, 64($17) \n\
- ldq $31, 64($18) \n\
- ldq $31, 64($19) \n\
- \n\
- ldq $31, 128($17) \n\
- ldq $31, 128($18) \n\
- ldq $31, 128($19) \n\
- \n\
- ldq $31, 192($17) \n\
- ldq $31, 192($18) \n\
- ldq $31, 192($19) \n\
- .align 4 \n\
-3: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,8($17) \n\
- \n\
- ldq $4,8($18) \n\
- ldq $6,16($17) \n\
- ldq $7,16($18) \n\
- ldq $21,24($17) \n\
- \n\
- ldq $22,24($18) \n\
- ldq $24,32($17) \n\
- ldq $25,32($18) \n\
- ldq $5,8($19) \n\
- \n\
- ldq $20,16($19) \n\
- ldq $23,24($19) \n\
- ldq $27,32($19) \n\
- nop \n\
- \n\
- xor $0,$1,$1 # 8 cycles from $0 load \n\
- xor $3,$4,$4 # 7 cycles from $4 load \n\
- xor $6,$7,$7 # 6 cycles from $7 load \n\
- xor $21,$22,$22 # 5 cycles from $22 load \n\
- \n\
- xor $1,$2,$2 # 9 cycles from $2 load \n\
- xor $24,$25,$25 # 5 cycles from $25 load \n\
- stq $2,0($17) \n\
- xor $4,$5,$5 # 6 cycles from $5 load \n\
- \n\
- stq $5,8($17) \n\
- xor $7,$20,$20 # 7 cycles from $20 load \n\
- stq $20,16($17) \n\
- xor $22,$23,$23 # 7 cycles from $23 load \n\
- \n\
- stq $23,24($17) \n\
- xor $25,$27,$27 # 7 cycles from $27 load \n\
- stq $27,32($17) \n\
- nop \n\
- \n\
- ldq $0,40($17) \n\
- ldq $1,40($18) \n\
- ldq $3,48($17) \n\
- ldq $4,48($18) \n\
- \n\
- ldq $6,56($17) \n\
- ldq $7,56($18) \n\
- ldq $2,40($19) \n\
- ldq $5,48($19) \n\
- \n\
- ldq $20,56($19) \n\
- ldq $31,256($17) \n\
- ldq $31,256($18) \n\
- ldq $31,256($19) \n\
- \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- xor $3,$4,$4 # 5 cycles from $4 load \n\
- xor $6,$7,$7 # 5 cycles from $7 load \n\
- xor $1,$2,$2 # 4 cycles from $2 load \n\
- \n\
- xor $4,$5,$5 # 5 cycles from $5 load \n\
- xor $7,$20,$20 # 4 cycles from $20 load \n\
- stq $2,40($17) \n\
- subq $16,1,$16 \n\
- \n\
- stq $5,48($17) \n\
- addq $19,64,$19 \n\
- stq $20,56($17) \n\
- addq $18,64,$18 \n\
- \n\
- addq $17,64,$17 \n\
- bgt $16,3b \n\
- ret \n\
- .end xor_alpha_prefetch_3 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_prefetch_4 \n\
-xor_alpha_prefetch_4: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- \n\
- ldq $31, 0($17) \n\
- ldq $31, 0($18) \n\
- ldq $31, 0($19) \n\
- ldq $31, 0($20) \n\
- \n\
- ldq $31, 64($17) \n\
- ldq $31, 64($18) \n\
- ldq $31, 64($19) \n\
- ldq $31, 64($20) \n\
- \n\
- ldq $31, 128($17) \n\
- ldq $31, 128($18) \n\
- ldq $31, 128($19) \n\
- ldq $31, 128($20) \n\
- \n\
- ldq $31, 192($17) \n\
- ldq $31, 192($18) \n\
- ldq $31, 192($19) \n\
- ldq $31, 192($20) \n\
- .align 4 \n\
-4: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,0($20) \n\
- \n\
- ldq $4,8($17) \n\
- ldq $5,8($18) \n\
- ldq $6,8($19) \n\
- ldq $7,8($20) \n\
- \n\
- ldq $21,16($17) \n\
- ldq $22,16($18) \n\
- ldq $23,16($19) \n\
- ldq $24,16($20) \n\
- \n\
- ldq $25,24($17) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- ldq $27,24($18) \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- ldq $0,24($19) \n\
- xor $1,$3,$3 \n\
- ldq $1,24($20) \n\
- xor $4,$5,$5 # 7 cycles from $5 load \n\
- \n\
- stq $3,0($17) \n\
- xor $6,$7,$7 \n\
- xor $21,$22,$22 # 7 cycles from $22 load \n\
- xor $5,$7,$7 \n\
- \n\
- stq $7,8($17) \n\
- xor $23,$24,$24 # 7 cycles from $24 load \n\
- ldq $2,32($17) \n\
- xor $22,$24,$24 \n\
- \n\
- ldq $3,32($18) \n\
- ldq $4,32($19) \n\
- ldq $5,32($20) \n\
- xor $25,$27,$27 # 8 cycles from $27 load \n\
- \n\
- ldq $6,40($17) \n\
- ldq $7,40($18) \n\
- ldq $21,40($19) \n\
- ldq $22,40($20) \n\
- \n\
- stq $24,16($17) \n\
- xor $0,$1,$1 # 9 cycles from $1 load \n\
- xor $2,$3,$3 # 5 cycles from $3 load \n\
- xor $27,$1,$1 \n\
- \n\
- stq $1,24($17) \n\
- xor $4,$5,$5 # 5 cycles from $5 load \n\
- ldq $23,48($17) \n\
- xor $3,$5,$5 \n\
- \n\
- ldq $24,48($18) \n\
- ldq $25,48($19) \n\
- ldq $27,48($20) \n\
- ldq $0,56($17) \n\
- \n\
- ldq $1,56($18) \n\
- ldq $2,56($19) \n\
- ldq $3,56($20) \n\
- xor $6,$7,$7 # 8 cycles from $6 load \n\
- \n\
- ldq $31,256($17) \n\
- xor $21,$22,$22 # 8 cycles from $22 load \n\
- ldq $31,256($18) \n\
- xor $7,$22,$22 \n\
- \n\
- ldq $31,256($19) \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- ldq $31,256($20) \n\
- xor $25,$27,$27 # 6 cycles from $27 load \n\
- \n\
- stq $5,32($17) \n\
- xor $24,$27,$27 \n\
- xor $0,$1,$1 # 7 cycles from $1 load \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- stq $22,40($17) \n\
- xor $1,$3,$3 \n\
- stq $27,48($17) \n\
- subq $16,1,$16 \n\
- \n\
- stq $3,56($17) \n\
- addq $20,64,$20 \n\
- addq $19,64,$19 \n\
- addq $18,64,$18 \n\
- \n\
- addq $17,64,$17 \n\
- bgt $16,4b \n\
- ret \n\
- .end xor_alpha_prefetch_4 \n\
- \n\
- .align 3 \n\
- .ent xor_alpha_prefetch_5 \n\
-xor_alpha_prefetch_5: \n\
- .prologue 0 \n\
- srl $16, 6, $16 \n\
- \n\
- ldq $31, 0($17) \n\
- ldq $31, 0($18) \n\
- ldq $31, 0($19) \n\
- ldq $31, 0($20) \n\
- ldq $31, 0($21) \n\
- \n\
- ldq $31, 64($17) \n\
- ldq $31, 64($18) \n\
- ldq $31, 64($19) \n\
- ldq $31, 64($20) \n\
- ldq $31, 64($21) \n\
- \n\
- ldq $31, 128($17) \n\
- ldq $31, 128($18) \n\
- ldq $31, 128($19) \n\
- ldq $31, 128($20) \n\
- ldq $31, 128($21) \n\
- \n\
- ldq $31, 192($17) \n\
- ldq $31, 192($18) \n\
- ldq $31, 192($19) \n\
- ldq $31, 192($20) \n\
- ldq $31, 192($21) \n\
- .align 4 \n\
-5: \n\
- ldq $0,0($17) \n\
- ldq $1,0($18) \n\
- ldq $2,0($19) \n\
- ldq $3,0($20) \n\
- \n\
- ldq $4,0($21) \n\
- ldq $5,8($17) \n\
- ldq $6,8($18) \n\
- ldq $7,8($19) \n\
- \n\
- ldq $22,8($20) \n\
- ldq $23,8($21) \n\
- ldq $24,16($17) \n\
- ldq $25,16($18) \n\
- \n\
- ldq $27,16($19) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- ldq $28,16($20) \n\
- xor $2,$3,$3 # 6 cycles from $3 load \n\
- \n\
- ldq $0,16($21) \n\
- xor $1,$3,$3 \n\
- ldq $1,24($17) \n\
- xor $3,$4,$4 # 7 cycles from $4 load \n\
- \n\
- stq $4,0($17) \n\
- xor $5,$6,$6 # 7 cycles from $6 load \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- xor $6,$23,$23 # 7 cycles from $23 load \n\
- \n\
- ldq $2,24($18) \n\
- xor $22,$23,$23 \n\
- ldq $3,24($19) \n\
- xor $24,$25,$25 # 8 cycles from $25 load \n\
- \n\
- stq $23,8($17) \n\
- xor $25,$27,$27 # 8 cycles from $27 load \n\
- ldq $4,24($20) \n\
- xor $28,$0,$0 # 7 cycles from $0 load \n\
- \n\
- ldq $5,24($21) \n\
- xor $27,$0,$0 \n\
- ldq $6,32($17) \n\
- ldq $7,32($18) \n\
- \n\
- stq $0,16($17) \n\
- xor $1,$2,$2 # 6 cycles from $2 load \n\
- ldq $22,32($19) \n\
- xor $3,$4,$4 # 4 cycles from $4 load \n\
- \n\
- ldq $23,32($20) \n\
- xor $2,$4,$4 \n\
- ldq $24,32($21) \n\
- ldq $25,40($17) \n\
- \n\
- ldq $27,40($18) \n\
- ldq $28,40($19) \n\
- ldq $0,40($20) \n\
- xor $4,$5,$5 # 7 cycles from $5 load \n\
- \n\
- stq $5,24($17) \n\
- xor $6,$7,$7 # 7 cycles from $7 load \n\
- ldq $1,40($21) \n\
- ldq $2,48($17) \n\
- \n\
- ldq $3,48($18) \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- ldq $4,48($19) \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- \n\
- ldq $5,48($20) \n\
- xor $22,$24,$24 \n\
- ldq $6,48($21) \n\
- xor $25,$27,$27 # 7 cycles from $27 load \n\
- \n\
- stq $24,32($17) \n\
- xor $27,$28,$28 # 8 cycles from $28 load \n\
- ldq $7,56($17) \n\
- xor $0,$1,$1 # 6 cycles from $1 load \n\
- \n\
- ldq $22,56($18) \n\
- ldq $23,56($19) \n\
- ldq $24,56($20) \n\
- ldq $25,56($21) \n\
- \n\
- ldq $31,256($17) \n\
- xor $28,$1,$1 \n\
- ldq $31,256($18) \n\
- xor $2,$3,$3 # 9 cycles from $3 load \n\
- \n\
- ldq $31,256($19) \n\
- xor $3,$4,$4 # 9 cycles from $4 load \n\
- ldq $31,256($20) \n\
- xor $5,$6,$6 # 8 cycles from $6 load \n\
- \n\
- stq $1,40($17) \n\
- xor $4,$6,$6 \n\
- xor $7,$22,$22 # 7 cycles from $22 load \n\
- xor $23,$24,$24 # 6 cycles from $24 load \n\
- \n\
- stq $6,48($17) \n\
- xor $22,$24,$24 \n\
- ldq $31,256($21) \n\
- xor $24,$25,$25 # 8 cycles from $25 load \n\
- \n\
- stq $25,56($17) \n\
- subq $16,1,$16 \n\
- addq $21,64,$21 \n\
- addq $20,64,$20 \n\
- \n\
- addq $19,64,$19 \n\
- addq $18,64,$18 \n\
- addq $17,64,$17 \n\
- bgt $16,5b \n\
- \n\
- ret \n\
- .end xor_alpha_prefetch_5 \n\
-");
-
-static struct xor_block_template xor_block_alpha = {
- .name = "alpha",
- .do_2 = xor_alpha_2,
- .do_3 = xor_alpha_3,
- .do_4 = xor_alpha_4,
- .do_5 = xor_alpha_5,
-};
-
-static struct xor_block_template xor_block_alpha_prefetch = {
- .name = "alpha prefetch",
- .do_2 = xor_alpha_prefetch_2,
- .do_3 = xor_alpha_prefetch_3,
- .do_4 = xor_alpha_prefetch_4,
- .do_5 = xor_alpha_prefetch_5,
-};
-
-/* For grins, also test the generic routines. */
+#include <asm/special_insns.h>
#include <asm-generic/xor.h>
+extern struct xor_block_template xor_block_alpha;
+extern struct xor_block_template xor_block_alpha_prefetch;
+
/*
* Force the use of alpha_prefetch if EV6, as it is significantly faster in the
* cold cache case.
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 89a944c9f990..6d03c27c37c7 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -7,3 +7,5 @@ xor-y += xor-8regs.o
xor-y += xor-32regs.o
xor-y += xor-8regs-prefetch.o
xor-y += xor-32regs-prefetch.o
+
+xor-$(CONFIG_ALPHA) += alpha/xor.o
diff --git a/lib/raid/xor/alpha/xor.c b/lib/raid/xor/alpha/xor.c
new file mode 100644
index 000000000000..0964ac420604
--- /dev/null
+++ b/lib/raid/xor/alpha/xor.c
@@ -0,0 +1,849 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Optimized XOR parity functions for alpha EV5 and EV6
+ */
+#include <linux/raid/xor_impl.h>
+#include <asm/xor.h>
+
+extern void
+xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2);
+extern void
+xor_alpha_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+extern void
+xor_alpha_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+extern void
+xor_alpha_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
+
+extern void
+xor_alpha_prefetch_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2);
+extern void
+xor_alpha_prefetch_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+extern void
+xor_alpha_prefetch_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+extern void
+xor_alpha_prefetch_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
+
+asm(" \n\
+ .text \n\
+ .align 3 \n\
+ .ent xor_alpha_2 \n\
+xor_alpha_2: \n\
+ .prologue 0 \n\
+ srl $16, 6, $16 \n\
+ .align 4 \n\
+2: \n\
+ ldq $0,0($17) \n\
+ ldq $1,0($18) \n\
+ ldq $2,8($17) \n\
+ ldq $3,8($18) \n\
+ \n\
+ ldq $4,16($17) \n\
+ ldq $5,16($18) \n\
+ ldq $6,24($17) \n\
+ ldq $7,24($18) \n\
+ \n\
+ ldq $19,32($17) \n\
+ ldq $20,32($18) \n\
+ ldq $21,40($17) \n\
+ ldq $22,40($18) \n\
+ \n\
+ ldq $23,48($17) \n\
+ ldq $24,48($18) \n\
+ ldq $25,56($17) \n\
+ xor $0,$1,$0 # 7 cycles from $1 load \n\
+ \n\
+ ldq $27,56($18) \n\
+ xor $2,$3,$2 \n\
+ stq $0,0($17) \n\
+ xor $4,$5,$4 \n\
+ \n\
+ stq $2,8($17) \n\
+ xor $6,$7,$6 \n\
+ stq $4,16($17) \n\
+ xor $19,$20,$19 \n\
+ \n\
+ stq $6,24($17) \n\
+ xor $21,$22,$21 \n\
+ stq $19,32($17) \n\
+ xor $23,$24,$23 \n\
+ \n\
+ stq $21,40($17) \n\
+ xor $25,$27,$25 \n\
+ stq $23,48($17) \n\
+ subq $16,1,$16 \n\
+ \n\
+ stq $25,56($17) \n\
+ addq $17,64,$17 \n\
+ addq $18,64,$18 \n\
+ bgt $16,2b \n\
+ \n\
+ ret \n\
+ .end xor_alpha_2 \n\
+ \n\
+ .align 3 \n\
+ .ent xor_alpha_3 \n\
+xor_alpha_3: \n\
+ .prologue 0 \n\
+ srl $16, 6, $16 \n\
+ .align 4 \n\
+3: \n\
+ ldq $0,0($17) \n\
+ ldq $1,0($18) \n\
+ ldq $2,0($19) \n\
+ ldq $3,8($17) \n\
+ \n\
+ ldq $4,8($18) \n\
+ ldq $6,16($17) \n\
+ ldq $7,16($18) \n\
+ ldq $21,24($17) \n\
+ \n\
+ ldq $22,24($18) \n\
+ ldq $24,32($17) \n\
+ ldq $25,32($18) \n\
+ ldq $5,8($19) \n\
+ \n\
+ ldq $20,16($19) \n\
+ ldq $23,24($19) \n\
+ ldq $27,32($19) \n\
+ nop \n\
+ \n\
+ xor $0,$1,$1 # 8 cycles from $0 load \n\
+ xor $3,$4,$4 # 6 cycles from $4 load \n\
+ xor $6,$7,$7 # 6 cycles from $7 load \n\
+ xor $21,$22,$22 # 5 cycles from $22 load \n\
+ \n\
+ xor $1,$2,$2 # 9 cycles from $2 load \n\
+ xor $24,$25,$25 # 5 cycles from $25 load \n\
+ stq $2,0($17) \n\
+ xor $4,$5,$5 # 6 cycles from $5 load \n\
+ \n\
+ stq $5,8($17) \n\
+ xor $7,$20,$20 # 7 cycles from $20 load \n\
+ stq $20,16($17) \n\
+ xor $22,$23,$23 # 7 cycles from $23 load \n\
+ \n\
+ stq $23,24($17) \n\
+ xor $25,$27,$27 # 7 cycles from $27 load \n\
+ stq $27,32($17) \n\
+ nop \n\
+ \n\
+ ldq $0,40($17) \n\
+ ldq $1,40($18) \n\
+ ldq $3,48($17) \n\
+ ldq $4,48($18) \n\
+ \n\
+ ldq $6,56($17) \n\
+ ldq $7,56($18) \n\
+ ldq $2,40($19) \n\
+ ldq $5,48($19) \n\
+ \n\
+ ldq $20,56($19) \n\
+ xor $0,$1,$1 # 4 cycles from $1 load \n\
+ xor $3,$4,$4 # 5 cycles from $4 load \n\
+ xor $6,$7,$7 # 5 cycles from $7 load \n\
+ \n\
+ xor $1,$2,$2 # 4 cycles from $2 load \n\
+ xor $4,$5,$5 # 5 cycles from $5 load \n\
+ stq $2,40($17) \n\
+ xor $7,$20,$20 # 4 cycles from $20 load \n\
+ \n\
+ stq $5,48($17) \n\
+ subq $16,1,$16 \n\
+ stq $20,56($17) \n\
+ addq $19,64,$19 \n\
+ \n\
+ addq $18,64,$18 \n\
+ addq $17,64,$17 \n\
+ bgt $16,3b \n\
+ ret \n\
+ .end xor_alpha_3 \n\
+ \n\
+ .align 3 \n\
+ .ent xor_alpha_4 \n\
+xor_alpha_4: \n\
+ .prologue 0 \n\
+ srl $16, 6, $16 \n\
+ .align 4 \n\
+4: \n\
+ ldq $0,0($17) \n\
+ ldq $1,0($18) \n\
+ ldq $2,0($19) \n\
+ ldq $3,0($20) \n\
+ \n\
+ ldq $4,8($17) \n\
+ ldq $5,8($18) \n\
+ ldq $6,8($19) \n\
+ ldq $7,8($20) \n\
+ \n\
+ ldq $21,16($17) \n\
+ ldq $22,16($18) \n\
+ ldq $23,16($19) \n\
+ ldq $24,16($20) \n\
+ \n\
+ ldq $25,24($17) \n\
+ xor $0,$1,$1 # 6 cycles from $1 load \n\
+ ldq $27,24($18) \n\
+ xor $2,$3,$3 # 6 cycles from $3 load \n\
+ \n\
+ ldq $0,24($19) \n\
+ xor $1,$3,$3 \n\
+ ldq $1,24($20) \n\
+ xor $4,$5,$5 # 7 cycles from $5 load \n\
+ \n\
+ stq $3,0($17) \n\
+ xor $6,$7,$7 \n\
+ xor $21,$22,$22 # 7 cycles from $22 load \n\
+ xor $5,$7,$7 \n\
+ \n\
+ stq $7,8($17) \n\
+ xor $23,$24,$24 # 7 cycles from $24 load \n\
+ ldq $2,32($17) \n\
+ xor $22,$24,$24 \n\
+ \n\
+ ldq $3,32($18) \n\
+ ldq $4,32($19) \n\
+ ldq $5,32($20) \n\
+ xor $25,$27,$27 # 8 cycles from $27 load \n\
+ \n\
+ ldq $6,40($17) \n\
+ ldq $7,40($18) \n\
+ ldq $21,40($19) \n\
+ ldq $22,40($20) \n\
+ \n\
+ stq $24,16($17) \n\
+ xor $0,$1,$1 # 9 cycles from $1 load \n\
+ xor $2,$3,$3 # 5 cycles from $3 load \n\
+ xor $27,$1,$1 \n\
+ \n\
+ stq $1,24($17) \n\
+ xor $4,$5,$5 # 5 cycles from $5 load \n\
+ ldq $23,48($17) \n\
+ ldq $24,48($18) \n\
+ \n\
+ ldq $25,48($19) \n\
+ xor $3,$5,$5 \n\
+ ldq $27,48($20) \n\
+ ldq $0,56($17) \n\
+ \n\
+ ldq $1,56($18) \n\
+ ldq $2,56($19) \n\
+ xor $6,$7,$7 # 8 cycles from $6 load \n\
+ ldq $3,56($20) \n\
+ \n\
+ stq $5,32($17) \n\
+ xor $21,$22,$22 # 8 cycles from $22 load \n\
+ xor $7,$22,$22 \n\
+ xor $23,$24,$24 # 5 cycles from $24 load \n\
+ \n\
+ stq $22,40($17) \n\
+ xor $25,$27,$27 # 5 cycles from $27 load \n\
+ xor $24,$27,$27 \n\
+ xor $0,$1,$1 # 5 cycles from $1 load \n\
+ \n\
+ stq $27,48($17) \n\
+ xor $2,$3,$3 # 4 cycles from $3 load \n\
+ xor $1,$3,$3 \n\
+ subq $16,1,$16 \n\
+ \n\
+ stq $3,56($17) \n\
+ addq $20,64,$20 \n\
+ addq $19,64,$19 \n\
+ addq $18,64,$18 \n\
+ \n\
+ addq $17,64,$17 \n\
+ bgt $16,4b \n\
+ ret \n\
+ .end xor_alpha_4 \n\
+ \n\
+ .align 3 \n\
+ .ent xor_alpha_5 \n\
+xor_alpha_5: \n\
+ .prologue 0 \n\
+ srl $16, 6, $16 \n\
+ .align 4 \n\
+5: \n\
+ ldq $0,0($17) \n\
+ ldq $1,0($18) \n\
+ ldq $2,0($19) \n\
+ ldq $3,0($20) \n\
+ \n\
+ ldq $4,0($21) \n\
+ ldq $5,8($17) \n\
+ ldq $6,8($18) \n\
+ ldq $7,8($19) \n\
+ \n\
+ ldq $22,8($20) \n\
+ ldq $23,8($21) \n\
+ ldq $24,16($17) \n\
+ ldq $25,16($18) \n\
+ \n\
+ ldq $27,16($19) \n\
+ xor $0,$1,$1 # 6 cycles from $1 load \n\
+ ldq $28,16($20) \n\
+ xor $2,$3,$3 # 6 cycles from $3 load \n\
+ \n\
+ ldq $0,16($21) \n\
+ xor $1,$3,$3 \n\
+ ldq $1,24($17) \n\
+ xor $3,$4,$4 # 7 cycles from $4 load \n\
+ \n\
+ stq $4,0($17) \n\
+ xor $5,$6,$6 # 7 cycles from $6 load \n\
+ xor $7,$22,$22 # 7 cycles from $22 load \n\
+ xor $6,$23,$23 # 7 cycles from $23 load \n\
+ \n\
+ ldq $2,24($18) \n\
+ xor $22,$23,$23 \n\
+ ldq $3,24($19) \n\
+ xor $24,$25,$25 # 8 cycles from $25 load \n\
+ \n\
+ stq $23,8($17) \n\
+ xor $25,$27,$27 # 8 cycles from $27 load \n\
+ ldq $4,24($20) \n\
+ xor $28,$0,$0 # 7 cycles from $0 load \n\
+ \n\
+ ldq $5,24($21) \n\
+ xor $27,$0,$0 \n\
+ ldq $6,32($17) \n\
+ ldq $7,32($18) \n\
+ \n\
+ stq $0,16($17) \n\
+ xor $1,$2,$2 # 6 cycles from $2 load \n\
+ ldq $22,32($19) \n\
+ xor $3,$4,$4 # 4 cycles from $4 load \n\
+ \n\
+ ldq $23,32($20) \n\
+ xor $2,$4,$4 \n\
+ ldq $24,32($21) \n\
+ ldq $25,40($17) \n\
+ \n\
+ ldq $27,40($18) \n\
+ ldq $28,40($19) \n\
+ ldq $0,40($20) \n\
+ xor $4,$5,$5 # 7 cycles from $5 load \n\
+ \n\
+ stq $5,24($17) \n\
+ xor $6,$7,$7 # 7 cycles from $7 load \n\
+ ldq $1,40($21) \n\
+ ldq $2,48($17) \n\
+ \n\
+ ldq $3,48($18) \n\
+ xor $7,$22,$22 # 7 cycles from $22 load \n\
+ ldq $4,48($19) \n\
+ xor $23,$24,$24 # 6 cycles from $24 load \n\
+ \n\
+ ldq $5,48($20) \n\
+ xor $22,$24,$24 \n\
+ ldq $6,48($21) \n\
+ xor $25,$27,$27 # 7 cycles from $27 load \n\
+ \n\
+ stq $24,32($17) \n\
+ xor $27,$28,$28 # 8 cycles from $28 load \n\
+ ldq $7,56($17) \n\
+ xor $0,$1,$1 # 6 cycles from $1 load \n\
+ \n\
+ ldq $22,56($18) \n\
+ ldq $23,56($19) \n\
+ ldq $24,56($20) \n\
+ ldq $25,56($21) \n\
+ \n\
+ xor $28,$1,$1 \n\
+ xor $2,$3,$3 # 9 cycles from $3 load \n\
+ xor $3,$4,$4 # 9 cycles from $4 load \n\
+ xor $5,$6,$6 # 8 cycles from $6 load \n\
+ \n\
+ stq $1,40($17) \n\
+ xor $4,$6,$6 \n\
+ xor $7,$22,$22 # 7 cycles from $22 load \n\
+ xor $23,$24,$24 # 6 cycles from $24 load \n\
+ \n\
+ stq $6,48($17) \n\
+ xor $22,$24,$24 \n\
+ subq $16,1,$16 \n\
+ xor $24,$25,$25 # 8 cycles from $25 load \n\
+ \n\
+ stq $25,56($17) \n\
+ addq $21,64,$21 \n\
+ addq $20,64,$20 \n\
+ addq $19,64,$19 \n\
+ \n\
+ addq $18,64,$18 \n\
+ addq $17,64,$17 \n\
+ bgt $16,5b \n\
+ ret \n\
+ .end xor_alpha_5 \n\
+ \n\
+ .align 3 \n\
+ .ent xor_alpha_prefetch_2 \n\
+xor_alpha_prefetch_2: \n\
+ .prologue 0 \n\
+ srl $16, 6, $16 \n\
+ \n\
+ ldq $31, 0($17) \n\
+ ldq $31, 0($18) \n\
+ \n\
+ ldq $31, 64($17) \n\
+ ldq $31, 64($18) \n\
+ \n\
+ ldq $31, 128($17) \n\
+ ldq $31, 128($18) \n\
+ \n\
+ ldq $31, 192($17) \n\
+ ldq $31, 192($18) \n\
+ .align 4 \n\
+2: \n\
+ ldq $0,0($17) \n\
+ ldq $1,0($18) \n\
+ ldq $2,8($17) \n\
+ ldq $3,8($18) \n\
+ \n\
+ ldq $4,16($17) \n\
+ ldq $5,16($18) \n\
+ ldq $6,24($17) \n\
+ ldq $7,24($18) \n\
+ \n\
+ ldq $19,32($17) \n\
+ ldq $20,32($18) \n\
+ ldq $21,40($17) \n\
+ ldq $22,40($18) \n\
+ \n\
+ ldq $23,48($17) \n\
+ ldq $24,48($18) \n\
+ ldq $25,56($17) \n\
+ ldq $27,56($18) \n\
+ \n\
+ ldq $31,256($17) \n\
+ xor $0,$1,$0 # 8 cycles from $1 load \n\
+ ldq $31,256($18) \n\
+ xor $2,$3,$2 \n\
+ \n\
+ stq $0,0($17) \n\
+ xor $4,$5,$4 \n\
+ stq $2,8($17) \n\
+ xor $6,$7,$6 \n\
+ \n\
+ stq $4,16($17) \n\
+ xor $19,$20,$19 \n\
+ stq $6,24($17) \n\
+ xor $21,$22,$21 \n\
+ \n\
+ stq $19,32($17) \n\
+ xor $23,$24,$23 \n\
+ stq $21,40($17) \n\
+ xor $25,$27,$25 \n\
+ \n\
+ stq $23,48($17) \n\
+ subq $16,1,$16 \n\
+ stq $25,56($17) \n\
+ addq $17,64,$17 \n\
+ \n\
+ addq $18,64,$18 \n\
+ bgt $16,2b \n\
+ ret \n\
+ .end xor_alpha_prefetch_2 \n\
+ \n\
+ .align 3 \n\
+ .ent xor_alpha_prefetch_3 \n\
+xor_alpha_prefetch_3: \n\
+ .prologue 0 \n\
+ srl $16, 6, $16 \n\
+ \n\
+ ldq $31, 0($17) \n\
+ ldq $31, 0($18) \n\
+ ldq $31, 0($19) \n\
+ \n\
+ ldq $31, 64($17) \n\
+ ldq $31, 64($18) \n\
+ ldq $31, 64($19) \n\
+ \n\
+ ldq $31, 128($17) \n\
+ ldq $31, 128($18) \n\
+ ldq $31, 128($19) \n\
+ \n\
+ ldq $31, 192($17) \n\
+ ldq $31, 192($18) \n\
+ ldq $31, 192($19) \n\
+ .align 4 \n\
+3: \n\
+ ldq $0,0($17) \n\
+ ldq $1,0($18) \n\
+ ldq $2,0($19) \n\
+ ldq $3,8($17) \n\
+ \n\
+ ldq $4,8($18) \n\
+ ldq $6,16($17) \n\
+ ldq $7,16($18) \n\
+ ldq $21,24($17) \n\
+ \n\
+ ldq $22,24($18) \n\
+ ldq $24,32($17) \n\
+ ldq $25,32($18) \n\
+ ldq $5,8($19) \n\
+ \n\
+ ldq $20,16($19) \n\
+ ldq $23,24($19) \n\
+ ldq $27,32($19) \n\
+ nop \n\
+ \n\
+ xor $0,$1,$1 # 8 cycles from $0 load \n\
+ xor $3,$4,$4 # 7 cycles from $4 load \n\
+ xor $6,$7,$7 # 6 cycles from $7 load \n\
+ xor $21,$22,$22 # 5 cycles from $22 load \n\
+ \n\
+ xor $1,$2,$2 # 9 cycles from $2 load \n\
+ xor $24,$25,$25 # 5 cycles from $25 load \n\
+ stq $2,0($17) \n\
+ xor $4,$5,$5 # 6 cycles from $5 load \n\
+ \n\
+ stq $5,8($17) \n\
+ xor $7,$20,$20 # 7 cycles from $20 load \n\
+ stq $20,16($17) \n\
+ xor $22,$23,$23 # 7 cycles from $23 load \n\
+ \n\
+ stq $23,24($17) \n\
+ xor $25,$27,$27 # 7 cycles from $27 load \n\
+ stq $27,32($17) \n\
+ nop \n\
+ \n\
+ ldq $0,40($17) \n\
+ ldq $1,40($18) \n\
+ ldq $3,48($17) \n\
+ ldq $4,48($18) \n\
+ \n\
+ ldq $6,56($17) \n\
+ ldq $7,56($18) \n\
+ ldq $2,40($19) \n\
+ ldq $5,48($19) \n\
+ \n\
+ ldq $20,56($19) \n\
+ ldq $31,256($17) \n\
+ ldq $31,256($18) \n\
+ ldq $31,256($19) \n\
+ \n\
+ xor $0,$1,$1 # 6 cycles from $1 load \n\
+ xor $3,$4,$4 # 5 cycles from $4 load \n\
+ xor $6,$7,$7 # 5 cycles from $7 load \n\
+ xor $1,$2,$2 # 4 cycles from $2 load \n\
+ \n\
+ xor $4,$5,$5 # 5 cycles from $5 load \n\
+ xor $7,$20,$20 # 4 cycles from $20 load \n\
+ stq $2,40($17) \n\
+ subq $16,1,$16 \n\
+ \n\
+ stq $5,48($17) \n\
+ addq $19,64,$19 \n\
+ stq $20,56($17) \n\
+ addq $18,64,$18 \n\
+ \n\
+ addq $17,64,$17 \n\
+ bgt $16,3b \n\
+ ret \n\
+ .end xor_alpha_prefetch_3 \n\
+ \n\
+ .align 3 \n\
+ .ent xor_alpha_prefetch_4 \n\
+xor_alpha_prefetch_4: \n\
+ .prologue 0 \n\
+ srl $16, 6, $16 \n\
+ \n\
+ ldq $31, 0($17) \n\
+ ldq $31, 0($18) \n\
+ ldq $31, 0($19) \n\
+ ldq $31, 0($20) \n\
+ \n\
+ ldq $31, 64($17) \n\
+ ldq $31, 64($18) \n\
+ ldq $31, 64($19) \n\
+ ldq $31, 64($20) \n\
+ \n\
+ ldq $31, 128($17) \n\
+ ldq $31, 128($18) \n\
+ ldq $31, 128($19) \n\
+ ldq $31, 128($20) \n\
+ \n\
+ ldq $31, 192($17) \n\
+ ldq $31, 192($18) \n\
+ ldq $31, 192($19) \n\
+ ldq $31, 192($20) \n\
+ .align 4 \n\
+4: \n\
+ ldq $0,0($17) \n\
+ ldq $1,0($18) \n\
+ ldq $2,0($19) \n\
+ ldq $3,0($20) \n\
+ \n\
+ ldq $4,8($17) \n\
+ ldq $5,8($18) \n\
+ ldq $6,8($19) \n\
+ ldq $7,8($20) \n\
+ \n\
+ ldq $21,16($17) \n\
+ ldq $22,16($18) \n\
+ ldq $23,16($19) \n\
+ ldq $24,16($20) \n\
+ \n\
+ ldq $25,24($17) \n\
+ xor $0,$1,$1 # 6 cycles from $1 load \n\
+ ldq $27,24($18) \n\
+ xor $2,$3,$3 # 6 cycles from $3 load \n\
+ \n\
+ ldq $0,24($19) \n\
+ xor $1,$3,$3 \n\
+ ldq $1,24($20) \n\
+ xor $4,$5,$5 # 7 cycles from $5 load \n\
+ \n\
+ stq $3,0($17) \n\
+ xor $6,$7,$7 \n\
+ xor $21,$22,$22 # 7 cycles from $22 load \n\
+ xor $5,$7,$7 \n\
+ \n\
+ stq $7,8($17) \n\
+ xor $23,$24,$24 # 7 cycles from $24 load \n\
+ ldq $2,32($17) \n\
+ xor $22,$24,$24 \n\
+ \n\
+ ldq $3,32($18) \n\
+ ldq $4,32($19) \n\
+ ldq $5,32($20) \n\
+ xor $25,$27,$27 # 8 cycles from $27 load \n\
+ \n\
+ ldq $6,40($17) \n\
+ ldq $7,40($18) \n\
+ ldq $21,40($19) \n\
+ ldq $22,40($20) \n\
+ \n\
+ stq $24,16($17) \n\
+ xor $0,$1,$1 # 9 cycles from $1 load \n\
+ xor $2,$3,$3 # 5 cycles from $3 load \n\
+ xor $27,$1,$1 \n\
+ \n\
+ stq $1,24($17) \n\
+ xor $4,$5,$5 # 5 cycles from $5 load \n\
+ ldq $23,48($17) \n\
+ xor $3,$5,$5 \n\
+ \n\
+ ldq $24,48($18) \n\
+ ldq $25,48($19) \n\
+ ldq $27,48($20) \n\
+ ldq $0,56($17) \n\
+ \n\
+ ldq $1,56($18) \n\
+ ldq $2,56($19) \n\
+ ldq $3,56($20) \n\
+ xor $6,$7,$7 # 8 cycles from $6 load \n\
+ \n\
+ ldq $31,256($17) \n\
+ xor $21,$22,$22 # 8 cycles from $22 load \n\
+ ldq $31,256($18) \n\
+ xor $7,$22,$22 \n\
+ \n\
+ ldq $31,256($19) \n\
+ xor $23,$24,$24 # 6 cycles from $24 load \n\
+ ldq $31,256($20) \n\
+ xor $25,$27,$27 # 6 cycles from $27 load \n\
+ \n\
+ stq $5,32($17) \n\
+ xor $24,$27,$27 \n\
+ xor $0,$1,$1 # 7 cycles from $1 load \n\
+ xor $2,$3,$3 # 6 cycles from $3 load \n\
+ \n\
+ stq $22,40($17) \n\
+ xor $1,$3,$3 \n\
+ stq $27,48($17) \n\
+ subq $16,1,$16 \n\
+ \n\
+ stq $3,56($17) \n\
+ addq $20,64,$20 \n\
+ addq $19,64,$19 \n\
+ addq $18,64,$18 \n\
+ \n\
+ addq $17,64,$17 \n\
+ bgt $16,4b \n\
+ ret \n\
+ .end xor_alpha_prefetch_4 \n\
+ \n\
+ .align 3 \n\
+ .ent xor_alpha_prefetch_5 \n\
+xor_alpha_prefetch_5: \n\
+ .prologue 0 \n\
+ srl $16, 6, $16 \n\
+ \n\
+ ldq $31, 0($17) \n\
+ ldq $31, 0($18) \n\
+ ldq $31, 0($19) \n\
+ ldq $31, 0($20) \n\
+ ldq $31, 0($21) \n\
+ \n\
+ ldq $31, 64($17) \n\
+ ldq $31, 64($18) \n\
+ ldq $31, 64($19) \n\
+ ldq $31, 64($20) \n\
+ ldq $31, 64($21) \n\
+ \n\
+ ldq $31, 128($17) \n\
+ ldq $31, 128($18) \n\
+ ldq $31, 128($19) \n\
+ ldq $31, 128($20) \n\
+ ldq $31, 128($21) \n\
+ \n\
+ ldq $31, 192($17) \n\
+ ldq $31, 192($18) \n\
+ ldq $31, 192($19) \n\
+ ldq $31, 192($20) \n\
+ ldq $31, 192($21) \n\
+ .align 4 \n\
+5: \n\
+ ldq $0,0($17) \n\
+ ldq $1,0($18) \n\
+ ldq $2,0($19) \n\
+ ldq $3,0($20) \n\
+ \n\
+ ldq $4,0($21) \n\
+ ldq $5,8($17) \n\
+ ldq $6,8($18) \n\
+ ldq $7,8($19) \n\
+ \n\
+ ldq $22,8($20) \n\
+ ldq $23,8($21) \n\
+ ldq $24,16($17) \n\
+ ldq $25,16($18) \n\
+ \n\
+ ldq $27,16($19) \n\
+ xor $0,$1,$1 # 6 cycles from $1 load \n\
+ ldq $28,16($20) \n\
+ xor $2,$3,$3 # 6 cycles from $3 load \n\
+ \n\
+ ldq $0,16($21) \n\
+ xor $1,$3,$3 \n\
+ ldq $1,24($17) \n\
+ xor $3,$4,$4 # 7 cycles from $4 load \n\
+ \n\
+ stq $4,0($17) \n\
+ xor $5,$6,$6 # 7 cycles from $6 load \n\
+ xor $7,$22,$22 # 7 cycles from $22 load \n\
+ xor $6,$23,$23 # 7 cycles from $23 load \n\
+ \n\
+ ldq $2,24($18) \n\
+ xor $22,$23,$23 \n\
+ ldq $3,24($19) \n\
+ xor $24,$25,$25 # 8 cycles from $25 load \n\
+ \n\
+ stq $23,8($17) \n\
+ xor $25,$27,$27 # 8 cycles from $27 load \n\
+ ldq $4,24($20) \n\
+ xor $28,$0,$0 # 7 cycles from $0 load \n\
+ \n\
+ ldq $5,24($21) \n\
+ xor $27,$0,$0 \n\
+ ldq $6,32($17) \n\
+ ldq $7,32($18) \n\
+ \n\
+ stq $0,16($17) \n\
+ xor $1,$2,$2 # 6 cycles from $2 load \n\
+ ldq $22,32($19) \n\
+ xor $3,$4,$4 # 4 cycles from $4 load \n\
+ \n\
+ ldq $23,32($20) \n\
+ xor $2,$4,$4 \n\
+ ldq $24,32($21) \n\
+ ldq $25,40($17) \n\
+ \n\
+ ldq $27,40($18) \n\
+ ldq $28,40($19) \n\
+ ldq $0,40($20) \n\
+ xor $4,$5,$5 # 7 cycles from $5 load \n\
+ \n\
+ stq $5,24($17) \n\
+ xor $6,$7,$7 # 7 cycles from $7 load \n\
+ ldq $1,40($21) \n\
+ ldq $2,48($17) \n\
+ \n\
+ ldq $3,48($18) \n\
+ xor $7,$22,$22 # 7 cycles from $22 load \n\
+ ldq $4,48($19) \n\
+ xor $23,$24,$24 # 6 cycles from $24 load \n\
+ \n\
+ ldq $5,48($20) \n\
+ xor $22,$24,$24 \n\
+ ldq $6,48($21) \n\
+ xor $25,$27,$27 # 7 cycles from $27 load \n\
+ \n\
+ stq $24,32($17) \n\
+ xor $27,$28,$28 # 8 cycles from $28 load \n\
+ ldq $7,56($17) \n\
+ xor $0,$1,$1 # 6 cycles from $1 load \n\
+ \n\
+ ldq $22,56($18) \n\
+ ldq $23,56($19) \n\
+ ldq $24,56($20) \n\
+ ldq $25,56($21) \n\
+ \n\
+ ldq $31,256($17) \n\
+ xor $28,$1,$1 \n\
+ ldq $31,256($18) \n\
+ xor $2,$3,$3 # 9 cycles from $3 load \n\
+ \n\
+ ldq $31,256($19) \n\
+ xor $3,$4,$4 # 9 cycles from $4 load \n\
+ ldq $31,256($20) \n\
+ xor $5,$6,$6 # 8 cycles from $6 load \n\
+ \n\
+ stq $1,40($17) \n\
+ xor $4,$6,$6 \n\
+ xor $7,$22,$22 # 7 cycles from $22 load \n\
+ xor $23,$24,$24 # 6 cycles from $24 load \n\
+ \n\
+ stq $6,48($17) \n\
+ xor $22,$24,$24 \n\
+ ldq $31,256($21) \n\
+ xor $24,$25,$25 # 8 cycles from $25 load \n\
+ \n\
+ stq $25,56($17) \n\
+ subq $16,1,$16 \n\
+ addq $21,64,$21 \n\
+ addq $20,64,$20 \n\
+ \n\
+ addq $19,64,$19 \n\
+ addq $18,64,$18 \n\
+ addq $17,64,$17 \n\
+ bgt $16,5b \n\
+ \n\
+ ret \n\
+ .end xor_alpha_prefetch_5 \n\
+");
+
+struct xor_block_template xor_block_alpha = {
+ .name = "alpha",
+ .do_2 = xor_alpha_2,
+ .do_3 = xor_alpha_3,
+ .do_4 = xor_alpha_4,
+ .do_5 = xor_alpha_5,
+};
+
+struct xor_block_template xor_block_alpha_prefetch = {
+ .name = "alpha prefetch",
+ .do_2 = xor_alpha_prefetch_2,
+ .do_3 = xor_alpha_prefetch_3,
+ .do_4 = xor_alpha_prefetch_4,
+ .do_5 = xor_alpha_prefetch_5,
+};
--
2.47.3
^ permalink raw reply related
* [PATCH 09/26] xor: move generic implementations out of asm-generic/xor.h
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the generic implementations from asm-generic/xor.h to
per-implementaion .c files in lib/raid. This will build them
unconditionally even when an architecture forces a specific
implementation, but as we'll need at least one generic version
for the static_call optimization later on we'll pay that price.
Note that this would cause the second xor_block_8regs instance created by
arch/arm/lib/xor-neon.c to be generated instead of discarded as dead
code, so add a NO_TEMPLATE symbol to disable it for this case.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/arm/lib/xor-neon.c | 4 +-
include/asm-generic/xor.h | 727 +----------------------------
lib/raid/xor/Makefile | 4 +
lib/raid/xor/xor-32regs-prefetch.c | 268 +++++++++++
lib/raid/xor/xor-32regs.c | 219 +++++++++
lib/raid/xor/xor-8regs-prefetch.c | 146 ++++++
lib/raid/xor/xor-8regs.c | 105 +++++
7 files changed, 748 insertions(+), 725 deletions(-)
create mode 100644 lib/raid/xor/xor-32regs-prefetch.c
create mode 100644 lib/raid/xor/xor-32regs.c
create mode 100644 lib/raid/xor/xor-8regs-prefetch.c
create mode 100644 lib/raid/xor/xor-8regs.c
diff --git a/arch/arm/lib/xor-neon.c b/arch/arm/lib/xor-neon.c
index 282980b9bf2a..b5be50567991 100644
--- a/arch/arm/lib/xor-neon.c
+++ b/arch/arm/lib/xor-neon.c
@@ -26,8 +26,8 @@ MODULE_LICENSE("GPL");
#pragma GCC optimize "tree-vectorize"
#endif
-#pragma GCC diagnostic ignored "-Wunused-variable"
-#include <asm-generic/xor.h>
+#define NO_TEMPLATE
+#include "../../../lib/raid/xor/xor-8regs.c"
struct xor_block_template const xor_block_neon_inner = {
.name = "__inner_neon__",
diff --git a/include/asm-generic/xor.h b/include/asm-generic/xor.h
index 79c0096aa9d9..fc151fdc45ab 100644
--- a/include/asm-generic/xor.h
+++ b/include/asm-generic/xor.h
@@ -5,726 +5,7 @@
* Generic optimized RAID-5 checksumming functions.
*/
-#include <linux/prefetch.h>
-
-static void
-xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- long lines = bytes / (sizeof (long)) / 8;
-
- do {
- p1[0] ^= p2[0];
- p1[1] ^= p2[1];
- p1[2] ^= p2[2];
- p1[3] ^= p2[3];
- p1[4] ^= p2[4];
- p1[5] ^= p2[5];
- p1[6] ^= p2[6];
- p1[7] ^= p2[7];
- p1 += 8;
- p2 += 8;
- } while (--lines > 0);
-}
-
-static void
-xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- long lines = bytes / (sizeof (long)) / 8;
-
- do {
- p1[0] ^= p2[0] ^ p3[0];
- p1[1] ^= p2[1] ^ p3[1];
- p1[2] ^= p2[2] ^ p3[2];
- p1[3] ^= p2[3] ^ p3[3];
- p1[4] ^= p2[4] ^ p3[4];
- p1[5] ^= p2[5] ^ p3[5];
- p1[6] ^= p2[6] ^ p3[6];
- p1[7] ^= p2[7] ^ p3[7];
- p1 += 8;
- p2 += 8;
- p3 += 8;
- } while (--lines > 0);
-}
-
-static void
-xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- long lines = bytes / (sizeof (long)) / 8;
-
- do {
- p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
- p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
- p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
- p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
- p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
- p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
- p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
- p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- } while (--lines > 0);
-}
-
-static void
-xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- long lines = bytes / (sizeof (long)) / 8;
-
- do {
- p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
- p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
- p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
- p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
- p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
- p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
- p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
- p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- p5 += 8;
- } while (--lines > 0);
-}
-
-static void
-xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- long lines = bytes / (sizeof (long)) / 8;
-
- do {
- register long d0, d1, d2, d3, d4, d5, d6, d7;
- d0 = p1[0]; /* Pull the stuff into registers */
- d1 = p1[1]; /* ... in bursts, if possible. */
- d2 = p1[2];
- d3 = p1[3];
- d4 = p1[4];
- d5 = p1[5];
- d6 = p1[6];
- d7 = p1[7];
- d0 ^= p2[0];
- d1 ^= p2[1];
- d2 ^= p2[2];
- d3 ^= p2[3];
- d4 ^= p2[4];
- d5 ^= p2[5];
- d6 ^= p2[6];
- d7 ^= p2[7];
- p1[0] = d0; /* Store the result (in bursts) */
- p1[1] = d1;
- p1[2] = d2;
- p1[3] = d3;
- p1[4] = d4;
- p1[5] = d5;
- p1[6] = d6;
- p1[7] = d7;
- p1 += 8;
- p2 += 8;
- } while (--lines > 0);
-}
-
-static void
-xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- long lines = bytes / (sizeof (long)) / 8;
-
- do {
- register long d0, d1, d2, d3, d4, d5, d6, d7;
- d0 = p1[0]; /* Pull the stuff into registers */
- d1 = p1[1]; /* ... in bursts, if possible. */
- d2 = p1[2];
- d3 = p1[3];
- d4 = p1[4];
- d5 = p1[5];
- d6 = p1[6];
- d7 = p1[7];
- d0 ^= p2[0];
- d1 ^= p2[1];
- d2 ^= p2[2];
- d3 ^= p2[3];
- d4 ^= p2[4];
- d5 ^= p2[5];
- d6 ^= p2[6];
- d7 ^= p2[7];
- d0 ^= p3[0];
- d1 ^= p3[1];
- d2 ^= p3[2];
- d3 ^= p3[3];
- d4 ^= p3[4];
- d5 ^= p3[5];
- d6 ^= p3[6];
- d7 ^= p3[7];
- p1[0] = d0; /* Store the result (in bursts) */
- p1[1] = d1;
- p1[2] = d2;
- p1[3] = d3;
- p1[4] = d4;
- p1[5] = d5;
- p1[6] = d6;
- p1[7] = d7;
- p1 += 8;
- p2 += 8;
- p3 += 8;
- } while (--lines > 0);
-}
-
-static void
-xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- long lines = bytes / (sizeof (long)) / 8;
-
- do {
- register long d0, d1, d2, d3, d4, d5, d6, d7;
- d0 = p1[0]; /* Pull the stuff into registers */
- d1 = p1[1]; /* ... in bursts, if possible. */
- d2 = p1[2];
- d3 = p1[3];
- d4 = p1[4];
- d5 = p1[5];
- d6 = p1[6];
- d7 = p1[7];
- d0 ^= p2[0];
- d1 ^= p2[1];
- d2 ^= p2[2];
- d3 ^= p2[3];
- d4 ^= p2[4];
- d5 ^= p2[5];
- d6 ^= p2[6];
- d7 ^= p2[7];
- d0 ^= p3[0];
- d1 ^= p3[1];
- d2 ^= p3[2];
- d3 ^= p3[3];
- d4 ^= p3[4];
- d5 ^= p3[5];
- d6 ^= p3[6];
- d7 ^= p3[7];
- d0 ^= p4[0];
- d1 ^= p4[1];
- d2 ^= p4[2];
- d3 ^= p4[3];
- d4 ^= p4[4];
- d5 ^= p4[5];
- d6 ^= p4[6];
- d7 ^= p4[7];
- p1[0] = d0; /* Store the result (in bursts) */
- p1[1] = d1;
- p1[2] = d2;
- p1[3] = d3;
- p1[4] = d4;
- p1[5] = d5;
- p1[6] = d6;
- p1[7] = d7;
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- } while (--lines > 0);
-}
-
-static void
-xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- long lines = bytes / (sizeof (long)) / 8;
-
- do {
- register long d0, d1, d2, d3, d4, d5, d6, d7;
- d0 = p1[0]; /* Pull the stuff into registers */
- d1 = p1[1]; /* ... in bursts, if possible. */
- d2 = p1[2];
- d3 = p1[3];
- d4 = p1[4];
- d5 = p1[5];
- d6 = p1[6];
- d7 = p1[7];
- d0 ^= p2[0];
- d1 ^= p2[1];
- d2 ^= p2[2];
- d3 ^= p2[3];
- d4 ^= p2[4];
- d5 ^= p2[5];
- d6 ^= p2[6];
- d7 ^= p2[7];
- d0 ^= p3[0];
- d1 ^= p3[1];
- d2 ^= p3[2];
- d3 ^= p3[3];
- d4 ^= p3[4];
- d5 ^= p3[5];
- d6 ^= p3[6];
- d7 ^= p3[7];
- d0 ^= p4[0];
- d1 ^= p4[1];
- d2 ^= p4[2];
- d3 ^= p4[3];
- d4 ^= p4[4];
- d5 ^= p4[5];
- d6 ^= p4[6];
- d7 ^= p4[7];
- d0 ^= p5[0];
- d1 ^= p5[1];
- d2 ^= p5[2];
- d3 ^= p5[3];
- d4 ^= p5[4];
- d5 ^= p5[5];
- d6 ^= p5[6];
- d7 ^= p5[7];
- p1[0] = d0; /* Store the result (in bursts) */
- p1[1] = d1;
- p1[2] = d2;
- p1[3] = d3;
- p1[4] = d4;
- p1[5] = d5;
- p1[6] = d6;
- p1[7] = d7;
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- p5 += 8;
- } while (--lines > 0);
-}
-
-static void
-xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- long lines = bytes / (sizeof (long)) / 8 - 1;
- prefetchw(p1);
- prefetch(p2);
-
- do {
- prefetchw(p1+8);
- prefetch(p2+8);
- once_more:
- p1[0] ^= p2[0];
- p1[1] ^= p2[1];
- p1[2] ^= p2[2];
- p1[3] ^= p2[3];
- p1[4] ^= p2[4];
- p1[5] ^= p2[5];
- p1[6] ^= p2[6];
- p1[7] ^= p2[7];
- p1 += 8;
- p2 += 8;
- } while (--lines > 0);
- if (lines == 0)
- goto once_more;
-}
-
-static void
-xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- long lines = bytes / (sizeof (long)) / 8 - 1;
- prefetchw(p1);
- prefetch(p2);
- prefetch(p3);
-
- do {
- prefetchw(p1+8);
- prefetch(p2+8);
- prefetch(p3+8);
- once_more:
- p1[0] ^= p2[0] ^ p3[0];
- p1[1] ^= p2[1] ^ p3[1];
- p1[2] ^= p2[2] ^ p3[2];
- p1[3] ^= p2[3] ^ p3[3];
- p1[4] ^= p2[4] ^ p3[4];
- p1[5] ^= p2[5] ^ p3[5];
- p1[6] ^= p2[6] ^ p3[6];
- p1[7] ^= p2[7] ^ p3[7];
- p1 += 8;
- p2 += 8;
- p3 += 8;
- } while (--lines > 0);
- if (lines == 0)
- goto once_more;
-}
-
-static void
-xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- long lines = bytes / (sizeof (long)) / 8 - 1;
-
- prefetchw(p1);
- prefetch(p2);
- prefetch(p3);
- prefetch(p4);
-
- do {
- prefetchw(p1+8);
- prefetch(p2+8);
- prefetch(p3+8);
- prefetch(p4+8);
- once_more:
- p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
- p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
- p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
- p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
- p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
- p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
- p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
- p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- } while (--lines > 0);
- if (lines == 0)
- goto once_more;
-}
-
-static void
-xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- long lines = bytes / (sizeof (long)) / 8 - 1;
-
- prefetchw(p1);
- prefetch(p2);
- prefetch(p3);
- prefetch(p4);
- prefetch(p5);
-
- do {
- prefetchw(p1+8);
- prefetch(p2+8);
- prefetch(p3+8);
- prefetch(p4+8);
- prefetch(p5+8);
- once_more:
- p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
- p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
- p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
- p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
- p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
- p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
- p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
- p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- p5 += 8;
- } while (--lines > 0);
- if (lines == 0)
- goto once_more;
-}
-
-static void
-xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- long lines = bytes / (sizeof (long)) / 8 - 1;
-
- prefetchw(p1);
- prefetch(p2);
-
- do {
- register long d0, d1, d2, d3, d4, d5, d6, d7;
-
- prefetchw(p1+8);
- prefetch(p2+8);
- once_more:
- d0 = p1[0]; /* Pull the stuff into registers */
- d1 = p1[1]; /* ... in bursts, if possible. */
- d2 = p1[2];
- d3 = p1[3];
- d4 = p1[4];
- d5 = p1[5];
- d6 = p1[6];
- d7 = p1[7];
- d0 ^= p2[0];
- d1 ^= p2[1];
- d2 ^= p2[2];
- d3 ^= p2[3];
- d4 ^= p2[4];
- d5 ^= p2[5];
- d6 ^= p2[6];
- d7 ^= p2[7];
- p1[0] = d0; /* Store the result (in bursts) */
- p1[1] = d1;
- p1[2] = d2;
- p1[3] = d3;
- p1[4] = d4;
- p1[5] = d5;
- p1[6] = d6;
- p1[7] = d7;
- p1 += 8;
- p2 += 8;
- } while (--lines > 0);
- if (lines == 0)
- goto once_more;
-}
-
-static void
-xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- long lines = bytes / (sizeof (long)) / 8 - 1;
-
- prefetchw(p1);
- prefetch(p2);
- prefetch(p3);
-
- do {
- register long d0, d1, d2, d3, d4, d5, d6, d7;
-
- prefetchw(p1+8);
- prefetch(p2+8);
- prefetch(p3+8);
- once_more:
- d0 = p1[0]; /* Pull the stuff into registers */
- d1 = p1[1]; /* ... in bursts, if possible. */
- d2 = p1[2];
- d3 = p1[3];
- d4 = p1[4];
- d5 = p1[5];
- d6 = p1[6];
- d7 = p1[7];
- d0 ^= p2[0];
- d1 ^= p2[1];
- d2 ^= p2[2];
- d3 ^= p2[3];
- d4 ^= p2[4];
- d5 ^= p2[5];
- d6 ^= p2[6];
- d7 ^= p2[7];
- d0 ^= p3[0];
- d1 ^= p3[1];
- d2 ^= p3[2];
- d3 ^= p3[3];
- d4 ^= p3[4];
- d5 ^= p3[5];
- d6 ^= p3[6];
- d7 ^= p3[7];
- p1[0] = d0; /* Store the result (in bursts) */
- p1[1] = d1;
- p1[2] = d2;
- p1[3] = d3;
- p1[4] = d4;
- p1[5] = d5;
- p1[6] = d6;
- p1[7] = d7;
- p1 += 8;
- p2 += 8;
- p3 += 8;
- } while (--lines > 0);
- if (lines == 0)
- goto once_more;
-}
-
-static void
-xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- long lines = bytes / (sizeof (long)) / 8 - 1;
-
- prefetchw(p1);
- prefetch(p2);
- prefetch(p3);
- prefetch(p4);
-
- do {
- register long d0, d1, d2, d3, d4, d5, d6, d7;
-
- prefetchw(p1+8);
- prefetch(p2+8);
- prefetch(p3+8);
- prefetch(p4+8);
- once_more:
- d0 = p1[0]; /* Pull the stuff into registers */
- d1 = p1[1]; /* ... in bursts, if possible. */
- d2 = p1[2];
- d3 = p1[3];
- d4 = p1[4];
- d5 = p1[5];
- d6 = p1[6];
- d7 = p1[7];
- d0 ^= p2[0];
- d1 ^= p2[1];
- d2 ^= p2[2];
- d3 ^= p2[3];
- d4 ^= p2[4];
- d5 ^= p2[5];
- d6 ^= p2[6];
- d7 ^= p2[7];
- d0 ^= p3[0];
- d1 ^= p3[1];
- d2 ^= p3[2];
- d3 ^= p3[3];
- d4 ^= p3[4];
- d5 ^= p3[5];
- d6 ^= p3[6];
- d7 ^= p3[7];
- d0 ^= p4[0];
- d1 ^= p4[1];
- d2 ^= p4[2];
- d3 ^= p4[3];
- d4 ^= p4[4];
- d5 ^= p4[5];
- d6 ^= p4[6];
- d7 ^= p4[7];
- p1[0] = d0; /* Store the result (in bursts) */
- p1[1] = d1;
- p1[2] = d2;
- p1[3] = d3;
- p1[4] = d4;
- p1[5] = d5;
- p1[6] = d6;
- p1[7] = d7;
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- } while (--lines > 0);
- if (lines == 0)
- goto once_more;
-}
-
-static void
-xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- long lines = bytes / (sizeof (long)) / 8 - 1;
-
- prefetchw(p1);
- prefetch(p2);
- prefetch(p3);
- prefetch(p4);
- prefetch(p5);
-
- do {
- register long d0, d1, d2, d3, d4, d5, d6, d7;
-
- prefetchw(p1+8);
- prefetch(p2+8);
- prefetch(p3+8);
- prefetch(p4+8);
- prefetch(p5+8);
- once_more:
- d0 = p1[0]; /* Pull the stuff into registers */
- d1 = p1[1]; /* ... in bursts, if possible. */
- d2 = p1[2];
- d3 = p1[3];
- d4 = p1[4];
- d5 = p1[5];
- d6 = p1[6];
- d7 = p1[7];
- d0 ^= p2[0];
- d1 ^= p2[1];
- d2 ^= p2[2];
- d3 ^= p2[3];
- d4 ^= p2[4];
- d5 ^= p2[5];
- d6 ^= p2[6];
- d7 ^= p2[7];
- d0 ^= p3[0];
- d1 ^= p3[1];
- d2 ^= p3[2];
- d3 ^= p3[3];
- d4 ^= p3[4];
- d5 ^= p3[5];
- d6 ^= p3[6];
- d7 ^= p3[7];
- d0 ^= p4[0];
- d1 ^= p4[1];
- d2 ^= p4[2];
- d3 ^= p4[3];
- d4 ^= p4[4];
- d5 ^= p4[5];
- d6 ^= p4[6];
- d7 ^= p4[7];
- d0 ^= p5[0];
- d1 ^= p5[1];
- d2 ^= p5[2];
- d3 ^= p5[3];
- d4 ^= p5[4];
- d5 ^= p5[5];
- d6 ^= p5[6];
- d7 ^= p5[7];
- p1[0] = d0; /* Store the result (in bursts) */
- p1[1] = d1;
- p1[2] = d2;
- p1[3] = d3;
- p1[4] = d4;
- p1[5] = d5;
- p1[6] = d6;
- p1[7] = d7;
- p1 += 8;
- p2 += 8;
- p3 += 8;
- p4 += 8;
- p5 += 8;
- } while (--lines > 0);
- if (lines == 0)
- goto once_more;
-}
-
-static struct xor_block_template xor_block_8regs = {
- .name = "8regs",
- .do_2 = xor_8regs_2,
- .do_3 = xor_8regs_3,
- .do_4 = xor_8regs_4,
- .do_5 = xor_8regs_5,
-};
-
-static struct xor_block_template xor_block_32regs = {
- .name = "32regs",
- .do_2 = xor_32regs_2,
- .do_3 = xor_32regs_3,
- .do_4 = xor_32regs_4,
- .do_5 = xor_32regs_5,
-};
-
-static struct xor_block_template xor_block_8regs_p __maybe_unused = {
- .name = "8regs_prefetch",
- .do_2 = xor_8regs_p_2,
- .do_3 = xor_8regs_p_3,
- .do_4 = xor_8regs_p_4,
- .do_5 = xor_8regs_p_5,
-};
-
-static struct xor_block_template xor_block_32regs_p __maybe_unused = {
- .name = "32regs_prefetch",
- .do_2 = xor_32regs_p_2,
- .do_3 = xor_32regs_p_3,
- .do_4 = xor_32regs_p_4,
- .do_5 = xor_32regs_p_5,
-};
+extern struct xor_block_template xor_block_8regs;
+extern struct xor_block_template xor_block_32regs;
+extern struct xor_block_template xor_block_8regs_p;
+extern struct xor_block_template xor_block_32regs_p;
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 7bca0ce8e90a..89a944c9f990 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -3,3 +3,7 @@
obj-$(CONFIG_XOR_BLOCKS) += xor.o
xor-y += xor-core.o
+xor-y += xor-8regs.o
+xor-y += xor-32regs.o
+xor-y += xor-8regs-prefetch.o
+xor-y += xor-32regs-prefetch.o
diff --git a/lib/raid/xor/xor-32regs-prefetch.c b/lib/raid/xor/xor-32regs-prefetch.c
new file mode 100644
index 000000000000..8666c287f777
--- /dev/null
+++ b/lib/raid/xor/xor-32regs-prefetch.c
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/prefetch.h>
+#include <linux/raid/xor_impl.h>
+#include <asm-generic/xor.h>
+
+static void
+xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ long lines = bytes / (sizeof (long)) / 8 - 1;
+
+ prefetchw(p1);
+ prefetch(p2);
+
+ do {
+ register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+ prefetchw(p1+8);
+ prefetch(p2+8);
+ once_more:
+ d0 = p1[0]; /* Pull the stuff into registers */
+ d1 = p1[1]; /* ... in bursts, if possible. */
+ d2 = p1[2];
+ d3 = p1[3];
+ d4 = p1[4];
+ d5 = p1[5];
+ d6 = p1[6];
+ d7 = p1[7];
+ d0 ^= p2[0];
+ d1 ^= p2[1];
+ d2 ^= p2[2];
+ d3 ^= p2[3];
+ d4 ^= p2[4];
+ d5 ^= p2[5];
+ d6 ^= p2[6];
+ d7 ^= p2[7];
+ p1[0] = d0; /* Store the result (in bursts) */
+ p1[1] = d1;
+ p1[2] = d2;
+ p1[3] = d3;
+ p1[4] = d4;
+ p1[5] = d5;
+ p1[6] = d6;
+ p1[7] = d7;
+ p1 += 8;
+ p2 += 8;
+ } while (--lines > 0);
+ if (lines == 0)
+ goto once_more;
+}
+
+static void
+xor_32regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ long lines = bytes / (sizeof (long)) / 8 - 1;
+
+ prefetchw(p1);
+ prefetch(p2);
+ prefetch(p3);
+
+ do {
+ register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+ prefetchw(p1+8);
+ prefetch(p2+8);
+ prefetch(p3+8);
+ once_more:
+ d0 = p1[0]; /* Pull the stuff into registers */
+ d1 = p1[1]; /* ... in bursts, if possible. */
+ d2 = p1[2];
+ d3 = p1[3];
+ d4 = p1[4];
+ d5 = p1[5];
+ d6 = p1[6];
+ d7 = p1[7];
+ d0 ^= p2[0];
+ d1 ^= p2[1];
+ d2 ^= p2[2];
+ d3 ^= p2[3];
+ d4 ^= p2[4];
+ d5 ^= p2[5];
+ d6 ^= p2[6];
+ d7 ^= p2[7];
+ d0 ^= p3[0];
+ d1 ^= p3[1];
+ d2 ^= p3[2];
+ d3 ^= p3[3];
+ d4 ^= p3[4];
+ d5 ^= p3[5];
+ d6 ^= p3[6];
+ d7 ^= p3[7];
+ p1[0] = d0; /* Store the result (in bursts) */
+ p1[1] = d1;
+ p1[2] = d2;
+ p1[3] = d3;
+ p1[4] = d4;
+ p1[5] = d5;
+ p1[6] = d6;
+ p1[7] = d7;
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ } while (--lines > 0);
+ if (lines == 0)
+ goto once_more;
+}
+
+static void
+xor_32regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ long lines = bytes / (sizeof (long)) / 8 - 1;
+
+ prefetchw(p1);
+ prefetch(p2);
+ prefetch(p3);
+ prefetch(p4);
+
+ do {
+ register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+ prefetchw(p1+8);
+ prefetch(p2+8);
+ prefetch(p3+8);
+ prefetch(p4+8);
+ once_more:
+ d0 = p1[0]; /* Pull the stuff into registers */
+ d1 = p1[1]; /* ... in bursts, if possible. */
+ d2 = p1[2];
+ d3 = p1[3];
+ d4 = p1[4];
+ d5 = p1[5];
+ d6 = p1[6];
+ d7 = p1[7];
+ d0 ^= p2[0];
+ d1 ^= p2[1];
+ d2 ^= p2[2];
+ d3 ^= p2[3];
+ d4 ^= p2[4];
+ d5 ^= p2[5];
+ d6 ^= p2[6];
+ d7 ^= p2[7];
+ d0 ^= p3[0];
+ d1 ^= p3[1];
+ d2 ^= p3[2];
+ d3 ^= p3[3];
+ d4 ^= p3[4];
+ d5 ^= p3[5];
+ d6 ^= p3[6];
+ d7 ^= p3[7];
+ d0 ^= p4[0];
+ d1 ^= p4[1];
+ d2 ^= p4[2];
+ d3 ^= p4[3];
+ d4 ^= p4[4];
+ d5 ^= p4[5];
+ d6 ^= p4[6];
+ d7 ^= p4[7];
+ p1[0] = d0; /* Store the result (in bursts) */
+ p1[1] = d1;
+ p1[2] = d2;
+ p1[3] = d3;
+ p1[4] = d4;
+ p1[5] = d5;
+ p1[6] = d6;
+ p1[7] = d7;
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ p4 += 8;
+ } while (--lines > 0);
+ if (lines == 0)
+ goto once_more;
+}
+
+static void
+xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ long lines = bytes / (sizeof (long)) / 8 - 1;
+
+ prefetchw(p1);
+ prefetch(p2);
+ prefetch(p3);
+ prefetch(p4);
+ prefetch(p5);
+
+ do {
+ register long d0, d1, d2, d3, d4, d5, d6, d7;
+
+ prefetchw(p1+8);
+ prefetch(p2+8);
+ prefetch(p3+8);
+ prefetch(p4+8);
+ prefetch(p5+8);
+ once_more:
+ d0 = p1[0]; /* Pull the stuff into registers */
+ d1 = p1[1]; /* ... in bursts, if possible. */
+ d2 = p1[2];
+ d3 = p1[3];
+ d4 = p1[4];
+ d5 = p1[5];
+ d6 = p1[6];
+ d7 = p1[7];
+ d0 ^= p2[0];
+ d1 ^= p2[1];
+ d2 ^= p2[2];
+ d3 ^= p2[3];
+ d4 ^= p2[4];
+ d5 ^= p2[5];
+ d6 ^= p2[6];
+ d7 ^= p2[7];
+ d0 ^= p3[0];
+ d1 ^= p3[1];
+ d2 ^= p3[2];
+ d3 ^= p3[3];
+ d4 ^= p3[4];
+ d5 ^= p3[5];
+ d6 ^= p3[6];
+ d7 ^= p3[7];
+ d0 ^= p4[0];
+ d1 ^= p4[1];
+ d2 ^= p4[2];
+ d3 ^= p4[3];
+ d4 ^= p4[4];
+ d5 ^= p4[5];
+ d6 ^= p4[6];
+ d7 ^= p4[7];
+ d0 ^= p5[0];
+ d1 ^= p5[1];
+ d2 ^= p5[2];
+ d3 ^= p5[3];
+ d4 ^= p5[4];
+ d5 ^= p5[5];
+ d6 ^= p5[6];
+ d7 ^= p5[7];
+ p1[0] = d0; /* Store the result (in bursts) */
+ p1[1] = d1;
+ p1[2] = d2;
+ p1[3] = d3;
+ p1[4] = d4;
+ p1[5] = d5;
+ p1[6] = d6;
+ p1[7] = d7;
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ p4 += 8;
+ p5 += 8;
+ } while (--lines > 0);
+ if (lines == 0)
+ goto once_more;
+}
+
+struct xor_block_template xor_block_32regs_p = {
+ .name = "32regs_prefetch",
+ .do_2 = xor_32regs_p_2,
+ .do_3 = xor_32regs_p_3,
+ .do_4 = xor_32regs_p_4,
+ .do_5 = xor_32regs_p_5,
+};
diff --git a/lib/raid/xor/xor-32regs.c b/lib/raid/xor/xor-32regs.c
new file mode 100644
index 000000000000..58d4fac43eb4
--- /dev/null
+++ b/lib/raid/xor/xor-32regs.c
@@ -0,0 +1,219 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/raid/xor_impl.h>
+#include <asm-generic/xor.h>
+
+static void
+xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ long lines = bytes / (sizeof (long)) / 8;
+
+ do {
+ register long d0, d1, d2, d3, d4, d5, d6, d7;
+ d0 = p1[0]; /* Pull the stuff into registers */
+ d1 = p1[1]; /* ... in bursts, if possible. */
+ d2 = p1[2];
+ d3 = p1[3];
+ d4 = p1[4];
+ d5 = p1[5];
+ d6 = p1[6];
+ d7 = p1[7];
+ d0 ^= p2[0];
+ d1 ^= p2[1];
+ d2 ^= p2[2];
+ d3 ^= p2[3];
+ d4 ^= p2[4];
+ d5 ^= p2[5];
+ d6 ^= p2[6];
+ d7 ^= p2[7];
+ p1[0] = d0; /* Store the result (in bursts) */
+ p1[1] = d1;
+ p1[2] = d2;
+ p1[3] = d3;
+ p1[4] = d4;
+ p1[5] = d5;
+ p1[6] = d6;
+ p1[7] = d7;
+ p1 += 8;
+ p2 += 8;
+ } while (--lines > 0);
+}
+
+static void
+xor_32regs_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ long lines = bytes / (sizeof (long)) / 8;
+
+ do {
+ register long d0, d1, d2, d3, d4, d5, d6, d7;
+ d0 = p1[0]; /* Pull the stuff into registers */
+ d1 = p1[1]; /* ... in bursts, if possible. */
+ d2 = p1[2];
+ d3 = p1[3];
+ d4 = p1[4];
+ d5 = p1[5];
+ d6 = p1[6];
+ d7 = p1[7];
+ d0 ^= p2[0];
+ d1 ^= p2[1];
+ d2 ^= p2[2];
+ d3 ^= p2[3];
+ d4 ^= p2[4];
+ d5 ^= p2[5];
+ d6 ^= p2[6];
+ d7 ^= p2[7];
+ d0 ^= p3[0];
+ d1 ^= p3[1];
+ d2 ^= p3[2];
+ d3 ^= p3[3];
+ d4 ^= p3[4];
+ d5 ^= p3[5];
+ d6 ^= p3[6];
+ d7 ^= p3[7];
+ p1[0] = d0; /* Store the result (in bursts) */
+ p1[1] = d1;
+ p1[2] = d2;
+ p1[3] = d3;
+ p1[4] = d4;
+ p1[5] = d5;
+ p1[6] = d6;
+ p1[7] = d7;
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ } while (--lines > 0);
+}
+
+static void
+xor_32regs_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ long lines = bytes / (sizeof (long)) / 8;
+
+ do {
+ register long d0, d1, d2, d3, d4, d5, d6, d7;
+ d0 = p1[0]; /* Pull the stuff into registers */
+ d1 = p1[1]; /* ... in bursts, if possible. */
+ d2 = p1[2];
+ d3 = p1[3];
+ d4 = p1[4];
+ d5 = p1[5];
+ d6 = p1[6];
+ d7 = p1[7];
+ d0 ^= p2[0];
+ d1 ^= p2[1];
+ d2 ^= p2[2];
+ d3 ^= p2[3];
+ d4 ^= p2[4];
+ d5 ^= p2[5];
+ d6 ^= p2[6];
+ d7 ^= p2[7];
+ d0 ^= p3[0];
+ d1 ^= p3[1];
+ d2 ^= p3[2];
+ d3 ^= p3[3];
+ d4 ^= p3[4];
+ d5 ^= p3[5];
+ d6 ^= p3[6];
+ d7 ^= p3[7];
+ d0 ^= p4[0];
+ d1 ^= p4[1];
+ d2 ^= p4[2];
+ d3 ^= p4[3];
+ d4 ^= p4[4];
+ d5 ^= p4[5];
+ d6 ^= p4[6];
+ d7 ^= p4[7];
+ p1[0] = d0; /* Store the result (in bursts) */
+ p1[1] = d1;
+ p1[2] = d2;
+ p1[3] = d3;
+ p1[4] = d4;
+ p1[5] = d5;
+ p1[6] = d6;
+ p1[7] = d7;
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ p4 += 8;
+ } while (--lines > 0);
+}
+
+static void
+xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ long lines = bytes / (sizeof (long)) / 8;
+
+ do {
+ register long d0, d1, d2, d3, d4, d5, d6, d7;
+ d0 = p1[0]; /* Pull the stuff into registers */
+ d1 = p1[1]; /* ... in bursts, if possible. */
+ d2 = p1[2];
+ d3 = p1[3];
+ d4 = p1[4];
+ d5 = p1[5];
+ d6 = p1[6];
+ d7 = p1[7];
+ d0 ^= p2[0];
+ d1 ^= p2[1];
+ d2 ^= p2[2];
+ d3 ^= p2[3];
+ d4 ^= p2[4];
+ d5 ^= p2[5];
+ d6 ^= p2[6];
+ d7 ^= p2[7];
+ d0 ^= p3[0];
+ d1 ^= p3[1];
+ d2 ^= p3[2];
+ d3 ^= p3[3];
+ d4 ^= p3[4];
+ d5 ^= p3[5];
+ d6 ^= p3[6];
+ d7 ^= p3[7];
+ d0 ^= p4[0];
+ d1 ^= p4[1];
+ d2 ^= p4[2];
+ d3 ^= p4[3];
+ d4 ^= p4[4];
+ d5 ^= p4[5];
+ d6 ^= p4[6];
+ d7 ^= p4[7];
+ d0 ^= p5[0];
+ d1 ^= p5[1];
+ d2 ^= p5[2];
+ d3 ^= p5[3];
+ d4 ^= p5[4];
+ d5 ^= p5[5];
+ d6 ^= p5[6];
+ d7 ^= p5[7];
+ p1[0] = d0; /* Store the result (in bursts) */
+ p1[1] = d1;
+ p1[2] = d2;
+ p1[3] = d3;
+ p1[4] = d4;
+ p1[5] = d5;
+ p1[6] = d6;
+ p1[7] = d7;
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ p4 += 8;
+ p5 += 8;
+ } while (--lines > 0);
+}
+
+struct xor_block_template xor_block_32regs = {
+ .name = "32regs",
+ .do_2 = xor_32regs_2,
+ .do_3 = xor_32regs_3,
+ .do_4 = xor_32regs_4,
+ .do_5 = xor_32regs_5,
+};
diff --git a/lib/raid/xor/xor-8regs-prefetch.c b/lib/raid/xor/xor-8regs-prefetch.c
new file mode 100644
index 000000000000..67061e35a0a6
--- /dev/null
+++ b/lib/raid/xor/xor-8regs-prefetch.c
@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/prefetch.h>
+#include <linux/raid/xor_impl.h>
+#include <asm-generic/xor.h>
+
+static void
+xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ long lines = bytes / (sizeof (long)) / 8 - 1;
+ prefetchw(p1);
+ prefetch(p2);
+
+ do {
+ prefetchw(p1+8);
+ prefetch(p2+8);
+ once_more:
+ p1[0] ^= p2[0];
+ p1[1] ^= p2[1];
+ p1[2] ^= p2[2];
+ p1[3] ^= p2[3];
+ p1[4] ^= p2[4];
+ p1[5] ^= p2[5];
+ p1[6] ^= p2[6];
+ p1[7] ^= p2[7];
+ p1 += 8;
+ p2 += 8;
+ } while (--lines > 0);
+ if (lines == 0)
+ goto once_more;
+}
+
+static void
+xor_8regs_p_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ long lines = bytes / (sizeof (long)) / 8 - 1;
+ prefetchw(p1);
+ prefetch(p2);
+ prefetch(p3);
+
+ do {
+ prefetchw(p1+8);
+ prefetch(p2+8);
+ prefetch(p3+8);
+ once_more:
+ p1[0] ^= p2[0] ^ p3[0];
+ p1[1] ^= p2[1] ^ p3[1];
+ p1[2] ^= p2[2] ^ p3[2];
+ p1[3] ^= p2[3] ^ p3[3];
+ p1[4] ^= p2[4] ^ p3[4];
+ p1[5] ^= p2[5] ^ p3[5];
+ p1[6] ^= p2[6] ^ p3[6];
+ p1[7] ^= p2[7] ^ p3[7];
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ } while (--lines > 0);
+ if (lines == 0)
+ goto once_more;
+}
+
+static void
+xor_8regs_p_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ long lines = bytes / (sizeof (long)) / 8 - 1;
+
+ prefetchw(p1);
+ prefetch(p2);
+ prefetch(p3);
+ prefetch(p4);
+
+ do {
+ prefetchw(p1+8);
+ prefetch(p2+8);
+ prefetch(p3+8);
+ prefetch(p4+8);
+ once_more:
+ p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
+ p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
+ p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
+ p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
+ p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
+ p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
+ p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
+ p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ p4 += 8;
+ } while (--lines > 0);
+ if (lines == 0)
+ goto once_more;
+}
+
+static void
+xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ long lines = bytes / (sizeof (long)) / 8 - 1;
+
+ prefetchw(p1);
+ prefetch(p2);
+ prefetch(p3);
+ prefetch(p4);
+ prefetch(p5);
+
+ do {
+ prefetchw(p1+8);
+ prefetch(p2+8);
+ prefetch(p3+8);
+ prefetch(p4+8);
+ prefetch(p5+8);
+ once_more:
+ p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
+ p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
+ p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
+ p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
+ p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
+ p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
+ p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
+ p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ p4 += 8;
+ p5 += 8;
+ } while (--lines > 0);
+ if (lines == 0)
+ goto once_more;
+}
+
+struct xor_block_template xor_block_8regs_p = {
+ .name = "8regs_prefetch",
+ .do_2 = xor_8regs_p_2,
+ .do_3 = xor_8regs_p_3,
+ .do_4 = xor_8regs_p_4,
+ .do_5 = xor_8regs_p_5,
+};
diff --git a/lib/raid/xor/xor-8regs.c b/lib/raid/xor/xor-8regs.c
new file mode 100644
index 000000000000..769f796ab2cf
--- /dev/null
+++ b/lib/raid/xor/xor-8regs.c
@@ -0,0 +1,105 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+#include <linux/raid/xor_impl.h>
+#include <asm-generic/xor.h>
+
+static void
+xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ long lines = bytes / (sizeof (long)) / 8;
+
+ do {
+ p1[0] ^= p2[0];
+ p1[1] ^= p2[1];
+ p1[2] ^= p2[2];
+ p1[3] ^= p2[3];
+ p1[4] ^= p2[4];
+ p1[5] ^= p2[5];
+ p1[6] ^= p2[6];
+ p1[7] ^= p2[7];
+ p1 += 8;
+ p2 += 8;
+ } while (--lines > 0);
+}
+
+static void
+xor_8regs_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ long lines = bytes / (sizeof (long)) / 8;
+
+ do {
+ p1[0] ^= p2[0] ^ p3[0];
+ p1[1] ^= p2[1] ^ p3[1];
+ p1[2] ^= p2[2] ^ p3[2];
+ p1[3] ^= p2[3] ^ p3[3];
+ p1[4] ^= p2[4] ^ p3[4];
+ p1[5] ^= p2[5] ^ p3[5];
+ p1[6] ^= p2[6] ^ p3[6];
+ p1[7] ^= p2[7] ^ p3[7];
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ } while (--lines > 0);
+}
+
+static void
+xor_8regs_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ long lines = bytes / (sizeof (long)) / 8;
+
+ do {
+ p1[0] ^= p2[0] ^ p3[0] ^ p4[0];
+ p1[1] ^= p2[1] ^ p3[1] ^ p4[1];
+ p1[2] ^= p2[2] ^ p3[2] ^ p4[2];
+ p1[3] ^= p2[3] ^ p3[3] ^ p4[3];
+ p1[4] ^= p2[4] ^ p3[4] ^ p4[4];
+ p1[5] ^= p2[5] ^ p3[5] ^ p4[5];
+ p1[6] ^= p2[6] ^ p3[6] ^ p4[6];
+ p1[7] ^= p2[7] ^ p3[7] ^ p4[7];
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ p4 += 8;
+ } while (--lines > 0);
+}
+
+static void
+xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ long lines = bytes / (sizeof (long)) / 8;
+
+ do {
+ p1[0] ^= p2[0] ^ p3[0] ^ p4[0] ^ p5[0];
+ p1[1] ^= p2[1] ^ p3[1] ^ p4[1] ^ p5[1];
+ p1[2] ^= p2[2] ^ p3[2] ^ p4[2] ^ p5[2];
+ p1[3] ^= p2[3] ^ p3[3] ^ p4[3] ^ p5[3];
+ p1[4] ^= p2[4] ^ p3[4] ^ p4[4] ^ p5[4];
+ p1[5] ^= p2[5] ^ p3[5] ^ p4[5] ^ p5[5];
+ p1[6] ^= p2[6] ^ p3[6] ^ p4[6] ^ p5[6];
+ p1[7] ^= p2[7] ^ p3[7] ^ p4[7] ^ p5[7];
+ p1 += 8;
+ p2 += 8;
+ p3 += 8;
+ p4 += 8;
+ p5 += 8;
+ } while (--lines > 0);
+}
+
+#ifndef NO_TEMPLATE
+struct xor_block_template xor_block_8regs = {
+ .name = "8regs",
+ .do_2 = xor_8regs_2,
+ .do_3 = xor_8regs_3,
+ .do_4 = xor_8regs_4,
+ .do_5 = xor_8regs_5,
+};
+#endif /* NO_TEMPLATE */
--
2.47.3
^ permalink raw reply related
* [PATCH 08/26] xor: remove macro abuse for XOR implementation registrations
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Drop the pretty confusing historic XOR_TRY_TEMPLATES and
XOR_SELECT_TEMPLATE, and instead let the architectures provide a
arch_xor_init that calls either xor_register to register candidates
or xor_force to force a specific implementation.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/alpha/include/asm/xor.h | 29 ++++++++++++----------
arch/arm/include/asm/xor.h | 25 +++++++++----------
arch/arm64/include/asm/xor.h | 18 +++++++-------
arch/loongarch/include/asm/xor.h | 42 ++++++++++++--------------------
arch/powerpc/include/asm/xor.h | 31 ++++++++++-------------
arch/riscv/include/asm/xor.h | 19 ++++++++-------
arch/s390/include/asm/xor.h | 12 ++++-----
arch/sparc/include/asm/xor_32.h | 14 +++++------
arch/sparc/include/asm/xor_64.h | 31 +++++++++++------------
arch/x86/include/asm/xor.h | 3 ---
arch/x86/include/asm/xor_32.h | 36 ++++++++++++++-------------
arch/x86/include/asm/xor_64.h | 18 ++++++++------
arch/x86/include/asm/xor_avx.h | 9 -------
include/asm-generic/xor.h | 8 ------
include/linux/raid/xor_impl.h | 5 ++++
lib/raid/xor/xor-core.c | 41 +++++++++++++++++++++++--------
16 files changed, 168 insertions(+), 173 deletions(-)
diff --git a/arch/alpha/include/asm/xor.h b/arch/alpha/include/asm/xor.h
index e0de0c233ab9..4c8085711df1 100644
--- a/arch/alpha/include/asm/xor.h
+++ b/arch/alpha/include/asm/xor.h
@@ -851,16 +851,19 @@ static struct xor_block_template xor_block_alpha_prefetch = {
/* For grins, also test the generic routines. */
#include <asm-generic/xor.h>
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_alpha); \
- xor_speed(&xor_block_alpha_prefetch); \
- } while (0)
-
-/* Force the use of alpha_prefetch if EV6, as it is significantly
- faster in the cold cache case. */
-#define XOR_SELECT_TEMPLATE(FASTEST) \
- (implver() == IMPLVER_EV6 ? &xor_block_alpha_prefetch : FASTEST)
+/*
+ * Force the use of alpha_prefetch if EV6, as it is significantly faster in the
+ * cold cache case.
+ */
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ if (implver() == IMPLVER_EV6) {
+ xor_force(&xor_block_alpha_prefetch);
+ } else {
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_32regs);
+ xor_register(&xor_block_alpha);
+ xor_register(&xor_block_alpha_prefetch);
+ }
+}
diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h
index bca2a6514746..b2dcd49186e2 100644
--- a/arch/arm/include/asm/xor.h
+++ b/arch/arm/include/asm/xor.h
@@ -138,15 +138,6 @@ static struct xor_block_template xor_block_arm4regs = {
.do_5 = xor_arm4regs_5,
};
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_arm4regs); \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- NEON_TEMPLATES; \
- } while (0)
-
#ifdef CONFIG_KERNEL_MODE_NEON
extern struct xor_block_template const xor_block_neon_inner;
@@ -201,8 +192,16 @@ static struct xor_block_template xor_block_neon = {
.do_5 = xor_neon_5
};
-#define NEON_TEMPLATES \
- do { if (cpu_has_neon()) xor_speed(&xor_block_neon); } while (0)
-#else
-#define NEON_TEMPLATES
+#endif /* CONFIG_KERNEL_MODE_NEON */
+
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ xor_register(&xor_block_arm4regs);
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_32regs);
+#ifdef CONFIG_KERNEL_MODE_NEON
+ if (cpu_has_neon())
+ xor_register(&xor_block_neon);
#endif
+}
diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h
index c38e3d017a79..bfa6122f55ce 100644
--- a/arch/arm64/include/asm/xor.h
+++ b/arch/arm64/include/asm/xor.h
@@ -60,14 +60,14 @@ static struct xor_block_template xor_block_arm64 = {
.do_4 = xor_neon_4,
.do_5 = xor_neon_5
};
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- if (cpu_has_neon()) { \
- xor_speed(&xor_block_arm64);\
- } \
- } while (0)
+
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_32regs);
+ if (cpu_has_neon())
+ xor_register(&xor_block_arm64);
+}
#endif /* ! CONFIG_KERNEL_MODE_NEON */
diff --git a/arch/loongarch/include/asm/xor.h b/arch/loongarch/include/asm/xor.h
index 12467fffee46..d17c0e3b047f 100644
--- a/arch/loongarch/include/asm/xor.h
+++ b/arch/loongarch/include/asm/xor.h
@@ -16,14 +16,6 @@ static struct xor_block_template xor_block_lsx = {
.do_4 = xor_lsx_4,
.do_5 = xor_lsx_5,
};
-
-#define XOR_SPEED_LSX() \
- do { \
- if (cpu_has_lsx) \
- xor_speed(&xor_block_lsx); \
- } while (0)
-#else /* CONFIG_CPU_HAS_LSX */
-#define XOR_SPEED_LSX()
#endif /* CONFIG_CPU_HAS_LSX */
#ifdef CONFIG_CPU_HAS_LASX
@@ -34,14 +26,6 @@ static struct xor_block_template xor_block_lasx = {
.do_4 = xor_lasx_4,
.do_5 = xor_lasx_5,
};
-
-#define XOR_SPEED_LASX() \
- do { \
- if (cpu_has_lasx) \
- xor_speed(&xor_block_lasx); \
- } while (0)
-#else /* CONFIG_CPU_HAS_LASX */
-#define XOR_SPEED_LASX()
#endif /* CONFIG_CPU_HAS_LASX */
/*
@@ -54,15 +38,21 @@ static struct xor_block_template xor_block_lasx = {
*/
#include <asm-generic/xor.h>
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_8regs_p); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_32regs_p); \
- XOR_SPEED_LSX(); \
- XOR_SPEED_LASX(); \
-} while (0)
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_8regs_p);
+ xor_register(&xor_block_32regs);
+ xor_register(&xor_block_32regs_p);
+#ifdef CONFIG_CPU_HAS_LSX
+ if (cpu_has_lsx)
+ xor_register(&xor_block_lsx);
+#endif
+#ifdef CONFIG_CPU_HAS_LASX
+ if (cpu_has_lasx)
+ xor_register(&xor_block_lasx);
+#endif
+}
#endif /* _ASM_LOONGARCH_XOR_H */
diff --git a/arch/powerpc/include/asm/xor.h b/arch/powerpc/include/asm/xor.h
index 37d05c11d09c..30224c5279c4 100644
--- a/arch/powerpc/include/asm/xor.h
+++ b/arch/powerpc/include/asm/xor.h
@@ -21,27 +21,22 @@ static struct xor_block_template xor_block_altivec = {
.do_4 = xor_altivec_4,
.do_5 = xor_altivec_5,
};
-
-#define XOR_SPEED_ALTIVEC() \
- do { \
- if (cpu_has_feature(CPU_FTR_ALTIVEC)) \
- xor_speed(&xor_block_altivec); \
- } while (0)
-#else
-#define XOR_SPEED_ALTIVEC()
-#endif
+#endif /* CONFIG_ALTIVEC */
/* Also try the generic routines. */
#include <asm-generic/xor.h>
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_8regs_p); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_32regs_p); \
- XOR_SPEED_ALTIVEC(); \
-} while (0)
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_8regs_p);
+ xor_register(&xor_block_32regs);
+ xor_register(&xor_block_32regs_p);
+#ifdef CONFIG_ALTIVEC
+ if (cpu_has_feature(CPU_FTR_ALTIVEC))
+ xor_register(&xor_block_altivec);
+#endif
+}
#endif /* _ASM_POWERPC_XOR_H */
diff --git a/arch/riscv/include/asm/xor.h b/arch/riscv/include/asm/xor.h
index 96011861e46b..ed5f27903efc 100644
--- a/arch/riscv/include/asm/xor.h
+++ b/arch/riscv/include/asm/xor.h
@@ -55,14 +55,15 @@ static struct xor_block_template xor_block_rvv = {
.do_4 = xor_vector_4,
.do_5 = xor_vector_5
};
+#endif /* CONFIG_RISCV_ISA_V */
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- if (has_vector()) { \
- xor_speed(&xor_block_rvv);\
- } \
- } while (0)
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_32regs);
+#ifdef CONFIG_RISCV_ISA_V
+ if (has_vector())
+ xor_register(&xor_block_rvv);
#endif
+}
diff --git a/arch/s390/include/asm/xor.h b/arch/s390/include/asm/xor.h
index 857d6759b67f..4e2233f64da9 100644
--- a/arch/s390/include/asm/xor.h
+++ b/arch/s390/include/asm/xor.h
@@ -10,12 +10,10 @@
extern struct xor_block_template xor_block_xc;
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- xor_speed(&xor_block_xc); \
-} while (0)
-
-#define XOR_SELECT_TEMPLATE(FASTEST) (&xor_block_xc)
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ xor_force(&xor_block_xc);
+}
#endif /* _ASM_S390_XOR_H */
diff --git a/arch/sparc/include/asm/xor_32.h b/arch/sparc/include/asm/xor_32.h
index 0351813cf3af..8fbf0c07ec28 100644
--- a/arch/sparc/include/asm/xor_32.h
+++ b/arch/sparc/include/asm/xor_32.h
@@ -259,10 +259,10 @@ static struct xor_block_template xor_block_SPARC = {
/* For grins, also test the generic routines. */
#include <asm-generic/xor.h>
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_SPARC); \
- } while (0)
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_32regs);
+ xor_register(&xor_block_SPARC);
+}
diff --git a/arch/sparc/include/asm/xor_64.h b/arch/sparc/include/asm/xor_64.h
index caaddea8ad79..e0482ecc0a68 100644
--- a/arch/sparc/include/asm/xor_64.h
+++ b/arch/sparc/include/asm/xor_64.h
@@ -60,20 +60,17 @@ static struct xor_block_template xor_block_niagara = {
.do_5 = xor_niagara_5,
};
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_VIS); \
- xor_speed(&xor_block_niagara); \
- } while (0)
-
-/* For VIS for everything except Niagara. */
-#define XOR_SELECT_TEMPLATE(FASTEST) \
- ((tlb_type == hypervisor && \
- (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 || \
- sun4v_chip_type == SUN4V_CHIP_NIAGARA2 || \
- sun4v_chip_type == SUN4V_CHIP_NIAGARA3 || \
- sun4v_chip_type == SUN4V_CHIP_NIAGARA4 || \
- sun4v_chip_type == SUN4V_CHIP_NIAGARA5)) ? \
- &xor_block_niagara : \
- &xor_block_VIS)
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ /* Force VIS for everything except Niagara. */
+ if (tlb_type == hypervisor &&
+ (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 ||
+ sun4v_chip_type == SUN4V_CHIP_NIAGARA2 ||
+ sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
+ sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
+ sun4v_chip_type == SUN4V_CHIP_NIAGARA5))
+ xor_force(&xor_block_niagara);
+ else
+ xor_force(&xor_block_VIS);
+}
diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h
index 7b0307acc410..33f5620d8d69 100644
--- a/arch/x86/include/asm/xor.h
+++ b/arch/x86/include/asm/xor.h
@@ -496,7 +496,4 @@ static struct xor_block_template xor_block_sse_pf64 = {
# include <asm/xor_64.h>
#endif
-#define XOR_SELECT_TEMPLATE(FASTEST) \
- AVX_SELECT(FASTEST)
-
#endif /* _ASM_X86_XOR_H */
diff --git a/arch/x86/include/asm/xor_32.h b/arch/x86/include/asm/xor_32.h
index 7a6b9474591e..ee32d08c27bc 100644
--- a/arch/x86/include/asm/xor_32.h
+++ b/arch/x86/include/asm/xor_32.h
@@ -552,22 +552,24 @@ static struct xor_block_template xor_block_pIII_sse = {
/* We force the use of the SSE xor block because it can write around L2.
We may also be able to load into the L1 only depending on how the cpu
deals with a load to a line that is being prefetched. */
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- AVX_XOR_SPEED; \
- if (boot_cpu_has(X86_FEATURE_XMM)) { \
- xor_speed(&xor_block_pIII_sse); \
- xor_speed(&xor_block_sse_pf64); \
- } else if (boot_cpu_has(X86_FEATURE_MMX)) { \
- xor_speed(&xor_block_pII_mmx); \
- xor_speed(&xor_block_p5_mmx); \
- } else { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_8regs_p); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_32regs_p); \
- } \
-} while (0)
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ if (boot_cpu_has(X86_FEATURE_AVX) &&
+ boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+ xor_force(&xor_block_avx);
+ } else if (boot_cpu_has(X86_FEATURE_XMM)) {
+ xor_register(&xor_block_pIII_sse);
+ xor_register(&xor_block_sse_pf64);
+ } else if (boot_cpu_has(X86_FEATURE_MMX)) {
+ xor_register(&xor_block_pII_mmx);
+ xor_register(&xor_block_p5_mmx);
+ } else {
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_8regs_p);
+ xor_register(&xor_block_32regs);
+ xor_register(&xor_block_32regs_p);
+ }
+}
#endif /* _ASM_X86_XOR_32_H */
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h
index 0307e4ec5044..2d2ceb241866 100644
--- a/arch/x86/include/asm/xor_64.h
+++ b/arch/x86/include/asm/xor_64.h
@@ -17,12 +17,16 @@ static struct xor_block_template xor_block_sse = {
/* We force the use of the SSE xor block because it can write around L2.
We may also be able to load into the L1 only depending on how the cpu
deals with a load to a line that is being prefetched. */
-#undef XOR_TRY_TEMPLATES
-#define XOR_TRY_TEMPLATES \
-do { \
- AVX_XOR_SPEED; \
- xor_speed(&xor_block_sse_pf64); \
- xor_speed(&xor_block_sse); \
-} while (0)
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ if (boot_cpu_has(X86_FEATURE_AVX) &&
+ boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+ xor_force(&xor_block_avx);
+ } else {
+ xor_register(&xor_block_sse_pf64);
+ xor_register(&xor_block_sse);
+ }
+}
#endif /* _ASM_X86_XOR_64_H */
diff --git a/arch/x86/include/asm/xor_avx.h b/arch/x86/include/asm/xor_avx.h
index 7f81dd5897f4..c600888436bb 100644
--- a/arch/x86/include/asm/xor_avx.h
+++ b/arch/x86/include/asm/xor_avx.h
@@ -166,13 +166,4 @@ static struct xor_block_template xor_block_avx = {
.do_5 = xor_avx_5,
};
-#define AVX_XOR_SPEED \
-do { \
- if (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_OSXSAVE)) \
- xor_speed(&xor_block_avx); \
-} while (0)
-
-#define AVX_SELECT(FASTEST) \
- (boot_cpu_has(X86_FEATURE_AVX) && boot_cpu_has(X86_FEATURE_OSXSAVE) ? &xor_block_avx : FASTEST)
-
#endif
diff --git a/include/asm-generic/xor.h b/include/asm-generic/xor.h
index 44509d48fca2..79c0096aa9d9 100644
--- a/include/asm-generic/xor.h
+++ b/include/asm-generic/xor.h
@@ -728,11 +728,3 @@ static struct xor_block_template xor_block_32regs_p __maybe_unused = {
.do_4 = xor_32regs_p_4,
.do_5 = xor_32regs_p_5,
};
-
-#define XOR_TRY_TEMPLATES \
- do { \
- xor_speed(&xor_block_8regs); \
- xor_speed(&xor_block_8regs_p); \
- xor_speed(&xor_block_32regs); \
- xor_speed(&xor_block_32regs_p); \
- } while (0)
diff --git a/include/linux/raid/xor_impl.h b/include/linux/raid/xor_impl.h
index a1890cd66812..6ed4c445ab24 100644
--- a/include/linux/raid/xor_impl.h
+++ b/include/linux/raid/xor_impl.h
@@ -2,6 +2,8 @@
#ifndef _XOR_IMPL_H
#define _XOR_IMPL_H
+#include <linux/init.h>
+
struct xor_block_template {
struct xor_block_template *next;
const char *name;
@@ -22,4 +24,7 @@ struct xor_block_template {
const unsigned long * __restrict);
};
+void __init xor_register(struct xor_block_template *tmpl);
+void __init xor_force(struct xor_block_template *tmpl);
+
#endif /* _XOR_IMPL_H */
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index 58f2d0c16420..01a42995b7a5 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -14,10 +14,6 @@
#include <linux/preempt.h>
#include <asm/xor.h>
-#ifndef XOR_SELECT_TEMPLATE
-#define XOR_SELECT_TEMPLATE(x) (x)
-#endif
-
/* The xor routines to use. */
static struct xor_block_template *active_template;
@@ -55,12 +51,33 @@ EXPORT_SYMBOL(xor_blocks);
static struct xor_block_template *__initdata template_list;
static bool __initdata xor_forced = false;
-static void __init do_xor_register(struct xor_block_template *tmpl)
+/**
+ * xor_register - register a XOR template
+ * @tmpl: template to register
+ *
+ * Register a XOR implementation with the core. Registered implementations
+ * will be measured by a trivial benchmark, and the fastest one is chosen
+ * unless an implementation is forced using xor_force().
+ */
+void __init xor_register(struct xor_block_template *tmpl)
{
tmpl->next = template_list;
template_list = tmpl;
}
+/**
+ * xor_force - force use of a XOR template
+ * @tmpl: template to register
+ *
+ * Register a XOR implementation with the core and force using it. Forcing
+ * an implementation will make the core ignore any template registered using
+ * xor_register(), or any previous implementation forced using xor_force().
+ */
+void __init xor_force(struct xor_block_template *tmpl)
+{
+ active_template = tmpl;
+}
+
#define BENCH_SIZE 4096
#define REPS 800U
@@ -126,11 +143,19 @@ static int __init calibrate_xor_blocks(void)
static int __init xor_init(void)
{
+#ifdef arch_xor_init
+ arch_xor_init();
+#else
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_8regs_p);
+ xor_register(&xor_block_32regs);
+ xor_register(&xor_block_32regs_p);
+#endif
+
/*
* If this arch/cpu has a short-circuited selection, don't loop through
* all the possible functions, just use the best one.
*/
- active_template = XOR_SELECT_TEMPLATE(NULL);
if (active_template) {
pr_info("xor: automatically using best checksumming function %-10s\n",
active_template->name);
@@ -138,10 +163,6 @@ static int __init xor_init(void)
return 0;
}
-#define xor_speed do_xor_register
- XOR_TRY_TEMPLATES;
-#undef xor_speed
-
#ifdef MODULE
return calibrate_xor_blocks();
#else
--
2.47.3
^ permalink raw reply related
* [PATCH 07/26] xor: split xor.h
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Keep xor.h for the public API, and split the struct xor_block_template
definition that is only needed by the xor.ko core and
architecture-specific optimizations into a separate xor_impl.h header.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/arm/lib/xor-neon.c | 1 +
arch/arm64/lib/xor-neon.c | 1 +
arch/s390/lib/xor.c | 2 +-
include/linux/raid/xor.h | 22 +---------------------
include/linux/raid/xor_impl.h | 25 +++++++++++++++++++++++++
lib/raid/xor/xor-core.c | 1 +
6 files changed, 30 insertions(+), 22 deletions(-)
create mode 100644 include/linux/raid/xor_impl.h
diff --git a/arch/arm/lib/xor-neon.c b/arch/arm/lib/xor-neon.c
index cf57fca97908..282980b9bf2a 100644
--- a/arch/arm/lib/xor-neon.c
+++ b/arch/arm/lib/xor-neon.c
@@ -6,6 +6,7 @@
*/
#include <linux/raid/xor.h>
+#include <linux/raid/xor_impl.h>
#include <linux/module.h>
MODULE_DESCRIPTION("NEON accelerated XOR implementation");
diff --git a/arch/arm64/lib/xor-neon.c b/arch/arm64/lib/xor-neon.c
index 8fffebfa17b2..351aba92d932 100644
--- a/arch/arm64/lib/xor-neon.c
+++ b/arch/arm64/lib/xor-neon.c
@@ -7,6 +7,7 @@
*/
#include <linux/raid/xor.h>
+#include <linux/raid/xor_impl.h>
#include <linux/module.h>
#include <asm/neon-intrinsics.h>
diff --git a/arch/s390/lib/xor.c b/arch/s390/lib/xor.c
index 5363e4c2462d..3bbe21b40e66 100644
--- a/arch/s390/lib/xor.c
+++ b/arch/s390/lib/xor.c
@@ -8,7 +8,7 @@
#include <linux/types.h>
#include <linux/export.h>
-#include <linux/raid/xor.h>
+#include <linux/raid/xor_impl.h>
#include <asm/xor.h>
static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/include/linux/raid/xor.h b/include/linux/raid/xor.h
index 51b811b62322..02bda8d99534 100644
--- a/include/linux/raid/xor.h
+++ b/include/linux/raid/xor.h
@@ -7,24 +7,4 @@
extern void xor_blocks(unsigned int count, unsigned int bytes,
void *dest, void **srcs);
-struct xor_block_template {
- struct xor_block_template *next;
- const char *name;
- int speed;
- void (*do_2)(unsigned long, unsigned long * __restrict,
- const unsigned long * __restrict);
- void (*do_3)(unsigned long, unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict);
- void (*do_4)(unsigned long, unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict);
- void (*do_5)(unsigned long, unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict);
-};
-
-#endif
+#endif /* _XOR_H */
diff --git a/include/linux/raid/xor_impl.h b/include/linux/raid/xor_impl.h
new file mode 100644
index 000000000000..a1890cd66812
--- /dev/null
+++ b/include/linux/raid/xor_impl.h
@@ -0,0 +1,25 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _XOR_IMPL_H
+#define _XOR_IMPL_H
+
+struct xor_block_template {
+ struct xor_block_template *next;
+ const char *name;
+ int speed;
+ void (*do_2)(unsigned long, unsigned long * __restrict,
+ const unsigned long * __restrict);
+ void (*do_3)(unsigned long, unsigned long * __restrict,
+ const unsigned long * __restrict,
+ const unsigned long * __restrict);
+ void (*do_4)(unsigned long, unsigned long * __restrict,
+ const unsigned long * __restrict,
+ const unsigned long * __restrict,
+ const unsigned long * __restrict);
+ void (*do_5)(unsigned long, unsigned long * __restrict,
+ const unsigned long * __restrict,
+ const unsigned long * __restrict,
+ const unsigned long * __restrict,
+ const unsigned long * __restrict);
+};
+
+#endif /* _XOR_IMPL_H */
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index 88667a89b75b..58f2d0c16420 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -9,6 +9,7 @@
#include <linux/module.h>
#include <linux/gfp.h>
#include <linux/raid/xor.h>
+#include <linux/raid/xor_impl.h>
#include <linux/jiffies.h>
#include <linux/preempt.h>
#include <asm/xor.h>
--
2.47.3
^ permalink raw reply related
* [PATCH 06/26] xor: cleanup registration and probing
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Originally, the XOR code benchmarked all algorithms at load time, but
it has since then been hacked multiple times to allow forcing an
algorithm, and then commit 524ccdbdfb52 ("crypto: xor - defer load time
benchmark to a later time") changed the logic to a two-step process
or registration and benchmarking, but only when built-in.
Rework this, so that the XOR_TRY_TEMPLATES macro magic now always just
deals with adding the templates to the list, and benchmarking is always
done in a second pass; for modular builds from module_init, and for the
built-in case using a separate init call level.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
lib/raid/xor/xor-core.c | 98 ++++++++++++++++++++---------------------
1 file changed, 48 insertions(+), 50 deletions(-)
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index edb4e498da60..88667a89b75b 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -52,29 +52,14 @@ EXPORT_SYMBOL(xor_blocks);
/* Set of all registered templates. */
static struct xor_block_template *__initdata template_list;
+static bool __initdata xor_forced = false;
-#ifndef MODULE
static void __init do_xor_register(struct xor_block_template *tmpl)
{
tmpl->next = template_list;
template_list = tmpl;
}
-static int __init register_xor_blocks(void)
-{
- active_template = XOR_SELECT_TEMPLATE(NULL);
-
- if (!active_template) {
-#define xor_speed do_xor_register
- // register all the templates and pick the first as the default
- XOR_TRY_TEMPLATES;
-#undef xor_speed
- active_template = template_list;
- }
- return 0;
-}
-#endif
-
#define BENCH_SIZE 4096
#define REPS 800U
@@ -85,9 +70,6 @@ do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
unsigned long reps;
ktime_t min, start, t0;
- tmpl->next = template_list;
- template_list = tmpl;
-
preempt_disable();
reps = 0;
@@ -111,63 +93,79 @@ do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
pr_info(" %-16s: %5d MB/sec\n", tmpl->name, speed);
}
-static int __init
-calibrate_xor_blocks(void)
+static int __init calibrate_xor_blocks(void)
{
void *b1, *b2;
struct xor_block_template *f, *fastest;
- fastest = XOR_SELECT_TEMPLATE(NULL);
-
- if (fastest) {
- printk(KERN_INFO "xor: automatically using best "
- "checksumming function %-10s\n",
- fastest->name);
- goto out;
- }
+ if (xor_forced)
+ return 0;
b1 = (void *) __get_free_pages(GFP_KERNEL, 2);
if (!b1) {
- printk(KERN_WARNING "xor: Yikes! No memory available.\n");
+ pr_warn("xor: Yikes! No memory available.\n");
return -ENOMEM;
}
b2 = b1 + 2*PAGE_SIZE + BENCH_SIZE;
- /*
- * If this arch/cpu has a short-circuited selection, don't loop through
- * all the possible functions, just test the best one
- */
-
-#define xor_speed(templ) do_xor_speed((templ), b1, b2)
-
- printk(KERN_INFO "xor: measuring software checksum speed\n");
- template_list = NULL;
- XOR_TRY_TEMPLATES;
+ pr_info("xor: measuring software checksum speed\n");
fastest = template_list;
- for (f = fastest; f; f = f->next)
+ for (f = template_list; f; f = f->next) {
+ do_xor_speed(f, b1, b2);
if (f->speed > fastest->speed)
fastest = f;
-
+ }
+ active_template = fastest;
pr_info("xor: using function: %s (%d MB/sec)\n",
fastest->name, fastest->speed);
+ free_pages((unsigned long)b1, 2);
+ return 0;
+}
+
+static int __init xor_init(void)
+{
+ /*
+ * If this arch/cpu has a short-circuited selection, don't loop through
+ * all the possible functions, just use the best one.
+ */
+ active_template = XOR_SELECT_TEMPLATE(NULL);
+ if (active_template) {
+ pr_info("xor: automatically using best checksumming function %-10s\n",
+ active_template->name);
+ xor_forced = true;
+ return 0;
+ }
+
+#define xor_speed do_xor_register
+ XOR_TRY_TEMPLATES;
#undef xor_speed
- free_pages((unsigned long)b1, 2);
-out:
- active_template = fastest;
+#ifdef MODULE
+ return calibrate_xor_blocks();
+#else
+ /*
+ * Pick the first template as the temporary default until calibration
+ * happens.
+ */
+ active_template = template_list;
return 0;
+#endif
}
-static __exit void xor_exit(void) { }
+static __exit void xor_exit(void)
+{
+}
MODULE_DESCRIPTION("RAID-5 checksumming functions");
MODULE_LICENSE("GPL");
+/*
+ * When built-in we must register the default template before md, but we don't
+ * want calibration to run that early as that would delay the boot process.
+ */
#ifndef MODULE
-/* when built-in xor.o must initialize before drivers/md/md.o */
-core_initcall(register_xor_blocks);
+__initcall(calibrate_xor_blocks);
#endif
-
-module_init(calibrate_xor_blocks);
+core_initcall(xor_init);
module_exit(xor_exit);
--
2.47.3
^ permalink raw reply related
* [PATCH 05/26] xor: small cleanups
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Update the to of file comment to be correct and non-redundant, and drop
the unused BH_TRACE define.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
lib/raid/xor/xor-core.c | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index 676d7401af43..edb4e498da60 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -1,14 +1,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
- * xor.c : Multiple Devices driver for Linux
- *
* Copyright (C) 1996, 1997, 1998, 1999, 2000,
* Ingo Molnar, Matti Aarnio, Jakub Jelinek, Richard Henderson.
*
- * Dispatch optimized RAID-5 checksumming functions.
+ * Dispatch optimized XOR parity functions.
*/
-#define BH_TRACE 0
#include <linux/module.h>
#include <linux/gfp.h>
#include <linux/raid/xor.h>
--
2.47.3
^ permalink raw reply related
* [PATCH 04/26] xor: move to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the RAID XOR code to lib/raid/ as it has nothing to do with the
crypto API.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
crypto/Kconfig | 2 --
crypto/Makefile | 1 -
lib/Kconfig | 1 +
lib/Makefile | 2 +-
lib/raid/Kconfig | 4 ++++
lib/raid/Makefile | 3 +++
lib/raid/xor/Makefile | 5 +++++
crypto/xor.c => lib/raid/xor/xor-core.c | 0
8 files changed, 14 insertions(+), 4 deletions(-)
create mode 100644 lib/raid/Kconfig
create mode 100644 lib/raid/Makefile
create mode 100644 lib/raid/xor/Makefile
rename crypto/xor.c => lib/raid/xor/xor-core.c (100%)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index b4bb85e8e226..97d73136a0b4 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -2,8 +2,6 @@
#
# Generic algorithms support
#
-config XOR_BLOCKS
- tristate
#
# async_tx api: hardware offloaded memory transfer/transform support
diff --git a/crypto/Makefile b/crypto/Makefile
index 04e269117589..795c2eea51fe 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -196,7 +196,6 @@ obj-$(CONFIG_CRYPTO_ECRDSA) += ecrdsa_generic.o
#
# generic algorithms and the async_tx api
#
-obj-$(CONFIG_XOR_BLOCKS) += xor.o
obj-$(CONFIG_ASYNC_CORE) += async_tx/
obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
crypto_simd-y := simd.o
diff --git a/lib/Kconfig b/lib/Kconfig
index 0f2fb9610647..5be57adcd454 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -138,6 +138,7 @@ config TRACE_MMIO_ACCESS
source "lib/crc/Kconfig"
source "lib/crypto/Kconfig"
+source "lib/raid/Kconfig"
config XXHASH
tristate
diff --git a/lib/Makefile b/lib/Makefile
index 1b9ee167517f..84da412a044f 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -120,7 +120,7 @@ endif
obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o
CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any)
-obj-y += math/ crc/ crypto/ tests/ vdso/
+obj-y += math/ crc/ crypto/ tests/ vdso/ raid/
obj-$(CONFIG_GENERIC_IOMAP) += iomap.o
obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
new file mode 100644
index 000000000000..01b73a1c303f
--- /dev/null
+++ b/lib/raid/Kconfig
@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: GPL-2.0
+
+config XOR_BLOCKS
+ tristate
diff --git a/lib/raid/Makefile b/lib/raid/Makefile
new file mode 100644
index 000000000000..3540fe846dc4
--- /dev/null
+++ b/lib/raid/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-y += xor/
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
new file mode 100644
index 000000000000..7bca0ce8e90a
--- /dev/null
+++ b/lib/raid/xor/Makefile
@@ -0,0 +1,5 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_XOR_BLOCKS) += xor.o
+
+xor-y += xor-core.o
diff --git a/crypto/xor.c b/lib/raid/xor/xor-core.c
similarity index 100%
rename from crypto/xor.c
rename to lib/raid/xor/xor-core.c
--
2.47.3
^ permalink raw reply related
* [PATCH 03/26] um/xor: cleanup xor.h
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Since commit c055e3eae0f1 ("crypto: xor - use ktime for template
benchmarking") the benchmarking works just fine even for TT_MODE_INFCPU,
so drop the workarounds. Note that for CPUs supporting AVX2, which
includes almost everything built in the last 10 years, the AVX2
implementation is forced anyway.
CONFIG_X86_32 is always correctly set for UM in arch/x86/um/Kconfig,
so don't override it either.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Richard Weinberger <richard@nod.at>
---
arch/um/include/asm/xor.h | 16 ----------------
1 file changed, 16 deletions(-)
diff --git a/arch/um/include/asm/xor.h b/arch/um/include/asm/xor.h
index 647fae200c5d..99e5c7e1f475 100644
--- a/arch/um/include/asm/xor.h
+++ b/arch/um/include/asm/xor.h
@@ -2,23 +2,7 @@
#ifndef _ASM_UM_XOR_H
#define _ASM_UM_XOR_H
-#ifdef CONFIG_64BIT
-#undef CONFIG_X86_32
-#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_sse_pf64))
-#else
-#define CONFIG_X86_32 1
-#define TT_CPU_INF_XOR_DEFAULT (AVX_SELECT(&xor_block_8regs))
-#endif
-
#include <asm/cpufeature.h>
#include <../../x86/include/asm/xor.h>
-#include <linux/time-internal.h>
-
-#ifdef CONFIG_UML_TIME_TRAVEL_SUPPORT
-#undef XOR_SELECT_TEMPLATE
-/* pick an arbitrary one - measuring isn't possible with inf-cpu */
-#define XOR_SELECT_TEMPLATE(x) \
- (time_travel_mode == TT_MODE_INFCPU ? TT_CPU_INF_XOR_DEFAULT : x)
-#endif
#endif
--
2.47.3
^ permalink raw reply related
* [PATCH 02/26] arm/xor: remove in_interrupt() handling
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
xor_blocks can't be called from interrupt context, so remove the
handling for that.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/arm/include/asm/xor.h | 41 +++++++++++---------------------------
1 file changed, 12 insertions(+), 29 deletions(-)
diff --git a/arch/arm/include/asm/xor.h b/arch/arm/include/asm/xor.h
index 934b549905f5..bca2a6514746 100644
--- a/arch/arm/include/asm/xor.h
+++ b/arch/arm/include/asm/xor.h
@@ -4,7 +4,6 @@
*
* Copyright (C) 2001 Russell King
*/
-#include <linux/hardirq.h>
#include <asm-generic/xor.h>
#include <asm/hwcap.h>
#include <asm/neon.h>
@@ -156,13 +155,9 @@ static void
xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
- if (in_interrupt()) {
- xor_arm4regs_2(bytes, p1, p2);
- } else {
- kernel_neon_begin();
- xor_block_neon_inner.do_2(bytes, p1, p2);
- kernel_neon_end();
- }
+ kernel_neon_begin();
+ xor_block_neon_inner.do_2(bytes, p1, p2);
+ kernel_neon_end();
}
static void
@@ -170,13 +165,9 @@ xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
- if (in_interrupt()) {
- xor_arm4regs_3(bytes, p1, p2, p3);
- } else {
- kernel_neon_begin();
- xor_block_neon_inner.do_3(bytes, p1, p2, p3);
- kernel_neon_end();
- }
+ kernel_neon_begin();
+ xor_block_neon_inner.do_3(bytes, p1, p2, p3);
+ kernel_neon_end();
}
static void
@@ -185,13 +176,9 @@ xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
{
- if (in_interrupt()) {
- xor_arm4regs_4(bytes, p1, p2, p3, p4);
- } else {
- kernel_neon_begin();
- xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
- kernel_neon_end();
- }
+ kernel_neon_begin();
+ xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
+ kernel_neon_end();
}
static void
@@ -201,13 +188,9 @@ xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5)
{
- if (in_interrupt()) {
- xor_arm4regs_5(bytes, p1, p2, p3, p4, p5);
- } else {
- kernel_neon_begin();
- xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
- kernel_neon_end();
- }
+ kernel_neon_begin();
+ xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
+ kernel_neon_end();
}
static struct xor_block_template xor_block_neon = {
--
2.47.3
^ permalink raw reply related
* [PATCH 01/26] xor: assert that xor_blocks is not from preemptible user context
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Most of the optimized xor_blocks versions require FPU/vector registers,
which generally are not supported in interrupt context.
Both callers already are in user context, so enforce this at the highest
level.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
crypto/xor.c | 2 ++
1 file changed, 2 insertions(+)
diff --git a/crypto/xor.c b/crypto/xor.c
index f39621a57bb3..676d7401af43 100644
--- a/crypto/xor.c
+++ b/crypto/xor.c
@@ -28,6 +28,8 @@ xor_blocks(unsigned int src_count, unsigned int bytes, void *dest, void **srcs)
{
unsigned long *p1, *p2, *p3, *p4;
+ lockdep_assert_preemption_enabled();
+
p1 = (unsigned long *) srcs[0];
if (src_count == 1) {
active_template->do_2(bytes, dest, p1);
--
2.47.3
^ permalink raw reply related
* cleanup the RAID5 XOR library v3
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
Hi all,
the XOR library used for the RAID5 parity is a bit of a mess right now.
The main file sits in crypto/ despite not being cryptography and not
using the crypto API, with the generic implementations sitting in
include/asm-generic and the arch implementations sitting in an asm/
header in theory. The latter doesn't work for many cases, so
architectures often build the code directly into the core kernel, or
create another module for the architecture code.
Changes this to a single module in lib/ that also contains the
architecture optimizations, similar to the library work Eric Biggers
has done for the CRC and crypto libraries later. After that it changes
to better calling conventions that allow for smarter architecture
implementations (although none is contained here yet), and uses
static_call to avoid indirection function call overhead.
A git tree is also available here:
git://git.infradead.org/users/hch/misc.git xor-improvements
Gitweb:
https://git.infradead.org/?p=users/hch/misc.git;a=shortlog;h=refs/heads/xor-improvements
Changes since v2:
- drop use of CONFIG_KERNEL_MODE_NEON for arm64
- drop the new __limit_random_u32_below for the unit test
- require 64-bit alignment because sparc64 requires it
- use DEFINE_STATIC_CALL_NULL to avoid exposing a specific xor_gen
routine
- keep CONFIG_XOR_BLOCKS_ARCH self-contained in lib/raid/
- don't select library option from kunit test and add a .kunitconfig
instead
- fix the module description for the kunit test
- add a case where buffers are at the end of the allocation in the kunit test
- use separate src/dst alignment in the kunit test
- fix and improve the kunit assert message
Changes since v1:
- use lockdep_assert_preemption_enabled()
- improve the commit message for the initial um xor.h cleanup
- further clean up the um arch specific header
- add SPDX identifier to the new build system files
- use bool for xor_forced
- fix an incorrect printk level conversion from warn to info
- include xor_impl.h in xor-neon.c
- remove unused exports for riscv
- simply move the sparc code instead of splititng it
- simplify the makefile for the x86-specific implementations
- remove stray references to xor_blocks in crypto/async_tx
- rework __DO_XOR_BLOCKS to avoid (theoretical) out of bounds references
- improve the kerneldoc API documentration for xor_gen()
- spell the name of the srcs argument to xor_gen correctly in xor.h
- add a kunit test, and a new random helper for it.
Diffstat:
arch/arm64/include/asm/xor.h | 73 --
arch/loongarch/include/asm/xor.h | 68 --
arch/loongarch/include/asm/xor_simd.h | 34 -
arch/loongarch/lib/xor_simd_glue.c | 72 --
arch/powerpc/include/asm/xor.h | 47 -
arch/powerpc/include/asm/xor_altivec.h | 22
arch/powerpc/lib/xor_vmx.h | 22
arch/powerpc/lib/xor_vmx_glue.c | 63 --
arch/riscv/include/asm/xor.h | 68 --
arch/s390/include/asm/xor.h | 21
arch/sparc/include/asm/xor.h | 9
arch/um/include/asm/xor.h | 24
arch/x86/include/asm/xor_64.h | 28 -
b/arch/arm/lib/Makefile | 5
b/arch/arm64/lib/Makefile | 6
b/arch/loongarch/lib/Makefile | 2
b/arch/powerpc/lib/Makefile | 5
b/arch/riscv/lib/Makefile | 1
b/arch/s390/lib/Makefile | 2
b/arch/sparc/include/asm/asm-prototypes.h | 1
b/arch/sparc/lib/Makefile | 2
b/crypto/Kconfig | 2
b/crypto/Makefile | 1
b/crypto/async_tx/async_xor.c | 34 -
b/fs/btrfs/raid56.c | 27 -
b/include/asm-generic/Kbuild | 1
b/include/linux/raid/xor.h | 27 -
b/lib/Kconfig | 1
b/lib/Makefile | 2
b/lib/raid/.kunitconfig | 3
b/lib/raid/Kconfig | 30 +
b/lib/raid/Makefile | 3
b/lib/raid/xor/Makefile | 42 +
b/lib/raid/xor/alpha/xor.c | 46 -
b/lib/raid/xor/alpha/xor_arch.h | 22
b/lib/raid/xor/arm/xor-neon-glue.c | 19
b/lib/raid/xor/arm/xor-neon.c | 22
b/lib/raid/xor/arm/xor.c | 105 ----
b/lib/raid/xor/arm/xor_arch.h | 22
b/lib/raid/xor/arm64/xor-neon-glue.c | 26 +
b/lib/raid/xor/arm64/xor-neon.c | 94 +--
b/lib/raid/xor/arm64/xor-neon.h | 6
b/lib/raid/xor/arm64/xor_arch.h | 21
b/lib/raid/xor/loongarch/xor_arch.h | 33 +
b/lib/raid/xor/loongarch/xor_simd_glue.c | 37 +
b/lib/raid/xor/powerpc/xor_arch.h | 22
b/lib/raid/xor/powerpc/xor_vmx.c | 40 -
b/lib/raid/xor/powerpc/xor_vmx.h | 10
b/lib/raid/xor/powerpc/xor_vmx_glue.c | 28 +
b/lib/raid/xor/riscv/xor-glue.c | 25 +
b/lib/raid/xor/riscv/xor.S | 4
b/lib/raid/xor/riscv/xor_arch.h | 17
b/lib/raid/xor/s390/xor.c | 15
b/lib/raid/xor/s390/xor_arch.h | 13
b/lib/raid/xor/sparc/xor-sparc32.c | 32 -
b/lib/raid/xor/sparc/xor-sparc64-glue.c | 48 -
b/lib/raid/xor/sparc/xor-sparc64.S | 10
b/lib/raid/xor/sparc/xor_arch.h | 35 +
b/lib/raid/xor/tests/Makefile | 3
b/lib/raid/xor/tests/xor_kunit.c | 187 +++++++
b/lib/raid/xor/um/xor_arch.h | 2
b/lib/raid/xor/x86/xor-avx.c | 52 --
b/lib/raid/xor/x86/xor-mmx.c | 120 +---
b/lib/raid/xor/x86/xor-sse.c | 105 +---
b/lib/raid/xor/x86/xor_arch.h | 36 +
b/lib/raid/xor/xor-32regs-prefetch.c | 267 ++++++++++
b/lib/raid/xor/xor-32regs.c | 217 ++++++++
b/lib/raid/xor/xor-8regs-prefetch.c | 146 +++++
b/lib/raid/xor/xor-8regs.c | 103 ++++
b/lib/raid/xor/xor-core.c | 192 +++++++
b/lib/raid/xor/xor_impl.h | 56 ++
crypto/xor.c | 174 -------
include/asm-generic/xor.h | 738 ------------------------------
73 files changed, 1831 insertions(+), 2067 deletions(-)
^ permalink raw reply
* [PATCH 2/2] nvme-multipath: enable PCI P2PDMA for multipath devices
From: Chaitanya Kulkarni @ 2026-03-23 23:44 UTC (permalink / raw)
To: song, yukuai, linan122, kbusch, axboe, hch, sagi
Cc: linux-raid, linux-nvme, Chaitanya Kulkarni, Kiran Kumar Modukuri
In-Reply-To: <20260323234416.46944-1-kch@nvidia.com>
NVMe multipath does not expose BLK_FEAT_PCI_P2PDMA on the head disk
even when the underlying controller supports it.
Set BLK_FEAT_PCI_P2PDMA in nvme_mpath_alloc_disk() when the controller
advertises P2PDMA support via ctrl->ops->supports_pci_p2pdma.
Signed-off-by: Chaitanya Kulkarni <kch@nvidia.com>
Signed-off-by: Kiran Kumar Modukuri <kmodukuri@nvidia.com>
---
drivers/nvme/host/multipath.c | 3 +++
1 file changed, 3 insertions(+)
diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index ba00f0b72b85..c49fca43ef19 100644
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -737,6 +737,9 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head)
BLK_FEAT_POLL | BLK_FEAT_ATOMIC_WRITES;
if (head->ids.csi == NVME_CSI_ZNS)
lim.features |= BLK_FEAT_ZONED;
+ if (ctrl->ops && ctrl->ops->supports_pci_p2pdma &&
+ ctrl->ops->supports_pci_p2pdma(ctrl))
+ lim.features |= BLK_FEAT_PCI_P2PDMA;
head->disk = blk_alloc_disk(&lim, ctrl->numa_node);
if (IS_ERR(head->disk))
--
2.39.5
^ permalink raw reply related
* [PATCH 1/2] md: Add PCI_P2PDMA support for MD RAID volumes
From: Chaitanya Kulkarni @ 2026-03-23 23:44 UTC (permalink / raw)
To: song, yukuai, linan122, kbusch, axboe, hch, sagi
Cc: linux-raid, linux-nvme, Chaitanya Kulkarni, Kiran Kumar Modukuri
In-Reply-To: <20260323234416.46944-1-kch@nvidia.com>
MD RAID does not propagate BLK_FEAT_PCI_P2PDMA from member devices to
the RAID device, preventing peer-to-peer DMA through the RAID layer even
when all underlying devices support it.
Enable BLK_FEAT_PCI_P2PDMA by default in md_init_stacking_limits() and
clear it in mddev_stack_rdev_limits() during array init and
mddev_stack_new_rdev() during hot-add if any member device lacks support.
Tested with RAID arrays containing multiple NVMe devices with P2PDMA
support, confirming that peer-to-peer transfers work correctly through
the RAID layer.
Signed-off-by: Chaitanya Kulkarni <kch@nvidia.com>
Signed-off-by: Kiran Kumar Modukuri <kmodukuri@nvidia.com>
---
drivers/md/md.c | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 521d9b34cd9e..a151ea86d844 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -6176,6 +6176,8 @@ int mddev_stack_rdev_limits(struct mddev *mddev, struct queue_limits *lim,
if ((flags & MDDEV_STACK_INTEGRITY) &&
!queue_limits_stack_integrity_bdev(lim, rdev->bdev))
return -EINVAL;
+ if (!blk_queue_pci_p2pdma(rdev->bdev->bd_disk->queue))
+ lim->features &= ~BLK_FEAT_PCI_P2PDMA;
}
/*
@@ -6231,6 +6233,8 @@ int mddev_stack_new_rdev(struct mddev *mddev, struct md_rdev *rdev)
lim = queue_limits_start_update(mddev->gendisk->queue);
queue_limits_stack_bdev(&lim, rdev->bdev, rdev->data_offset,
mddev->gendisk->disk_name);
+ if (!blk_queue_pci_p2pdma(rdev->bdev->bd_disk->queue))
+ lim.features &= ~BLK_FEAT_PCI_P2PDMA;
if (!queue_limits_stack_integrity_bdev(&lim, rdev->bdev)) {
pr_err("%s: incompatible integrity profile for %pg\n",
@@ -6272,7 +6276,8 @@ void md_init_stacking_limits(struct queue_limits *lim)
{
blk_set_stacking_limits(lim);
lim->features = BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA |
- BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT;
+ BLK_FEAT_IO_STAT | BLK_FEAT_NOWAIT |
+ BLK_FEAT_PCI_P2PDMA;
}
EXPORT_SYMBOL_GPL(md_init_stacking_limits);
--
2.39.5
^ permalink raw reply related
* [PATCH 0/2] Enable PCI P2PDMA support for RAID0 and NVMe Multipath
From: Chaitanya Kulkarni @ 2026-03-23 23:44 UTC (permalink / raw)
To: song, yukuai, linan122, kbusch, axboe, hch, sagi
Cc: linux-raid, linux-nvme, Chaitanya Kulkarni
Hi,
This patch series extends PCI peer-to-peer DMA (P2PDMA) support to enable
direct data transfers between PCIe devices through RAID and NVMe multipath
block layers.
Background
==========
Current Linux kernel P2PDMA infrastructure supports direct peer-to-peer
transfers, but this support is not propagated through certain storage
stacks like MD RAID and NVMe multipath.
Patch Overview
==============
Patch 1/2: MD RAID0 PCI_P2PDMA support
---------------------------------------
Enables PCI_P2PDMA for MD RAID volumes by propagating the BLK_FEAT_PCI_P2PDMA
feature to the RAID device when all underlying member devices support P2PDMA.
This follows the same pattern as the NOWAIT flag handling in the MD layer.
Without this patch, even if all underlying NVMe devices in a RAID0 array
support P2PDMA, the RAID device itself does not advertise this capability,
preventing direct device-to-device transfers through the RAID layer.
Patch 2/2: NVMe Multipath PCI_P2PDMA support
--------------------------------------------
Adds PCI_P2PDMA support for NVMe multipath devices by setting
BLK_FEAT_PCI_P2PDMA in the queue limits during nvme_mpath_alloc_disk()
when the controller supports P2PDMA operations.
NVMe multipath provides high availability by creating a single block device
(/dev/nvmeXn1) that aggregates multiple paths to the same namespace. This
patch ensures P2PDMA capability is exposed through the multipath device,
enabling peer-to-peer DMA operations on multipath configurations.
Summary
=======
All four test scenarios demonstrate that P2PDMA capabilities are correctly
propagated through both the MD RAID0 layer (patch 1/2) and NVMe multipath
layer (patch 2/2). Direct peer-to-peer transfers complete successfully with
full data integrity verification, confirming that:
1. RAID0 devices properly inherit P2PDMA capability from member devices
2. NVMe multipath devices correctly expose P2PDMA support
3. P2P memory buffers can be used for transfers involving both types
4. Data integrity is maintained across all transfer combinations
I've added the blktest log as well at the end.
Repo:-
git://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
Branch HEAD:-
commit e7cbe110ab3c38b07e5bed91808b7f6a2c328ad6 (origin/for-next)
Merge: 4107f06b64ed 67807fbaf127
Author: Jens Axboe <axboe@kernel.dk>
Date: Mon Mar 23 07:58:45 2026 -0600
Merge branch 'for-7.1/block' into for-next
* for-7.1/block:
block: fix bio_alloc_bioset slowpath GFP handling
-ck
Chaitanya Kulkarni (2):
md: Add PCI_P2PDMA support for MD RAID volumes
nvme-multipath: enable PCI P2PDMA for multipath devices
drivers/md/md.c | 7 ++++++-
drivers/nvme/host/multipath.c | 3 +++
2 files changed, 9 insertions(+), 1 deletion(-)
* P2P Tests :-
lab@vm70:~/p2pmem-test$ lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS
vda 253:0 0 50G 0 disk
├─vda1 253:1 0 49G 0 part /
├─vda14 253:14 0 4M 0 part
├─vda15 253:15 0 106M 0 part
└─vda16 259:0 0 913M 0 part
nvme1n1 259:2 0 10G 0 disk
nvme2n1 259:3 0 10G 0 disk
└─md127 9:127 0 20G 0 raid0
nvme4n1 259:4 0 10G 0 disk
nvme3n1 259:5 0 10G 0 disk
└─md127 9:127 0 20G 0 raid0
lab@vm70:~/p2pmem-test$ sudo ./p2pmem-test /dev/nvme2n1 /dev/nvme4n1
/sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate -c 10 -s 1M --check
Running p2pmem-test: reading /dev/nvme2n1 (10.74GB): writing /dev/nvme4n1 (10.74GB): p2pmem buffer /sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate.
chunk size = 1048576 : number of chunks = 10: total = 10.49MB : thread(s) = 1 : overlap = OFF.
skip-read = OFF : skip-write = OFF : duration = INF sec.
buffer = 0x7f6ad2125000 (p2pmem): mmap = 1.049MB
PAGE_SIZE = 4096B
checking data with seed = 1773335040
MATCH on data check, 0x3c4f165f = 0x3c4f165f.
Transfer:
10.49MB in 191.4 ms 54.80MB/s
lab@vm70:~/p2pmem-test$ sudo ./p2pmem-test /dev/nvme3n1 /dev/nvme4n1
/sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate -c 10 -s 1M --check
Running p2pmem-test: reading /dev/nvme3n1 (10.74GB): writing /dev/nvme4n1 (10.74GB): p2pmem buffer /sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate.
chunk size = 1048576 : number of chunks = 10: total = 10.49MB : thread(s) = 1 : overlap = OFF.
skip-read = OFF : skip-write = OFF : duration = INF sec.
buffer = 0x7f9a2f478000 (p2pmem): mmap = 1.049MB
PAGE_SIZE = 4096B
checking data with seed = 1773335062
MATCH on data check, 0x6db73ca7 = 0x6db73ca7.
Transfer:
10.49MB in 489.3 ms 21.43MB/s
lab@vm70:~/p2pmem-test$ sudo ./p2pmem-test /dev/nvme1n1 /dev/nvme3n1
/sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate -c 10 -s 1M --check
Running p2pmem-test: reading /dev/nvme1n1 (10.74GB): writing /dev/nvme3n1 (10.74GB): p2pmem buffer /sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate.
chunk size = 1048576 : number of chunks = 10: total = 10.49MB : thread(s) = 1 : overlap = OFF.
skip-read = OFF : skip-write = OFF : duration = INF sec.
buffer = 0x7fdc0cd93000 (p2pmem): mmap = 1.049MB
PAGE_SIZE = 4096B
checking data with seed = 1773334971
* pread: Remote I/O error *
lab@vm70:~/p2pmem-test$ sudo ./p2pmem-test /dev/md127 /dev/nvme4n1
/sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate -c 10 -s 1M --check
Running p2pmem-test: reading /dev/md127 (21.46GB): writing /dev/nvme4n1 (10.74GB): p2pmem buffer /sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate.
chunk size = 1048576 : number of chunks = 10: total = 10.49MB : thread(s) = 1 : overlap = OFF.
skip-read = OFF : skip-write = OFF : duration = INF sec.
buffer = 0x7fdddb745000 (p2pmem): mmap = 1.049MB
PAGE_SIZE = 4096B
checking data with seed = 1773334985
* pread: Remote I/O error *
# 7.0 with with PCI_P2PDMA patches
lab@vm70:~$ uname -r
7.0.0-rc2-p2pdma
lab@vm70:~$ lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS
vda 253:0 0 50G 0 disk
├─vda1 253:1 0 49G 0 part /
├─vda14 253:14 0 4M 0 part
├─vda15 253:15 0 106M 0 part
└─vda16 259:0 0 913M 0 part
nvme0n1 259:2 0 10G 0 disk
nvme2n1 259:3 0 10G 0 disk
└─md0 9:0 0 20G 0 raid0
nvme3n1 259:4 0 10G 0 disk
└─md0 9:0 0 20G 0 raid0
nvme4n1 259:5 0 10G 0 disk
lab@vm70:~/p2pmem-test$ sudo ./p2pmem-test /dev/nvme0n1 /dev/nvme3n1
/sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate -c 10 -s 1M --check
Running p2pmem-test: reading /dev/nvme0n1 (10.74GB): writing /dev/nvme3n1 (10.74GB): p2pmem buffer /sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate.
chunk size = 1048576 : number of chunks = 10: total = 10.49MB : thread(s) = 1 : overlap = OFF.
skip-read = OFF : skip-write = OFF : duration = INF sec.
buffer = 0x7f2ec7006000 (p2pmem): mmap = 1.049MB
PAGE_SIZE = 4096B
checking data with seed = 1773337740
MATCH on data check, 0x1a4324f = 0x1a4324f.
Transfer:
10.49MB in 318.0 ms 32.98MB/s
lab@vm70:~/p2pmem-test$ sudo ./p2pmem-test /dev/nvme2n1 /dev/nvme3n1
/sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate -c 10 -s 1M --check
Running p2pmem-test: reading /dev/nvme2n1 (10.74GB): writing /dev/nvme3n1 (10.74GB): p2pmem buffer /sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate.
chunk size = 1048576 : number of chunks = 10: total = 10.49MB : thread(s) = 1 : overlap = OFF.
skip-read = OFF : skip-write = OFF : duration = INF sec.
buffer = 0x7f194de7e000 (p2pmem): mmap = 1.049MB
PAGE_SIZE = 4096B
checking data with seed = 1773337752
MATCH on data check, 0x57f470e7 = 0x57f470e7.
Transfer:
10.49MB in 179.1 ms 58.55MB/s
lab@vm70:~/p2pmem-test$ sudo ./p2pmem-test /dev/md0 /dev/nvme3n1
/sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate -c 10 -s 1M --check
Running p2pmem-test: reading /dev/md0 (21.46GB): writing /dev/nvme3n1 (10.74GB): p2pmem buffer /sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate.
chunk size = 1048576 : number of chunks = 10: total = 10.49MB : thread(s) = 1 : overlap = OFF.
skip-read = OFF : skip-write = OFF : duration = INF sec.
buffer = 0x7f9972431000 (p2pmem): mmap = 1.049MB
PAGE_SIZE = 4096B
checking data with seed = 1773337761
MATCH on data check, 0x286fdcdb = 0x286fdcdb.
Transfer:
10.49MB in 243.9 ms 42.99MB/s
lab@vm70:~/p2pmem-test$ sudo ./p2pmem-test /dev/md0 /dev/nvme4n1
/sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate -c 10 -s 1M --check
Running p2pmem-test: reading /dev/md0 (21.46GB): writing /dev/nvme4n1 (10.74GB): p2pmem buffer /sys/bus/pci/devices/0000:0c:00.0/p2pmem/allocate.
chunk size = 1048576 : number of chunks = 10: total = 10.49MB : thread(s) = 1 : overlap = OFF.
skip-read = OFF : skip-write = OFF : duration = INF sec.
buffer = 0x7f73a098e000 (p2pmem): mmap = 1.049MB
PAGE_SIZE = 4096B
checking data with seed = 1773337770
MATCH on data check, 0x5518d20f = 0x5518d20f.
Transfer:
10.49MB in 251.7 ms 41.66MB/s
* blktest with these patches :-
29ec80128cdc (HEAD -> for-next) nvme-multipath: enable PCI P2PDMA for multipath devices
b4086566d2f7 md: Add PCI_P2PDMA support for MD RAID volumes
p2pdma-nvme-mpath-md (for-next) #
blktests (master) # ./test-nvme.sh
++ for t in loop tcp
++ echo '################NVMET_TRTYPES=loop############'
################NVMET_TRTYPES=loop############
++ NVME_IMG_SIZE=1G
++ NVME_NUM_ITER=1
++ NVMET_TRTYPES=loop
++ ./check nvme
nvme/002 (tr=loop) (create many subsystems and test discovery) [passed]
runtime 35.359s ... 34.600s
nvme/003 (tr=loop) (test if we're sending keep-alives to a discovery controller) [passed]
runtime 10.232s ... 10.214s
nvme/004 (tr=loop) (test nvme and nvmet UUID NS descriptors) [passed]
runtime 0.690s ... 0.643s
nvme/005 (tr=loop) (reset local loopback target) [passed]
runtime 1.041s ... 0.995s
nvme/006 (tr=loop bd=device) (create an NVMeOF target) [passed]
runtime 0.093s ... 0.089s
nvme/006 (tr=loop bd=file) (create an NVMeOF target) [passed]
runtime 0.085s ... 0.083s
nvme/008 (tr=loop bd=device) (create an NVMeOF host) [passed]
runtime 0.777s ... 0.648s
nvme/008 (tr=loop bd=file) (create an NVMeOF host) [passed]
runtime 0.728s ... 0.644s
nvme/010 (tr=loop bd=device) (run data verification fio job) [passed]
runtime 9.037s ... 8.917s
nvme/010 (tr=loop bd=file) (run data verification fio job) [passed]
runtime 46.257s ... 43.209s
nvme/012 (tr=loop bd=device) (run mkfs and data verification fio) [passed]
runtime 47.039s ... 46.860s
nvme/012 (tr=loop bd=file) (run mkfs and data verification fio) [passed]
runtime 40.123s ... 40.060s
nvme/014 (tr=loop bd=device) (flush a command from host) [passed]
runtime 8.451s ... 8.331s
nvme/014 (tr=loop bd=file) (flush a command from host) [passed]
runtime 7.805s ... 7.822s
nvme/016 (tr=loop) (create/delete many NVMeOF block device-backed ns and test discovery) [passed]
runtime 0.132s ... 0.139s
nvme/017 (tr=loop) (create/delete many file-ns and test discovery) [passed]
runtime 0.146s ... 0.146s
nvme/018 (tr=loop) (unit test NVMe-oF out of range access on a file backend) [passed]
runtime 0.657s ... 0.631s
nvme/019 (tr=loop bd=device) (test NVMe DSM Discard command) [passed]
runtime 0.642s ... 0.654s
nvme/019 (tr=loop bd=file) (test NVMe DSM Discard command) [passed]
runtime 0.648s ... 0.626s
nvme/021 (tr=loop bd=device) (test NVMe list command) [passed]
runtime 0.679s ... 0.631s
nvme/021 (tr=loop bd=file) (test NVMe list command) [passed]
runtime 0.635s ... 0.646s
nvme/022 (tr=loop bd=device) (test NVMe reset command) [passed]
runtime 1.007s ... 0.993s
nvme/022 (tr=loop bd=file) (test NVMe reset command) [passed]
runtime 0.999s ... 1.005s
nvme/023 (tr=loop bd=device) (test NVMe smart-log command) [passed]
runtime 0.650s ... 0.635s
nvme/023 (tr=loop bd=file) (test NVMe smart-log command) [passed]
runtime 0.662s ... 0.646s
nvme/025 (tr=loop bd=device) (test NVMe effects-log) [passed]
runtime 0.665s ... 0.646s
nvme/025 (tr=loop bd=file) (test NVMe effects-log) [passed]
runtime 0.671s ... 0.639s
nvme/026 (tr=loop bd=device) (test NVMe ns-descs) [passed]
runtime 0.716s ... 0.631s
nvme/026 (tr=loop bd=file) (test NVMe ns-descs) [passed]
runtime 0.617s ... 0.613s
nvme/027 (tr=loop bd=device) (test NVMe ns-rescan command) [passed]
runtime 0.656s ... 0.646s
nvme/027 (tr=loop bd=file) (test NVMe ns-rescan command) [passed]
runtime 0.644s ... 0.659s
nvme/028 (tr=loop bd=device) (test NVMe list-subsys) [passed]
runtime 0.636s ... 0.623s
nvme/028 (tr=loop bd=file) (test NVMe list-subsys) [passed]
runtime 0.635s ... 0.621s
nvme/029 (tr=loop) (test userspace IO via nvme-cli read/write interface) [passed]
runtime 0.954s ... 0.946s
nvme/030 (tr=loop) (ensure the discovery generation counter is updated appropriately) [passed]
runtime 0.428s ... 0.421s
nvme/031 (tr=loop) (test deletion of NVMeOF controllers immediately after setup) [passed]
runtime 5.984s ... 5.933s
nvme/038 (tr=loop) (test deletion of NVMeOF subsystem without enabling) [passed]
runtime 0.034s ... 0.033s
nvme/040 (tr=loop) (test nvme fabrics controller reset/disconnect operation during I/O) [passed]
runtime 7.068s ... 7.012s
nvme/041 (tr=loop) (Create authenticated connections) [passed]
runtime 0.705s ... 0.949s
nvme/042 (tr=loop) (Test dhchap key types for authenticated connections) [passed]
runtime 4.007s ... 4.295s
nvme/043 (tr=loop) (Test hash and DH group variations for authenticated connections) [passed]
runtime 5.191s ... 6.744s
nvme/044 (tr=loop) (Test bi-directional authentication) [passed]
runtime 1.307s ... 2.293s
nvme/045 (tr=loop) (Test re-authentication) [passed]
runtime 1.639s ... 1.624s
nvme/047 (tr=loop) (test different queue types for fabric transports) [not run]
nvme_trtype=loop is not supported in this test
nvme/048 (tr=loop) (Test queue count changes on reconnect) [not run]
nvme_trtype=loop is not supported in this test
nvme/051 (tr=loop) (test nvmet concurrent ns enable/disable) [passed]
runtime 1.395s ... 2.673s
nvme/052 (tr=loop) (Test file-ns creation/deletion under one subsystem) [passed]
runtime 6.288s ... 6.832s
nvme/054 (tr=loop) (Test the NVMe reservation feature) [passed]
runtime 0.806s ... 1.287s
nvme/055 (tr=loop) (Test nvme write to a loop target ns just after ns is disabled) [not run]
kernel option DEBUG_ATOMIC_SLEEP has not been enabled
nvme/056 (tr=loop) (enable zero copy offload and run rw traffic) [not run]
Remote target required but NVME_TARGET_CONTROL is not set
nvme_trtype=loop is not supported in this test
kernel option ULP_DDP has not been enabled
module nvme_tcp does not have parameter ddp_offload
KERNELSRC not set
Kernel sources do not have tools/net/ynl/cli.py
NVME_IFACE not set
nvme/057 (tr=loop) (test nvme fabrics controller ANA failover during I/O) [passed]
runtime 27.321s ... 27.487s
nvme/058 (tr=loop) (test rapid namespace remapping) [passed]
runtime 4.709s ... 4.932s
nvme/060 (tr=loop) (test nvme fabrics target reset) [not run]
nvme_trtype=loop is not supported in this test
nvme/061 (tr=loop) (test fabric target teardown and setup during I/O) [not run]
nvme_trtype=loop is not supported in this test
nvme/062 (tr=loop) (Create TLS-encrypted connections) [not run]
nvme_trtype=loop is not supported in this test
nvme/063 (tr=loop) (Create authenticated TCP connections with secure concatenation) [not run]
nvme_trtype=loop is not supported in this test
nvme/065 (test unmap write zeroes sysfs interface with nvmet devices) [passed]
runtime 2.349s ... 2.495s
++ for t in loop tcp
++ echo '################NVMET_TRTYPES=tcp############'
################NVMET_TRTYPES=tcp############
++ NVME_IMG_SIZE=1G
++ NVME_NUM_ITER=1
++ NVMET_TRTYPES=tcp
++ ./check nvme
nvme/002 (tr=tcp) (create many subsystems and test discovery) [not run]
nvme_trtype=tcp is not supported in this test
nvme/003 (tr=tcp) (test if we're sending keep-alives to a discovery controller) [passed]
runtime 10.244s ... 10.249s
nvme/004 (tr=tcp) (test nvme and nvmet UUID NS descriptors) [passed]
runtime 0.379s ... 0.380s
nvme/005 (tr=tcp) (reset local loopback target) [passed]
runtime 0.451s ... 0.437s
nvme/006 (tr=tcp bd=device) (create an NVMeOF target) [passed]
runtime 0.097s ... 0.101s
nvme/006 (tr=tcp bd=file) (create an NVMeOF target) [passed]
runtime 0.093s ... 0.097s
nvme/008 (tr=tcp bd=device) (create an NVMeOF host) [passed]
runtime 0.389s ... 0.369s
nvme/008 (tr=tcp bd=file) (create an NVMeOF host) [passed]
runtime 0.377s ... 0.379s
nvme/010 (tr=tcp bd=device) (run data verification fio job) [passed]
runtime 77.447s ... 76.703s
nvme/010 (tr=tcp bd=file) (run data verification fio job) [passed]
runtime 120.190s ... 121.711s
nvme/012 (tr=tcp bd=device) (run mkfs and data verification fio) [passed]
runtime 82.928s ... 82.706s
nvme/012 (tr=tcp bd=file) (run mkfs and data verification fio) [passed]
runtime ... 118.897s
nvme/014 (tr=tcp bd=device) (flush a command from host) [passed]
runtime 7.825s ... 8.880s
nvme/014 (tr=tcp bd=file) (flush a command from host) [passed]
runtime 7.862s ... 8.690s
nvme/016 (tr=tcp) (create/delete many NVMeOF block device-backed ns and test discovery) [not run]
nvme_trtype=tcp is not supported in this test
nvme/017 (tr=tcp) (create/delete many file-ns and test discovery) [not run]
nvme_trtype=tcp is not supported in this test
nvme/018 (tr=tcp) (unit test NVMe-oF out of range access on a file backend) [passed]
runtime 0.355s ... 0.357s
nvme/019 (tr=tcp bd=device) (test NVMe DSM Discard command) [passed]
runtime 0.364s ... 0.369s
nvme/019 (tr=tcp bd=file) (test NVMe DSM Discard command) [passed]
runtime 0.352s ... 0.345s
nvme/021 (tr=tcp bd=device) (test NVMe list command) [passed]
runtime 0.366s ... 0.384s
nvme/021 (tr=tcp bd=file) (test NVMe list command) [passed]
runtime 0.375s ... 0.377s
nvme/022 (tr=tcp bd=device) (test NVMe reset command) [passed]
runtime 0.464s ... 0.465s
nvme/022 (tr=tcp bd=file) (test NVMe reset command) [passed]
runtime 0.452s ... 0.460s
nvme/023 (tr=tcp bd=device) (test NVMe smart-log command) [passed]
runtime 0.377s ... 0.375s
nvme/023 (tr=tcp bd=file) (test NVMe smart-log command) [passed]
runtime 0.352s ... 0.355s
nvme/025 (tr=tcp bd=device) (test NVMe effects-log) [passed]
runtime 0.382s ... 0.376s
nvme/025 (tr=tcp bd=file) (test NVMe effects-log) [passed]
runtime 0.365s ... 0.378s
nvme/026 (tr=tcp bd=device) (test NVMe ns-descs) [passed]
runtime 0.361s ... 0.361s
nvme/026 (tr=tcp bd=file) (test NVMe ns-descs) [passed]
runtime 0.362s ... 0.354s
nvme/027 (tr=tcp bd=device) (test NVMe ns-rescan command) [passed]
runtime 0.385s ... 0.390s
nvme/027 (tr=tcp bd=file) (test NVMe ns-rescan command) [passed]
runtime 0.390s ... 0.398s
nvme/028 (tr=tcp bd=device) (test NVMe list-subsys) [passed]
runtime 0.353s ... 0.368s
nvme/028 (tr=tcp bd=file) (test NVMe list-subsys) [passed]
runtime 0.357s ... 0.346s
nvme/029 (tr=tcp) (test userspace IO via nvme-cli read/write interface) [passed]
runtime 0.695s ... 0.729s
nvme/030 (tr=tcp) (ensure the discovery generation counter is updated appropriately) [passed]
runtime 0.400s ... 0.401s
nvme/031 (tr=tcp) (test deletion of NVMeOF controllers immediately after setup) [passed]
runtime 3.016s ... 3.074s
nvme/038 (tr=tcp) (test deletion of NVMeOF subsystem without enabling) [passed]
runtime 0.042s ... 0.042s
nvme/040 (tr=tcp) (test nvme fabrics controller reset/disconnect operation during I/O) [passed]
runtime 6.452s ... 6.454s
nvme/041 (tr=tcp) (Create authenticated connections) [passed]
runtime 0.417s ... 0.426s
nvme/042 (tr=tcp) (Test dhchap key types for authenticated connections) [passed]
runtime 1.805s ... 1.820s
nvme/043 (tr=tcp) (Test hash and DH group variations for authenticated connections) [passed]
runtime 2.411s ... 2.432s
nvme/044 (tr=tcp) (Test bi-directional authentication) [passed]
runtime 0.739s ... 0.741s
nvme/045 (tr=tcp) (Test re-authentication) [passed]
runtime 1.338s ... 1.326s
nvme/047 (tr=tcp) (test different queue types for fabric transports) [passed]
runtime 1.864s ... 1.856s
nvme/048 (tr=tcp) (Test queue count changes on reconnect) [passed]
runtime 5.535s ... 5.553s
nvme/051 (tr=tcp) (test nvmet concurrent ns enable/disable) [passed]
runtime 1.344s ... 1.432s
nvme/052 (tr=tcp) (Test file-ns creation/deletion under one subsystem) [not run]
nvme_trtype=tcp is not supported in this test
nvme/054 (tr=tcp) (Test the NVMe reservation feature) [passed]
runtime 0.484s ... 0.508s
nvme/055 (tr=tcp) (Test nvme write to a loop target ns just after ns is disabled) [not run]
nvme_trtype=tcp is not supported in this test
kernel option DEBUG_ATOMIC_SLEEP has not been enabled
nvme/056 (tr=tcp) (enable zero copy offload and run rw traffic) [not run]
Remote target required but NVME_TARGET_CONTROL is not set
kernel option ULP_DDP has not been enabled
module nvme_tcp does not have parameter ddp_offload
KERNELSRC not set
Kernel sources do not have tools/net/ynl/cli.py
NVME_IFACE not set
nvme/057 (tr=tcp) (test nvme fabrics controller ANA failover during I/O) [passed]
runtime 25.951s ... 25.962s
nvme/058 (tr=tcp) (test rapid namespace remapping) [passed]
runtime 2.935s ... 3.025s
nvme/060 (tr=tcp) (test nvme fabrics target reset) [passed]
runtime 19.400s ... 19.329s
nvme/061 (tr=tcp) (test fabric target teardown and setup during I/O) [passed]
runtime 8.566s ... 8.596s
nvme/062 (tr=tcp) (Create TLS-encrypted connections) [failed]
runtime 5.221s ... 5.232s
--- tests/nvme/062.out 2026-01-28 12:04:48.888356244 -0800
+++ /mnt/sda/blktests/results/nodev_tr_tcp/nvme/062.out.bad 2026-03-23 16:32:31.377957706 -0700
@@ -2,9 +2,13 @@
Test unencrypted connection w/ tls not required
disconnected 1 controller(s)
Test encrypted connection w/ tls not required
-disconnected 1 controller(s)
+FAIL: nvme connect return error code
+WARNING: connection is not encrypted
+disconnected 0 controller(s)
...
(Run 'diff -u tests/nvme/062.out /mnt/sda/blktests/results/nodev_tr_tcp/nvme/062.out.bad' to see the entire diff)
nvme/063 (tr=tcp) (Create authenticated TCP connections with secure concatenation) [passed]
runtime 1.907s ... 1.969s
nvme/065 (test unmap write zeroes sysfs interface with nvmet devices) [passed]
runtime 2.495s ... 2.314s
++ ./manage-rdma-nvme.sh --cleanup
====== RDMA NVMe Cleanup ======
[INFO] Disconnecting NVMe RDMA controllers...
[INFO] No NVMe RDMA controllers to disconnect
[INFO] Removing RDMA links...
[INFO] No RDMA links to remove
[INFO] Unloading NVMe RDMA modules...
[INFO] NVMe RDMA modules unloaded successfully
[INFO] Unloading soft-RDMA modules...
[INFO] Soft-RDMA modules unloaded successfully
[INFO] Verifying cleanup...
[INFO] Verification passed
[INFO] RDMA cleanup completed successfully
====== RDMA Network Configuration Status ======
Loaded Modules:
None
RDMA Links:
None
Network Interfaces (RDMA-capable):
None
blktests Configuration:
Not configured (run --setup first)
NVMe RDMA Controllers:
None
=================================================
++ ./manage-rdma-nvme.sh --setup
====== RDMA NVMe Setup ======
RDMA Type: siw
Interface: auto-detect
[INFO] Checking prerequisites...
[INFO] Prerequisites check passed
[INFO] Loading RDMA module: siw
[INFO] Module siw loaded successfully
[INFO] Creating RDMA links...
[INFO] Creating RDMA link: ens5_siw
[INFO] Created RDMA link: ens5_siw -> ens5
++ ./manage-rdma-nvme.sh --status
====== RDMA Configuration Status ======
====== RDMA Network Configuration Status ======
Loaded Modules:
siw 217088
RDMA Links:
link ens5_siw/1 state ACTIVE physical_state LINK_UP netdev ens5
Network Interfaces (RDMA-capable):
Interface: ens5
IPv4: 192.168.0.46
IPv6: fe80::5054:98ff:fe76:5440%ens5
blktests Configuration:
Transport Address: 192.168.0.46:4420
Transport Type: rdma
Command: NVMET_TRTYPES=rdma ./check nvme/
NVMe RDMA Controllers:
None
=================================================
++ echo '################NVMET_TRTYPES=rdma############'
################NVMET_TRTYPES=rdma############
++ NVME_IMG_SIZE=1G
++ NVME_NUM_ITER=1
++ nvme_trtype=rdma
++ ./check nvme
nvme/002 (tr=rdma) (create many subsystems and test discovery) [not run]
nvme_trtype=rdma is not supported in this test
nvme/003 (tr=rdma) (test if we're sending keep-alives to a discovery controller) [passed]
runtime 10.327s ... 10.331s
nvme/004 (tr=rdma) (test nvme and nvmet UUID NS descriptors) [passed]
runtime 0.689s ... 0.689s
nvme/005 (tr=rdma) (reset local loopback target) [passed]
runtime 0.979s ... 1.006s
nvme/006 (tr=rdma bd=device) (create an NVMeOF target) [passed]
runtime 0.147s ... 0.143s
nvme/006 (tr=rdma bd=file) (create an NVMeOF target) [passed]
runtime 0.136s ... 0.137s
nvme/008 (tr=rdma bd=device) (create an NVMeOF host) [passed]
runtime 0.691s ... 0.702s
nvme/008 (tr=rdma bd=file) (create an NVMeOF host) [passed]
runtime 0.671s ... 0.698s
nvme/010 (tr=rdma bd=device) (run data verification fio job) [passed]
runtime 29.999s ... 34.266s
nvme/010 (tr=rdma bd=file) (run data verification fio job) [passed]
runtime 59.734s ... 62.762s
nvme/012 (tr=rdma bd=device) (run mkfs and data verification fio) [passed]
runtime 34.220s ... 41.940s
nvme/012 (tr=rdma bd=file) (run mkfs and data verification fio) [passed]
runtime 58.929s ... 71.674s
nvme/014 (tr=rdma bd=device) (flush a command from host) [passed]
runtime 7.554s ... 8.508s
nvme/014 (tr=rdma bd=file) (flush a command from host) [passed]
runtime 7.404s ... 8.398s
nvme/016 (tr=rdma) (create/delete many NVMeOF block device-backed ns and test discovery) [not run]
nvme_trtype=rdma is not supported in this test
nvme/017 (tr=rdma) (create/delete many file-ns and test discovery) [not run]
nvme_trtype=rdma is not supported in this test
nvme/018 (tr=rdma) (unit test NVMe-oF out of range access on a file backend) [passed]
runtime 0.671s ... 0.671s
nvme/019 (tr=rdma bd=device) (test NVMe DSM Discard command) [passed]
runtime 0.679s ... 0.687s
nvme/019 (tr=rdma bd=file) (test NVMe DSM Discard command) [passed]
runtime 0.672s ... 0.671s
nvme/021 (tr=rdma bd=device) (test NVMe list command) [passed]
runtime 0.647s ... 0.686s
nvme/021 (tr=rdma bd=file) (test NVMe list command) [passed]
runtime 0.662s ... 0.704s
nvme/022 (tr=rdma bd=device) (test NVMe reset command) [passed]
runtime 1.009s ... 1.029s
nvme/022 (tr=rdma bd=file) (test NVMe reset command) [passed]
runtime 0.980s ... 1.028s
nvme/023 (tr=rdma bd=device) (test NVMe smart-log command) [passed]
runtime 0.658s ... 0.671s
nvme/023 (tr=rdma bd=file) (test NVMe smart-log command) [passed]
runtime 0.649s ... 0.665s
nvme/025 (tr=rdma bd=device) (test NVMe effects-log) [passed]
runtime 0.683s ... 0.698s
nvme/025 (tr=rdma bd=file) (test NVMe effects-log) [passed]
runtime 0.658s ... 0.694s
nvme/026 (tr=rdma bd=device) (test NVMe ns-descs) [passed]
runtime 0.645s ... 0.680s
nvme/026 (tr=rdma bd=file) (test NVMe ns-descs) [passed]
runtime 0.662s ... 0.679s
nvme/027 (tr=rdma bd=device) (test NVMe ns-rescan command) [passed]
runtime 0.702s ... 0.708s
nvme/027 (tr=rdma bd=file) (test NVMe ns-rescan command) [passed]
runtime 0.699s ... 0.713s
nvme/028 (tr=rdma bd=device) (test NVMe list-subsys) [passed]
runtime 0.639s ... 0.670s
nvme/028 (tr=rdma bd=file) (test NVMe list-subsys) [passed]
runtime 0.651s ... 0.658s
nvme/029 (tr=rdma) (test userspace IO via nvme-cli read/write interface) [passed]
runtime 1.048s ... 1.045s
nvme/030 (tr=rdma) (ensure the discovery generation counter is updated appropriately) [passed]
runtime 0.508s ... 0.552s
nvme/031 (tr=rdma) (test deletion of NVMeOF controllers immediately after setup) [passed]
runtime 5.746s ... 5.820s
nvme/038 (tr=rdma) (test deletion of NVMeOF subsystem without enabling) [passed]
runtime 0.086s ... 0.086s
nvme/040 (tr=rdma) (test nvme fabrics controller reset/disconnect operation during I/O) [passed]
runtime 7.008s ... 7.010s
nvme/041 (tr=rdma) (Create authenticated connections) [passed]
runtime 0.732s ... 0.783s
nvme/042 (tr=rdma) (Test dhchap key types for authenticated connections) [passed]
runtime 3.669s ... 3.780s
nvme/043 (tr=rdma) (Test hash and DH group variations for authenticated connections) [passed]
runtime 4.683s ... 4.689s
nvme/044 (tr=rdma) (Test bi-directional authentication) [passed]
runtime 1.342s ... 1.306s
nvme/045 (tr=rdma) (Test re-authentication) [passed]
runtime 1.809s ... 1.825s
nvme/047 (tr=rdma) (test different queue types for fabric transports) [passed]
runtime 2.675s ... 2.638s
nvme/048 (tr=rdma) (Test queue count changes on reconnect) [passed]
runtime 6.818s ... 5.799s
nvme/051 (tr=rdma) (test nvmet concurrent ns enable/disable) [passed]
runtime 1.389s ... 1.465s
nvme/052 (tr=rdma) (Test file-ns creation/deletion under one subsystem) [not run]
nvme_trtype=rdma is not supported in this test
nvme/054 (tr=rdma) (Test the NVMe reservation feature) [passed]
runtime 0.802s ... 0.804s
nvme/055 (tr=rdma) (Test nvme write to a loop target ns just after ns is disabled) [not run]
nvme_trtype=rdma is not supported in this test
kernel option DEBUG_ATOMIC_SLEEP has not been enabled
nvme/056 (tr=rdma) (enable zero copy offload and run rw traffic) [not run]
Remote target required but NVME_TARGET_CONTROL is not set
nvme_trtype=rdma is not supported in this test
kernel option ULP_DDP has not been enabled
module nvme_tcp does not have parameter ddp_offload
KERNELSRC not set
Kernel sources do not have tools/net/ynl/cli.py
NVME_IFACE not set
nvme/057 (tr=rdma) (test nvme fabrics controller ANA failover during I/O) [passed]
runtime 26.945s ... 26.949s
nvme/058 (tr=rdma) (test rapid namespace remapping) [passed]
runtime 4.293s ... 4.336s
nvme/060 (tr=rdma) (test nvme fabrics target reset) [passed]
runtime 20.730s ... 20.756s
nvme/061 (tr=rdma) (test fabric target teardown and setup during I/O) [passed]
runtime 15.686s ... 15.514s
nvme/062 (tr=rdma) (Create TLS-encrypted connections) [not run]
nvme_trtype=rdma is not supported in this test
nvme/063 (tr=rdma) (Create authenticated TCP connections with secure concatenation) [not run]
nvme_trtype=rdma is not supported in this test
nvme/065 (test unmap write zeroes sysfs interface with nvmet devices) [passed]
runtime 2.314s ... 2.336s
++ ./manage-rdma-nvme.sh --cleanup
====== RDMA NVMe Cleanup ======
[INFO] Disconnecting NVMe RDMA controllers...
[INFO] No NVMe RDMA controllers to disconnect
[INFO] Removing RDMA links...
[INFO] No RDMA links to remove
[INFO] Unloading NVMe RDMA modules...
[INFO] NVMe RDMA modules unloaded successfully
[INFO] Unloading soft-RDMA modules...
[INFO] Soft-RDMA modules unloaded successfully
[INFO] Verifying cleanup...
[INFO] Verification passed
[INFO] RDMA cleanup completed successfully
====== RDMA Network Configuration Status ======
Loaded Modules:
None
RDMA Links:
None
Network Interfaces (RDMA-capable):
None
blktests Configuration:
Not configured (run --setup first)
NVMe RDMA Controllers:
None
==========================
--
2.39.5
^ permalink raw reply
* [RFC PATCH] md/raid5: Fix UAF on IO across the reshape position
From: Benjamin Marzinski @ 2026-03-23 22:58 UTC (permalink / raw)
To: Song Liu, Yu Kuai, Li Nan; +Cc: linux-raid, dm-devel, Nigel Croxon, Xiao Ni
If make_stripe_request() returns STRIPE_WAIT_RESHAPE,
raid5_make_request() will free the cloned bio. But raid5_make_request()
can call make_stripe_request() multiple times, writing to the various
stripes. If that bio got added to the toread or towrite lists of a
stripe disk in an earlier call to make_stripe_request(), then it's not
safe to just free the bio if a later part of it is found to cross the
reshape position. Doing so can lead to a UAF error, when bio_endio()
is called on the bio for the earlier stripes.
Instead, raid5_make_request() needs to wait until all parts of the bio
have called bio_endio(). To do this, bios that cross the reshape
position while the reshape can't make progress are flagged as needing a
retry, and mddev tracks the number of bios needing a retry which have
not yet completed. When raid5_make_request() has a bio that failed
make_stripe_request() with STRIPE_WAIT_RESHAPE, it waits for this
counter to reach zero. When the bio_endio() is called for the last time
on a bio needing a retry, it decrements mddev's count of outstanding
bios needing a retry. This guarantees that raid5_make_request() doesn't
return until the cloned bio needing a retry for io across the reshape
boundary is safely cleaned up.
There is a simple reproducer available at [1]. Compile the kernel with
KASAN for more useful reporting when the error is triggered (this is not
necessary to see the bug).
[1] https://gist.github.com/bmarzins/e48598824305cf2171289e47d7241fa5
Signed-off-by: Benjamin Marzinski <bmarzins@redhat.com>
---
I've tested this for regressions with the lvm2-testsuite raid tests. I
have not run any md-specific tests on it.
drivers/md/md.c | 30 +++++++++---------------------
drivers/md/md.h | 5 ++++-
drivers/md/raid5.c | 8 +++++++-
3 files changed, 20 insertions(+), 23 deletions(-)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 3ce6f9e9d38e..5ec116b9da32 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -776,9 +776,11 @@ int mddev_init(struct mddev *mddev)
atomic_set(&mddev->active, 1);
atomic_set(&mddev->openers, 0);
atomic_set(&mddev->sync_seq, 0);
+ atomic_set(&mddev->pending_retry_bios, 0);
spin_lock_init(&mddev->lock);
init_waitqueue_head(&mddev->sb_wait);
init_waitqueue_head(&mddev->recovery_wait);
+ init_waitqueue_head(&mddev->retry_bios_wait);
mddev->reshape_position = MaxSector;
mddev->reshape_backwards = 0;
mddev->last_sync_action = ACTION_IDLE;
@@ -9218,6 +9220,7 @@ static void md_end_clone_io(struct bio *bio)
struct md_io_clone *md_io_clone = bio->bi_private;
struct bio *orig_bio = md_io_clone->orig_bio;
struct mddev *mddev = md_io_clone->mddev;
+ unsigned int must_retry = md_io_clone->must_retry;
if (bio_data_dir(orig_bio) == WRITE && md_bitmap_enabled(mddev, false))
md_bitmap_end(mddev, md_io_clone);
@@ -9229,7 +9232,11 @@ static void md_end_clone_io(struct bio *bio)
bio_end_io_acct(orig_bio, md_io_clone->start_time);
bio_put(bio);
- bio_endio(orig_bio);
+ if (unlikely(must_retry)) {
+ if (atomic_dec_and_test(&mddev->pending_retry_bios))
+ wake_up(&mddev->retry_bios_wait);
+ } else
+ bio_endio(orig_bio);
percpu_ref_put(&mddev->active_io);
}
@@ -9243,6 +9250,7 @@ static void md_clone_bio(struct mddev *mddev, struct bio **bio)
md_io_clone = container_of(clone, struct md_io_clone, bio_clone);
md_io_clone->orig_bio = *bio;
md_io_clone->mddev = mddev;
+ md_io_clone->must_retry = 0;
if (blk_queue_io_stat(bdev->bd_disk->queue))
md_io_clone->start_time = bio_start_io_acct(*bio);
@@ -9265,26 +9273,6 @@ void md_account_bio(struct mddev *mddev, struct bio **bio)
}
EXPORT_SYMBOL_GPL(md_account_bio);
-void md_free_cloned_bio(struct bio *bio)
-{
- struct md_io_clone *md_io_clone = bio->bi_private;
- struct bio *orig_bio = md_io_clone->orig_bio;
- struct mddev *mddev = md_io_clone->mddev;
-
- if (bio_data_dir(orig_bio) == WRITE && md_bitmap_enabled(mddev, false))
- md_bitmap_end(mddev, md_io_clone);
-
- if (bio->bi_status && !orig_bio->bi_status)
- orig_bio->bi_status = bio->bi_status;
-
- if (md_io_clone->start_time)
- bio_end_io_acct(orig_bio, md_io_clone->start_time);
-
- bio_put(bio);
- percpu_ref_put(&mddev->active_io);
-}
-EXPORT_SYMBOL_GPL(md_free_cloned_bio);
-
/* md_allow_write(mddev)
* Calling this ensures that the array is marked 'active' so that writes
* may proceed without blocking. It is important to call this before
diff --git a/drivers/md/md.h b/drivers/md/md.h
index ac84289664cd..49a231f11676 100644
--- a/drivers/md/md.h
+++ b/drivers/md/md.h
@@ -626,6 +626,9 @@ struct mddev {
/* The sequence number for sync thread */
atomic_t sync_seq;
+
+ wait_queue_head_t retry_bios_wait;
+ atomic_t pending_retry_bios;
};
enum recovery_flags {
@@ -877,6 +880,7 @@ struct md_io_clone {
sector_t offset;
unsigned long sectors;
enum stat_group rw;
+ unsigned int must_retry;
struct bio bio_clone;
};
@@ -917,7 +921,6 @@ extern void md_finish_reshape(struct mddev *mddev);
void md_submit_discard_bio(struct mddev *mddev, struct md_rdev *rdev,
struct bio *bio, sector_t start, sector_t size);
void md_account_bio(struct mddev *mddev, struct bio **bio);
-void md_free_cloned_bio(struct bio *bio);
extern bool __must_check md_flush_request(struct mddev *mddev, struct bio *bio);
void md_write_metadata(struct mddev *mddev, struct md_rdev *rdev,
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index a8e8d431071b..fb78a757f2fd 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -6217,7 +6217,13 @@ static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
mempool_free(ctx, conf->ctx_pool);
if (res == STRIPE_WAIT_RESHAPE) {
- md_free_cloned_bio(bi);
+ struct md_io_clone *md_io_clone = bi->bi_private;
+
+ md_io_clone->must_retry = 1;
+ atomic_inc(&mddev->pending_retry_bios);
+ bio_endio(bi);
+ wait_event(mddev->retry_bios_wait,
+ atomic_read(&mddev->pending_retry_bios)==0);
return false;
}
--
2.53.0
^ permalink raw reply related
* [PATCH v3 3/3] md/md-llbitmap: optimize initial sync with write_zeroes_unmap support
From: Yu Kuai @ 2026-03-23 5:46 UTC (permalink / raw)
To: linux-raid, song; +Cc: linan122, xni
In-Reply-To: <20260323054644.3351791-1-yukuai@fnnas.com>
For RAID-456 arrays with llbitmap, if all underlying disks support
write_zeroes with unmap, issue write_zeroes to zero all disk data
regions and initialize the bitmap to BitCleanUnwritten instead of
BitUnwritten.
This optimization skips the initial XOR parity building because:
1. write_zeroes with unmap guarantees zeroed reads after the operation
2. For RAID-456, when all data is zero, parity is automatically
consistent (0 XOR 0 XOR ... = 0)
3. BitCleanUnwritten indicates parity is valid but no user data
has been written
The implementation adds two helper functions:
- llbitmap_all_disks_support_wzeroes_unmap(): Checks if all active
disks support write_zeroes with unmap
- llbitmap_zero_all_disks(): Issues blkdev_issue_zeroout() to each
rdev's data region to zero all disks
The zeroing and bitmap state setting happens in llbitmap_init_state()
during bitmap initialization. If any disk fails to zero, we fall back
to BitUnwritten and normal lazy recovery.
This significantly reduces array initialization time for RAID-456
arrays built on modern NVMe SSDs or other devices that support
write_zeroes with unmap.
Signed-off-by: Yu Kuai <yukuai@fnnas.com>
Reviewed-by: Xiao Ni <xni@redhat.com>
---
drivers/md/md-llbitmap.c | 62 +++++++++++++++++++++++++++++++++++++++-
1 file changed, 61 insertions(+), 1 deletion(-)
diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c
index f10374242c9a..9e7e6b1a6f15 100644
--- a/drivers/md/md-llbitmap.c
+++ b/drivers/md/md-llbitmap.c
@@ -654,13 +654,73 @@ static int llbitmap_cache_pages(struct llbitmap *llbitmap)
return 0;
}
+/*
+ * Check if all underlying disks support write_zeroes with unmap.
+ */
+static bool llbitmap_all_disks_support_wzeroes_unmap(struct llbitmap *llbitmap)
+{
+ struct mddev *mddev = llbitmap->mddev;
+ struct md_rdev *rdev;
+
+ rdev_for_each(rdev, mddev) {
+ if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags))
+ continue;
+
+ if (bdev_write_zeroes_unmap_sectors(rdev->bdev) == 0)
+ return false;
+ }
+
+ return true;
+}
+
+/*
+ * Issue write_zeroes to all underlying disks to zero their data regions.
+ * This ensures parity consistency for RAID-456 (0 XOR 0 = 0).
+ * Returns true if all disks were successfully zeroed.
+ */
+static bool llbitmap_zero_all_disks(struct llbitmap *llbitmap)
+{
+ struct mddev *mddev = llbitmap->mddev;
+ struct md_rdev *rdev;
+ sector_t dev_sectors = mddev->dev_sectors;
+ int ret;
+
+ rdev_for_each(rdev, mddev) {
+ if (rdev->raid_disk < 0 || test_bit(Faulty, &rdev->flags))
+ continue;
+
+ ret = blkdev_issue_zeroout(rdev->bdev,
+ rdev->data_offset,
+ dev_sectors,
+ GFP_KERNEL, 0);
+ if (ret) {
+ pr_warn("md/llbitmap: failed to zero disk %pg: %d\n",
+ rdev->bdev, ret);
+ return false;
+ }
+ }
+
+ return true;
+}
+
static void llbitmap_init_state(struct llbitmap *llbitmap)
{
+ struct mddev *mddev = llbitmap->mddev;
enum llbitmap_state state = BitUnwritten;
unsigned long i;
- if (test_and_clear_bit(BITMAP_CLEAN, &llbitmap->flags))
+ if (test_and_clear_bit(BITMAP_CLEAN, &llbitmap->flags)) {
state = BitClean;
+ } else if (raid_is_456(mddev) &&
+ llbitmap_all_disks_support_wzeroes_unmap(llbitmap)) {
+ /*
+ * All disks support write_zeroes with unmap. Zero all disks
+ * to ensure parity consistency, then set BitCleanUnwritten
+ * to skip initial sync.
+ */
+ if (llbitmap_zero_all_disks(llbitmap))
+ state = BitCleanUnwritten;
+ }
for (i = 0; i < llbitmap->chunks; i++)
llbitmap_write(llbitmap, state, i);
--
2.51.0
^ permalink raw reply related
* [PATCH v3 2/3] md/md-llbitmap: add CleanUnwritten state for RAID-5 proactive parity building
From: Yu Kuai @ 2026-03-23 5:46 UTC (permalink / raw)
To: linux-raid, song; +Cc: linan122, xni
In-Reply-To: <20260323054644.3351791-1-yukuai@fnnas.com>
Add new states to the llbitmap state machine to support proactive XOR
parity building for RAID-5 arrays. This allows users to pre-build parity
data for unwritten regions before any user data is written.
New states added:
- BitNeedSyncUnwritten: Transitional state when proactive sync is triggered
via sysfs on Unwritten regions.
- BitSyncingUnwritten: Proactive sync in progress for unwritten region.
- BitCleanUnwritten: XOR parity has been pre-built, but no user data
written yet. When user writes to this region, it transitions to BitDirty.
New actions added:
- BitmapActionProactiveSync: Trigger for proactive XOR parity building.
- BitmapActionClearUnwritten: Convert CleanUnwritten/NeedSyncUnwritten/
SyncingUnwritten states back to Unwritten before recovery starts.
State flows:
- Current (lazy): Unwritten -> (write) -> NeedSync -> (sync) -> Dirty -> Clean
- New (proactive): Unwritten -> (sysfs) -> NeedSyncUnwritten -> (sync) -> CleanUnwritten
- On write to CleanUnwritten: CleanUnwritten -> (write) -> Dirty -> Clean
- On disk replacement: CleanUnwritten regions are converted to Unwritten
before recovery starts, so recovery only rebuilds regions with user data
A new sysfs interface is added at /sys/block/mdX/md/llbitmap/proactive_sync
(write-only) to trigger proactive sync. This only works for RAID-456 arrays.
Signed-off-by: Yu Kuai <yukuai@fnnas.com>
---
drivers/md/md-llbitmap.c | 140 +++++++++++++++++++++++++++++++++++----
1 file changed, 128 insertions(+), 12 deletions(-)
diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c
index cdfecaca216b..f10374242c9a 100644
--- a/drivers/md/md-llbitmap.c
+++ b/drivers/md/md-llbitmap.c
@@ -208,6 +208,20 @@ enum llbitmap_state {
BitNeedSync,
/* data is synchronizing */
BitSyncing,
+ /*
+ * Proactive sync requested for unwritten region (raid456 only).
+ * Triggered via sysfs when user wants to pre-build XOR parity
+ * for regions that have never been written.
+ */
+ BitNeedSyncUnwritten,
+ /* Proactive sync in progress for unwritten region */
+ BitSyncingUnwritten,
+ /*
+ * XOR parity has been pre-built for a region that has never had
+ * user data written. When user writes to this region, it transitions
+ * to BitDirty.
+ */
+ BitCleanUnwritten,
BitStateCount,
BitNone = 0xff,
};
@@ -232,6 +246,12 @@ enum llbitmap_action {
* BitNeedSync.
*/
BitmapActionStale,
+ /*
+ * Proactive sync trigger for raid456 - builds XOR parity for
+ * Unwritten regions without requiring user data write first.
+ */
+ BitmapActionProactiveSync,
+ BitmapActionClearUnwritten,
BitmapActionCount,
/* Init state is BitUnwritten */
BitmapActionInit,
@@ -304,6 +324,8 @@ static char state_machine[BitStateCount][BitmapActionCount] = {
[BitmapActionDaemon] = BitNone,
[BitmapActionDiscard] = BitNone,
[BitmapActionStale] = BitNone,
+ [BitmapActionProactiveSync] = BitNeedSyncUnwritten,
+ [BitmapActionClearUnwritten] = BitNone,
},
[BitClean] = {
[BitmapActionStartwrite] = BitDirty,
@@ -314,6 +336,8 @@ static char state_machine[BitStateCount][BitmapActionCount] = {
[BitmapActionDaemon] = BitNone,
[BitmapActionDiscard] = BitUnwritten,
[BitmapActionStale] = BitNeedSync,
+ [BitmapActionProactiveSync] = BitNone,
+ [BitmapActionClearUnwritten] = BitNone,
},
[BitDirty] = {
[BitmapActionStartwrite] = BitNone,
@@ -324,6 +348,8 @@ static char state_machine[BitStateCount][BitmapActionCount] = {
[BitmapActionDaemon] = BitClean,
[BitmapActionDiscard] = BitUnwritten,
[BitmapActionStale] = BitNeedSync,
+ [BitmapActionProactiveSync] = BitNone,
+ [BitmapActionClearUnwritten] = BitNone,
},
[BitNeedSync] = {
[BitmapActionStartwrite] = BitNone,
@@ -334,6 +360,8 @@ static char state_machine[BitStateCount][BitmapActionCount] = {
[BitmapActionDaemon] = BitNone,
[BitmapActionDiscard] = BitUnwritten,
[BitmapActionStale] = BitNone,
+ [BitmapActionProactiveSync] = BitNone,
+ [BitmapActionClearUnwritten] = BitNone,
},
[BitSyncing] = {
[BitmapActionStartwrite] = BitNone,
@@ -344,6 +372,44 @@ static char state_machine[BitStateCount][BitmapActionCount] = {
[BitmapActionDaemon] = BitNone,
[BitmapActionDiscard] = BitUnwritten,
[BitmapActionStale] = BitNeedSync,
+ [BitmapActionProactiveSync] = BitNone,
+ [BitmapActionClearUnwritten] = BitNone,
+ },
+ [BitNeedSyncUnwritten] = {
+ [BitmapActionStartwrite] = BitNeedSync,
+ [BitmapActionStartsync] = BitSyncingUnwritten,
+ [BitmapActionEndsync] = BitNone,
+ [BitmapActionAbortsync] = BitUnwritten,
+ [BitmapActionReload] = BitUnwritten,
+ [BitmapActionDaemon] = BitNone,
+ [BitmapActionDiscard] = BitUnwritten,
+ [BitmapActionStale] = BitUnwritten,
+ [BitmapActionProactiveSync] = BitNone,
+ [BitmapActionClearUnwritten] = BitUnwritten,
+ },
+ [BitSyncingUnwritten] = {
+ [BitmapActionStartwrite] = BitSyncing,
+ [BitmapActionStartsync] = BitSyncingUnwritten,
+ [BitmapActionEndsync] = BitCleanUnwritten,
+ [BitmapActionAbortsync] = BitUnwritten,
+ [BitmapActionReload] = BitUnwritten,
+ [BitmapActionDaemon] = BitNone,
+ [BitmapActionDiscard] = BitUnwritten,
+ [BitmapActionStale] = BitUnwritten,
+ [BitmapActionProactiveSync] = BitNone,
+ [BitmapActionClearUnwritten] = BitUnwritten,
+ },
+ [BitCleanUnwritten] = {
+ [BitmapActionStartwrite] = BitDirty,
+ [BitmapActionStartsync] = BitNone,
+ [BitmapActionEndsync] = BitNone,
+ [BitmapActionAbortsync] = BitNone,
+ [BitmapActionReload] = BitNone,
+ [BitmapActionDaemon] = BitNone,
+ [BitmapActionDiscard] = BitUnwritten,
+ [BitmapActionStale] = BitUnwritten,
+ [BitmapActionProactiveSync] = BitNone,
+ [BitmapActionClearUnwritten] = BitUnwritten,
},
};
@@ -376,6 +442,7 @@ static void llbitmap_infect_dirty_bits(struct llbitmap *llbitmap,
pctl->state[pos] = level_456 ? BitNeedSync : BitDirty;
break;
case BitClean:
+ case BitCleanUnwritten:
pctl->state[pos] = BitDirty;
break;
}
@@ -383,7 +450,7 @@ static void llbitmap_infect_dirty_bits(struct llbitmap *llbitmap,
}
static void llbitmap_set_page_dirty(struct llbitmap *llbitmap, int idx,
- int offset)
+ int offset, bool infect)
{
struct llbitmap_page_ctl *pctl = llbitmap->pctl[idx];
unsigned int io_size = llbitmap->io_size;
@@ -398,7 +465,7 @@ static void llbitmap_set_page_dirty(struct llbitmap *llbitmap, int idx,
* resync all the dirty bits, hence skip infect new dirty bits to
* prevent resync unnecessary data.
*/
- if (llbitmap->mddev->degraded) {
+ if (llbitmap->mddev->degraded || !infect) {
set_bit(block, pctl->dirty);
return;
}
@@ -438,7 +505,9 @@ static void llbitmap_write(struct llbitmap *llbitmap, enum llbitmap_state state,
llbitmap->pctl[idx]->state[bit] = state;
if (state == BitDirty || state == BitNeedSync)
- llbitmap_set_page_dirty(llbitmap, idx, bit);
+ llbitmap_set_page_dirty(llbitmap, idx, bit, true);
+ else if (state == BitNeedSyncUnwritten)
+ llbitmap_set_page_dirty(llbitmap, idx, bit, false);
}
static struct page *llbitmap_read_page(struct llbitmap *llbitmap, int idx)
@@ -627,11 +696,10 @@ static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap,
goto write_bitmap;
}
- if (c == BitNeedSync)
+ if (c == BitNeedSync || c == BitNeedSyncUnwritten)
need_resync = !mddev->degraded;
state = state_machine[c][action];
-
write_bitmap:
if (unlikely(mddev->degraded)) {
/* For degraded array, mark new data as need sync. */
@@ -658,8 +726,7 @@ static enum llbitmap_state llbitmap_state_machine(struct llbitmap *llbitmap,
}
llbitmap_write(llbitmap, state, start);
-
- if (state == BitNeedSync)
+ if (state == BitNeedSync || state == BitNeedSyncUnwritten)
need_resync = !mddev->degraded;
else if (state == BitDirty &&
!timer_pending(&llbitmap->pending_timer))
@@ -1229,7 +1296,7 @@ static bool llbitmap_blocks_synced(struct mddev *mddev, sector_t offset)
unsigned long p = offset >> llbitmap->chunkshift;
enum llbitmap_state c = llbitmap_read(llbitmap, p);
- return c == BitClean || c == BitDirty;
+ return c == BitClean || c == BitDirty || c == BitCleanUnwritten;
}
static sector_t llbitmap_skip_sync_blocks(struct mddev *mddev, sector_t offset)
@@ -1243,6 +1310,10 @@ static sector_t llbitmap_skip_sync_blocks(struct mddev *mddev, sector_t offset)
if (c == BitUnwritten)
return blocks;
+ /* Skip CleanUnwritten - no user data, will be reset after recovery */
+ if (c == BitCleanUnwritten)
+ return blocks;
+
/* For degraded array, don't skip */
if (mddev->degraded)
return 0;
@@ -1261,14 +1332,25 @@ static bool llbitmap_start_sync(struct mddev *mddev, sector_t offset,
{
struct llbitmap *llbitmap = mddev->bitmap;
unsigned long p = offset >> llbitmap->chunkshift;
+ enum llbitmap_state state;
+
+ /*
+ * Before recovery starts, convert CleanUnwritten to Unwritten.
+ * This ensures the new disk won't have stale parity data.
+ */
+ if (offset == 0 && test_bit(MD_RECOVERY_RECOVER, &mddev->recovery) &&
+ !test_bit(MD_RECOVERY_LAZY_RECOVER, &mddev->recovery))
+ llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1,
+ BitmapActionClearUnwritten);
+
/*
* Handle one bit at a time, this is much simpler. And it doesn't matter
* if md_do_sync() loop more times.
*/
*blocks = llbitmap->chunksize - (offset & (llbitmap->chunksize - 1));
- return llbitmap_state_machine(llbitmap, p, p,
- BitmapActionStartsync) == BitSyncing;
+ state = llbitmap_state_machine(llbitmap, p, p, BitmapActionStartsync);
+ return state == BitSyncing || state == BitSyncingUnwritten;
}
/* Something is wrong, sync_thread stop at @offset */
@@ -1474,9 +1556,15 @@ static ssize_t bits_show(struct mddev *mddev, char *page)
}
mutex_unlock(&mddev->bitmap_info.mutex);
- return sprintf(page, "unwritten %d\nclean %d\ndirty %d\nneed sync %d\nsyncing %d\n",
+ return sprintf(page,
+ "unwritten %d\nclean %d\ndirty %d\n"
+ "need sync %d\nsyncing %d\n"
+ "need sync unwritten %d\nsyncing unwritten %d\n"
+ "clean unwritten %d\n",
bits[BitUnwritten], bits[BitClean], bits[BitDirty],
- bits[BitNeedSync], bits[BitSyncing]);
+ bits[BitNeedSync], bits[BitSyncing],
+ bits[BitNeedSyncUnwritten], bits[BitSyncingUnwritten],
+ bits[BitCleanUnwritten]);
}
static struct md_sysfs_entry llbitmap_bits = __ATTR_RO(bits);
@@ -1549,11 +1637,39 @@ barrier_idle_store(struct mddev *mddev, const char *buf, size_t len)
static struct md_sysfs_entry llbitmap_barrier_idle = __ATTR_RW(barrier_idle);
+static ssize_t
+proactive_sync_store(struct mddev *mddev, const char *buf, size_t len)
+{
+ struct llbitmap *llbitmap;
+
+ /* Only for RAID-456 */
+ if (!raid_is_456(mddev))
+ return -EINVAL;
+
+ mutex_lock(&mddev->bitmap_info.mutex);
+ llbitmap = mddev->bitmap;
+ if (!llbitmap || !llbitmap->pctl) {
+ mutex_unlock(&mddev->bitmap_info.mutex);
+ return -ENODEV;
+ }
+
+ /* Trigger proactive sync on all Unwritten regions */
+ llbitmap_state_machine(llbitmap, 0, llbitmap->chunks - 1,
+ BitmapActionProactiveSync);
+
+ mutex_unlock(&mddev->bitmap_info.mutex);
+ return len;
+}
+
+static struct md_sysfs_entry llbitmap_proactive_sync =
+ __ATTR(proactive_sync, 0200, NULL, proactive_sync_store);
+
static struct attribute *md_llbitmap_attrs[] = {
&llbitmap_bits.attr,
&llbitmap_metadata.attr,
&llbitmap_daemon_sleep.attr,
&llbitmap_barrier_idle.attr,
+ &llbitmap_proactive_sync.attr,
NULL
};
--
2.51.0
^ permalink raw reply related
* [PATCH v3 1/3] md: add fallback to correct bitmap_ops on version mismatch
From: Yu Kuai @ 2026-03-23 5:46 UTC (permalink / raw)
To: linux-raid, song; +Cc: linan122, xni
In-Reply-To: <20260323054644.3351791-1-yukuai@fnnas.com>
If default bitmap version and on-disk version doesn't match, and mdadm
is not the latest version to set bitmap_type, set bitmap_ops based on
the disk version.
Signed-off-by: Yu Kuai <yukuai@fnnas.com>
---
drivers/md/md.c | 111 +++++++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 110 insertions(+), 1 deletion(-)
diff --git a/drivers/md/md.c b/drivers/md/md.c
index 521d9b34cd9e..c9597c6bd341 100644
--- a/drivers/md/md.c
+++ b/drivers/md/md.c
@@ -6449,15 +6449,124 @@ static void md_safemode_timeout(struct timer_list *t)
static int start_dirty_degraded;
+/*
+ * Read bitmap superblock and return the bitmap_id based on disk version.
+ * This is used as fallback when default bitmap version and on-disk version
+ * doesn't match, and mdadm is not the latest version to set bitmap_type.
+ */
+static enum md_submodule_id md_bitmap_get_id_from_sb(struct mddev *mddev)
+{
+ struct md_rdev *rdev;
+ struct page *sb_page;
+ bitmap_super_t *sb;
+ enum md_submodule_id id = ID_BITMAP_NONE;
+ sector_t sector;
+ u32 version;
+
+ if (!mddev->bitmap_info.offset)
+ return ID_BITMAP_NONE;
+
+ sb_page = alloc_page(GFP_KERNEL);
+ if (!sb_page) {
+ pr_warn("md: %s: failed to allocate memory for bitmap\n",
+ mdname(mddev));
+ return ID_BITMAP_NONE;
+ }
+
+ sector = mddev->bitmap_info.offset;
+
+ rdev_for_each(rdev, mddev) {
+ u32 iosize;
+
+ if (!test_bit(In_sync, &rdev->flags) ||
+ test_bit(Faulty, &rdev->flags) ||
+ test_bit(Bitmap_sync, &rdev->flags))
+ continue;
+
+ iosize = roundup(sizeof(bitmap_super_t),
+ bdev_logical_block_size(rdev->bdev));
+ if (sync_page_io(rdev, sector, iosize, sb_page, REQ_OP_READ,
+ true))
+ goto read_ok;
+ }
+ pr_warn("md: %s: failed to read bitmap from any device\n",
+ mdname(mddev));
+ goto out;
+
+read_ok:
+ sb = kmap_local_page(sb_page);
+ if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) {
+ pr_warn("md: %s: invalid bitmap magic 0x%x\n",
+ mdname(mddev), le32_to_cpu(sb->magic));
+ goto out_unmap;
+ }
+
+ version = le32_to_cpu(sb->version);
+ switch (version) {
+ case BITMAP_MAJOR_LO:
+ case BITMAP_MAJOR_HI:
+ case BITMAP_MAJOR_CLUSTERED:
+ id = ID_BITMAP;
+ break;
+ case BITMAP_MAJOR_LOCKLESS:
+ id = ID_LLBITMAP;
+ break;
+ default:
+ pr_warn("md: %s: unknown bitmap version %u\n",
+ mdname(mddev), version);
+ break;
+ }
+
+out_unmap:
+ kunmap_local(sb);
+out:
+ __free_page(sb_page);
+ return id;
+}
+
static int md_bitmap_create(struct mddev *mddev)
{
+ enum md_submodule_id orig_id = mddev->bitmap_id;
+ enum md_submodule_id sb_id;
+ int err;
+
if (mddev->bitmap_id == ID_BITMAP_NONE)
return -EINVAL;
if (!mddev_set_bitmap_ops(mddev))
return -ENOENT;
- return mddev->bitmap_ops->create(mddev);
+ err = mddev->bitmap_ops->create(mddev);
+ if (!err)
+ return 0;
+
+ /*
+ * Create failed, if default bitmap version and on-disk version
+ * doesn't match, and mdadm is not the latest version to set
+ * bitmap_type, set bitmap_ops based on the disk version.
+ */
+ mddev_clear_bitmap_ops(mddev);
+
+ sb_id = md_bitmap_get_id_from_sb(mddev);
+ if (sb_id == ID_BITMAP_NONE || sb_id == orig_id)
+ return err;
+
+ pr_info("md: %s: bitmap version mismatch, switching from %d to %d\n",
+ mdname(mddev), orig_id, sb_id);
+
+ mddev->bitmap_id = sb_id;
+ if (!mddev_set_bitmap_ops(mddev)) {
+ mddev->bitmap_id = orig_id;
+ return -ENOENT;
+ }
+
+ err = mddev->bitmap_ops->create(mddev);
+ if (err) {
+ mddev_clear_bitmap_ops(mddev);
+ mddev->bitmap_id = orig_id;
+ }
+
+ return err;
}
static void md_bitmap_destroy(struct mddev *mddev)
--
2.51.0
^ permalink raw reply related
* [PATCH v3 0/3] md/md-llbitmap: add CleanUnwritten support
From: Yu Kuai @ 2026-03-23 5:46 UTC (permalink / raw)
To: linux-raid, song; +Cc: linan122, xni
Patch 1 improves compatibility with older mdadm versions by detecting
on-disk bitmap version and falling back to the correct bitmap_ops when
there's a version mismatch.
Patch 2 adds support for proactive XOR parity building in RAID-456 arrays.
This allows users to pre-build parity for unwritten regions via sysfs
before any user data is written, which can improve write performance for
workloads that will eventually use all storage. New states (CleanUnwritten,
NeedSyncUnwritten, SyncingUnwritten) are added to track these regions
separately from normal dirty/syncing states.
Patch 3 optimizes initial array sync for RAID-456 arrays on devices that
support write_zeroes with unmap. By zeroing all disks upfront, parity is
automatically consistent (0 XOR 0 = 0), allowing the bitmap to be
initialized to BitCleanUnwritten and skipping the initial sync entirely.
This significantly reduces array initialization time on modern NVMe SSDs.
Yu Kuai (3):
md: add fallback to correct bitmap_ops on version mismatch
md/md-llbitmap: add CleanUnwritten state for RAID-5 proactive parity
building
md/md-llbitmap: optimize initial sync with write_zeroes_unmap support
drivers/md/md-llbitmap.c | 202 ++++++++++++++++++++++++++++++++++++---
drivers/md/md.c | 111 ++++++++++++++++++++-
2 files changed, 299 insertions(+), 14 deletions(-)
--
2.51.0
^ permalink raw reply
* Re: [GIT PULL] md-7.1-20260323
From: Jens Axboe @ 2026-03-22 19:39 UTC (permalink / raw)
To: Yu Kuai
Cc: linux-raid, linux-block, Chen Cheng, FengWei Shih, Josh Hunt,
Xiao Ni
In-Reply-To: <20260322183911.2732398-1-yukuai@fnnas.com>
On 3/22/26 12:39 PM, Yu Kuai wrote:
> Hi Jens,
>
> Please consider pulling the following changes into your for-7.1/block branch.
>
> This pull request contains:
>
> Bug Fixes:
> - md: suppress spurious superblock update error message for dm-raid (Chen Cheng)
> - md/raid1: fix the comparing region of interval tree (Xiao Ni)
> - md/raid10: fix deadlock with check operation and nowait requests (Josh Hunt)
> - md/raid5: skip 2-failure compute when other disk is R5_LOCKED (FengWei Shih)
> - md/md-llbitmap: raise barrier before state machine transition (Yu Kuai)
> - md/md-llbitmap: skip reading rdevs that are not in_sync (Yu Kuai)
>
> Improvements:
> - md/raid5: set chunk_sectors to enable full stripe I/O splitting (Yu Kuai)
>
> Cleanups:
> - md: remove unused mddev argument from export_rdev (Chen Cheng)
> - md/raid5: remove stale md_raid5_kick_device() declaration (Chen Cheng)
> - md/raid5: move handle_stripe() comment to correct location (Chen Cheng)
Pulled, thanks.
--
Jens Axboe
^ permalink raw reply
* [GIT PULL] md-7.1-20260323
From: Yu Kuai @ 2026-03-22 18:39 UTC (permalink / raw)
To: Jens Axboe
Cc: linux-raid, linux-block, Chen Cheng, FengWei Shih, Josh Hunt,
Xiao Ni, Yu Kuai
Hi Jens,
Please consider pulling the following changes into your for-7.1/block branch.
This pull request contains:
Bug Fixes:
- md: suppress spurious superblock update error message for dm-raid (Chen Cheng)
- md/raid1: fix the comparing region of interval tree (Xiao Ni)
- md/raid10: fix deadlock with check operation and nowait requests (Josh Hunt)
- md/raid5: skip 2-failure compute when other disk is R5_LOCKED (FengWei Shih)
- md/md-llbitmap: raise barrier before state machine transition (Yu Kuai)
- md/md-llbitmap: skip reading rdevs that are not in_sync (Yu Kuai)
Improvements:
- md/raid5: set chunk_sectors to enable full stripe I/O splitting (Yu Kuai)
Cleanups:
- md: remove unused mddev argument from export_rdev (Chen Cheng)
- md/raid5: remove stale md_raid5_kick_device() declaration (Chen Cheng)
- md/raid5: move handle_stripe() comment to correct location (Chen Cheng)
Thanks,
Kuai
---
The following changes since commit 5d540162059598c3f79e12f96064825cc91f0f9c:
ublk: report BLK_SPLIT_INTERVAL_CAPABLE (2026-03-14 07:44:30 -0600)
are available in the Git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/mdraid/linux.git tags/md-7.1-20260323
for you to fetch changes up to 6f507eb2bb5491327fe634dc23558d4ca5d710b8:
md: remove unused mddev argument from export_rdev (2026-03-23 02:15:11 +0800)
----------------------------------------------------------------
Chen Cheng (4):
md: suppress spurious superblock update error message for dm-raid
md/raid5: remove stale md_raid5_kick_device() declaration
md/raid5: move handle_stripe() comment to correct location
md: remove unused mddev argument from export_rdev
FengWei Shih (1):
md/raid5: skip 2-failure compute when other disk is R5_LOCKED
Josh Hunt (1):
md/raid10: fix deadlock with check operation and nowait requests
Xiao Ni (1):
md/raid1: fix the comparing region of interval tree
Yu Kuai (3):
md/raid5: set chunk_sectors to enable full stripe I/O splitting
md/md-llbitmap: skip reading rdevs that are not in_sync
md/md-llbitmap: raise barrier before state machine transition
drivers/md/md-llbitmap.c | 11 ++++++-----
drivers/md/md.c | 32 +++++++++++++++++---------------
drivers/md/raid1.c | 4 ++--
drivers/md/raid10.c | 4 ++--
drivers/md/raid5.c | 29 +++++++++++++++--------------
drivers/md/raid5.h | 1 -
6 files changed, 42 insertions(+), 39 deletions(-)
^ permalink raw reply
* Re: [PATCH 2/2] md/raid5: move handle_stripe() comment to correct location
From: Yu Kuai @ 2026-03-22 18:33 UTC (permalink / raw)
To: Chen Cheng, Song Liu, yukuai; +Cc: Li Nan, linux-raid
In-Reply-To: <20260304111001.15767-1-chencheng@fnnas.com>
在 2026/3/4 19:10, Chen Cheng 写道:
> Move the handle_stripe() documentation comment from above
> analyse_stripe() to directly above handle_stripe() where it belongs.
>
> Signed-off-by: Chen Cheng<chencheng@fnnas.com>
> ---
> drivers/md/raid5.c | 26 ++++++++++++--------------
> 1 file changed, 12 insertions(+), 14 deletions(-)
Applied
--
Thansk,
Kuai
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox