* [PATCH 10/17] raid6: use static_call for gen_syndrom and xor_syndrom
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
Avoid indirect calls for P/Q parity generation.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
lib/raid/raid6/algos.c | 18 ++++++++++++------
1 file changed, 12 insertions(+), 6 deletions(-)
diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c
index f9e8a8752e2d..b81c7594f6c4 100644
--- a/lib/raid/raid6/algos.c
+++ b/lib/raid/raid6/algos.c
@@ -14,6 +14,7 @@
#include <linux/module.h>
#include <linux/gfp.h>
#include <linux/raid/pq.h>
+#include <linux/static_call.h>
#include <kunit/visibility.h>
#include "algos.h"
@@ -23,7 +24,8 @@ static unsigned int raid6_nr_algos;
static const struct raid6_recov_calls *raid6_recov_algo;
/* Selected algorithm */
-static struct raid6_calls raid6_call;
+DEFINE_STATIC_CALL_NULL(raid6_gen_syndrome_impl, *raid6_intx1.gen_syndrome);
+DEFINE_STATIC_CALL_NULL(raid6_xor_syndrome_impl, *raid6_intx1.xor_syndrome);
/**
* raid6_gen_syndrome - generate RAID6 P/Q parity
@@ -47,7 +49,7 @@ void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs)
lockdep_assert_preemption_enabled();
WARN_ON_ONCE(bytes & 511);
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ static_call(raid6_gen_syndrome_impl)(disks, bytes, ptrs);
}
EXPORT_SYMBOL_GPL(raid6_gen_syndrome);
@@ -83,7 +85,7 @@ void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes,
WARN_ON_ONCE(bytes & 511);
WARN_ON_ONCE(stop < start);
- raid6_call.xor_syndrome(disks, start, stop, bytes, ptrs);
+ static_call(raid6_xor_syndrome_impl)(disks, start, stop, bytes, ptrs);
}
EXPORT_SYMBOL_GPL(raid6_xor_syndrome);
@@ -94,7 +96,7 @@ EXPORT_SYMBOL_GPL(raid6_xor_syndrome);
*/
bool raid6_can_xor_syndrome(void)
{
- return !!raid6_call.xor_syndrome;
+ return !!static_call_query(raid6_xor_syndrome_impl);
}
EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome);
@@ -193,7 +195,8 @@ static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS],
return -EINVAL;
}
- raid6_call = *best;
+ static_call_update(raid6_gen_syndrome_impl, best->gen_syndrome);
+ static_call_update(raid6_xor_syndrome_impl, best->xor_syndrome);
pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
best->name,
@@ -238,7 +241,10 @@ static int __init raid6_select_algo(void)
if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) {
pr_info("raid6: skipped pq benchmark and selected %s\n",
raid6_algos[0]->name);
- raid6_call = *raid6_algos[0];
+ static_call_update(raid6_gen_syndrome_impl,
+ raid6_algos[0]->gen_syndrome);
+ static_call_update(raid6_xor_syndrome_impl,
+ raid6_algos[0]->xor_syndrome);
return 0;
}
--
2.47.3
^ permalink raw reply related
* [PATCH 09/17] raid6: rework the init helpers
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
Match the xor version with two initcalls for the built-in case to delay
calibrartion. This prepares for adding non-calibration init code.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
lib/raid/Kconfig | 11 +
lib/raid/raid6/Makefile | 4 +
lib/raid/raid6/algos.c | 309 ++++++++----------
lib/raid/raid6/algos.h | 67 +---
lib/raid/raid6/arm/neon.c | 6 -
lib/raid/raid6/arm/pq_arch.h | 22 ++
lib/raid/raid6/arm/recov_neon.c | 7 -
lib/raid/raid6/loongarch/loongarch_simd.c | 12 -
lib/raid/raid6/loongarch/pq_arch.h | 23 ++
.../raid6/loongarch/recov_loongarch_simd.c | 14 -
lib/raid/raid6/powerpc/altivec.uc | 10 -
lib/raid/raid6/powerpc/pq_arch.h | 31 ++
lib/raid/raid6/powerpc/vpermxor.uc | 11 -
lib/raid/raid6/recov.c | 2 -
lib/raid/raid6/riscv/pq_arch.h | 21 ++
lib/raid/raid6/riscv/recov_rvv.c | 2 -
lib/raid/raid6/riscv/rvv.h | 6 -
lib/raid/raid6/s390/pq_arch.h | 15 +
lib/raid/raid6/s390/recov_s390xc.c | 2 -
lib/raid/raid6/s390/s390vx.uc | 7 -
lib/raid/raid6/tests/raid6_kunit.c | 23 +-
lib/raid/raid6/x86/avx2.c | 14 -
lib/raid/raid6/x86/avx512.c | 19 --
lib/raid/raid6/x86/mmx.c | 8 -
lib/raid/raid6/x86/pq_arch.h | 96 ++++++
lib/raid/raid6/x86/recov_avx2.c | 8 -
lib/raid/raid6/x86/recov_avx512.c | 12 -
lib/raid/raid6/x86/recov_ssse3.c | 9 -
lib/raid/raid6/x86/sse1.c | 12 -
lib/raid/raid6/x86/sse2.c | 15 -
30 files changed, 389 insertions(+), 409 deletions(-)
create mode 100644 lib/raid/raid6/arm/pq_arch.h
create mode 100644 lib/raid/raid6/loongarch/pq_arch.h
create mode 100644 lib/raid/raid6/powerpc/pq_arch.h
create mode 100644 lib/raid/raid6/riscv/pq_arch.h
create mode 100644 lib/raid/raid6/s390/pq_arch.h
create mode 100644 lib/raid/raid6/x86/pq_arch.h
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
index 9bea599d66da..6795cfefb43b 100644
--- a/lib/raid/Kconfig
+++ b/lib/raid/Kconfig
@@ -32,6 +32,17 @@ config XOR_KUNIT_TEST
config RAID6_PQ
tristate
+# selected by architectures that provide an optimized PQ implementation
+config RAID6_PQ_ARCH
+ depends on RAID6_PQ
+ default y if CONFIG_KERNEL_MODE_NEON # arm32/arm64
+ default y if LOONGARCH
+ default y if ALTIVEC # powerpc
+ default y if RISCV_ISA_V
+ default y if S390
+ default y if X86
+ bool
+
config RAID6_PQ_KUNIT_TEST
tristate "KUnit tests for raid6 PQ functions" if !KUNIT_ALL_TESTS
depends on KUNIT
diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile
index fc46c3e61c82..db2844a314b0 100644
--- a/lib/raid/raid6/Makefile
+++ b/lib/raid/raid6/Makefile
@@ -2,6 +2,10 @@
ccflags-y += -I $(src)
+ifeq ($(CONFIG_RAID6_PQ_ARCH),y)
+CFLAGS_algos.o += -I$(src)/$(SRCARCH)
+endif
+
obj-$(CONFIG_RAID6_PQ) += raid6_pq.o tests/
raid6_pq-y += algos.o tables.o
diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c
index 406efdcc03b8..f9e8a8752e2d 100644
--- a/lib/raid/raid6/algos.c
+++ b/lib/raid/raid6/algos.c
@@ -17,6 +17,9 @@
#include <kunit/visibility.h>
#include "algos.h"
+#define RAID6_MAX_ALGOS 16
+static const struct raid6_calls *raid6_algos[RAID6_MAX_ALGOS];
+static unsigned int raid6_nr_algos;
static const struct raid6_recov_calls *raid6_recov_algo;
/* Selected algorithm */
@@ -95,71 +98,6 @@ bool raid6_can_xor_syndrome(void)
}
EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome);
-const struct raid6_calls * const raid6_algos[] = {
-#if defined(__i386__) && !defined(__arch_um__)
- &raid6_avx512x2,
- &raid6_avx512x1,
- &raid6_avx2x2,
- &raid6_avx2x1,
- &raid6_sse2x2,
- &raid6_sse2x1,
- &raid6_sse1x2,
- &raid6_sse1x1,
- &raid6_mmxx2,
- &raid6_mmxx1,
-#endif
-#if defined(__x86_64__) && !defined(__arch_um__)
- &raid6_avx512x4,
- &raid6_avx512x2,
- &raid6_avx512x1,
- &raid6_avx2x4,
- &raid6_avx2x2,
- &raid6_avx2x1,
- &raid6_sse2x4,
- &raid6_sse2x2,
- &raid6_sse2x1,
-#endif
-#ifdef CONFIG_ALTIVEC
- &raid6_vpermxor8,
- &raid6_vpermxor4,
- &raid6_vpermxor2,
- &raid6_vpermxor1,
- &raid6_altivec8,
- &raid6_altivec4,
- &raid6_altivec2,
- &raid6_altivec1,
-#endif
-#if defined(CONFIG_S390)
- &raid6_s390vx8,
-#endif
-#ifdef CONFIG_KERNEL_MODE_NEON
- &raid6_neonx8,
- &raid6_neonx4,
- &raid6_neonx2,
- &raid6_neonx1,
-#endif
-#ifdef CONFIG_LOONGARCH
-#ifdef CONFIG_CPU_HAS_LASX
- &raid6_lasx,
-#endif
-#ifdef CONFIG_CPU_HAS_LSX
- &raid6_lsx,
-#endif
-#endif
-#ifdef CONFIG_RISCV_ISA_V
- &raid6_rvvx1,
- &raid6_rvvx2,
- &raid6_rvvx4,
- &raid6_rvvx8,
-#endif
- &raid6_intx8,
- &raid6_intx4,
- &raid6_intx2,
- &raid6_intx1,
- NULL
-};
-EXPORT_SYMBOL_IF_KUNIT(raid6_algos);
-
/**
* raid6_recov_2data - recover two missing data disks
* @disks: number of "disks" to operate on including parity
@@ -213,119 +151,57 @@ void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs)
}
EXPORT_SYMBOL_GPL(raid6_recov_datap);
-const struct raid6_recov_calls *const raid6_recov_algos[] = {
-#ifdef CONFIG_X86
- &raid6_recov_avx512,
- &raid6_recov_avx2,
- &raid6_recov_ssse3,
-#endif
-#ifdef CONFIG_S390
- &raid6_recov_s390xc,
-#endif
-#if defined(CONFIG_KERNEL_MODE_NEON)
- &raid6_recov_neon,
-#endif
-#ifdef CONFIG_LOONGARCH
-#ifdef CONFIG_CPU_HAS_LASX
- &raid6_recov_lasx,
-#endif
-#ifdef CONFIG_CPU_HAS_LSX
- &raid6_recov_lsx,
-#endif
-#endif
-#ifdef CONFIG_RISCV_ISA_V
- &raid6_recov_rvv,
-#endif
- &raid6_recov_intx1,
- NULL
-};
-EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algos);
-
#define RAID6_TIME_JIFFIES_LG2 4
#define RAID6_TEST_DISKS 8
#define RAID6_TEST_DISKS_ORDER 3
-static inline const struct raid6_recov_calls *raid6_choose_recov(void)
+static int raid6_choose_gen(void *(*const dptrs)[RAID6_TEST_DISKS],
+ const int disks)
{
- const struct raid6_recov_calls *const *algo;
- const struct raid6_recov_calls *best;
-
- for (best = NULL, algo = raid6_recov_algos; *algo; algo++)
- if (!best || (*algo)->priority > best->priority)
- if (!(*algo)->valid || (*algo)->valid())
- best = *algo;
+ /* work on the second half of the disks */
+ int start = (disks >> 1) - 1, stop = disks - 3;
+ const struct raid6_calls *best = NULL;
+ unsigned long bestgenperf = 0;
+ unsigned int i;
- if (best) {
- raid6_recov_algo = best;
+ for (i = 0; i < raid6_nr_algos; i++) {
+ const struct raid6_calls *algo = raid6_algos[i];
+ unsigned long perf = 0, j0, j1;
- pr_info("raid6: using %s recovery algorithm\n", best->name);
- } else
- pr_err("raid6: Yikes! No recovery algorithm found!\n");
-
- return best;
-}
+ preempt_disable();
+ j0 = jiffies;
+ while ((j1 = jiffies) == j0)
+ cpu_relax();
+ while (time_before(jiffies,
+ j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
+ algo->gen_syndrome(disks, PAGE_SIZE, *dptrs);
+ perf++;
+ }
+ preempt_enable();
-static inline const struct raid6_calls *raid6_choose_gen(
- void *(*const dptrs)[RAID6_TEST_DISKS], const int disks)
-{
- unsigned long perf, bestgenperf, j0, j1;
- int start = (disks>>1)-1, stop = disks-3; /* work on the second half of the disks */
- const struct raid6_calls *const *algo;
- const struct raid6_calls *best;
-
- for (bestgenperf = 0, best = NULL, algo = raid6_algos; *algo; algo++) {
- if (!best || (*algo)->priority >= best->priority) {
- if ((*algo)->valid && !(*algo)->valid())
- continue;
-
- if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
- best = *algo;
- break;
- }
-
- perf = 0;
-
- preempt_disable();
- j0 = jiffies;
- while ((j1 = jiffies) == j0)
- cpu_relax();
- while (time_before(jiffies,
- j1 + (1<<RAID6_TIME_JIFFIES_LG2))) {
- (*algo)->gen_syndrome(disks, PAGE_SIZE, *dptrs);
- perf++;
- }
- preempt_enable();
-
- if (perf > bestgenperf) {
- bestgenperf = perf;
- best = *algo;
- }
- pr_info("raid6: %-8s gen() %5ld MB/s\n", (*algo)->name,
- (perf * HZ * (disks-2)) >>
- (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
+ if (perf > bestgenperf) {
+ bestgenperf = perf;
+ best = algo;
}
+ pr_info("raid6: %-8s gen() %5ld MB/s\n", algo->name,
+ (perf * HZ * (disks-2)) >>
+ (20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
}
if (!best) {
pr_err("raid6: Yikes! No algorithm found!\n");
- goto out;
+ return -EINVAL;
}
raid6_call = *best;
- if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK)) {
- pr_info("raid6: skipped pq benchmark and selected %s\n",
- best->name);
- goto out;
- }
-
pr_info("raid6: using algorithm %s gen() %ld MB/s\n",
best->name,
(bestgenperf * HZ * (disks - 2)) >>
(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2));
if (best->xor_syndrome) {
- perf = 0;
+ unsigned long perf = 0, j0, j1;
preempt_disable();
j0 = jiffies;
@@ -344,8 +220,7 @@ static inline const struct raid6_calls *raid6_choose_gen(
(20 - PAGE_SHIFT + RAID6_TIME_JIFFIES_LG2 + 1));
}
-out:
- return best;
+ return 0;
}
@@ -355,12 +230,17 @@ static inline const struct raid6_calls *raid6_choose_gen(
static int __init raid6_select_algo(void)
{
const int disks = RAID6_TEST_DISKS;
-
- const struct raid6_calls *gen_best;
- const struct raid6_recov_calls *rec_best;
char *disk_ptr, *p;
void *dptrs[RAID6_TEST_DISKS];
int i, cycle;
+ int error;
+
+ if (!IS_ENABLED(CONFIG_RAID6_PQ_BENCHMARK) || raid6_nr_algos == 1) {
+ pr_info("raid6: skipped pq benchmark and selected %s\n",
+ raid6_algos[0]->name);
+ raid6_call = *raid6_algos[0];
+ return 0;
+ }
/* prepare the buffer and fill it circularly with gfmul table */
disk_ptr = (char *)__get_free_pages(GFP_KERNEL, RAID6_TEST_DISKS_ORDER);
@@ -383,22 +263,115 @@ static int __init raid6_select_algo(void)
memcpy(p, raid6_gfmul, (disks - 2) * PAGE_SIZE % 65536);
/* select raid gen_syndrome function */
- gen_best = raid6_choose_gen(&dptrs, disks);
-
- /* select raid recover functions */
- rec_best = raid6_choose_recov();
+ error = raid6_choose_gen(&dptrs, disks);
free_pages((unsigned long)disk_ptr, RAID6_TEST_DISKS_ORDER);
- return gen_best && rec_best ? 0 : -EINVAL;
+ return error;
+}
+
+void __init raid6_algo_add(const struct raid6_calls *algo)
+{
+ if (WARN_ON_ONCE(raid6_nr_algos == RAID6_MAX_ALGOS))
+ return;
+ raid6_algos[raid6_nr_algos++] = algo;
+}
+
+void __init raid6_algo_add_default(void)
+{
+ raid6_algo_add(&raid6_intx1);
+ raid6_algo_add(&raid6_intx2);
+ raid6_algo_add(&raid6_intx4);
+ raid6_algo_add(&raid6_intx8);
+}
+
+void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo)
+{
+ if (WARN_ON_ONCE(raid6_recov_algo))
+ return;
+ raid6_recov_algo = algo;
+}
+
+#ifdef CONFIG_RAID6_PQ_ARCH
+#include "pq_arch.h"
+#else
+static inline void arch_raid6_init(void)
+{
+ raid6_algo_add_default();
+}
+#endif /* CONFIG_RAID6_PQ_ARCH */
+
+static int __init raid6_init(void)
+{
+ /*
+ * Architectures providing arch_raid6_init must add all PQ generation
+ * algorithms they want to consider in arch_raid6_init(), including
+ * the generic ones using raid6_algo_add_default() if wanted.
+ */
+ arch_raid6_init();
+
+ /*
+ * Architectures don't have to set a recovery algorithm, we'll just pick
+ * the generic integer one if none was set.
+ */
+ if (!raid6_recov_algo)
+ raid6_recov_algo = &raid6_recov_intx1;
+ pr_info("raid6: using %s recovery algorithm\n", raid6_recov_algo->name);
+
+#ifdef MODULE
+ return raid6_select_algo();
+#else
+ return 0;
+#endif
}
-static void raid6_exit(void)
+static void __exit raid6_exit(void)
{
- do { } while (0);
}
-subsys_initcall(raid6_select_algo);
+/*
+ * When built-in we must register the default template before md, but we don't
+ * want calibration to run that early as that would delay the boot process.
+ */
+#ifndef MODULE
+__initcall(raid6_select_algo);
+#endif
+core_initcall(raid6_init);
module_exit(raid6_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("RAID6 Q-syndrome calculations");
+
+#if IS_ENABLED(CONFIG_RAID6_PQ_KUNIT_TEST)
+const struct raid6_calls *raid6_algo_find(unsigned int idx)
+{
+ if (idx >= raid6_nr_algos) {
+ /*
+ * Always include the simplest generic integer implementation in
+ * the unit tests as a baseline.
+ */
+ if (idx == raid6_nr_algos &&
+ raid6_algos[0] != &raid6_intx1)
+ return &raid6_intx1;
+ return NULL;
+ }
+ return raid6_algos[idx];
+}
+EXPORT_SYMBOL_IF_KUNIT(raid6_algo_find);
+
+const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx)
+{
+ switch (idx) {
+ case 0:
+ /* always test the generic integer implementation */
+ return &raid6_recov_intx1;
+ case 1:
+ /* test the optimized implementation if there is one */
+ if (raid6_recov_algo != &raid6_recov_intx1)
+ return raid6_recov_algo;
+ return NULL;
+ default:
+ return NULL;
+ }
+}
+EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algo_find);
+#endif /* CONFIG_RAID6_PQ_KUNIT_TEST */
diff --git a/lib/raid/raid6/algos.h b/lib/raid/raid6/algos.h
index e5f1098d2179..43f636be183f 100644
--- a/lib/raid/raid6/algos.h
+++ b/lib/raid/raid6/algos.h
@@ -5,6 +5,7 @@
#ifndef _PQ_IMPL_H
#define _PQ_IMPL_H
+#include <linux/init.h>
#include <linux/raid/pq_tables.h>
/* Routine choices */
@@ -13,70 +14,28 @@ struct raid6_calls {
void (*gen_syndrome)(int disks, size_t bytes, void **ptrs);
void (*xor_syndrome)(int disks, int start, int stop, size_t bytes,
void **ptrs);
- int (*valid)(void); /* Returns 1 if this routine set is usable */
- int priority; /* Relative priority ranking if non-zero */
};
-/* Various routine sets */
-extern const struct raid6_calls raid6_intx1;
-extern const struct raid6_calls raid6_intx2;
-extern const struct raid6_calls raid6_intx4;
-extern const struct raid6_calls raid6_intx8;
-extern const struct raid6_calls raid6_mmxx1;
-extern const struct raid6_calls raid6_mmxx2;
-extern const struct raid6_calls raid6_sse1x1;
-extern const struct raid6_calls raid6_sse1x2;
-extern const struct raid6_calls raid6_sse2x1;
-extern const struct raid6_calls raid6_sse2x2;
-extern const struct raid6_calls raid6_sse2x4;
-extern const struct raid6_calls raid6_altivec1;
-extern const struct raid6_calls raid6_altivec2;
-extern const struct raid6_calls raid6_altivec4;
-extern const struct raid6_calls raid6_altivec8;
-extern const struct raid6_calls raid6_avx2x1;
-extern const struct raid6_calls raid6_avx2x2;
-extern const struct raid6_calls raid6_avx2x4;
-extern const struct raid6_calls raid6_avx512x1;
-extern const struct raid6_calls raid6_avx512x2;
-extern const struct raid6_calls raid6_avx512x4;
-extern const struct raid6_calls raid6_s390vx8;
-extern const struct raid6_calls raid6_vpermxor1;
-extern const struct raid6_calls raid6_vpermxor2;
-extern const struct raid6_calls raid6_vpermxor4;
-extern const struct raid6_calls raid6_vpermxor8;
-extern const struct raid6_calls raid6_lsx;
-extern const struct raid6_calls raid6_lasx;
-extern const struct raid6_calls raid6_rvvx1;
-extern const struct raid6_calls raid6_rvvx2;
-extern const struct raid6_calls raid6_rvvx4;
-extern const struct raid6_calls raid6_rvvx8;
-
struct raid6_recov_calls {
const char *name;
void (*data2)(int disks, size_t bytes, int faila, int failb,
void **ptrs);
void (*datap)(int disks, size_t bytes, int faila, void **ptrs);
- int (*valid)(void);
- int priority;
};
-extern const struct raid6_recov_calls raid6_recov_intx1;
-extern const struct raid6_recov_calls raid6_recov_ssse3;
-extern const struct raid6_recov_calls raid6_recov_avx2;
-extern const struct raid6_recov_calls raid6_recov_avx512;
-extern const struct raid6_recov_calls raid6_recov_s390xc;
-extern const struct raid6_recov_calls raid6_recov_neon;
-extern const struct raid6_recov_calls raid6_recov_lsx;
-extern const struct raid6_recov_calls raid6_recov_lasx;
-extern const struct raid6_recov_calls raid6_recov_rvv;
+void __init raid6_algo_add(const struct raid6_calls *algo);
+void __init raid6_algo_add_default(void);
+void __init raid6_recov_algo_add(const struct raid6_recov_calls *algo);
-extern const struct raid6_calls raid6_neonx1;
-extern const struct raid6_calls raid6_neonx2;
-extern const struct raid6_calls raid6_neonx4;
-extern const struct raid6_calls raid6_neonx8;
+/* for the kunit test */
+const struct raid6_calls *raid6_algo_find(unsigned int idx);
+const struct raid6_recov_calls *raid6_recov_algo_find(unsigned int idx);
-/* Algorithm list */
-extern const struct raid6_calls * const raid6_algos[];
-extern const struct raid6_recov_calls *const raid6_recov_algos[];
+/* generic implementations */
+extern const struct raid6_calls raid6_intx1;
+extern const struct raid6_calls raid6_intx2;
+extern const struct raid6_calls raid6_intx4;
+extern const struct raid6_calls raid6_intx8;
+extern const struct raid6_recov_calls raid6_recov_intx1;
#endif /* _PQ_IMPL_H */
diff --git a/lib/raid/raid6/arm/neon.c b/lib/raid/raid6/arm/neon.c
index bd4ec4c86ee8..341c61af675e 100644
--- a/lib/raid/raid6/arm/neon.c
+++ b/lib/raid/raid6/arm/neon.c
@@ -42,15 +42,9 @@
struct raid6_calls const raid6_neonx ## _n = { \
.gen_syndrome = raid6_neon ## _n ## _gen_syndrome, \
.xor_syndrome = raid6_neon ## _n ## _xor_syndrome, \
- .valid = raid6_have_neon, \
.name = "neonx" #_n, \
}
-static int raid6_have_neon(void)
-{
- return cpu_has_neon();
-}
-
RAID6_NEON_WRAPPER(1);
RAID6_NEON_WRAPPER(2);
RAID6_NEON_WRAPPER(4);
diff --git a/lib/raid/raid6/arm/pq_arch.h b/lib/raid/raid6/arm/pq_arch.h
new file mode 100644
index 000000000000..f2bbef00cd80
--- /dev/null
+++ b/lib/raid/raid6/arm/pq_arch.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+extern const struct raid6_calls raid6_neonx1;
+extern const struct raid6_calls raid6_neonx2;
+extern const struct raid6_calls raid6_neonx4;
+extern const struct raid6_calls raid6_neonx8;
+extern const struct raid6_recov_calls raid6_recov_neon;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+ if (cpu_has_neon()) {
+ raid6_algo_add(&raid6_neonx1);
+ raid6_algo_add(&raid6_neonx2);
+ raid6_algo_add(&raid6_neonx4);
+ raid6_algo_add(&raid6_neonx8);
+ }
+
+ raid6_algo_add_default();
+
+ if (cpu_has_neon())
+ raid6_recov_algo_add(&raid6_recov_neon);
+}
diff --git a/lib/raid/raid6/arm/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c
index 2a2afbd9ead5..f7f7e83ea7ea 100644
--- a/lib/raid/raid6/arm/recov_neon.c
+++ b/lib/raid/raid6/arm/recov_neon.c
@@ -10,11 +10,6 @@
#include "algos.h"
#include "neon.h"
-static int raid6_has_neon(void)
-{
- return cpu_has_neon();
-}
-
static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
int failb, void **ptrs)
{
@@ -87,7 +82,5 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_neon = {
.data2 = raid6_2data_recov_neon,
.datap = raid6_datap_recov_neon,
- .valid = raid6_has_neon,
.name = "neon",
- .priority = 10,
};
diff --git a/lib/raid/raid6/loongarch/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c
index f77d11ce676e..c1eb53fafd27 100644
--- a/lib/raid/raid6/loongarch/loongarch_simd.c
+++ b/lib/raid/raid6/loongarch/loongarch_simd.c
@@ -26,11 +26,6 @@
#ifdef CONFIG_CPU_HAS_LSX
#define NSIZE 16
-static int raid6_has_lsx(void)
-{
- return cpu_has_lsx;
-}
-
static void raid6_lsx_gen_syndrome(int disks, size_t bytes, void **ptrs)
{
u8 **dptr = (u8 **)ptrs;
@@ -246,7 +241,6 @@ static void raid6_lsx_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_lsx = {
.gen_syndrome = raid6_lsx_gen_syndrome,
.xor_syndrome = raid6_lsx_xor_syndrome,
- .valid = raid6_has_lsx,
.name = "lsx",
};
@@ -256,11 +250,6 @@ const struct raid6_calls raid6_lsx = {
#ifdef CONFIG_CPU_HAS_LASX
#define NSIZE 32
-static int raid6_has_lasx(void)
-{
- return cpu_has_lasx;
-}
-
static void raid6_lasx_gen_syndrome(int disks, size_t bytes, void **ptrs)
{
u8 **dptr = (u8 **)ptrs;
@@ -414,7 +403,6 @@ static void raid6_lasx_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_lasx = {
.gen_syndrome = raid6_lasx_gen_syndrome,
.xor_syndrome = raid6_lasx_xor_syndrome,
- .valid = raid6_has_lasx,
.name = "lasx",
};
#undef NSIZE
diff --git a/lib/raid/raid6/loongarch/pq_arch.h b/lib/raid/raid6/loongarch/pq_arch.h
new file mode 100644
index 000000000000..cd4ee2df3f37
--- /dev/null
+++ b/lib/raid/raid6/loongarch/pq_arch.h
@@ -0,0 +1,23 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/cpu-features.h>
+
+extern const struct raid6_calls raid6_lsx;
+extern const struct raid6_calls raid6_lasx;
+
+extern const struct raid6_recov_calls raid6_recov_lsx;
+extern const struct raid6_recov_calls raid6_recov_lasx;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+ if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx)
+ raid6_algo_add(&raid6_lsx);
+ if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx)
+ raid6_algo_add(&raid6_lasx);
+ raid6_algo_add_default();
+
+ if (IS_ENABLED(CONFIG_CPU_HAS_LASX) && cpu_has_lasx)
+ raid6_recov_algo_add(&raid6_recov_lasx);
+ else if (IS_ENABLED(CONFIG_CPU_HAS_LSX) && cpu_has_lsx)
+ raid6_recov_algo_add(&raid6_recov_lsx);
+}
diff --git a/lib/raid/raid6/loongarch/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
index 0bbdc8b5c2e7..87a2313bbb4f 100644
--- a/lib/raid/raid6/loongarch/recov_loongarch_simd.c
+++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
@@ -24,11 +24,6 @@
*/
#ifdef CONFIG_CPU_HAS_LSX
-static int raid6_has_lsx(void)
-{
- return cpu_has_lsx;
-}
-
static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila,
int failb, void **ptrs)
{
@@ -291,18 +286,11 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_lsx = {
.data2 = raid6_2data_recov_lsx,
.datap = raid6_datap_recov_lsx,
- .valid = raid6_has_lsx,
.name = "lsx",
- .priority = 1,
};
#endif /* CONFIG_CPU_HAS_LSX */
#ifdef CONFIG_CPU_HAS_LASX
-static int raid6_has_lasx(void)
-{
- return cpu_has_lasx;
-}
-
static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila,
int failb, void **ptrs)
{
@@ -509,8 +497,6 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_lasx = {
.data2 = raid6_2data_recov_lasx,
.datap = raid6_datap_recov_lasx,
- .valid = raid6_has_lasx,
.name = "lasx",
- .priority = 2,
};
#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/lib/raid/raid6/powerpc/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc
index eb4a448cc88e..c5429fb71dd6 100644
--- a/lib/raid/raid6/powerpc/altivec.uc
+++ b/lib/raid/raid6/powerpc/altivec.uc
@@ -104,17 +104,7 @@ static void raid6_altivec$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
preempt_enable();
}
-int raid6_have_altivec(void);
-#if $# == 1
-int raid6_have_altivec(void)
-{
- /* This assumes either all CPUs have Altivec or none does */
- return cpu_has_feature(CPU_FTR_ALTIVEC);
-}
-#endif
-
const struct raid6_calls raid6_altivec$# = {
.gen_syndrome = raid6_altivec$#_gen_syndrome,
- .valid = raid6_have_altivec,
.name = "altivecx$#",
};
diff --git a/lib/raid/raid6/powerpc/pq_arch.h b/lib/raid/raid6/powerpc/pq_arch.h
new file mode 100644
index 000000000000..ecae7d3be131
--- /dev/null
+++ b/lib/raid/raid6/powerpc/pq_arch.h
@@ -0,0 +1,31 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/cputable.h>
+
+extern const struct raid6_calls raid6_altivec1;
+extern const struct raid6_calls raid6_altivec2;
+extern const struct raid6_calls raid6_altivec4;
+extern const struct raid6_calls raid6_altivec8;
+extern const struct raid6_calls raid6_vpermxor1;
+extern const struct raid6_calls raid6_vpermxor2;
+extern const struct raid6_calls raid6_vpermxor4;
+extern const struct raid6_calls raid6_vpermxor8;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+ /* This assumes either all CPUs have Altivec or none does */
+ if (cpu_has_feature(CPU_FTR_ALTIVEC)) {
+ raid6_algo_add(&raid6_altivec1);
+ raid6_algo_add(&raid6_altivec2);
+ raid6_algo_add(&raid6_altivec4);
+ raid6_algo_add(&raid6_altivec8);
+ }
+ if (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) &&
+ cpu_has_feature(CPU_FTR_ARCH_207S)) {
+ raid6_algo_add(&raid6_vpermxor1);
+ raid6_algo_add(&raid6_vpermxor2);
+ raid6_algo_add(&raid6_vpermxor4);
+ raid6_algo_add(&raid6_vpermxor8);
+ }
+ raid6_algo_add_default();
+}
diff --git a/lib/raid/raid6/powerpc/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc
index ec61f30bec11..e8964361aaef 100644
--- a/lib/raid/raid6/powerpc/vpermxor.uc
+++ b/lib/raid/raid6/powerpc/vpermxor.uc
@@ -76,18 +76,7 @@ static void raid6_vpermxor$#_gen_syndrome(int disks, size_t bytes, void **ptrs)
preempt_enable();
}
-int raid6_have_altivec_vpermxor(void);
-#if $# == 1
-int raid6_have_altivec_vpermxor(void)
-{
- /* Check if arch has both altivec and the vpermxor instructions */
- return (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) &&
- cpu_has_feature(CPU_FTR_ARCH_207S));
-}
-#endif
-
const struct raid6_calls raid6_vpermxor$# = {
.gen_syndrome = raid6_vpermxor$#_gen_syndrome,
- .valid = raid6_have_altivec_vpermxor,
.name = "vpermxor$#",
};
diff --git a/lib/raid/raid6/recov.c b/lib/raid/raid6/recov.c
index 735ab4013771..76eb2aef3667 100644
--- a/lib/raid/raid6/recov.c
+++ b/lib/raid/raid6/recov.c
@@ -97,7 +97,5 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_intx1 = {
.data2 = raid6_2data_recov_intx1,
.datap = raid6_datap_recov_intx1,
- .valid = NULL,
.name = "intx1",
- .priority = 0,
};
diff --git a/lib/raid/raid6/riscv/pq_arch.h b/lib/raid/raid6/riscv/pq_arch.h
new file mode 100644
index 000000000000..52dd01e9fc42
--- /dev/null
+++ b/lib/raid/raid6/riscv/pq_arch.h
@@ -0,0 +1,21 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/vector.h>
+
+extern const struct raid6_calls raid6_rvvx1;
+extern const struct raid6_calls raid6_rvvx2;
+extern const struct raid6_calls raid6_rvvx4;
+extern const struct raid6_calls raid6_rvvx8;
+extern const struct raid6_recov_calls raid6_recov_rvv;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+ if (has_vector()) {
+ raid6_algo_add(&raid6_rvvx1);
+ raid6_algo_add(&raid6_rvvx2);
+ raid6_algo_add(&raid6_rvvx4);
+ raid6_algo_add(&raid6_rvvx8);
+ raid6_recov_algo_add(&raid6_recov_rvv);
+ }
+ raid6_algo_add_default();
+}
diff --git a/lib/raid/raid6/riscv/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c
index 02120d245e22..2305940276dd 100644
--- a/lib/raid/raid6/riscv/recov_rvv.c
+++ b/lib/raid/raid6/riscv/recov_rvv.c
@@ -218,7 +218,5 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_rvv = {
.data2 = raid6_2data_recov_rvv,
.datap = raid6_datap_recov_rvv,
- .valid = rvv_has_vector,
.name = "rvv",
- .priority = 1,
};
diff --git a/lib/raid/raid6/riscv/rvv.h b/lib/raid/raid6/riscv/rvv.h
index c293130d798b..3a7c2468b1ea 100644
--- a/lib/raid/raid6/riscv/rvv.h
+++ b/lib/raid/raid6/riscv/rvv.h
@@ -10,11 +10,6 @@
#include <asm/vector.h>
#include "algos.h"
-static int rvv_has_vector(void)
-{
- return has_vector();
-}
-
#define RAID6_RVV_WRAPPER(_n) \
static void raid6_rvv ## _n ## _gen_syndrome(int disks, \
size_t bytes, void **ptrs) \
@@ -41,6 +36,5 @@ static int rvv_has_vector(void)
struct raid6_calls const raid6_rvvx ## _n = { \
.gen_syndrome = raid6_rvv ## _n ## _gen_syndrome, \
.xor_syndrome = raid6_rvv ## _n ## _xor_syndrome, \
- .valid = rvv_has_vector, \
.name = "rvvx" #_n, \
}
diff --git a/lib/raid/raid6/s390/pq_arch.h b/lib/raid/raid6/s390/pq_arch.h
new file mode 100644
index 000000000000..95d14c342306
--- /dev/null
+++ b/lib/raid/raid6/s390/pq_arch.h
@@ -0,0 +1,15 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <linux/cpufeature.h>
+
+extern const struct raid6_calls raid6_s390vx8;
+extern const struct raid6_recov_calls raid6_recov_s390xc;
+
+static __always_inline void __init arch_raid6_init(void)
+{
+ if (cpu_has_vx())
+ raid6_algo_add(&raid6_s390vx8);
+ else
+ raid6_algo_add_default();
+ raid6_recov_algo_add(&raid6_recov_s390xc);
+}
diff --git a/lib/raid/raid6/s390/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c
index e7b3409f21e2..08d56896e5ea 100644
--- a/lib/raid/raid6/s390/recov_s390xc.c
+++ b/lib/raid/raid6/s390/recov_s390xc.c
@@ -112,7 +112,5 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_s390xc = {
.data2 = raid6_2data_recov_s390xc,
.datap = raid6_datap_recov_s390xc,
- .valid = NULL,
.name = "s390xc",
- .priority = 1,
};
diff --git a/lib/raid/raid6/s390/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc
index aba3515eacac..e5cf9054be2a 100644
--- a/lib/raid/raid6/s390/s390vx.uc
+++ b/lib/raid/raid6/s390/s390vx.uc
@@ -121,15 +121,8 @@ static void raid6_s390vx$#_xor_syndrome(int disks, int start, int stop,
kernel_fpu_end(&vxstate, KERNEL_VXR);
}
-static int raid6_s390vx$#_valid(void)
-{
- return cpu_has_vx();
-}
-
const struct raid6_calls raid6_s390vx$# = {
.gen_syndrome = raid6_s390vx$#_gen_syndrome,
.xor_syndrome = raid6_s390vx$#_xor_syndrome,
- .valid = raid6_s390vx$#_valid,
.name = "vx128x$#",
- .priority = 1,
};
diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c
index 1a6168096456..abc854cf4a98 100644
--- a/lib/raid/raid6/tests/raid6_kunit.c
+++ b/lib/raid/raid6/tests/raid6_kunit.c
@@ -91,19 +91,20 @@ static void test_disks(struct kunit *test, const struct raid6_calls *calls,
static void raid6_test(struct kunit *test)
{
- const struct raid6_calls *const *algo;
- const struct raid6_recov_calls *const *ra;
int i, j, p1, p2;
+ unsigned int r, g;
- for (ra = raid6_recov_algos; *ra; ra++) {
- if ((*ra)->valid && !(*ra)->valid())
- continue;
+ for (r = 0; ; r++) {
+ const struct raid6_recov_calls *ra = raid6_recov_algo_find(r);
- for (algo = raid6_algos; *algo; algo++) {
- const struct raid6_calls *calls = *algo;
+ if (!ra)
+ break;
- if (calls->valid && !calls->valid())
- continue;
+ for (g = 0; ; g++) {
+ const struct raid6_calls *calls = raid6_algo_find(g);
+
+ if (!calls)
+ break;
/* Nuke syndromes */
memset(data[NDISKS - 2], 0xee, PAGE_SIZE);
@@ -115,7 +116,7 @@ static void raid6_test(struct kunit *test)
for (i = 0; i < NDISKS-1; i++)
for (j = i+1; j < NDISKS; j++)
- test_disks(test, calls, *ra, i, j);
+ test_disks(test, calls, ra, i, j);
if (!calls->xor_syndrome)
continue;
@@ -133,7 +134,7 @@ static void raid6_test(struct kunit *test)
for (i = 0; i < NDISKS-1; i++)
for (j = i+1; j < NDISKS; j++)
test_disks(test, calls,
- *ra, i, j);
+ ra, i, j);
}
}
diff --git a/lib/raid/raid6/x86/avx2.c b/lib/raid/raid6/x86/avx2.c
index 0bf831799082..7efd94e6a87a 100644
--- a/lib/raid/raid6/x86/avx2.c
+++ b/lib/raid/raid6/x86/avx2.c
@@ -24,11 +24,6 @@ static const struct raid6_avx2_constants {
0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
};
-static int raid6_have_avx2(void)
-{
- return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX);
-}
-
/*
* Plain AVX2 implementation
*/
@@ -131,10 +126,7 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_avx2x1 = {
.gen_syndrome = raid6_avx21_gen_syndrome,
.xor_syndrome = raid6_avx21_xor_syndrome,
- .valid = raid6_have_avx2,
.name = "avx2x1",
- /* Prefer AVX2 over priority 1 (SSE2 and others) */
- .priority = 2,
};
/*
@@ -262,10 +254,7 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_avx2x2 = {
.gen_syndrome = raid6_avx22_gen_syndrome,
.xor_syndrome = raid6_avx22_xor_syndrome,
- .valid = raid6_have_avx2,
.name = "avx2x2",
- /* Prefer AVX2 over priority 1 (SSE2 and others) */
- .priority = 2,
};
#ifdef CONFIG_X86_64
@@ -466,9 +455,6 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_avx2x4 = {
.gen_syndrome = raid6_avx24_gen_syndrome,
.xor_syndrome = raid6_avx24_xor_syndrome,
- .valid = raid6_have_avx2,
.name = "avx2x4",
- /* Prefer AVX2 over priority 1 (SSE2 and others) */
- .priority = 2,
};
#endif /* CONFIG_X86_64 */
diff --git a/lib/raid/raid6/x86/avx512.c b/lib/raid/raid6/x86/avx512.c
index 98ed42fb0a46..0772e798b742 100644
--- a/lib/raid/raid6/x86/avx512.c
+++ b/lib/raid/raid6/x86/avx512.c
@@ -30,16 +30,6 @@ static const struct raid6_avx512_constants {
0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL,},
};
-static int raid6_have_avx512(void)
-{
- return boot_cpu_has(X86_FEATURE_AVX2) &&
- boot_cpu_has(X86_FEATURE_AVX) &&
- boot_cpu_has(X86_FEATURE_AVX512F) &&
- boot_cpu_has(X86_FEATURE_AVX512BW) &&
- boot_cpu_has(X86_FEATURE_AVX512VL) &&
- boot_cpu_has(X86_FEATURE_AVX512DQ);
-}
-
static void raid6_avx5121_gen_syndrome(int disks, size_t bytes, void **ptrs)
{
u8 **dptr = (u8 **)ptrs;
@@ -159,10 +149,7 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_avx512x1 = {
.gen_syndrome = raid6_avx5121_gen_syndrome,
.xor_syndrome = raid6_avx5121_xor_syndrome,
- .valid = raid6_have_avx512,
.name = "avx512x1",
- /* Prefer AVX512 over priority 1 (SSE2 and others) */
- .priority = 2,
};
/*
@@ -317,10 +304,7 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_avx512x2 = {
.gen_syndrome = raid6_avx5122_gen_syndrome,
.xor_syndrome = raid6_avx5122_xor_syndrome,
- .valid = raid6_have_avx512,
.name = "avx512x2",
- /* Prefer AVX512 over priority 1 (SSE2 and others) */
- .priority = 2,
};
#ifdef CONFIG_X86_64
@@ -556,9 +540,6 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_avx512x4 = {
.gen_syndrome = raid6_avx5124_gen_syndrome,
.xor_syndrome = raid6_avx5124_xor_syndrome,
- .valid = raid6_have_avx512,
.name = "avx512x4",
- /* Prefer AVX512 over priority 1 (SSE2 and others) */
- .priority = 2,
};
#endif
diff --git a/lib/raid/raid6/x86/mmx.c b/lib/raid/raid6/x86/mmx.c
index 052d9f010bfe..3228c335965a 100644
--- a/lib/raid/raid6/x86/mmx.c
+++ b/lib/raid/raid6/x86/mmx.c
@@ -22,12 +22,6 @@ const struct raid6_mmx_constants {
0x1d1d1d1d1d1d1d1dULL,
};
-static int raid6_have_mmx(void)
-{
- /* Not really "boot_cpu" but "all_cpus" */
- return boot_cpu_has(X86_FEATURE_MMX);
-}
-
/*
* Plain MMX implementation
*/
@@ -70,7 +64,6 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_mmxx1 = {
.gen_syndrome = raid6_mmx1_gen_syndrome,
- .valid = raid6_have_mmx,
.name = "mmxx1",
};
@@ -127,6 +120,5 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_mmxx2 = {
.gen_syndrome = raid6_mmx2_gen_syndrome,
- .valid = raid6_have_mmx,
.name = "mmxx2",
};
diff --git a/lib/raid/raid6/x86/pq_arch.h b/lib/raid/raid6/x86/pq_arch.h
new file mode 100644
index 000000000000..09838cad630b
--- /dev/null
+++ b/lib/raid/raid6/x86/pq_arch.h
@@ -0,0 +1,96 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+
+#include <asm/cpufeature.h>
+
+extern const struct raid6_calls raid6_mmxx1;
+extern const struct raid6_calls raid6_mmxx2;
+extern const struct raid6_calls raid6_sse1x1;
+extern const struct raid6_calls raid6_sse1x2;
+extern const struct raid6_calls raid6_sse2x1;
+extern const struct raid6_calls raid6_sse2x2;
+extern const struct raid6_calls raid6_sse2x4;
+extern const struct raid6_calls raid6_avx2x1;
+extern const struct raid6_calls raid6_avx2x2;
+extern const struct raid6_calls raid6_avx2x4;
+extern const struct raid6_calls raid6_avx512x1;
+extern const struct raid6_calls raid6_avx512x2;
+extern const struct raid6_calls raid6_avx512x4;
+
+extern const struct raid6_recov_calls raid6_recov_ssse3;
+extern const struct raid6_recov_calls raid6_recov_avx2;
+extern const struct raid6_recov_calls raid6_recov_avx512;
+
+static inline int raid6_has_avx512(void)
+{
+ return boot_cpu_has(X86_FEATURE_AVX2) &&
+ boot_cpu_has(X86_FEATURE_AVX) &&
+ boot_cpu_has(X86_FEATURE_AVX512F) &&
+ boot_cpu_has(X86_FEATURE_AVX512BW) &&
+ boot_cpu_has(X86_FEATURE_AVX512VL) &&
+ boot_cpu_has(X86_FEATURE_AVX512DQ);
+}
+
+static inline bool raid6_has_avx2(void)
+{
+ return boot_cpu_has(X86_FEATURE_AVX2) && boot_cpu_has(X86_FEATURE_AVX);
+}
+
+static inline bool raid6_has_ssse3(void)
+{
+ return boot_cpu_has(X86_FEATURE_XMM) &&
+ boot_cpu_has(X86_FEATURE_XMM2) &&
+ boot_cpu_has(X86_FEATURE_SSSE3);
+}
+
+static inline bool raid6_has_sse2(void)
+{
+ return boot_cpu_has(X86_FEATURE_MMX) &&
+ boot_cpu_has(X86_FEATURE_FXSR) &&
+ boot_cpu_has(X86_FEATURE_XMM) &&
+ boot_cpu_has(X86_FEATURE_XMM2);
+}
+
+static inline bool raid6_has_sse1_or_mmxext(void)
+{
+ return boot_cpu_has(X86_FEATURE_MMX) &&
+ (boot_cpu_has(X86_FEATURE_XMM) ||
+ boot_cpu_has(X86_FEATURE_MMXEXT));
+}
+
+static __always_inline void __init arch_raid6_init(void)
+{
+ if (raid6_has_avx2()) {
+ if (raid6_has_avx512()) {
+ raid6_algo_add(&raid6_avx512x1);
+ raid6_algo_add(&raid6_avx512x2);
+ if (IS_ENABLED(CONFIG_X86_64))
+ raid6_algo_add(&raid6_avx512x4);
+ }
+ raid6_algo_add(&raid6_avx2x1);
+ raid6_algo_add(&raid6_avx2x2);
+ if (IS_ENABLED(CONFIG_X86_64))
+ raid6_algo_add(&raid6_avx2x4);
+ } else if (IS_ENABLED(CONFIG_X86_64) || raid6_has_sse2()) {
+ raid6_algo_add(&raid6_sse2x1);
+ raid6_algo_add(&raid6_sse2x2);
+ if (IS_ENABLED(CONFIG_X86_64))
+ raid6_algo_add(&raid6_sse2x4);
+ } else if (raid6_has_sse1_or_mmxext()) {
+ raid6_algo_add(&raid6_sse1x1);
+ raid6_algo_add(&raid6_sse1x2);
+ raid6_algo_add_default();
+ } else {
+ if (boot_cpu_has(X86_FEATURE_MMX)) {
+ raid6_algo_add(&raid6_mmxx1);
+ raid6_algo_add(&raid6_mmxx2);
+ }
+ raid6_algo_add_default();
+ }
+
+ if (raid6_has_avx512())
+ raid6_recov_algo_add(&raid6_recov_avx512);
+ else if (raid6_has_avx2())
+ raid6_recov_algo_add(&raid6_recov_avx2);
+ else if (raid6_has_ssse3())
+ raid6_recov_algo_add(&raid6_recov_ssse3);
+}
diff --git a/lib/raid/raid6/x86/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c
index 06c6e05763bc..a714a780a2d8 100644
--- a/lib/raid/raid6/x86/recov_avx2.c
+++ b/lib/raid/raid6/x86/recov_avx2.c
@@ -9,12 +9,6 @@
#include <asm/fpu/api.h>
#include "algos.h"
-static int raid6_has_avx2(void)
-{
- return boot_cpu_has(X86_FEATURE_AVX2) &&
- boot_cpu_has(X86_FEATURE_AVX);
-}
-
static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila,
int failb, void **ptrs)
{
@@ -305,11 +299,9 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_avx2 = {
.data2 = raid6_2data_recov_avx2,
.datap = raid6_datap_recov_avx2,
- .valid = raid6_has_avx2,
#ifdef CONFIG_X86_64
.name = "avx2x2",
#else
.name = "avx2x1",
#endif
- .priority = 2,
};
diff --git a/lib/raid/raid6/x86/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c
index 850bb962b514..ec72d5a30c01 100644
--- a/lib/raid/raid6/x86/recov_avx512.c
+++ b/lib/raid/raid6/x86/recov_avx512.c
@@ -11,16 +11,6 @@
#include <asm/fpu/api.h>
#include "algos.h"
-static int raid6_has_avx512(void)
-{
- return boot_cpu_has(X86_FEATURE_AVX2) &&
- boot_cpu_has(X86_FEATURE_AVX) &&
- boot_cpu_has(X86_FEATURE_AVX512F) &&
- boot_cpu_has(X86_FEATURE_AVX512BW) &&
- boot_cpu_has(X86_FEATURE_AVX512VL) &&
- boot_cpu_has(X86_FEATURE_AVX512DQ);
-}
-
static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
int failb, void **ptrs)
{
@@ -369,11 +359,9 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_avx512 = {
.data2 = raid6_2data_recov_avx512,
.datap = raid6_datap_recov_avx512,
- .valid = raid6_has_avx512,
#ifdef CONFIG_X86_64
.name = "avx512x2",
#else
.name = "avx512x1",
#endif
- .priority = 3,
};
diff --git a/lib/raid/raid6/x86/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c
index 95589c33003a..700bd2c865ec 100644
--- a/lib/raid/raid6/x86/recov_ssse3.c
+++ b/lib/raid/raid6/x86/recov_ssse3.c
@@ -8,13 +8,6 @@
#include <asm/fpu/api.h>
#include "algos.h"
-static int raid6_has_ssse3(void)
-{
- return boot_cpu_has(X86_FEATURE_XMM) &&
- boot_cpu_has(X86_FEATURE_XMM2) &&
- boot_cpu_has(X86_FEATURE_SSSE3);
-}
-
static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila,
int failb, void **ptrs)
{
@@ -320,11 +313,9 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila,
const struct raid6_recov_calls raid6_recov_ssse3 = {
.data2 = raid6_2data_recov_ssse3,
.datap = raid6_datap_recov_ssse3,
- .valid = raid6_has_ssse3,
#ifdef CONFIG_X86_64
.name = "ssse3x2",
#else
.name = "ssse3x1",
#endif
- .priority = 1,
};
diff --git a/lib/raid/raid6/x86/sse1.c b/lib/raid/raid6/x86/sse1.c
index 7004255a0bb1..6ebdcf824e00 100644
--- a/lib/raid/raid6/x86/sse1.c
+++ b/lib/raid/raid6/x86/sse1.c
@@ -25,14 +25,6 @@ extern const struct raid6_mmx_constants {
u64 x1d;
} raid6_mmx_constants;
-static int raid6_have_sse1_or_mmxext(void)
-{
- /* Not really boot_cpu but "all_cpus" */
- return boot_cpu_has(X86_FEATURE_MMX) &&
- (boot_cpu_has(X86_FEATURE_XMM) ||
- boot_cpu_has(X86_FEATURE_MMXEXT));
-}
-
/*
* Plain SSE1 implementation
*/
@@ -86,9 +78,7 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_sse1x1 = {
.gen_syndrome = raid6_sse11_gen_syndrome,
- .valid = raid6_have_sse1_or_mmxext,
.name = "sse1x1",
- .priority = 1, /* Has cache hints */
};
/*
@@ -148,7 +138,5 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs)
const struct raid6_calls raid6_sse1x2 = {
.gen_syndrome = raid6_sse12_gen_syndrome,
- .valid = raid6_have_sse1_or_mmxext,
.name = "sse1x2",
- .priority = 1, /* Has cache hints */
};
diff --git a/lib/raid/raid6/x86/sse2.c b/lib/raid/raid6/x86/sse2.c
index f30be4ee14d0..7049c8512f35 100644
--- a/lib/raid/raid6/x86/sse2.c
+++ b/lib/raid/raid6/x86/sse2.c
@@ -22,15 +22,6 @@ static const struct raid6_sse_constants {
{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
};
-static int raid6_have_sse2(void)
-{
- /* Not really boot_cpu but "all_cpus" */
- return boot_cpu_has(X86_FEATURE_MMX) &&
- boot_cpu_has(X86_FEATURE_FXSR) &&
- boot_cpu_has(X86_FEATURE_XMM) &&
- boot_cpu_has(X86_FEATURE_XMM2);
-}
-
/*
* Plain SSE2 implementation
*/
@@ -136,9 +127,7 @@ static void raid6_sse21_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_sse2x1 = {
.gen_syndrome = raid6_sse21_gen_syndrome,
.xor_syndrome = raid6_sse21_xor_syndrome,
- .valid = raid6_have_sse2,
.name = "sse2x1",
- .priority = 1, /* Has cache hints */
};
/*
@@ -266,9 +255,7 @@ static void raid6_sse22_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_sse2x2 = {
.gen_syndrome = raid6_sse22_gen_syndrome,
.xor_syndrome = raid6_sse22_xor_syndrome,
- .valid = raid6_have_sse2,
.name = "sse2x2",
- .priority = 1, /* Has cache hints */
};
#ifdef CONFIG_X86_64
@@ -473,9 +460,7 @@ static void raid6_sse24_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_sse2x4 = {
.gen_syndrome = raid6_sse24_gen_syndrome,
.xor_syndrome = raid6_sse24_xor_syndrome,
- .valid = raid6_have_sse2,
.name = "sse2x4",
- .priority = 1, /* Has cache hints */
};
#endif /* CONFIG_X86_64 */
--
2.47.3
^ permalink raw reply related
* [PATCH 08/17] raid6: hide internals
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
Split out two new headers from the public pq.h:
- lib/raid/raid6/algos.h contains the algorithm lists private to
lib/raid/raid6
- include/linux/raid/pq_tables.h contains the tables also used by
async_tx providers.
The public include/linux/pq.h is now limited to the public interface for
the consumers of the RAID6 PQ API.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
crypto/async_tx/async_pq.c | 1 +
crypto/async_tx/async_raid6_recov.c | 1 +
drivers/dma/bcm-sba-raid.c | 1 +
include/linux/raid/pq.h | 96 ++-----------------
include/linux/raid/pq_tables.h | 19 ++++
lib/raid/raid6/Makefile | 2 +
lib/raid/raid6/algos.c | 3 +-
lib/raid/raid6/algos.h | 82 ++++++++++++++++
lib/raid/raid6/arm/neon.c | 2 +-
lib/raid/raid6/arm/recov_neon.c | 2 +
lib/raid/raid6/int.uc | 2 +-
lib/raid/raid6/loongarch/loongarch_simd.c | 2 +-
.../raid6/loongarch/recov_loongarch_simd.c | 2 +
lib/raid/raid6/mktables.c | 2 +-
lib/raid/raid6/powerpc/altivec.uc | 2 +-
lib/raid/raid6/powerpc/vpermxor.uc | 2 +-
lib/raid/raid6/recov.c | 2 +
lib/raid/raid6/riscv/recov_rvv.c | 2 +
lib/raid/raid6/riscv/rvv.h | 2 +-
lib/raid/raid6/s390/recov_s390xc.c | 2 +
lib/raid/raid6/s390/s390vx.uc | 2 +-
lib/raid/raid6/tests/raid6_kunit.c | 2 +-
lib/raid/raid6/x86/avx2.c | 3 +-
lib/raid/raid6/x86/avx512.c | 3 +-
lib/raid/raid6/x86/mmx.c | 3 +-
lib/raid/raid6/x86/recov_avx2.c | 2 +
lib/raid/raid6/x86/recov_avx512.c | 2 +
lib/raid/raid6/x86/recov_ssse3.c | 2 +
lib/raid/raid6/x86/sse1.c | 3 +-
lib/raid/raid6/x86/sse2.c | 3 +-
30 files changed, 151 insertions(+), 103 deletions(-)
create mode 100644 include/linux/raid/pq_tables.h
create mode 100644 lib/raid/raid6/algos.h
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
index f3574f80d1df..27f99349e310 100644
--- a/crypto/async_tx/async_pq.c
+++ b/crypto/async_tx/async_pq.c
@@ -8,6 +8,7 @@
#include <linux/module.h>
#include <linux/dma-mapping.h>
#include <linux/raid/pq.h>
+#include <linux/raid/pq_tables.h>
#include <linux/async_tx.h>
#include <linux/gfp.h>
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
index 305ea1421a3e..e53870d84bc5 100644
--- a/crypto/async_tx/async_raid6_recov.c
+++ b/crypto/async_tx/async_raid6_recov.c
@@ -11,6 +11,7 @@
#include <linux/module.h>
#include <linux/dma-mapping.h>
#include <linux/raid/pq.h>
+#include <linux/raid/pq_tables.h>
#include <linux/async_tx.h>
#include <linux/dmaengine.h>
diff --git a/drivers/dma/bcm-sba-raid.c b/drivers/dma/bcm-sba-raid.c
index ed037fa883f6..0de03611252e 100644
--- a/drivers/dma/bcm-sba-raid.c
+++ b/drivers/dma/bcm-sba-raid.c
@@ -40,6 +40,7 @@
#include <linux/platform_device.h>
#include <linux/slab.h>
#include <linux/raid/pq.h>
+#include <linux/raid/pq_tables.h>
#include "dmaengine.h"
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 425a227591c0..1b4da8b98828 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -1,15 +1,13 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* -*- linux-c -*- ------------------------------------------------------- *
+/*
+ * Copyright 2003 H. Peter Anvin - All Rights Reserved
*
- * Copyright 2003 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-#ifndef LINUX_RAID_RAID6_H
-#define LINUX_RAID_RAID6_H
+ * Public interface to the RAID6 P/Q calculation and recovery library.
+ */
+#ifndef LINUX_RAID_PQ_H
+#define LINUX_RAID_PQ_H
-#include <linux/blkdev.h>
-#include <linux/mm.h>
+#include <linux/types.h>
void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs);
void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes,
@@ -21,82 +19,4 @@ void raid6_recov_2data(int disks, size_t bytes, int faila, int failb,
void raid6_recov_datap(int disks, size_t bytes, int faila,
void **ptrs);
-/* Routine choices */
-struct raid6_calls {
- void (*gen_syndrome)(int, size_t, void **);
- void (*xor_syndrome)(int, int, int, size_t, void **);
- int (*valid)(void); /* Returns 1 if this routine set is usable */
- const char *name; /* Name of this routine set */
- int priority; /* Relative priority ranking if non-zero */
-};
-
-/* Various routine sets */
-extern const struct raid6_calls raid6_intx1;
-extern const struct raid6_calls raid6_intx2;
-extern const struct raid6_calls raid6_intx4;
-extern const struct raid6_calls raid6_intx8;
-extern const struct raid6_calls raid6_mmxx1;
-extern const struct raid6_calls raid6_mmxx2;
-extern const struct raid6_calls raid6_sse1x1;
-extern const struct raid6_calls raid6_sse1x2;
-extern const struct raid6_calls raid6_sse2x1;
-extern const struct raid6_calls raid6_sse2x2;
-extern const struct raid6_calls raid6_sse2x4;
-extern const struct raid6_calls raid6_altivec1;
-extern const struct raid6_calls raid6_altivec2;
-extern const struct raid6_calls raid6_altivec4;
-extern const struct raid6_calls raid6_altivec8;
-extern const struct raid6_calls raid6_avx2x1;
-extern const struct raid6_calls raid6_avx2x2;
-extern const struct raid6_calls raid6_avx2x4;
-extern const struct raid6_calls raid6_avx512x1;
-extern const struct raid6_calls raid6_avx512x2;
-extern const struct raid6_calls raid6_avx512x4;
-extern const struct raid6_calls raid6_s390vx8;
-extern const struct raid6_calls raid6_vpermxor1;
-extern const struct raid6_calls raid6_vpermxor2;
-extern const struct raid6_calls raid6_vpermxor4;
-extern const struct raid6_calls raid6_vpermxor8;
-extern const struct raid6_calls raid6_lsx;
-extern const struct raid6_calls raid6_lasx;
-extern const struct raid6_calls raid6_rvvx1;
-extern const struct raid6_calls raid6_rvvx2;
-extern const struct raid6_calls raid6_rvvx4;
-extern const struct raid6_calls raid6_rvvx8;
-
-struct raid6_recov_calls {
- void (*data2)(int, size_t, int, int, void **);
- void (*datap)(int, size_t, int, void **);
- int (*valid)(void);
- const char *name;
- int priority;
-};
-
-extern const struct raid6_recov_calls raid6_recov_intx1;
-extern const struct raid6_recov_calls raid6_recov_ssse3;
-extern const struct raid6_recov_calls raid6_recov_avx2;
-extern const struct raid6_recov_calls raid6_recov_avx512;
-extern const struct raid6_recov_calls raid6_recov_s390xc;
-extern const struct raid6_recov_calls raid6_recov_neon;
-extern const struct raid6_recov_calls raid6_recov_lsx;
-extern const struct raid6_recov_calls raid6_recov_lasx;
-extern const struct raid6_recov_calls raid6_recov_rvv;
-
-extern const struct raid6_calls raid6_neonx1;
-extern const struct raid6_calls raid6_neonx2;
-extern const struct raid6_calls raid6_neonx4;
-extern const struct raid6_calls raid6_neonx8;
-
-/* Algorithm list */
-extern const struct raid6_calls * const raid6_algos[];
-extern const struct raid6_recov_calls *const raid6_recov_algos[];
-
-/* Galois field tables */
-extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256)));
-extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256)));
-extern const u8 raid6_gfexp[256] __attribute__((aligned(256)));
-extern const u8 raid6_gflog[256] __attribute__((aligned(256)));
-extern const u8 raid6_gfinv[256] __attribute__((aligned(256)));
-extern const u8 raid6_gfexi[256] __attribute__((aligned(256)));
-
-#endif /* LINUX_RAID_RAID6_H */
+#endif /* LINUX_RAID_PQ_H */
diff --git a/include/linux/raid/pq_tables.h b/include/linux/raid/pq_tables.h
new file mode 100644
index 000000000000..7b1ebe675677
--- /dev/null
+++ b/include/linux/raid/pq_tables.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright 2003 H. Peter Anvin - All Rights Reserved
+ *
+ * Galois field tables for the Linux RAID6 P/Q parity algorithm.
+ */
+#ifndef _LINUX_RAID_PQ_TABLES_H
+#define _LINUX_RAID_PQ_TABLES_H
+
+#include <linux/types.h>
+
+extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256)));
+extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256)));
+extern const u8 raid6_gfexp[256] __attribute__((aligned(256)));
+extern const u8 raid6_gflog[256] __attribute__((aligned(256)));
+extern const u8 raid6_gfinv[256] __attribute__((aligned(256)));
+extern const u8 raid6_gfexi[256] __attribute__((aligned(256)));
+
+#endif /* _LINUX_RAID_PQ_TABLES_H */
diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile
index f155eba06fe3..fc46c3e61c82 100644
--- a/lib/raid/raid6/Makefile
+++ b/lib/raid/raid6/Makefile
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
+ccflags-y += -I $(src)
+
obj-$(CONFIG_RAID6_PQ) += raid6_pq.o tests/
raid6_pq-y += algos.o tables.o
diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c
index 4958c58e3f69..406efdcc03b8 100644
--- a/lib/raid/raid6/algos.c
+++ b/lib/raid/raid6/algos.c
@@ -11,10 +11,11 @@
* Algorithm list and algorithm selection for RAID-6
*/
-#include <linux/raid/pq.h>
#include <linux/module.h>
#include <linux/gfp.h>
+#include <linux/raid/pq.h>
#include <kunit/visibility.h>
+#include "algos.h"
static const struct raid6_recov_calls *raid6_recov_algo;
diff --git a/lib/raid/raid6/algos.h b/lib/raid/raid6/algos.h
new file mode 100644
index 000000000000..e5f1098d2179
--- /dev/null
+++ b/lib/raid/raid6/algos.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0-or-later */
+/*
+ * Copyright 2003 H. Peter Anvin - All Rights Reserved
+ */
+#ifndef _PQ_IMPL_H
+#define _PQ_IMPL_H
+
+#include <linux/raid/pq_tables.h>
+
+/* Routine choices */
+struct raid6_calls {
+ const char *name;
+ void (*gen_syndrome)(int disks, size_t bytes, void **ptrs);
+ void (*xor_syndrome)(int disks, int start, int stop, size_t bytes,
+ void **ptrs);
+ int (*valid)(void); /* Returns 1 if this routine set is usable */
+ int priority; /* Relative priority ranking if non-zero */
+};
+
+/* Various routine sets */
+extern const struct raid6_calls raid6_intx1;
+extern const struct raid6_calls raid6_intx2;
+extern const struct raid6_calls raid6_intx4;
+extern const struct raid6_calls raid6_intx8;
+extern const struct raid6_calls raid6_mmxx1;
+extern const struct raid6_calls raid6_mmxx2;
+extern const struct raid6_calls raid6_sse1x1;
+extern const struct raid6_calls raid6_sse1x2;
+extern const struct raid6_calls raid6_sse2x1;
+extern const struct raid6_calls raid6_sse2x2;
+extern const struct raid6_calls raid6_sse2x4;
+extern const struct raid6_calls raid6_altivec1;
+extern const struct raid6_calls raid6_altivec2;
+extern const struct raid6_calls raid6_altivec4;
+extern const struct raid6_calls raid6_altivec8;
+extern const struct raid6_calls raid6_avx2x1;
+extern const struct raid6_calls raid6_avx2x2;
+extern const struct raid6_calls raid6_avx2x4;
+extern const struct raid6_calls raid6_avx512x1;
+extern const struct raid6_calls raid6_avx512x2;
+extern const struct raid6_calls raid6_avx512x4;
+extern const struct raid6_calls raid6_s390vx8;
+extern const struct raid6_calls raid6_vpermxor1;
+extern const struct raid6_calls raid6_vpermxor2;
+extern const struct raid6_calls raid6_vpermxor4;
+extern const struct raid6_calls raid6_vpermxor8;
+extern const struct raid6_calls raid6_lsx;
+extern const struct raid6_calls raid6_lasx;
+extern const struct raid6_calls raid6_rvvx1;
+extern const struct raid6_calls raid6_rvvx2;
+extern const struct raid6_calls raid6_rvvx4;
+extern const struct raid6_calls raid6_rvvx8;
+
+struct raid6_recov_calls {
+ const char *name;
+ void (*data2)(int disks, size_t bytes, int faila, int failb,
+ void **ptrs);
+ void (*datap)(int disks, size_t bytes, int faila, void **ptrs);
+ int (*valid)(void);
+ int priority;
+};
+
+extern const struct raid6_recov_calls raid6_recov_intx1;
+extern const struct raid6_recov_calls raid6_recov_ssse3;
+extern const struct raid6_recov_calls raid6_recov_avx2;
+extern const struct raid6_recov_calls raid6_recov_avx512;
+extern const struct raid6_recov_calls raid6_recov_s390xc;
+extern const struct raid6_recov_calls raid6_recov_neon;
+extern const struct raid6_recov_calls raid6_recov_lsx;
+extern const struct raid6_recov_calls raid6_recov_lasx;
+extern const struct raid6_recov_calls raid6_recov_rvv;
+
+extern const struct raid6_calls raid6_neonx1;
+extern const struct raid6_calls raid6_neonx2;
+extern const struct raid6_calls raid6_neonx4;
+extern const struct raid6_calls raid6_neonx8;
+
+/* Algorithm list */
+extern const struct raid6_calls * const raid6_algos[];
+extern const struct raid6_recov_calls *const raid6_recov_algos[];
+
+#endif /* _PQ_IMPL_H */
diff --git a/lib/raid/raid6/arm/neon.c b/lib/raid/raid6/arm/neon.c
index c21da59ab48f..bd4ec4c86ee8 100644
--- a/lib/raid/raid6/arm/neon.c
+++ b/lib/raid/raid6/arm/neon.c
@@ -5,8 +5,8 @@
* Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
-#include <linux/raid/pq.h>
#include <asm/simd.h>
+#include "algos.h"
/*
* There are 2 reasons these wrappers are kept in a separate compilation unit
diff --git a/lib/raid/raid6/arm/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c
index 21c26da6290a..2a2afbd9ead5 100644
--- a/lib/raid/raid6/arm/recov_neon.c
+++ b/lib/raid/raid6/arm/recov_neon.c
@@ -4,8 +4,10 @@
* Copyright (C) 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
*/
+#include <linux/mm.h>
#include <linux/raid/pq.h>
#include <asm/simd.h>
+#include "algos.h"
#include "neon.h"
static int raid6_has_neon(void)
diff --git a/lib/raid/raid6/int.uc b/lib/raid/raid6/int.uc
index 4f5f2869e21e..e63bd5a9c2ed 100644
--- a/lib/raid/raid6/int.uc
+++ b/lib/raid/raid6/int.uc
@@ -18,7 +18,7 @@
* This file is postprocessed using unroll.awk
*/
-#include <linux/raid/pq.h>
+#include "algos.h"
/*
* This is the C data type to use
diff --git a/lib/raid/raid6/loongarch/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c
index 1b4cd1512d05..f77d11ce676e 100644
--- a/lib/raid/raid6/loongarch/loongarch_simd.c
+++ b/lib/raid/raid6/loongarch/loongarch_simd.c
@@ -9,9 +9,9 @@
* Copyright 2002-2004 H. Peter Anvin
*/
-#include <linux/raid/pq.h>
#include <asm/cpu-features.h>
#include <asm/fpu.h>
+#include "algos.h"
/*
* The vector algorithms are currently priority 0, which means the generic
diff --git a/lib/raid/raid6/loongarch/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
index 7d4d349322b3..0bbdc8b5c2e7 100644
--- a/lib/raid/raid6/loongarch/recov_loongarch_simd.c
+++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
@@ -10,9 +10,11 @@
* Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
*/
+#include <linux/mm.h>
#include <linux/raid/pq.h>
#include <asm/cpu-features.h>
#include <asm/fpu.h>
+#include "algos.h"
/*
* Unlike with the syndrome calculation algorithms, there's no boot-time
diff --git a/lib/raid/raid6/mktables.c b/lib/raid/raid6/mktables.c
index 3de1dbf6846c..97a17493bbd8 100644
--- a/lib/raid/raid6/mktables.c
+++ b/lib/raid/raid6/mktables.c
@@ -57,7 +57,7 @@ int main(int argc, char *argv[])
uint8_t exptbl[256], invtbl[256];
printf("#include <linux/export.h>\n");
- printf("#include <linux/raid/pq.h>\n");
+ printf("#include \"algos.h\"\n");
/* Compute multiplication table */
printf("\nconst u8 __attribute__((aligned(256)))\n"
diff --git a/lib/raid/raid6/powerpc/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc
index 084ead768ddb..eb4a448cc88e 100644
--- a/lib/raid/raid6/powerpc/altivec.uc
+++ b/lib/raid/raid6/powerpc/altivec.uc
@@ -22,7 +22,7 @@
* bracked this with preempt_disable/enable or in a lock)
*/
-#include <linux/raid/pq.h>
+#include "algos.h"
#include <altivec.h>
#include <asm/cputable.h>
diff --git a/lib/raid/raid6/powerpc/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc
index bb2c3a316ae8..ec61f30bec11 100644
--- a/lib/raid/raid6/powerpc/vpermxor.uc
+++ b/lib/raid/raid6/powerpc/vpermxor.uc
@@ -20,11 +20,11 @@
* This instruction was introduced in POWER8 - ISA v2.07.
*/
-#include <linux/raid/pq.h>
#include <altivec.h>
#include <asm/ppc-opcode.h>
#include <asm/cputable.h>
#include <asm/switch_to.h>
+#include "algos.h"
typedef vector unsigned char unative_t;
#define NSIZE sizeof(unative_t)
diff --git a/lib/raid/raid6/recov.c b/lib/raid/raid6/recov.c
index cc7e4dc1eaa6..735ab4013771 100644
--- a/lib/raid/raid6/recov.c
+++ b/lib/raid/raid6/recov.c
@@ -13,7 +13,9 @@
* the syndrome.)
*/
+#include <linux/mm.h>
#include <linux/raid/pq.h>
+#include "algos.h"
/* Recover two failed data blocks. */
static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila,
diff --git a/lib/raid/raid6/riscv/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c
index 3ff39826e33f..02120d245e22 100644
--- a/lib/raid/raid6/riscv/recov_rvv.c
+++ b/lib/raid/raid6/riscv/recov_rvv.c
@@ -4,7 +4,9 @@
* Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
*/
+#include <linux/mm.h>
#include <linux/raid/pq.h>
+#include "algos.h"
#include "rvv.h"
static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp,
diff --git a/lib/raid/raid6/riscv/rvv.h b/lib/raid/raid6/riscv/rvv.h
index 0d430a4c5f08..c293130d798b 100644
--- a/lib/raid/raid6/riscv/rvv.h
+++ b/lib/raid/raid6/riscv/rvv.h
@@ -7,8 +7,8 @@
* Definitions for RISC-V RAID-6 code
*/
-#include <linux/raid/pq.h>
#include <asm/vector.h>
+#include "algos.h"
static int rvv_has_vector(void)
{
diff --git a/lib/raid/raid6/s390/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c
index 2bc4c85174de..e7b3409f21e2 100644
--- a/lib/raid/raid6/s390/recov_s390xc.c
+++ b/lib/raid/raid6/s390/recov_s390xc.c
@@ -6,7 +6,9 @@
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
*/
+#include <linux/mm.h>
#include <linux/raid/pq.h>
+#include "algos.h"
static inline void xor_block(u8 *p1, u8 *p2)
{
diff --git a/lib/raid/raid6/s390/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc
index 97c5d5d9dcf9..aba3515eacac 100644
--- a/lib/raid/raid6/s390/s390vx.uc
+++ b/lib/raid/raid6/s390/s390vx.uc
@@ -12,8 +12,8 @@
*/
#include <linux/cpufeature.h>
-#include <linux/raid/pq.h>
#include <asm/fpu.h>
+#include "algos.h"
#define NSIZE 16
diff --git a/lib/raid/raid6/tests/raid6_kunit.c b/lib/raid/raid6/tests/raid6_kunit.c
index 97e036b19049..1a6168096456 100644
--- a/lib/raid/raid6/tests/raid6_kunit.c
+++ b/lib/raid/raid6/tests/raid6_kunit.c
@@ -7,7 +7,7 @@
#include <kunit/test.h>
#include <linux/prandom.h>
-#include <linux/raid/pq.h>
+#include "../algos.h"
MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
diff --git a/lib/raid/raid6/x86/avx2.c b/lib/raid/raid6/x86/avx2.c
index aab8b624c635..0bf831799082 100644
--- a/lib/raid/raid6/x86/avx2.c
+++ b/lib/raid/raid6/x86/avx2.c
@@ -13,8 +13,9 @@
*
*/
-#include <linux/raid/pq.h>
+#include <asm/cpufeature.h>
#include <asm/fpu/api.h>
+#include "algos.h"
static const struct raid6_avx2_constants {
u64 x1d[4];
diff --git a/lib/raid/raid6/x86/avx512.c b/lib/raid/raid6/x86/avx512.c
index 47636b16632f..98ed42fb0a46 100644
--- a/lib/raid/raid6/x86/avx512.c
+++ b/lib/raid/raid6/x86/avx512.c
@@ -17,8 +17,9 @@
*
*/
-#include <linux/raid/pq.h>
+#include <asm/cpufeature.h>
#include <asm/fpu/api.h>
+#include "algos.h"
static const struct raid6_avx512_constants {
u64 x1d[8];
diff --git a/lib/raid/raid6/x86/mmx.c b/lib/raid/raid6/x86/mmx.c
index 22b9fdaa705f..052d9f010bfe 100644
--- a/lib/raid/raid6/x86/mmx.c
+++ b/lib/raid/raid6/x86/mmx.c
@@ -11,8 +11,9 @@
* MMX implementation of RAID-6 syndrome functions
*/
-#include <linux/raid/pq.h>
+#include <asm/cpufeature.h>
#include <asm/fpu/api.h>
+#include "algos.h"
/* Shared with raid6/sse1.c */
const struct raid6_mmx_constants {
diff --git a/lib/raid/raid6/x86/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c
index bef82a38d8eb..06c6e05763bc 100644
--- a/lib/raid/raid6/x86/recov_avx2.c
+++ b/lib/raid/raid6/x86/recov_avx2.c
@@ -4,8 +4,10 @@
* Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
*/
+#include <linux/mm.h>
#include <linux/raid/pq.h>
#include <asm/fpu/api.h>
+#include "algos.h"
static int raid6_has_avx2(void)
{
diff --git a/lib/raid/raid6/x86/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c
index 06c70e771eaa..850bb962b514 100644
--- a/lib/raid/raid6/x86/recov_avx512.c
+++ b/lib/raid/raid6/x86/recov_avx512.c
@@ -6,8 +6,10 @@
* Author: Megha Dey <megha.dey@linux.intel.com>
*/
+#include <linux/mm.h>
#include <linux/raid/pq.h>
#include <asm/fpu/api.h>
+#include "algos.h"
static int raid6_has_avx512(void)
{
diff --git a/lib/raid/raid6/x86/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c
index 5ca7d56f23d8..95589c33003a 100644
--- a/lib/raid/raid6/x86/recov_ssse3.c
+++ b/lib/raid/raid6/x86/recov_ssse3.c
@@ -3,8 +3,10 @@
* Copyright (C) 2012 Intel Corporation
*/
+#include <linux/mm.h>
#include <linux/raid/pq.h>
#include <asm/fpu/api.h>
+#include "algos.h"
static int raid6_has_ssse3(void)
{
diff --git a/lib/raid/raid6/x86/sse1.c b/lib/raid/raid6/x86/sse1.c
index fad214a430d8..7004255a0bb1 100644
--- a/lib/raid/raid6/x86/sse1.c
+++ b/lib/raid/raid6/x86/sse1.c
@@ -16,8 +16,9 @@
* worthwhile as a separate implementation.
*/
-#include <linux/raid/pq.h>
+#include <asm/cpufeature.h>
#include <asm/fpu/api.h>
+#include "algos.h"
/* Defined in raid6/mmx.c */
extern const struct raid6_mmx_constants {
diff --git a/lib/raid/raid6/x86/sse2.c b/lib/raid/raid6/x86/sse2.c
index 1b28e858a1d4..f30be4ee14d0 100644
--- a/lib/raid/raid6/x86/sse2.c
+++ b/lib/raid/raid6/x86/sse2.c
@@ -12,8 +12,9 @@
*
*/
-#include <linux/raid/pq.h>
+#include <asm/cpufeature.h>
#include <asm/fpu/api.h>
+#include "algos.h"
static const struct raid6_sse_constants {
u64 x1d[2];
--
2.47.3
^ permalink raw reply related
* [PATCH 07/17] raid6: improve the public interface
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
Stop directly calling into function pointers from users of the RAID6 PQ
API, and provide exported functions with proper documentation and
API gurantees asserts where applicable instead.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
Documentation/crypto/async-tx-api.rst | 4 +-
crypto/async_tx/async_pq.c | 6 +-
crypto/async_tx/async_raid6_recov.c | 4 +-
drivers/md/raid5.c | 4 +-
fs/btrfs/raid56.c | 8 +-
include/linux/raid/pq.h | 19 +--
lib/raid/raid6/algos.c | 137 +++++++++++++++++-
lib/raid/raid6/arm/recov_neon.c | 4 +-
.../raid6/loongarch/recov_loongarch_simd.c | 8 +-
lib/raid/raid6/recov.c | 4 +-
lib/raid/raid6/riscv/recov_rvv.c | 4 +-
lib/raid/raid6/s390/recov_s390xc.c | 4 +-
lib/raid/raid6/x86/recov_avx2.c | 4 +-
lib/raid/raid6/x86/recov_avx512.c | 4 +-
lib/raid/raid6/x86/recov_ssse3.c | 4 +-
15 files changed, 170 insertions(+), 48 deletions(-)
diff --git a/Documentation/crypto/async-tx-api.rst b/Documentation/crypto/async-tx-api.rst
index f88a7809385e..49fcfc66314a 100644
--- a/Documentation/crypto/async-tx-api.rst
+++ b/Documentation/crypto/async-tx-api.rst
@@ -82,9 +82,9 @@ xor_val xor a series of source buffers and set a flag if the
pq generate the p+q (raid6 syndrome) from a series of source buffers
pq_val validate that a p and or q buffer are in sync with a given series of
sources
-datap (raid6_datap_recov) recover a raid6 data block and the p block
+datap (raid6_recov_datap) recover a raid6 data block and the p block
from the given sources
-2data (raid6_2data_recov) recover 2 raid6 data blocks from the given
+2data (raid6_recov_2data) recover 2 raid6 data blocks from the given
sources
======== ====================================================================
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
index 0ce6f07b4e0d..f3574f80d1df 100644
--- a/crypto/async_tx/async_pq.c
+++ b/crypto/async_tx/async_pq.c
@@ -131,11 +131,11 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks,
}
}
if (submit->flags & ASYNC_TX_PQ_XOR_DST) {
- BUG_ON(!raid6_call.xor_syndrome);
+ BUG_ON(!raid6_can_xor_syndrome());
if (start >= 0)
- raid6_call.xor_syndrome(disks, start, stop, len, srcs);
+ raid6_xor_syndrome(disks, start, stop, len, srcs);
} else
- raid6_call.gen_syndrome(disks, len, srcs);
+ raid6_gen_syndrome(disks, len, srcs);
async_tx_sync_epilog(submit);
}
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
index f2dc6af6e6a7..305ea1421a3e 100644
--- a/crypto/async_tx/async_raid6_recov.c
+++ b/crypto/async_tx/async_raid6_recov.c
@@ -418,7 +418,7 @@ async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
else
ptrs[i] = page_address(blocks[i]) + offs[i];
- raid6_2data_recov(disks, bytes, faila, failb, ptrs);
+ raid6_recov_2data(disks, bytes, faila, failb, ptrs);
async_tx_sync_epilog(submit);
@@ -501,7 +501,7 @@ async_raid6_datap_recov(int disks, size_t bytes, int faila,
else
ptrs[i] = page_address(blocks[i]) + offs[i];
- raid6_datap_recov(disks, bytes, faila, ptrs);
+ raid6_recov_datap(disks, bytes, faila, ptrs);
async_tx_sync_epilog(submit);
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index a8e8d431071b..13943e2b3d51 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -6944,7 +6944,7 @@ raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len)
if (kstrtoul(page, 10, &new))
return -EINVAL;
- if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
+ if (new != PARITY_DISABLE_RMW && !raid6_can_xor_syndrome())
return -EINVAL;
if (new != PARITY_DISABLE_RMW &&
@@ -7635,7 +7635,7 @@ static struct r5conf *setup_conf(struct mddev *mddev)
conf->level = mddev->new_level;
if (conf->level == 6) {
conf->max_degraded = 2;
- if (raid6_call.xor_syndrome)
+ if (raid6_can_xor_syndrome())
conf->rmw_level = PARITY_ENABLE_RMW;
else
conf->rmw_level = PARITY_DISABLE_RMW;
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index ad091c56ce77..4ab3cccb11f1 100644
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -1410,7 +1410,7 @@ static void generate_pq_vertical_step(struct btrfs_raid_bio *rbio, unsigned int
rbio_qstripe_paddr(rbio, sector_nr, step_nr));
assert_rbio(rbio);
- raid6_call.gen_syndrome(rbio->real_stripes, step, pointers);
+ raid6_gen_syndrome(rbio->real_stripes, step, pointers);
} else {
/* raid5 */
memcpy(pointers[rbio->nr_data], pointers[0], step);
@@ -1992,10 +1992,10 @@ static void recover_vertical_step(struct btrfs_raid_bio *rbio,
}
if (failb == rbio->real_stripes - 2) {
- raid6_datap_recov(rbio->real_stripes, step,
+ raid6_recov_datap(rbio->real_stripes, step,
faila, pointers);
} else {
- raid6_2data_recov(rbio->real_stripes, step,
+ raid6_recov_2data(rbio->real_stripes, step,
faila, failb, pointers);
}
} else {
@@ -2649,7 +2649,7 @@ static bool verify_one_parity_step(struct btrfs_raid_bio *rbio,
if (has_qstripe) {
assert_rbio(rbio);
/* RAID6, call the library function to fill in our P/Q. */
- raid6_call.gen_syndrome(rbio->real_stripes, step, pointers);
+ raid6_gen_syndrome(rbio->real_stripes, step, pointers);
} else {
/* RAID5. */
memcpy(pointers[nr_data], pointers[0], step);
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index f27a866c287f..425a227591c0 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -11,6 +11,16 @@
#include <linux/blkdev.h>
#include <linux/mm.h>
+void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs);
+void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes,
+ void **ptrs);
+bool raid6_can_xor_syndrome(void);
+
+void raid6_recov_2data(int disks, size_t bytes, int faila, int failb,
+ void **ptrs);
+void raid6_recov_datap(int disks, size_t bytes, int faila,
+ void **ptrs);
+
/* Routine choices */
struct raid6_calls {
void (*gen_syndrome)(int, size_t, void **);
@@ -20,9 +30,6 @@ struct raid6_calls {
int priority; /* Relative priority ranking if non-zero */
};
-/* Selected algorithm */
-extern struct raid6_calls raid6_call;
-
/* Various routine sets */
extern const struct raid6_calls raid6_intx1;
extern const struct raid6_calls raid6_intx2;
@@ -92,10 +99,4 @@ extern const u8 raid6_gflog[256] __attribute__((aligned(256)));
extern const u8 raid6_gfinv[256] __attribute__((aligned(256)));
extern const u8 raid6_gfexi[256] __attribute__((aligned(256)));
-/* Recovery routines */
-extern void (*raid6_2data_recov)(int disks, size_t bytes, int faila, int failb,
- void **ptrs);
-extern void (*raid6_datap_recov)(int disks, size_t bytes, int faila,
- void **ptrs);
-
#endif /* LINUX_RAID_RAID6_H */
diff --git a/lib/raid/raid6/algos.c b/lib/raid/raid6/algos.c
index 985c60bb00a4..4958c58e3f69 100644
--- a/lib/raid/raid6/algos.c
+++ b/lib/raid/raid6/algos.c
@@ -16,8 +16,83 @@
#include <linux/gfp.h>
#include <kunit/visibility.h>
-struct raid6_calls raid6_call;
-EXPORT_SYMBOL_GPL(raid6_call);
+static const struct raid6_recov_calls *raid6_recov_algo;
+
+/* Selected algorithm */
+static struct raid6_calls raid6_call;
+
+/**
+ * raid6_gen_syndrome - generate RAID6 P/Q parity
+ * @disks: number of "disks" to operate on including parity
+ * @bytes: length in bytes of each vector
+ * @ptrs: @disks size array of memory pointers
+ *
+ * Generate @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and
+ * @ptrs[@disks - 1] respectively from the memory pointed to by @ptrs[0] to
+ * @ptrs[@disks - 3].
+ *
+ * @disks must be at least 3, and the memory pointed to by each member of @ptrs
+ * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of
+ * 512.
+ *
+ * See https://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf for underlying
+ * algorithm.
+ */
+void raid6_gen_syndrome(int disks, size_t bytes, void **ptrs)
+{
+ lockdep_assert_preemption_enabled();
+ WARN_ON_ONCE(bytes & 511);
+
+ raid6_call.gen_syndrome(disks, bytes, ptrs);
+}
+EXPORT_SYMBOL_GPL(raid6_gen_syndrome);
+
+/**
+ * raid6_xor_syndrome - update RAID6 P/Q parity
+ * @disks: number of "disks" to operate on including parity
+ * @start: first index into @disk to update
+ * @stop: last index into @disk to update
+ * @bytes: length in bytes of each vector
+ * @ptrs: @disks size array of memory pointers
+ *
+ * Update @bytes worth of RAID6 P and Q parity in @ptrs[@disks - 2] and
+ * @ptrs[@disks - 1] respectively for the memory pointed to by
+ * @ptrs[@start..@stop].
+ *
+ * This is used to update parity in place using the following sequence:
+ *
+ * 1) call raid6_xor_syndrome(disk, start, stop, ...) for the existing data.
+ * 2) update the the data in @ptrs[@start..@stop].
+ * 3) call raid6_xor_syndrome(disk, start, stop, ...) for the new data.
+ *
+ * Data between @start and @stop that is not changed should be filled
+ * with a pointer to the kernel zero page.
+ *
+ * @disks must be at least 3, and the memory pointed to by each member of @ptrs
+ * must be at least 64-byte aligned. @bytes must be non-zero and a multiple of
+ * 512. @stop must be larger or equal to @start.
+ */
+void raid6_xor_syndrome(int disks, int start, int stop, size_t bytes,
+ void **ptrs)
+{
+ lockdep_assert_preemption_enabled();
+ WARN_ON_ONCE(bytes & 511);
+ WARN_ON_ONCE(stop < start);
+
+ raid6_call.xor_syndrome(disks, start, stop, bytes, ptrs);
+}
+EXPORT_SYMBOL_GPL(raid6_xor_syndrome);
+
+/*
+ * raid6_can_xor_syndrome - check if raid6_xor_syndrome() can be used
+ *
+ * Returns %true if raid6_can_xor_syndrome() can be used, else %false.
+ */
+bool raid6_can_xor_syndrome(void)
+{
+ return !!raid6_call.xor_syndrome;
+}
+EXPORT_SYMBOL_GPL(raid6_can_xor_syndrome);
const struct raid6_calls * const raid6_algos[] = {
#if defined(__i386__) && !defined(__arch_um__)
@@ -84,11 +159,58 @@ const struct raid6_calls * const raid6_algos[] = {
};
EXPORT_SYMBOL_IF_KUNIT(raid6_algos);
-void (*raid6_2data_recov)(int, size_t, int, int, void **);
-EXPORT_SYMBOL_GPL(raid6_2data_recov);
+/**
+ * raid6_recov_2data - recover two missing data disks
+ * @disks: number of "disks" to operate on including parity
+ * @bytes: length in bytes of each vector
+ * @faila: first failed data disk index
+ * @failb: second failed data disk index
+ * @ptrs: @disks size array of memory pointers
+ *
+ * Rebuild @bytes of missing data in @ptrs[@faila] and @ptrs[@failb] from the
+ * data in the remaining disks and the two parities pointed to by the other
+ * indices between 0 and @disks - 1 in @ptrs. @disks includes the data disks
+ * and the two parities. @faila must be smaller than @failb.
+ *
+ * Memory pointed to by each pointer in @ptrs must be page aligned and is
+ * limited to %PAGE_SIZE.
+ */
+void raid6_recov_2data(int disks, size_t bytes, int faila, int failb,
+ void **ptrs)
+{
+ lockdep_assert_preemption_enabled();
+ WARN_ON_ONCE(bytes & 511);
+ WARN_ON_ONCE(bytes > PAGE_SIZE);
+ WARN_ON_ONCE(failb <= faila);
+
+ raid6_recov_algo->data2(disks, bytes, faila, failb, ptrs);
+}
+EXPORT_SYMBOL_GPL(raid6_recov_2data);
+
+/**
+ * raid6_recov_datap - recover a missing data disk and missing P-parity
+ * @disks: number of "disks" to operate on including parity
+ * @bytes: length in bytes of each vector
+ * @faila: failed data disk index
+ * @ptrs: @disks size array of memory pointers
+ *
+ * Rebuild @bytes of missing data in @ptrs[@faila] and the missing P-parity in
+ * @ptrs[@disks - 2] from the data in the remaining disks and the Q-parity
+ * pointed to by the other indices between 0 and @disks - 1 in @ptrs. @disks
+ * includes the data disks and the two parities.
+ *
+ * Memory pointed to by each pointer in @ptrs must be page aligned and is
+ * limited to %PAGE_SIZE.
+ */
+void raid6_recov_datap(int disks, size_t bytes, int faila, void **ptrs)
+{
+ lockdep_assert_preemption_enabled();
+ WARN_ON_ONCE(bytes & 511);
+ WARN_ON_ONCE(bytes > PAGE_SIZE);
-void (*raid6_datap_recov)(int, size_t, int, void **);
-EXPORT_SYMBOL_GPL(raid6_datap_recov);
+ raid6_recov_algo->datap(disks, bytes, faila, ptrs);
+}
+EXPORT_SYMBOL_GPL(raid6_recov_datap);
const struct raid6_recov_calls *const raid6_recov_algos[] = {
#ifdef CONFIG_X86
@@ -133,8 +255,7 @@ static inline const struct raid6_recov_calls *raid6_choose_recov(void)
best = *algo;
if (best) {
- raid6_2data_recov = best->data2;
- raid6_datap_recov = best->datap;
+ raid6_recov_algo = best;
pr_info("raid6: using %s recovery algorithm\n", best->name);
} else
diff --git a/lib/raid/raid6/arm/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c
index 461bdb98b5cc..21c26da6290a 100644
--- a/lib/raid/raid6/arm/recov_neon.c
+++ b/lib/raid/raid6/arm/recov_neon.c
@@ -35,7 +35,7 @@ static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -69,7 +69,7 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
diff --git a/lib/raid/raid6/loongarch/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
index 4d4563209647..7d4d349322b3 100644
--- a/lib/raid/raid6/loongarch/recov_loongarch_simd.c
+++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
@@ -49,7 +49,7 @@ static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -201,7 +201,7 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
@@ -323,7 +323,7 @@ static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -440,7 +440,7 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
diff --git a/lib/raid/raid6/recov.c b/lib/raid/raid6/recov.c
index 211e1df28963..cc7e4dc1eaa6 100644
--- a/lib/raid/raid6/recov.c
+++ b/lib/raid/raid6/recov.c
@@ -37,7 +37,7 @@ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -75,7 +75,7 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
diff --git a/lib/raid/raid6/riscv/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c
index f77d9c430687..3ff39826e33f 100644
--- a/lib/raid/raid6/riscv/recov_rvv.c
+++ b/lib/raid/raid6/riscv/recov_rvv.c
@@ -164,7 +164,7 @@ static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -199,7 +199,7 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
diff --git a/lib/raid/raid6/s390/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c
index 0f32217b7123..2bc4c85174de 100644
--- a/lib/raid/raid6/s390/recov_s390xc.c
+++ b/lib/raid/raid6/s390/recov_s390xc.c
@@ -40,7 +40,7 @@ static void raid6_2data_recov_s390xc(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -84,7 +84,7 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
diff --git a/lib/raid/raid6/x86/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c
index 325310c81e1c..bef82a38d8eb 100644
--- a/lib/raid/raid6/x86/recov_avx2.c
+++ b/lib/raid/raid6/x86/recov_avx2.c
@@ -34,7 +34,7 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -199,7 +199,7 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
diff --git a/lib/raid/raid6/x86/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c
index 08de77fcb8bd..06c70e771eaa 100644
--- a/lib/raid/raid6/x86/recov_avx512.c
+++ b/lib/raid/raid6/x86/recov_avx512.c
@@ -43,7 +43,7 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -241,7 +241,7 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
diff --git a/lib/raid/raid6/x86/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c
index 002bef1e0847..5ca7d56f23d8 100644
--- a/lib/raid/raid6/x86/recov_ssse3.c
+++ b/lib/raid/raid6/x86/recov_ssse3.c
@@ -36,7 +36,7 @@ static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila,
ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dp;
@@ -206,7 +206,7 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila,
ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
- raid6_call.gen_syndrome(disks, bytes, ptrs);
+ raid6_gen_syndrome(disks, bytes, ptrs);
/* Restore pointer table */
ptrs[faila] = dq;
--
2.47.3
^ permalink raw reply related
* [PATCH 06/17] raid6: use named initializers for struct raid6_calls
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
lib/raid/raid6/arm/neon.c | 9 +++----
lib/raid/raid6/int.uc | 8 +++---
lib/raid/raid6/loongarch/loongarch_simd.c | 18 ++++++-------
lib/raid/raid6/powerpc/altivec.uc | 8 +++---
lib/raid/raid6/powerpc/vpermxor.uc | 8 +++---
lib/raid/raid6/riscv/rvv.h | 9 +++----
lib/raid/raid6/s390/s390vx.uc | 10 +++----
lib/raid/raid6/x86/avx2.c | 33 ++++++++++++-----------
lib/raid/raid6/x86/avx512.c | 33 ++++++++++++-----------
lib/raid/raid6/x86/mmx.c | 16 +++++------
lib/raid/raid6/x86/sse1.c | 18 ++++++-------
lib/raid/raid6/x86/sse2.c | 30 ++++++++++-----------
12 files changed, 95 insertions(+), 105 deletions(-)
diff --git a/lib/raid/raid6/arm/neon.c b/lib/raid/raid6/arm/neon.c
index 47b8bb0afc65..c21da59ab48f 100644
--- a/lib/raid/raid6/arm/neon.c
+++ b/lib/raid/raid6/arm/neon.c
@@ -40,11 +40,10 @@
start, stop, (unsigned long)bytes, ptrs);\
} \
struct raid6_calls const raid6_neonx ## _n = { \
- raid6_neon ## _n ## _gen_syndrome, \
- raid6_neon ## _n ## _xor_syndrome, \
- raid6_have_neon, \
- "neonx" #_n, \
- 0 \
+ .gen_syndrome = raid6_neon ## _n ## _gen_syndrome, \
+ .xor_syndrome = raid6_neon ## _n ## _xor_syndrome, \
+ .valid = raid6_have_neon, \
+ .name = "neonx" #_n, \
}
static int raid6_have_neon(void)
diff --git a/lib/raid/raid6/int.uc b/lib/raid/raid6/int.uc
index 1ba56c3fa482..4f5f2869e21e 100644
--- a/lib/raid/raid6/int.uc
+++ b/lib/raid/raid6/int.uc
@@ -139,9 +139,7 @@ static void raid6_int$#_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_intx$# = {
- raid6_int$#_gen_syndrome,
- raid6_int$#_xor_syndrome,
- NULL, /* always valid */
- "int" NSTRING "x$#",
- 0
+ .gen_syndrome = raid6_int$#_gen_syndrome,
+ .xor_syndrome = raid6_int$#_xor_syndrome,
+ .name = "int" NSTRING "x$#",
};
diff --git a/lib/raid/raid6/loongarch/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c
index 72f4d92d4876..1b4cd1512d05 100644
--- a/lib/raid/raid6/loongarch/loongarch_simd.c
+++ b/lib/raid/raid6/loongarch/loongarch_simd.c
@@ -244,11 +244,10 @@ static void raid6_lsx_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_lsx = {
- raid6_lsx_gen_syndrome,
- raid6_lsx_xor_syndrome,
- raid6_has_lsx,
- "lsx",
- .priority = 0 /* see the comment near the top of the file for reason */
+ .gen_syndrome = raid6_lsx_gen_syndrome,
+ .xor_syndrome = raid6_lsx_xor_syndrome,
+ .valid = raid6_has_lsx,
+ .name = "lsx",
};
#undef NSIZE
@@ -413,11 +412,10 @@ static void raid6_lasx_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_lasx = {
- raid6_lasx_gen_syndrome,
- raid6_lasx_xor_syndrome,
- raid6_has_lasx,
- "lasx",
- .priority = 0 /* see the comment near the top of the file for reason */
+ .gen_syndrome = raid6_lasx_gen_syndrome,
+ .xor_syndrome = raid6_lasx_xor_syndrome,
+ .valid = raid6_has_lasx,
+ .name = "lasx",
};
#undef NSIZE
#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/lib/raid/raid6/powerpc/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc
index 130d3d3dd42c..084ead768ddb 100644
--- a/lib/raid/raid6/powerpc/altivec.uc
+++ b/lib/raid/raid6/powerpc/altivec.uc
@@ -114,9 +114,7 @@ int raid6_have_altivec(void)
#endif
const struct raid6_calls raid6_altivec$# = {
- raid6_altivec$#_gen_syndrome,
- NULL, /* XOR not yet implemented */
- raid6_have_altivec,
- "altivecx$#",
- 0
+ .gen_syndrome = raid6_altivec$#_gen_syndrome,
+ .valid = raid6_have_altivec,
+ .name = "altivecx$#",
};
diff --git a/lib/raid/raid6/powerpc/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc
index 595f20aaf4cf..bb2c3a316ae8 100644
--- a/lib/raid/raid6/powerpc/vpermxor.uc
+++ b/lib/raid/raid6/powerpc/vpermxor.uc
@@ -87,9 +87,7 @@ int raid6_have_altivec_vpermxor(void)
#endif
const struct raid6_calls raid6_vpermxor$# = {
- raid6_vpermxor$#_gen_syndrome,
- NULL,
- raid6_have_altivec_vpermxor,
- "vpermxor$#",
- 0
+ .gen_syndrome = raid6_vpermxor$#_gen_syndrome,
+ .valid = raid6_have_altivec_vpermxor,
+ .name = "vpermxor$#",
};
diff --git a/lib/raid/raid6/riscv/rvv.h b/lib/raid/raid6/riscv/rvv.h
index b0a71b375962..0d430a4c5f08 100644
--- a/lib/raid/raid6/riscv/rvv.h
+++ b/lib/raid/raid6/riscv/rvv.h
@@ -39,9 +39,8 @@ static int rvv_has_vector(void)
kernel_vector_end(); \
} \
struct raid6_calls const raid6_rvvx ## _n = { \
- raid6_rvv ## _n ## _gen_syndrome, \
- raid6_rvv ## _n ## _xor_syndrome, \
- rvv_has_vector, \
- "rvvx" #_n, \
- 0 \
+ .gen_syndrome = raid6_rvv ## _n ## _gen_syndrome, \
+ .xor_syndrome = raid6_rvv ## _n ## _xor_syndrome, \
+ .valid = rvv_has_vector, \
+ .name = "rvvx" #_n, \
}
diff --git a/lib/raid/raid6/s390/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc
index 8aa53eb2f395..97c5d5d9dcf9 100644
--- a/lib/raid/raid6/s390/s390vx.uc
+++ b/lib/raid/raid6/s390/s390vx.uc
@@ -127,9 +127,9 @@ static int raid6_s390vx$#_valid(void)
}
const struct raid6_calls raid6_s390vx$# = {
- raid6_s390vx$#_gen_syndrome,
- raid6_s390vx$#_xor_syndrome,
- raid6_s390vx$#_valid,
- "vx128x$#",
- 1
+ .gen_syndrome = raid6_s390vx$#_gen_syndrome,
+ .xor_syndrome = raid6_s390vx$#_xor_syndrome,
+ .valid = raid6_s390vx$#_valid,
+ .name = "vx128x$#",
+ .priority = 1,
};
diff --git a/lib/raid/raid6/x86/avx2.c b/lib/raid/raid6/x86/avx2.c
index a1a5213918af..aab8b624c635 100644
--- a/lib/raid/raid6/x86/avx2.c
+++ b/lib/raid/raid6/x86/avx2.c
@@ -128,11 +128,12 @@ static void raid6_avx21_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_avx2x1 = {
- raid6_avx21_gen_syndrome,
- raid6_avx21_xor_syndrome,
- raid6_have_avx2,
- "avx2x1",
- .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */
+ .gen_syndrome = raid6_avx21_gen_syndrome,
+ .xor_syndrome = raid6_avx21_xor_syndrome,
+ .valid = raid6_have_avx2,
+ .name = "avx2x1",
+ /* Prefer AVX2 over priority 1 (SSE2 and others) */
+ .priority = 2,
};
/*
@@ -258,11 +259,12 @@ static void raid6_avx22_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_avx2x2 = {
- raid6_avx22_gen_syndrome,
- raid6_avx22_xor_syndrome,
- raid6_have_avx2,
- "avx2x2",
- .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */
+ .gen_syndrome = raid6_avx22_gen_syndrome,
+ .xor_syndrome = raid6_avx22_xor_syndrome,
+ .valid = raid6_have_avx2,
+ .name = "avx2x2",
+ /* Prefer AVX2 over priority 1 (SSE2 and others) */
+ .priority = 2,
};
#ifdef CONFIG_X86_64
@@ -461,10 +463,11 @@ static void raid6_avx24_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_avx2x4 = {
- raid6_avx24_gen_syndrome,
- raid6_avx24_xor_syndrome,
- raid6_have_avx2,
- "avx2x4",
- .priority = 2 /* Prefer AVX2 over priority 1 (SSE2 and others) */
+ .gen_syndrome = raid6_avx24_gen_syndrome,
+ .xor_syndrome = raid6_avx24_xor_syndrome,
+ .valid = raid6_have_avx2,
+ .name = "avx2x4",
+ /* Prefer AVX2 over priority 1 (SSE2 and others) */
+ .priority = 2,
};
#endif /* CONFIG_X86_64 */
diff --git a/lib/raid/raid6/x86/avx512.c b/lib/raid/raid6/x86/avx512.c
index 874998bcd7d7..47636b16632f 100644
--- a/lib/raid/raid6/x86/avx512.c
+++ b/lib/raid/raid6/x86/avx512.c
@@ -156,11 +156,12 @@ static void raid6_avx5121_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_avx512x1 = {
- raid6_avx5121_gen_syndrome,
- raid6_avx5121_xor_syndrome,
- raid6_have_avx512,
- "avx512x1",
- .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
+ .gen_syndrome = raid6_avx5121_gen_syndrome,
+ .xor_syndrome = raid6_avx5121_xor_syndrome,
+ .valid = raid6_have_avx512,
+ .name = "avx512x1",
+ /* Prefer AVX512 over priority 1 (SSE2 and others) */
+ .priority = 2,
};
/*
@@ -313,11 +314,12 @@ static void raid6_avx5122_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_avx512x2 = {
- raid6_avx5122_gen_syndrome,
- raid6_avx5122_xor_syndrome,
- raid6_have_avx512,
- "avx512x2",
- .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
+ .gen_syndrome = raid6_avx5122_gen_syndrome,
+ .xor_syndrome = raid6_avx5122_xor_syndrome,
+ .valid = raid6_have_avx512,
+ .name = "avx512x2",
+ /* Prefer AVX512 over priority 1 (SSE2 and others) */
+ .priority = 2,
};
#ifdef CONFIG_X86_64
@@ -551,10 +553,11 @@ static void raid6_avx5124_xor_syndrome(int disks, int start, int stop,
kernel_fpu_end();
}
const struct raid6_calls raid6_avx512x4 = {
- raid6_avx5124_gen_syndrome,
- raid6_avx5124_xor_syndrome,
- raid6_have_avx512,
- "avx512x4",
- .priority = 2 /* Prefer AVX512 over priority 1 (SSE2 and others) */
+ .gen_syndrome = raid6_avx5124_gen_syndrome,
+ .xor_syndrome = raid6_avx5124_xor_syndrome,
+ .valid = raid6_have_avx512,
+ .name = "avx512x4",
+ /* Prefer AVX512 over priority 1 (SSE2 and others) */
+ .priority = 2,
};
#endif
diff --git a/lib/raid/raid6/x86/mmx.c b/lib/raid/raid6/x86/mmx.c
index 7e9810669347..22b9fdaa705f 100644
--- a/lib/raid/raid6/x86/mmx.c
+++ b/lib/raid/raid6/x86/mmx.c
@@ -68,11 +68,9 @@ static void raid6_mmx1_gen_syndrome(int disks, size_t bytes, void **ptrs)
}
const struct raid6_calls raid6_mmxx1 = {
- raid6_mmx1_gen_syndrome,
- NULL, /* XOR not yet implemented */
- raid6_have_mmx,
- "mmxx1",
- 0
+ .gen_syndrome = raid6_mmx1_gen_syndrome,
+ .valid = raid6_have_mmx,
+ .name = "mmxx1",
};
/*
@@ -127,9 +125,7 @@ static void raid6_mmx2_gen_syndrome(int disks, size_t bytes, void **ptrs)
}
const struct raid6_calls raid6_mmxx2 = {
- raid6_mmx2_gen_syndrome,
- NULL, /* XOR not yet implemented */
- raid6_have_mmx,
- "mmxx2",
- 0
+ .gen_syndrome = raid6_mmx2_gen_syndrome,
+ .valid = raid6_have_mmx,
+ .name = "mmxx2",
};
diff --git a/lib/raid/raid6/x86/sse1.c b/lib/raid/raid6/x86/sse1.c
index deecdd72ceec..fad214a430d8 100644
--- a/lib/raid/raid6/x86/sse1.c
+++ b/lib/raid/raid6/x86/sse1.c
@@ -84,11 +84,10 @@ static void raid6_sse11_gen_syndrome(int disks, size_t bytes, void **ptrs)
}
const struct raid6_calls raid6_sse1x1 = {
- raid6_sse11_gen_syndrome,
- NULL, /* XOR not yet implemented */
- raid6_have_sse1_or_mmxext,
- "sse1x1",
- 1 /* Has cache hints */
+ .gen_syndrome = raid6_sse11_gen_syndrome,
+ .valid = raid6_have_sse1_or_mmxext,
+ .name = "sse1x1",
+ .priority = 1, /* Has cache hints */
};
/*
@@ -147,9 +146,8 @@ static void raid6_sse12_gen_syndrome(int disks, size_t bytes, void **ptrs)
}
const struct raid6_calls raid6_sse1x2 = {
- raid6_sse12_gen_syndrome,
- NULL, /* XOR not yet implemented */
- raid6_have_sse1_or_mmxext,
- "sse1x2",
- 1 /* Has cache hints */
+ .gen_syndrome = raid6_sse12_gen_syndrome,
+ .valid = raid6_have_sse1_or_mmxext,
+ .name = "sse1x2",
+ .priority = 1, /* Has cache hints */
};
diff --git a/lib/raid/raid6/x86/sse2.c b/lib/raid/raid6/x86/sse2.c
index f9edf8a8d1c4..1b28e858a1d4 100644
--- a/lib/raid/raid6/x86/sse2.c
+++ b/lib/raid/raid6/x86/sse2.c
@@ -133,11 +133,11 @@ static void raid6_sse21_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_sse2x1 = {
- raid6_sse21_gen_syndrome,
- raid6_sse21_xor_syndrome,
- raid6_have_sse2,
- "sse2x1",
- 1 /* Has cache hints */
+ .gen_syndrome = raid6_sse21_gen_syndrome,
+ .xor_syndrome = raid6_sse21_xor_syndrome,
+ .valid = raid6_have_sse2,
+ .name = "sse2x1",
+ .priority = 1, /* Has cache hints */
};
/*
@@ -263,11 +263,11 @@ static void raid6_sse22_xor_syndrome(int disks, int start, int stop,
}
const struct raid6_calls raid6_sse2x2 = {
- raid6_sse22_gen_syndrome,
- raid6_sse22_xor_syndrome,
- raid6_have_sse2,
- "sse2x2",
- 1 /* Has cache hints */
+ .gen_syndrome = raid6_sse22_gen_syndrome,
+ .xor_syndrome = raid6_sse22_xor_syndrome,
+ .valid = raid6_have_sse2,
+ .name = "sse2x2",
+ .priority = 1, /* Has cache hints */
};
#ifdef CONFIG_X86_64
@@ -470,11 +470,11 @@ static void raid6_sse24_xor_syndrome(int disks, int start, int stop,
const struct raid6_calls raid6_sse2x4 = {
- raid6_sse24_gen_syndrome,
- raid6_sse24_xor_syndrome,
- raid6_have_sse2,
- "sse2x4",
- 1 /* Has cache hints */
+ .gen_syndrome = raid6_sse24_gen_syndrome,
+ .xor_syndrome = raid6_sse24_xor_syndrome,
+ .valid = raid6_have_sse2,
+ .name = "sse2x4",
+ .priority = 1, /* Has cache hints */
};
#endif /* CONFIG_X86_64 */
--
2.47.3
^ permalink raw reply related
* [PATCH 05/17] raid6: remove raid6_get_zero_page
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
Just open code it as in other places in the kernel.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
crypto/async_tx/async_pq.c | 2 +-
crypto/async_tx/async_raid6_recov.c | 4 ++--
include/linux/raid/pq.h | 6 ------
lib/raid/raid6/arm/recov_neon.c | 6 +++---
lib/raid/raid6/loongarch/recov_loongarch_simd.c | 12 ++++++------
lib/raid/raid6/recov.c | 6 +++---
lib/raid/raid6/riscv/recov_rvv.c | 6 +++---
lib/raid/raid6/s390/recov_s390xc.c | 6 +++---
lib/raid/raid6/x86/recov_avx2.c | 6 +++---
lib/raid/raid6/x86/recov_avx512.c | 6 +++---
lib/raid/raid6/x86/recov_ssse3.c | 6 +++---
11 files changed, 30 insertions(+), 36 deletions(-)
diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c
index 9e4bb7fbde25..0ce6f07b4e0d 100644
--- a/crypto/async_tx/async_pq.c
+++ b/crypto/async_tx/async_pq.c
@@ -119,7 +119,7 @@ do_sync_gen_syndrome(struct page **blocks, unsigned int *offsets, int disks,
for (i = 0; i < disks; i++) {
if (blocks[i] == NULL) {
BUG_ON(i > disks - 3); /* P or Q can't be zero */
- srcs[i] = raid6_get_zero_page();
+ srcs[i] = page_address(ZERO_PAGE(0));
} else {
srcs[i] = page_address(blocks[i]) + offsets[i];
diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c
index 539ea5b378dc..f2dc6af6e6a7 100644
--- a/crypto/async_tx/async_raid6_recov.c
+++ b/crypto/async_tx/async_raid6_recov.c
@@ -414,7 +414,7 @@ async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb,
async_tx_quiesce(&submit->depend_tx);
for (i = 0; i < disks; i++)
if (blocks[i] == NULL)
- ptrs[i] = raid6_get_zero_page();
+ ptrs[i] = page_address(ZERO_PAGE(0));
else
ptrs[i] = page_address(blocks[i]) + offs[i];
@@ -497,7 +497,7 @@ async_raid6_datap_recov(int disks, size_t bytes, int faila,
async_tx_quiesce(&submit->depend_tx);
for (i = 0; i < disks; i++)
if (blocks[i] == NULL)
- ptrs[i] = raid6_get_zero_page();
+ ptrs[i] = page_address(ZERO_PAGE(0));
else
ptrs[i] = page_address(blocks[i]) + offs[i];
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 5e7e743b83f5..f27a866c287f 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -11,12 +11,6 @@
#include <linux/blkdev.h>
#include <linux/mm.h>
-/* This should be const but the raid6 code is too convoluted for that. */
-static inline void *raid6_get_zero_page(void)
-{
- return page_address(ZERO_PAGE(0));
-}
-
/* Routine choices */
struct raid6_calls {
void (*gen_syndrome)(int, size_t, void **);
diff --git a/lib/raid/raid6/arm/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c
index 13d5df718c15..461bdb98b5cc 100644
--- a/lib/raid/raid6/arm/recov_neon.c
+++ b/lib/raid/raid6/arm/recov_neon.c
@@ -29,10 +29,10 @@ static void raid6_2data_recov_neon(int disks, size_t bytes, int faila,
* delta p and delta q
*/
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -66,7 +66,7 @@ static void raid6_datap_recov_neon(int disks, size_t bytes, int faila,
* Use the dead data page as temporary storage for delta q
*/
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
diff --git a/lib/raid/raid6/loongarch/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
index eb3a1e79f01f..4d4563209647 100644
--- a/lib/raid/raid6/loongarch/recov_loongarch_simd.c
+++ b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
@@ -43,10 +43,10 @@ static void raid6_2data_recov_lsx(int disks, size_t bytes, int faila,
* delta p and delta q
*/
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -198,7 +198,7 @@ static void raid6_datap_recov_lsx(int disks, size_t bytes, int faila,
* Use the dead data page as temporary storage for delta q
*/
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -317,10 +317,10 @@ static void raid6_2data_recov_lasx(int disks, size_t bytes, int faila,
* delta p and delta q
*/
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -437,7 +437,7 @@ static void raid6_datap_recov_lasx(int disks, size_t bytes, int faila,
* Use the dead data page as temporary storage for delta q
*/
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
diff --git a/lib/raid/raid6/recov.c b/lib/raid/raid6/recov.c
index 8d113196632e..211e1df28963 100644
--- a/lib/raid/raid6/recov.c
+++ b/lib/raid/raid6/recov.c
@@ -31,10 +31,10 @@ static void raid6_2data_recov_intx1(int disks, size_t bytes, int faila,
Use the dead data pages as temporary storage for
delta p and delta q */
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -72,7 +72,7 @@ static void raid6_datap_recov_intx1(int disks, size_t bytes, int faila,
/* Compute syndrome with zero for the missing data page
Use the dead data page as temporary storage for delta q */
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
diff --git a/lib/raid/raid6/riscv/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c
index 40c393206b6a..f77d9c430687 100644
--- a/lib/raid/raid6/riscv/recov_rvv.c
+++ b/lib/raid/raid6/riscv/recov_rvv.c
@@ -158,10 +158,10 @@ static void raid6_2data_recov_rvv(int disks, size_t bytes, int faila,
* delta p and delta q
*/
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -196,7 +196,7 @@ static void raid6_datap_recov_rvv(int disks, size_t bytes, int faila,
* Use the dead data page as temporary storage for delta q
*/
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks - 1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
diff --git a/lib/raid/raid6/s390/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c
index 487018f81192..0f32217b7123 100644
--- a/lib/raid/raid6/s390/recov_s390xc.c
+++ b/lib/raid/raid6/s390/recov_s390xc.c
@@ -34,10 +34,10 @@ static void raid6_2data_recov_s390xc(int disks, size_t bytes, int faila,
Use the dead data pages as temporary storage for
delta p and delta q */
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -81,7 +81,7 @@ static void raid6_datap_recov_s390xc(int disks, size_t bytes, int faila,
/* Compute syndrome with zero for the missing data page
Use the dead data page as temporary storage for delta q */
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
diff --git a/lib/raid/raid6/x86/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c
index 19fbd9c4dce6..325310c81e1c 100644
--- a/lib/raid/raid6/x86/recov_avx2.c
+++ b/lib/raid/raid6/x86/recov_avx2.c
@@ -28,10 +28,10 @@ static void raid6_2data_recov_avx2(int disks, size_t bytes, int faila,
Use the dead data pages as temporary storage for
delta p and delta q */
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -196,7 +196,7 @@ static void raid6_datap_recov_avx2(int disks, size_t bytes, int faila,
/* Compute syndrome with zero for the missing data page
Use the dead data page as temporary storage for delta q */
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
diff --git a/lib/raid/raid6/x86/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c
index 143f4976b2ad..08de77fcb8bd 100644
--- a/lib/raid/raid6/x86/recov_avx512.c
+++ b/lib/raid/raid6/x86/recov_avx512.c
@@ -37,10 +37,10 @@ static void raid6_2data_recov_avx512(int disks, size_t bytes, int faila,
*/
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -238,7 +238,7 @@ static void raid6_datap_recov_avx512(int disks, size_t bytes, int faila,
*/
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
diff --git a/lib/raid/raid6/x86/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c
index 146cdbf465bd..002bef1e0847 100644
--- a/lib/raid/raid6/x86/recov_ssse3.c
+++ b/lib/raid/raid6/x86/recov_ssse3.c
@@ -30,10 +30,10 @@ static void raid6_2data_recov_ssse3(int disks, size_t bytes, int faila,
Use the dead data pages as temporary storage for
delta p and delta q */
dp = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-2] = dp;
dq = (u8 *)ptrs[failb];
- ptrs[failb] = raid6_get_zero_page();
+ ptrs[failb] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
@@ -203,7 +203,7 @@ static void raid6_datap_recov_ssse3(int disks, size_t bytes, int faila,
/* Compute syndrome with zero for the missing data page
Use the dead data page as temporary storage for delta q */
dq = (u8 *)ptrs[faila];
- ptrs[faila] = raid6_get_zero_page();
+ ptrs[faila] = page_address(ZERO_PAGE(0));
ptrs[disks-1] = dq;
raid6_call.gen_syndrome(disks, bytes, ptrs);
--
2.47.3
^ permalink raw reply related
* [PATCH 04/17] raid6: remove unused defines in pq.h
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
These are not used anywhere in the kernel.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
include/linux/raid/pq.h | 6 ------
1 file changed, 6 deletions(-)
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index d26788fada58..5e7e743b83f5 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -90,12 +90,6 @@ extern const struct raid6_calls raid6_neonx8;
extern const struct raid6_calls * const raid6_algos[];
extern const struct raid6_recov_calls *const raid6_recov_algos[];
-/* Return values from chk_syndrome */
-#define RAID6_OK 0
-#define RAID6_P_BAD 1
-#define RAID6_Q_BAD 2
-#define RAID6_PQ_BAD 3
-
/* Galois field tables */
extern const u8 raid6_gfmul[256][256] __attribute__((aligned(256)));
extern const u8 raid6_vgfmul[256][32] __attribute__((aligned(256)));
--
2.47.3
^ permalink raw reply related
* [PATCH 03/17] raid6: move to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
Move the raid6 code to live in lib/raid/ with the XOR code, and change
the internal organization so that each architecture has a subdirectory
similar to the CRC, crypto and XOR libraries, and fix up the Makefile to
only build files actually needed.
Also move the kunit test case from the history test/ subdirectory to
tests/ and use the normal naming scheme for it.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
lib/Kconfig | 22 ----
lib/Makefile | 1 -
lib/raid/Kconfig | 22 ++++
lib/raid/Makefile | 2 +-
lib/{ => raid}/raid6/.gitignore | 0
lib/raid/raid6/Makefile | 118 ++++++++++++++++++
lib/{ => raid}/raid6/algos.c | 0
lib/{raid6 => raid/raid6/arm}/neon.c | 0
lib/{raid6 => raid/raid6/arm}/neon.h | 0
lib/{raid6 => raid/raid6/arm}/neon.uc | 0
lib/{raid6 => raid/raid6/arm}/recov_neon.c | 0
.../raid6/arm}/recov_neon_inner.c | 0
lib/{ => raid}/raid6/int.uc | 0
.../raid6/loongarch}/loongarch_simd.c | 0
.../raid6/loongarch}/recov_loongarch_simd.c | 0
lib/{ => raid}/raid6/mktables.c | 0
lib/{raid6 => raid/raid6/powerpc}/altivec.uc | 4 -
lib/{raid6 => raid/raid6/powerpc}/vpermxor.uc | 3 -
lib/{ => raid}/raid6/recov.c | 0
lib/{raid6 => raid/raid6/riscv}/recov_rvv.c | 0
lib/{raid6 => raid/raid6/riscv}/rvv.c | 0
lib/{raid6 => raid/raid6/riscv}/rvv.h | 0
lib/{raid6 => raid/raid6/s390}/recov_s390xc.c | 0
lib/{raid6 => raid/raid6/s390}/s390vx.uc | 0
lib/raid/raid6/tests/Makefile | 3 +
.../test.c => raid/raid6/tests/raid6_kunit.c} | 0
lib/{ => raid}/raid6/unroll.awk | 0
lib/{raid6 => raid/raid6/x86}/avx2.c | 0
lib/{raid6 => raid/raid6/x86}/avx512.c | 0
lib/{raid6 => raid/raid6/x86}/mmx.c | 4 -
lib/{raid6 => raid/raid6/x86}/recov_avx2.c | 0
lib/{raid6 => raid/raid6/x86}/recov_avx512.c | 0
lib/{raid6 => raid/raid6/x86}/recov_ssse3.c | 0
lib/{raid6 => raid/raid6/x86}/sse1.c | 4 -
lib/{raid6 => raid/raid6/x86}/sse2.c | 0
lib/raid6/Makefile | 83 ------------
lib/raid6/test/.gitignore | 3 -
lib/raid6/test/Makefile | 3 -
38 files changed, 144 insertions(+), 128 deletions(-)
rename lib/{ => raid}/raid6/.gitignore (100%)
create mode 100644 lib/raid/raid6/Makefile
rename lib/{ => raid}/raid6/algos.c (100%)
rename lib/{raid6 => raid/raid6/arm}/neon.c (100%)
rename lib/{raid6 => raid/raid6/arm}/neon.h (100%)
rename lib/{raid6 => raid/raid6/arm}/neon.uc (100%)
rename lib/{raid6 => raid/raid6/arm}/recov_neon.c (100%)
rename lib/{raid6 => raid/raid6/arm}/recov_neon_inner.c (100%)
rename lib/{ => raid}/raid6/int.uc (100%)
rename lib/{raid6 => raid/raid6/loongarch}/loongarch_simd.c (100%)
rename lib/{raid6 => raid/raid6/loongarch}/recov_loongarch_simd.c (100%)
rename lib/{ => raid}/raid6/mktables.c (100%)
rename lib/{raid6 => raid/raid6/powerpc}/altivec.uc (98%)
rename lib/{raid6 => raid/raid6/powerpc}/vpermxor.uc (98%)
rename lib/{ => raid}/raid6/recov.c (100%)
rename lib/{raid6 => raid/raid6/riscv}/recov_rvv.c (100%)
rename lib/{raid6 => raid/raid6/riscv}/rvv.c (100%)
rename lib/{raid6 => raid/raid6/riscv}/rvv.h (100%)
rename lib/{raid6 => raid/raid6/s390}/recov_s390xc.c (100%)
rename lib/{raid6 => raid/raid6/s390}/s390vx.uc (100%)
create mode 100644 lib/raid/raid6/tests/Makefile
rename lib/{raid6/test/test.c => raid/raid6/tests/raid6_kunit.c} (100%)
rename lib/{ => raid}/raid6/unroll.awk (100%)
rename lib/{raid6 => raid/raid6/x86}/avx2.c (100%)
rename lib/{raid6 => raid/raid6/x86}/avx512.c (100%)
rename lib/{raid6 => raid/raid6/x86}/mmx.c (99%)
rename lib/{raid6 => raid/raid6/x86}/recov_avx2.c (100%)
rename lib/{raid6 => raid/raid6/x86}/recov_avx512.c (100%)
rename lib/{raid6 => raid/raid6/x86}/recov_ssse3.c (100%)
rename lib/{raid6 => raid/raid6/x86}/sse1.c (99%)
rename lib/{raid6 => raid/raid6/x86}/sse2.c (100%)
delete mode 100644 lib/raid6/Makefile
delete mode 100644 lib/raid6/test/.gitignore
delete mode 100644 lib/raid6/test/Makefile
diff --git a/lib/Kconfig b/lib/Kconfig
index 716247fedaf0..67cc485484aa 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -8,28 +8,6 @@ config BINARY_PRINTF
menu "Library routines"
-config RAID6_PQ
- tristate
-
-config RAID6_PQ_KUNIT_TEST
- tristate "KUnit tests for raid6 PQ functions" if !KUNIT_ALL_TESTS
- depends on KUNIT
- depends on RAID6_PQ
- default KUNIT_ALL_TESTS
- help
- Unit tests for the RAID6 PQ library functions.
-
- This is intended to help people writing architecture-specific
- optimized versions. If unsure, say N.
-
-config RAID6_PQ_BENCHMARK
- bool "Automatically choose fastest RAID6 PQ functions"
- depends on RAID6_PQ
- default y
- help
- Benchmark all available RAID6 PQ functions on init and choose the
- fastest one.
-
config LINEAR_RANGES
tristate
diff --git a/lib/Makefile b/lib/Makefile
index 84da412a044f..b5275d922662 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -166,7 +166,6 @@ obj-$(CONFIG_LZ4_DECOMPRESS) += lz4/
obj-$(CONFIG_ZSTD_COMPRESS) += zstd/
obj-$(CONFIG_ZSTD_DECOMPRESS) += zstd/
obj-$(CONFIG_XZ_DEC) += xz/
-obj-$(CONFIG_RAID6_PQ) += raid6/
lib-$(CONFIG_DECOMPRESS_GZIP) += decompress_inflate.o
lib-$(CONFIG_DECOMPRESS_BZIP2) += decompress_bunzip2.o
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
index 1fc4b00e0d71..9bea599d66da 100644
--- a/lib/raid/Kconfig
+++ b/lib/raid/Kconfig
@@ -28,3 +28,25 @@ config XOR_KUNIT_TEST
This is intended to help people writing architecture-specific
optimized versions. If unsure, say N.
+
+config RAID6_PQ
+ tristate
+
+config RAID6_PQ_KUNIT_TEST
+ tristate "KUnit tests for raid6 PQ functions" if !KUNIT_ALL_TESTS
+ depends on KUNIT
+ depends on RAID6_PQ
+ default KUNIT_ALL_TESTS
+ help
+ Unit tests for the RAID6 PQ library functions.
+
+ This is intended to help people writing architecture-specific
+ optimized versions. If unsure, say N.
+
+config RAID6_PQ_BENCHMARK
+ bool "Automatically choose fastest RAID6 PQ functions"
+ depends on RAID6_PQ
+ default y
+ help
+ Benchmark all available RAID6 PQ functions on init and choose the
+ fastest one.
diff --git a/lib/raid/Makefile b/lib/raid/Makefile
index 3540fe846dc4..6fc5eeb53df0 100644
--- a/lib/raid/Makefile
+++ b/lib/raid/Makefile
@@ -1,3 +1,3 @@
# SPDX-License-Identifier: GPL-2.0
-obj-y += xor/
+obj-y += xor/ raid6/
diff --git a/lib/raid6/.gitignore b/lib/raid/raid6/.gitignore
similarity index 100%
rename from lib/raid6/.gitignore
rename to lib/raid/raid6/.gitignore
diff --git a/lib/raid/raid6/Makefile b/lib/raid/raid6/Makefile
new file mode 100644
index 000000000000..f155eba06fe3
--- /dev/null
+++ b/lib/raid/raid6/Makefile
@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_RAID6_PQ) += raid6_pq.o tests/
+
+raid6_pq-y += algos.o tables.o
+
+# generic integer generation and recovery implementation
+raid6_pq-y += int1.o int2.o int4.o int8.o
+raid6_pq-y += recov.o
+
+# architecture-specific generation and recovery implementations:
+raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += arm/neon.o \
+ arm/neon1.o \
+ arm/neon2.o \
+ arm/neon4.o \
+ arm/neon8.o \
+ arm/recov_neon.o \
+ arm/recov_neon_inner.o
+raid6_pq-$(CONFIG_LOONGARCH) += loongarch/loongarch_simd.o \
+ loongarch/recov_loongarch_simd.o
+raid6_pq-$(CONFIG_ALTIVEC) += powerpc/altivec1.o \
+ powerpc/altivec2.o \
+ powerpc/altivec4.o \
+ powerpc/altivec8.o \
+ powerpc/vpermxor1.o \
+ powerpc/vpermxor2.o \
+ powerpc/vpermxor4.o \
+ powerpc/vpermxor8.o
+raid6_pq-$(CONFIG_RISCV_ISA_V) += riscv/rvv.o \
+ riscv/recov_rvv.o
+raid6_pq-$(CONFIG_S390) += s390/s390vx8.o \
+ s390/recov_s390xc.o
+ifeq ($(CONFIG_X86),y)
+raid6_pq-$(CONFIG_X86_32) += x86/mmx.o \
+ x86/sse1.o
+endif
+raid6_pq-$(CONFIG_X86) += x86/sse2.o \
+ x86/avx2.o \
+ x86/avx512.o \
+ x86/recov_ssse3.o \
+ x86/recov_avx2.o \
+ x86/recov_avx512.o
+
+hostprogs += mktables
+
+ifeq ($(CONFIG_ALTIVEC),y)
+altivec_flags := -maltivec $(call cc-option,-mabi=altivec)
+# Enable <altivec.h>
+altivec_flags += -isystem $(shell $(CC) -print-file-name=include)
+
+ifdef CONFIG_CC_IS_CLANG
+# clang ppc port does not yet support -maltivec when -msoft-float is
+# enabled. A future release of clang will resolve this
+# https://llvm.org/pr31177
+CFLAGS_REMOVE_powerpc/altivec1.o += -msoft-float
+CFLAGS_REMOVE_powerpc/altivec2.o += -msoft-float
+CFLAGS_REMOVE_powerpc/altivec4.o += -msoft-float
+CFLAGS_REMOVE_powerpc/altivec8.o += -msoft-float
+CFLAGS_REMOVE_vpermxor1.o += -msoft-float
+CFLAGS_REMOVE_vpermxor2.o += -msoft-float
+CFLAGS_REMOVE_vpermxor4.o += -msoft-float
+CFLAGS_REMOVE_vpermxor8.o += -msoft-float
+endif
+endif
+
+quiet_cmd_unroll = UNROLL $@
+ cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@
+
+targets += int1.c int2.c int4.c int8.c
+$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE
+ $(call if_changed,unroll)
+
+CFLAGS_arm/neon1.o += $(CC_FLAGS_FPU)
+CFLAGS_arm/neon2.o += $(CC_FLAGS_FPU)
+CFLAGS_arm/neon4.o += $(CC_FLAGS_FPU)
+CFLAGS_arm/neon8.o += $(CC_FLAGS_FPU)
+CFLAGS_arm/recov_neon_inner.o += $(CC_FLAGS_FPU)
+CFLAGS_REMOVE_arm/neon1.o += $(CC_FLAGS_NO_FPU)
+CFLAGS_REMOVE_arm/neon2.o += $(CC_FLAGS_NO_FPU)
+CFLAGS_REMOVE_arm/neon4.o += $(CC_FLAGS_NO_FPU)
+CFLAGS_REMOVE_arm/neon8.o += $(CC_FLAGS_NO_FPU)
+CFLAGS_REMOVE_arm/recov_neon_inner.o += $(CC_FLAGS_NO_FPU)
+targets += arm/neon1.c arm/neon2.c arm/neon4.c arm/neon8.c
+$(obj)/arm/neon%.c: $(src)/arm/neon.uc $(src)/unroll.awk FORCE
+ $(call if_changed,unroll)
+
+CFLAGS_powerpc/altivec1.o += $(altivec_flags)
+CFLAGS_powerpc/altivec2.o += $(altivec_flags)
+CFLAGS_powerpc/altivec4.o += $(altivec_flags)
+CFLAGS_powerpc/altivec8.o += $(altivec_flags)
+targets += powerpc/altivec1.c \
+ powerpc/altivec2.c \
+ powerpc/altivec4.c \
+ powerpc/altivec8.c
+$(obj)/powerpc/altivec%.c: $(src)/powerpc/altivec.uc $(src)/unroll.awk FORCE
+ $(call if_changed,unroll)
+
+CFLAGS_powerpc/vpermxor1.o += $(altivec_flags)
+CFLAGS_powerpc/vpermxor2.o += $(altivec_flags)
+CFLAGS_powerpc/vpermxor4.o += $(altivec_flags)
+CFLAGS_powerpc/vpermxor8.o += $(altivec_flags)
+targets += powerpc/vpermxor1.c \
+ powerpc/vpermxor2.c \
+ powerpc/vpermxor4.c \
+ powerpc/vpermxor8.c
+$(obj)/powerpc/vpermxor%.c: $(src)/powerpc/vpermxor.uc $(src)/unroll.awk FORCE
+ $(call if_changed,unroll)
+
+targets += s390/s390vx8.c
+$(obj)/s390/s390vx%.c: $(src)/s390/s390vx.uc $(src)/unroll.awk FORCE
+ $(call if_changed,unroll)
+
+quiet_cmd_mktable = TABLE $@
+ cmd_mktable = $(obj)/mktables > $@
+
+targets += tables.c
+$(obj)/tables.c: $(obj)/mktables FORCE
+ $(call if_changed,mktable)
diff --git a/lib/raid6/algos.c b/lib/raid/raid6/algos.c
similarity index 100%
rename from lib/raid6/algos.c
rename to lib/raid/raid6/algos.c
diff --git a/lib/raid6/neon.c b/lib/raid/raid6/arm/neon.c
similarity index 100%
rename from lib/raid6/neon.c
rename to lib/raid/raid6/arm/neon.c
diff --git a/lib/raid6/neon.h b/lib/raid/raid6/arm/neon.h
similarity index 100%
rename from lib/raid6/neon.h
rename to lib/raid/raid6/arm/neon.h
diff --git a/lib/raid6/neon.uc b/lib/raid/raid6/arm/neon.uc
similarity index 100%
rename from lib/raid6/neon.uc
rename to lib/raid/raid6/arm/neon.uc
diff --git a/lib/raid6/recov_neon.c b/lib/raid/raid6/arm/recov_neon.c
similarity index 100%
rename from lib/raid6/recov_neon.c
rename to lib/raid/raid6/arm/recov_neon.c
diff --git a/lib/raid6/recov_neon_inner.c b/lib/raid/raid6/arm/recov_neon_inner.c
similarity index 100%
rename from lib/raid6/recov_neon_inner.c
rename to lib/raid/raid6/arm/recov_neon_inner.c
diff --git a/lib/raid6/int.uc b/lib/raid/raid6/int.uc
similarity index 100%
rename from lib/raid6/int.uc
rename to lib/raid/raid6/int.uc
diff --git a/lib/raid6/loongarch_simd.c b/lib/raid/raid6/loongarch/loongarch_simd.c
similarity index 100%
rename from lib/raid6/loongarch_simd.c
rename to lib/raid/raid6/loongarch/loongarch_simd.c
diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid/raid6/loongarch/recov_loongarch_simd.c
similarity index 100%
rename from lib/raid6/recov_loongarch_simd.c
rename to lib/raid/raid6/loongarch/recov_loongarch_simd.c
diff --git a/lib/raid6/mktables.c b/lib/raid/raid6/mktables.c
similarity index 100%
rename from lib/raid6/mktables.c
rename to lib/raid/raid6/mktables.c
diff --git a/lib/raid6/altivec.uc b/lib/raid/raid6/powerpc/altivec.uc
similarity index 98%
rename from lib/raid6/altivec.uc
rename to lib/raid/raid6/powerpc/altivec.uc
index 2c59963e58f9..130d3d3dd42c 100644
--- a/lib/raid6/altivec.uc
+++ b/lib/raid/raid6/powerpc/altivec.uc
@@ -24,8 +24,6 @@
#include <linux/raid/pq.h>
-#ifdef CONFIG_ALTIVEC
-
#include <altivec.h>
#include <asm/cputable.h>
#include <asm/switch_to.h>
@@ -122,5 +120,3 @@ const struct raid6_calls raid6_altivec$# = {
"altivecx$#",
0
};
-
-#endif /* CONFIG_ALTIVEC */
diff --git a/lib/raid6/vpermxor.uc b/lib/raid/raid6/powerpc/vpermxor.uc
similarity index 98%
rename from lib/raid6/vpermxor.uc
rename to lib/raid/raid6/powerpc/vpermxor.uc
index a8e76b1c956e..595f20aaf4cf 100644
--- a/lib/raid6/vpermxor.uc
+++ b/lib/raid/raid6/powerpc/vpermxor.uc
@@ -21,8 +21,6 @@
*/
#include <linux/raid/pq.h>
-#ifdef CONFIG_ALTIVEC
-
#include <altivec.h>
#include <asm/ppc-opcode.h>
#include <asm/cputable.h>
@@ -95,4 +93,3 @@ const struct raid6_calls raid6_vpermxor$# = {
"vpermxor$#",
0
};
-#endif
diff --git a/lib/raid6/recov.c b/lib/raid/raid6/recov.c
similarity index 100%
rename from lib/raid6/recov.c
rename to lib/raid/raid6/recov.c
diff --git a/lib/raid6/recov_rvv.c b/lib/raid/raid6/riscv/recov_rvv.c
similarity index 100%
rename from lib/raid6/recov_rvv.c
rename to lib/raid/raid6/riscv/recov_rvv.c
diff --git a/lib/raid6/rvv.c b/lib/raid/raid6/riscv/rvv.c
similarity index 100%
rename from lib/raid6/rvv.c
rename to lib/raid/raid6/riscv/rvv.c
diff --git a/lib/raid6/rvv.h b/lib/raid/raid6/riscv/rvv.h
similarity index 100%
rename from lib/raid6/rvv.h
rename to lib/raid/raid6/riscv/rvv.h
diff --git a/lib/raid6/recov_s390xc.c b/lib/raid/raid6/s390/recov_s390xc.c
similarity index 100%
rename from lib/raid6/recov_s390xc.c
rename to lib/raid/raid6/s390/recov_s390xc.c
diff --git a/lib/raid6/s390vx.uc b/lib/raid/raid6/s390/s390vx.uc
similarity index 100%
rename from lib/raid6/s390vx.uc
rename to lib/raid/raid6/s390/s390vx.uc
diff --git a/lib/raid/raid6/tests/Makefile b/lib/raid/raid6/tests/Makefile
new file mode 100644
index 000000000000..87a001b22847
--- /dev/null
+++ b/lib/raid/raid6/tests/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0
+
+obj-$(CONFIG_RAID6_PQ_KUNIT_TEST) += raid6_kunit.o
diff --git a/lib/raid6/test/test.c b/lib/raid/raid6/tests/raid6_kunit.c
similarity index 100%
rename from lib/raid6/test/test.c
rename to lib/raid/raid6/tests/raid6_kunit.c
diff --git a/lib/raid6/unroll.awk b/lib/raid/raid6/unroll.awk
similarity index 100%
rename from lib/raid6/unroll.awk
rename to lib/raid/raid6/unroll.awk
diff --git a/lib/raid6/avx2.c b/lib/raid/raid6/x86/avx2.c
similarity index 100%
rename from lib/raid6/avx2.c
rename to lib/raid/raid6/x86/avx2.c
diff --git a/lib/raid6/avx512.c b/lib/raid/raid6/x86/avx512.c
similarity index 100%
rename from lib/raid6/avx512.c
rename to lib/raid/raid6/x86/avx512.c
diff --git a/lib/raid6/mmx.c b/lib/raid/raid6/x86/mmx.c
similarity index 99%
rename from lib/raid6/mmx.c
rename to lib/raid/raid6/x86/mmx.c
index e411f0cfbd95..7e9810669347 100644
--- a/lib/raid6/mmx.c
+++ b/lib/raid/raid6/x86/mmx.c
@@ -11,8 +11,6 @@
* MMX implementation of RAID-6 syndrome functions
*/
-#ifdef CONFIG_X86_32
-
#include <linux/raid/pq.h>
#include <asm/fpu/api.h>
@@ -135,5 +133,3 @@ const struct raid6_calls raid6_mmxx2 = {
"mmxx2",
0
};
-
-#endif
diff --git a/lib/raid6/recov_avx2.c b/lib/raid/raid6/x86/recov_avx2.c
similarity index 100%
rename from lib/raid6/recov_avx2.c
rename to lib/raid/raid6/x86/recov_avx2.c
diff --git a/lib/raid6/recov_avx512.c b/lib/raid/raid6/x86/recov_avx512.c
similarity index 100%
rename from lib/raid6/recov_avx512.c
rename to lib/raid/raid6/x86/recov_avx512.c
diff --git a/lib/raid6/recov_ssse3.c b/lib/raid/raid6/x86/recov_ssse3.c
similarity index 100%
rename from lib/raid6/recov_ssse3.c
rename to lib/raid/raid6/x86/recov_ssse3.c
diff --git a/lib/raid6/sse1.c b/lib/raid/raid6/x86/sse1.c
similarity index 99%
rename from lib/raid6/sse1.c
rename to lib/raid/raid6/x86/sse1.c
index 794d5cfa0306..deecdd72ceec 100644
--- a/lib/raid6/sse1.c
+++ b/lib/raid/raid6/x86/sse1.c
@@ -16,8 +16,6 @@
* worthwhile as a separate implementation.
*/
-#ifdef CONFIG_X86_32
-
#include <linux/raid/pq.h>
#include <asm/fpu/api.h>
@@ -155,5 +153,3 @@ const struct raid6_calls raid6_sse1x2 = {
"sse1x2",
1 /* Has cache hints */
};
-
-#endif
diff --git a/lib/raid6/sse2.c b/lib/raid/raid6/x86/sse2.c
similarity index 100%
rename from lib/raid6/sse2.c
rename to lib/raid/raid6/x86/sse2.c
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
deleted file mode 100644
index 6fd048c127b6..000000000000
--- a/lib/raid6/Makefile
+++ /dev/null
@@ -1,83 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_RAID6_PQ) += raid6_pq.o test/
-
-raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \
- int8.o
-
-raid6_pq-$(CONFIG_X86) += recov_ssse3.o recov_avx2.o mmx.o sse1.o sse2.o avx2.o avx512.o recov_avx512.o
-raid6_pq-$(CONFIG_ALTIVEC) += altivec1.o altivec2.o altivec4.o altivec8.o \
- vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
-raid6_pq-$(CONFIG_KERNEL_MODE_NEON) += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
-raid6_pq-$(CONFIG_S390) += s390vx8.o recov_s390xc.o
-raid6_pq-$(CONFIG_LOONGARCH) += loongarch_simd.o recov_loongarch_simd.o
-raid6_pq-$(CONFIG_RISCV_ISA_V) += rvv.o recov_rvv.o
-
-hostprogs += mktables
-
-ifeq ($(CONFIG_ALTIVEC),y)
-altivec_flags := -maltivec $(call cc-option,-mabi=altivec)
-# Enable <altivec.h>
-altivec_flags += -isystem $(shell $(CC) -print-file-name=include)
-
-ifdef CONFIG_CC_IS_CLANG
-# clang ppc port does not yet support -maltivec when -msoft-float is
-# enabled. A future release of clang will resolve this
-# https://llvm.org/pr31177
-CFLAGS_REMOVE_altivec1.o += -msoft-float
-CFLAGS_REMOVE_altivec2.o += -msoft-float
-CFLAGS_REMOVE_altivec4.o += -msoft-float
-CFLAGS_REMOVE_altivec8.o += -msoft-float
-CFLAGS_REMOVE_vpermxor1.o += -msoft-float
-CFLAGS_REMOVE_vpermxor2.o += -msoft-float
-CFLAGS_REMOVE_vpermxor4.o += -msoft-float
-CFLAGS_REMOVE_vpermxor8.o += -msoft-float
-endif
-endif
-
-quiet_cmd_unroll = UNROLL $@
- cmd_unroll = $(AWK) -v N=$* -f $(src)/unroll.awk < $< > $@
-
-targets += int1.c int2.c int4.c int8.c
-$(obj)/int%.c: $(src)/int.uc $(src)/unroll.awk FORCE
- $(call if_changed,unroll)
-
-CFLAGS_altivec1.o += $(altivec_flags)
-CFLAGS_altivec2.o += $(altivec_flags)
-CFLAGS_altivec4.o += $(altivec_flags)
-CFLAGS_altivec8.o += $(altivec_flags)
-targets += altivec1.c altivec2.c altivec4.c altivec8.c
-$(obj)/altivec%.c: $(src)/altivec.uc $(src)/unroll.awk FORCE
- $(call if_changed,unroll)
-
-CFLAGS_vpermxor1.o += $(altivec_flags)
-CFLAGS_vpermxor2.o += $(altivec_flags)
-CFLAGS_vpermxor4.o += $(altivec_flags)
-CFLAGS_vpermxor8.o += $(altivec_flags)
-targets += vpermxor1.c vpermxor2.c vpermxor4.c vpermxor8.c
-$(obj)/vpermxor%.c: $(src)/vpermxor.uc $(src)/unroll.awk FORCE
- $(call if_changed,unroll)
-
-CFLAGS_neon1.o += $(CC_FLAGS_FPU)
-CFLAGS_neon2.o += $(CC_FLAGS_FPU)
-CFLAGS_neon4.o += $(CC_FLAGS_FPU)
-CFLAGS_neon8.o += $(CC_FLAGS_FPU)
-CFLAGS_recov_neon_inner.o += $(CC_FLAGS_FPU)
-CFLAGS_REMOVE_neon1.o += $(CC_FLAGS_NO_FPU)
-CFLAGS_REMOVE_neon2.o += $(CC_FLAGS_NO_FPU)
-CFLAGS_REMOVE_neon4.o += $(CC_FLAGS_NO_FPU)
-CFLAGS_REMOVE_neon8.o += $(CC_FLAGS_NO_FPU)
-CFLAGS_REMOVE_recov_neon_inner.o += $(CC_FLAGS_NO_FPU)
-targets += neon1.c neon2.c neon4.c neon8.c
-$(obj)/neon%.c: $(src)/neon.uc $(src)/unroll.awk FORCE
- $(call if_changed,unroll)
-
-targets += s390vx8.c
-$(obj)/s390vx%.c: $(src)/s390vx.uc $(src)/unroll.awk FORCE
- $(call if_changed,unroll)
-
-quiet_cmd_mktable = TABLE $@
- cmd_mktable = $(obj)/mktables > $@
-
-targets += tables.c
-$(obj)/tables.c: $(obj)/mktables FORCE
- $(call if_changed,mktable)
diff --git a/lib/raid6/test/.gitignore b/lib/raid6/test/.gitignore
deleted file mode 100644
index 1b68a77f348f..000000000000
--- a/lib/raid6/test/.gitignore
+++ /dev/null
@@ -1,3 +0,0 @@
-/int.uc
-/neon.uc
-/raid6test
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
deleted file mode 100644
index 268b085af4d3..000000000000
--- a/lib/raid6/test/Makefile
+++ /dev/null
@@ -1,3 +0,0 @@
-# SPDX-License-Identifier: GPL-2.0
-
-obj-$(CONFIG_RAID6_PQ_KUNIT_TEST) += test.o
--
2.47.3
^ permalink raw reply related
* [PATCH 02/17] raid6: remove __KERNEL__ ifdefs
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
With the test code ported to kernel space, none of this is required.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
include/linux/raid/pq.h | 90 --------------------------------
lib/raid6/algos.c | 12 -----
lib/raid6/altivec.uc | 10 +---
lib/raid6/avx2.c | 2 +-
lib/raid6/avx512.c | 2 +-
lib/raid6/loongarch.h | 38 --------------
lib/raid6/loongarch_simd.c | 3 +-
lib/raid6/mktables.c | 14 -----
lib/raid6/mmx.c | 2 +-
lib/raid6/neon.c | 6 ---
lib/raid6/recov_avx2.c | 2 +-
lib/raid6/recov_avx512.c | 2 +-
lib/raid6/recov_loongarch_simd.c | 3 +-
lib/raid6/recov_neon.c | 6 ---
lib/raid6/recov_ssse3.c | 2 +-
lib/raid6/rvv.h | 11 +---
lib/raid6/sse1.c | 2 +-
lib/raid6/sse2.c | 2 +-
lib/raid6/vpermxor.uc | 7 ---
lib/raid6/x86.h | 75 --------------------------
20 files changed, 15 insertions(+), 276 deletions(-)
delete mode 100644 lib/raid6/loongarch.h
delete mode 100644 lib/raid6/x86.h
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 08c5995ea980..d26788fada58 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -8,8 +8,6 @@
#ifndef LINUX_RAID_RAID6_H
#define LINUX_RAID_RAID6_H
-#ifdef __KERNEL__
-
#include <linux/blkdev.h>
#include <linux/mm.h>
@@ -19,59 +17,6 @@ static inline void *raid6_get_zero_page(void)
return page_address(ZERO_PAGE(0));
}
-#else /* ! __KERNEL__ */
-/* Used for testing in user space */
-
-#include <errno.h>
-#include <inttypes.h>
-#include <stddef.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/time.h>
-#include <sys/types.h>
-
-/* Not standard, but glibc defines it */
-#define BITS_PER_LONG __WORDSIZE
-
-typedef uint8_t u8;
-typedef uint16_t u16;
-typedef uint32_t u32;
-typedef uint64_t u64;
-
-#ifndef PAGE_SIZE
-# define PAGE_SIZE 4096
-#endif
-#ifndef PAGE_SHIFT
-# define PAGE_SHIFT 12
-#endif
-extern const char raid6_empty_zero_page[PAGE_SIZE];
-
-#define __init
-#define __exit
-#ifndef __attribute_const__
-# define __attribute_const__ __attribute__((const))
-#endif
-#define noinline __attribute__((noinline))
-
-#define preempt_enable()
-#define preempt_disable()
-#define cpu_has_feature(x) 1
-#define enable_kernel_altivec()
-#define disable_kernel_altivec()
-
-#undef EXPORT_SYMBOL
-#define EXPORT_SYMBOL(sym)
-#undef EXPORT_SYMBOL_GPL
-#define EXPORT_SYMBOL_GPL(sym)
-#define MODULE_LICENSE(licence)
-#define MODULE_DESCRIPTION(desc)
-#define subsys_initcall(x)
-#define module_exit(x)
-
-#define IS_ENABLED(x) (x)
-#define CONFIG_RAID6_PQ_BENCHMARK 1
-#endif /* __KERNEL__ */
-
/* Routine choices */
struct raid6_calls {
void (*gen_syndrome)(int, size_t, void **);
@@ -165,39 +110,4 @@ extern void (*raid6_2data_recov)(int disks, size_t bytes, int faila, int failb,
extern void (*raid6_datap_recov)(int disks, size_t bytes, int faila,
void **ptrs);
-/* Some definitions to allow code to be compiled for testing in userspace */
-#ifndef __KERNEL__
-
-# define jiffies raid6_jiffies()
-# define printk printf
-# define pr_err(format, ...) fprintf(stderr, format, ## __VA_ARGS__)
-# define pr_info(format, ...) fprintf(stdout, format, ## __VA_ARGS__)
-# define GFP_KERNEL 0
-# define __get_free_pages(x, y) ((unsigned long)mmap(NULL, PAGE_SIZE << (y), \
- PROT_READ|PROT_WRITE, \
- MAP_PRIVATE|MAP_ANONYMOUS,\
- 0, 0))
-# define free_pages(x, y) munmap((void *)(x), PAGE_SIZE << (y))
-
-static inline void cpu_relax(void)
-{
- /* Nothing */
-}
-
-#undef HZ
-#define HZ 1000
-static inline uint32_t raid6_jiffies(void)
-{
- struct timeval tv;
- gettimeofday(&tv, NULL);
- return tv.tv_sec*1000 + tv.tv_usec/1000;
-}
-
-static inline void *raid6_get_zero_page(void)
-{
- return raid6_empty_zero_page;
-}
-
-#endif /* ! __KERNEL__ */
-
#endif /* LINUX_RAID_RAID6_H */
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 5a9f4882e18d..985c60bb00a4 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -12,13 +12,8 @@
*/
#include <linux/raid/pq.h>
-#ifndef __KERNEL__
-#include <sys/mman.h>
-#include <stdio.h>
-#else
#include <linux/module.h>
#include <linux/gfp.h>
-#endif
#include <kunit/visibility.h>
struct raid6_calls raid6_call;
@@ -123,14 +118,7 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = {
};
EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algos);
-#ifdef __KERNEL__
#define RAID6_TIME_JIFFIES_LG2 4
-#else
-/* Need more time to be stable in userspace */
-#define RAID6_TIME_JIFFIES_LG2 9
-#define time_before(x, y) ((x) < (y))
-#endif
-
#define RAID6_TEST_DISKS 8
#define RAID6_TEST_DISKS_ORDER 3
diff --git a/lib/raid6/altivec.uc b/lib/raid6/altivec.uc
index d20ed0d11411..2c59963e58f9 100644
--- a/lib/raid6/altivec.uc
+++ b/lib/raid6/altivec.uc
@@ -27,10 +27,8 @@
#ifdef CONFIG_ALTIVEC
#include <altivec.h>
-#ifdef __KERNEL__
-# include <asm/cputable.h>
-# include <asm/switch_to.h>
-#endif /* __KERNEL__ */
+#include <asm/cputable.h>
+#include <asm/switch_to.h>
/*
* This is the C data type to use. We use a vector of
@@ -113,11 +111,7 @@ int raid6_have_altivec(void);
int raid6_have_altivec(void)
{
/* This assumes either all CPUs have Altivec or none does */
-# ifdef __KERNEL__
return cpu_has_feature(CPU_FTR_ALTIVEC);
-# else
- return 1;
-# endif
}
#endif
diff --git a/lib/raid6/avx2.c b/lib/raid6/avx2.c
index 059024234dce..a1a5213918af 100644
--- a/lib/raid6/avx2.c
+++ b/lib/raid6/avx2.c
@@ -14,7 +14,7 @@
*/
#include <linux/raid/pq.h>
-#include "x86.h"
+#include <asm/fpu/api.h>
static const struct raid6_avx2_constants {
u64 x1d[4];
diff --git a/lib/raid6/avx512.c b/lib/raid6/avx512.c
index 009bd0adeebf..874998bcd7d7 100644
--- a/lib/raid6/avx512.c
+++ b/lib/raid6/avx512.c
@@ -18,7 +18,7 @@
*/
#include <linux/raid/pq.h>
-#include "x86.h"
+#include <asm/fpu/api.h>
static const struct raid6_avx512_constants {
u64 x1d[8];
diff --git a/lib/raid6/loongarch.h b/lib/raid6/loongarch.h
deleted file mode 100644
index acfc33ce7056..000000000000
--- a/lib/raid6/loongarch.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- *
- * raid6/loongarch.h
- *
- * Definitions common to LoongArch RAID-6 code only
- */
-
-#ifndef _LIB_RAID6_LOONGARCH_H
-#define _LIB_RAID6_LOONGARCH_H
-
-#ifdef __KERNEL__
-
-#include <asm/cpu-features.h>
-#include <asm/fpu.h>
-
-#else /* for user-space testing */
-
-#include <sys/auxv.h>
-
-/* have to supply these defines for glibc 2.37- and musl */
-#ifndef HWCAP_LOONGARCH_LSX
-#define HWCAP_LOONGARCH_LSX (1 << 4)
-#endif
-#ifndef HWCAP_LOONGARCH_LASX
-#define HWCAP_LOONGARCH_LASX (1 << 5)
-#endif
-
-#define kernel_fpu_begin()
-#define kernel_fpu_end()
-
-#define cpu_has_lsx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LSX)
-#define cpu_has_lasx (getauxval(AT_HWCAP) & HWCAP_LOONGARCH_LASX)
-
-#endif /* __KERNEL__ */
-
-#endif /* _LIB_RAID6_LOONGARCH_H */
diff --git a/lib/raid6/loongarch_simd.c b/lib/raid6/loongarch_simd.c
index aa5d9f924ca3..72f4d92d4876 100644
--- a/lib/raid6/loongarch_simd.c
+++ b/lib/raid6/loongarch_simd.c
@@ -10,7 +10,8 @@
*/
#include <linux/raid/pq.h>
-#include "loongarch.h"
+#include <asm/cpu-features.h>
+#include <asm/fpu.h>
/*
* The vector algorithms are currently priority 0, which means the generic
diff --git a/lib/raid6/mktables.c b/lib/raid6/mktables.c
index 3be03793237c..3de1dbf6846c 100644
--- a/lib/raid6/mktables.c
+++ b/lib/raid6/mktables.c
@@ -56,9 +56,7 @@ int main(int argc, char *argv[])
uint8_t v;
uint8_t exptbl[256], invtbl[256];
- printf("#ifdef __KERNEL__\n");
printf("#include <linux/export.h>\n");
- printf("#endif\n");
printf("#include <linux/raid/pq.h>\n");
/* Compute multiplication table */
@@ -76,9 +74,7 @@ int main(int argc, char *argv[])
printf("\t},\n");
}
printf("};\n");
- printf("#ifdef __KERNEL__\n");
printf("EXPORT_SYMBOL(raid6_gfmul);\n");
- printf("#endif\n");
/* Compute vector multiplication table */
printf("\nconst u8 __attribute__((aligned(256)))\n"
@@ -101,9 +97,7 @@ int main(int argc, char *argv[])
printf("\t},\n");
}
printf("};\n");
- printf("#ifdef __KERNEL__\n");
printf("EXPORT_SYMBOL(raid6_vgfmul);\n");
- printf("#endif\n");
/* Compute power-of-2 table (exponent) */
v = 1;
@@ -120,9 +114,7 @@ int main(int argc, char *argv[])
}
}
printf("};\n");
- printf("#ifdef __KERNEL__\n");
printf("EXPORT_SYMBOL(raid6_gfexp);\n");
- printf("#endif\n");
/* Compute log-of-2 table */
printf("\nconst u8 __attribute__((aligned(256)))\n"
@@ -140,9 +132,7 @@ int main(int argc, char *argv[])
}
}
printf("};\n");
- printf("#ifdef __KERNEL__\n");
printf("EXPORT_SYMBOL(raid6_gflog);\n");
- printf("#endif\n");
/* Compute inverse table x^-1 == x^254 */
printf("\nconst u8 __attribute__((aligned(256)))\n"
@@ -155,9 +145,7 @@ int main(int argc, char *argv[])
}
}
printf("};\n");
- printf("#ifdef __KERNEL__\n");
printf("EXPORT_SYMBOL(raid6_gfinv);\n");
- printf("#endif\n");
/* Compute inv(2^x + 1) (exponent-xor-inverse) table */
printf("\nconst u8 __attribute__((aligned(256)))\n"
@@ -169,9 +157,7 @@ int main(int argc, char *argv[])
(j == 7) ? '\n' : ' ');
}
printf("};\n");
- printf("#ifdef __KERNEL__\n");
printf("EXPORT_SYMBOL(raid6_gfexi);\n");
- printf("#endif\n");
return 0;
}
diff --git a/lib/raid6/mmx.c b/lib/raid6/mmx.c
index 3a5bf53a297b..e411f0cfbd95 100644
--- a/lib/raid6/mmx.c
+++ b/lib/raid6/mmx.c
@@ -14,7 +14,7 @@
#ifdef CONFIG_X86_32
#include <linux/raid/pq.h>
-#include "x86.h"
+#include <asm/fpu/api.h>
/* Shared with raid6/sse1.c */
const struct raid6_mmx_constants {
diff --git a/lib/raid6/neon.c b/lib/raid6/neon.c
index 6d9474ce6da9..47b8bb0afc65 100644
--- a/lib/raid6/neon.c
+++ b/lib/raid6/neon.c
@@ -6,13 +6,7 @@
*/
#include <linux/raid/pq.h>
-
-#ifdef __KERNEL__
#include <asm/simd.h>
-#else
-#define scoped_ksimd()
-#define cpu_has_neon() (1)
-#endif
/*
* There are 2 reasons these wrappers are kept in a separate compilation unit
diff --git a/lib/raid6/recov_avx2.c b/lib/raid6/recov_avx2.c
index 97d598d2535c..19fbd9c4dce6 100644
--- a/lib/raid6/recov_avx2.c
+++ b/lib/raid6/recov_avx2.c
@@ -5,7 +5,7 @@
*/
#include <linux/raid/pq.h>
-#include "x86.h"
+#include <asm/fpu/api.h>
static int raid6_has_avx2(void)
{
diff --git a/lib/raid6/recov_avx512.c b/lib/raid6/recov_avx512.c
index 7986120ca444..143f4976b2ad 100644
--- a/lib/raid6/recov_avx512.c
+++ b/lib/raid6/recov_avx512.c
@@ -7,7 +7,7 @@
*/
#include <linux/raid/pq.h>
-#include "x86.h"
+#include <asm/fpu/api.h>
static int raid6_has_avx512(void)
{
diff --git a/lib/raid6/recov_loongarch_simd.c b/lib/raid6/recov_loongarch_simd.c
index 93dc515997a1..eb3a1e79f01f 100644
--- a/lib/raid6/recov_loongarch_simd.c
+++ b/lib/raid6/recov_loongarch_simd.c
@@ -11,7 +11,8 @@
*/
#include <linux/raid/pq.h>
-#include "loongarch.h"
+#include <asm/cpu-features.h>
+#include <asm/fpu.h>
/*
* Unlike with the syndrome calculation algorithms, there's no boot-time
diff --git a/lib/raid6/recov_neon.c b/lib/raid6/recov_neon.c
index 9d99aeabd31a..13d5df718c15 100644
--- a/lib/raid6/recov_neon.c
+++ b/lib/raid6/recov_neon.c
@@ -5,14 +5,8 @@
*/
#include <linux/raid/pq.h>
-
-#ifdef __KERNEL__
#include <asm/simd.h>
#include "neon.h"
-#else
-#define scoped_ksimd()
-#define cpu_has_neon() (1)
-#endif
static int raid6_has_neon(void)
{
diff --git a/lib/raid6/recov_ssse3.c b/lib/raid6/recov_ssse3.c
index 2e849185c32b..146cdbf465bd 100644
--- a/lib/raid6/recov_ssse3.c
+++ b/lib/raid6/recov_ssse3.c
@@ -4,7 +4,7 @@
*/
#include <linux/raid/pq.h>
-#include "x86.h"
+#include <asm/fpu/api.h>
static int raid6_has_ssse3(void)
{
diff --git a/lib/raid6/rvv.h b/lib/raid6/rvv.h
index 6d0708a2c8a4..b0a71b375962 100644
--- a/lib/raid6/rvv.h
+++ b/lib/raid6/rvv.h
@@ -7,17 +7,8 @@
* Definitions for RISC-V RAID-6 code
*/
-#ifdef __KERNEL__
-#include <asm/vector.h>
-#else
-#define kernel_vector_begin()
-#define kernel_vector_end()
-#include <sys/auxv.h>
-#include <asm/hwcap.h>
-#define has_vector() (getauxval(AT_HWCAP) & COMPAT_HWCAP_ISA_V)
-#endif
-
#include <linux/raid/pq.h>
+#include <asm/vector.h>
static int rvv_has_vector(void)
{
diff --git a/lib/raid6/sse1.c b/lib/raid6/sse1.c
index 692fa3a93bf0..794d5cfa0306 100644
--- a/lib/raid6/sse1.c
+++ b/lib/raid6/sse1.c
@@ -19,7 +19,7 @@
#ifdef CONFIG_X86_32
#include <linux/raid/pq.h>
-#include "x86.h"
+#include <asm/fpu/api.h>
/* Defined in raid6/mmx.c */
extern const struct raid6_mmx_constants {
diff --git a/lib/raid6/sse2.c b/lib/raid6/sse2.c
index 2930220249c9..f9edf8a8d1c4 100644
--- a/lib/raid6/sse2.c
+++ b/lib/raid6/sse2.c
@@ -13,7 +13,7 @@
*/
#include <linux/raid/pq.h>
-#include "x86.h"
+#include <asm/fpu/api.h>
static const struct raid6_sse_constants {
u64 x1d[2];
diff --git a/lib/raid6/vpermxor.uc b/lib/raid6/vpermxor.uc
index 1bfb127fbfe8..a8e76b1c956e 100644
--- a/lib/raid6/vpermxor.uc
+++ b/lib/raid6/vpermxor.uc
@@ -25,10 +25,8 @@
#include <altivec.h>
#include <asm/ppc-opcode.h>
-#ifdef __KERNEL__
#include <asm/cputable.h>
#include <asm/switch_to.h>
-#endif
typedef vector unsigned char unative_t;
#define NSIZE sizeof(unative_t)
@@ -85,13 +83,8 @@ int raid6_have_altivec_vpermxor(void);
int raid6_have_altivec_vpermxor(void)
{
/* Check if arch has both altivec and the vpermxor instructions */
-# ifdef __KERNEL__
return (cpu_has_feature(CPU_FTR_ALTIVEC_COMP) &&
cpu_has_feature(CPU_FTR_ARCH_207S));
-# else
- return 1;
-#endif
-
}
#endif
diff --git a/lib/raid6/x86.h b/lib/raid6/x86.h
deleted file mode 100644
index 9a6ff37115e7..000000000000
--- a/lib/raid6/x86.h
+++ /dev/null
@@ -1,75 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/* ----------------------------------------------------------------------- *
- *
- * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
-/*
- * raid6/x86.h
- *
- * Definitions common to x86 and x86-64 RAID-6 code only
- */
-
-#ifndef LINUX_RAID_RAID6X86_H
-#define LINUX_RAID_RAID6X86_H
-
-#if (defined(__i386__) || defined(__x86_64__)) && !defined(__arch_um__)
-
-#ifdef __KERNEL__ /* Real code */
-
-#include <asm/fpu/api.h>
-
-#else /* Dummy code for user space testing */
-
-static inline void kernel_fpu_begin(void)
-{
-}
-
-static inline void kernel_fpu_end(void)
-{
-}
-
-#define __aligned(x) __attribute__((aligned(x)))
-
-#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */
-#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions
- * (fast save and restore) */
-#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */
-#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */
-#define X86_FEATURE_XMM3 (4*32+ 0) /* "pni" SSE-3 */
-#define X86_FEATURE_SSSE3 (4*32+ 9) /* Supplemental SSE-3 */
-#define X86_FEATURE_AVX (4*32+28) /* Advanced Vector Extensions */
-#define X86_FEATURE_AVX2 (9*32+ 5) /* AVX2 instructions */
-#define X86_FEATURE_AVX512F (9*32+16) /* AVX-512 Foundation */
-#define X86_FEATURE_AVX512DQ (9*32+17) /* AVX-512 DQ (Double/Quad granular)
- * Instructions
- */
-#define X86_FEATURE_AVX512BW (9*32+30) /* AVX-512 BW (Byte/Word granular)
- * Instructions
- */
-#define X86_FEATURE_AVX512VL (9*32+31) /* AVX-512 VL (128/256 Vector Length)
- * Extensions
- */
-#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */
-
-/* Should work well enough on modern CPUs for testing */
-static inline int boot_cpu_has(int flag)
-{
- u32 eax, ebx, ecx, edx;
-
- eax = (flag & 0x100) ? 7 :
- (flag & 0x20) ? 0x80000001 : 1;
- ecx = 0;
-
- asm volatile("cpuid"
- : "+a" (eax), "=b" (ebx), "=d" (edx), "+c" (ecx));
-
- return ((flag & 0x100 ? ebx :
- (flag & 0x80) ? ecx : edx) >> (flag & 31)) & 1;
-}
-
-#endif /* ndef __KERNEL__ */
-
-#endif
-#endif
--
2.47.3
^ permalink raw reply related
* [PATCH 01/17] raid6: turn the userspace test harness into a kunit test
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
In-Reply-To: <20260324064115.3217136-1-hch@lst.de>
Currently the raid6 code can be compiled as userspace code to run the
test suite. Convert that to be a kunit case with minimal changes to
avoid mutating global state so that we can drop this requirement.
Note that this is not a good kunit test case yet and will need a lot more
work, but that is deferred until the raid6 code is moved to it's new
place, which is easier if the userspace makefile doesn't need adjustments
for the new location first.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
include/linux/raid/pq.h | 3 -
lib/Kconfig | 11 +++
lib/raid6/Makefile | 2 +-
lib/raid6/algos.c | 5 +-
lib/raid6/recov.c | 34 ---------
lib/raid6/test/Makefile | 155 +-------------------------------------
lib/raid6/test/test.c | 161 +++++++++++++++++++++-------------------
7 files changed, 103 insertions(+), 268 deletions(-)
diff --git a/include/linux/raid/pq.h b/include/linux/raid/pq.h
index 2467b3be15c9..08c5995ea980 100644
--- a/include/linux/raid/pq.h
+++ b/include/linux/raid/pq.h
@@ -144,7 +144,6 @@ extern const struct raid6_calls raid6_neonx8;
/* Algorithm list */
extern const struct raid6_calls * const raid6_algos[];
extern const struct raid6_recov_calls *const raid6_recov_algos[];
-int raid6_select_algo(void);
/* Return values from chk_syndrome */
#define RAID6_OK 0
@@ -165,8 +164,6 @@ extern void (*raid6_2data_recov)(int disks, size_t bytes, int faila, int failb,
void **ptrs);
extern void (*raid6_datap_recov)(int disks, size_t bytes, int faila,
void **ptrs);
-void raid6_dual_recov(int disks, size_t bytes, int faila, int failb,
- void **ptrs);
/* Some definitions to allow code to be compiled for testing in userspace */
#ifndef __KERNEL__
diff --git a/lib/Kconfig b/lib/Kconfig
index 5be57adcd454..716247fedaf0 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -11,6 +11,17 @@ menu "Library routines"
config RAID6_PQ
tristate
+config RAID6_PQ_KUNIT_TEST
+ tristate "KUnit tests for raid6 PQ functions" if !KUNIT_ALL_TESTS
+ depends on KUNIT
+ depends on RAID6_PQ
+ default KUNIT_ALL_TESTS
+ help
+ Unit tests for the RAID6 PQ library functions.
+
+ This is intended to help people writing architecture-specific
+ optimized versions. If unsure, say N.
+
config RAID6_PQ_BENCHMARK
bool "Automatically choose fastest RAID6 PQ functions"
depends on RAID6_PQ
diff --git a/lib/raid6/Makefile b/lib/raid6/Makefile
index 5be0a4e60ab1..6fd048c127b6 100644
--- a/lib/raid6/Makefile
+++ b/lib/raid6/Makefile
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_RAID6_PQ) += raid6_pq.o
+obj-$(CONFIG_RAID6_PQ) += raid6_pq.o test/
raid6_pq-y += algos.o recov.o tables.o int1.o int2.o int4.o \
int8.o
diff --git a/lib/raid6/algos.c b/lib/raid6/algos.c
index 799e0e5eac26..5a9f4882e18d 100644
--- a/lib/raid6/algos.c
+++ b/lib/raid6/algos.c
@@ -19,6 +19,7 @@
#include <linux/module.h>
#include <linux/gfp.h>
#endif
+#include <kunit/visibility.h>
struct raid6_calls raid6_call;
EXPORT_SYMBOL_GPL(raid6_call);
@@ -86,6 +87,7 @@ const struct raid6_calls * const raid6_algos[] = {
&raid6_intx1,
NULL
};
+EXPORT_SYMBOL_IF_KUNIT(raid6_algos);
void (*raid6_2data_recov)(int, size_t, int, int, void **);
EXPORT_SYMBOL_GPL(raid6_2data_recov);
@@ -119,6 +121,7 @@ const struct raid6_recov_calls *const raid6_recov_algos[] = {
&raid6_recov_intx1,
NULL
};
+EXPORT_SYMBOL_IF_KUNIT(raid6_recov_algos);
#ifdef __KERNEL__
#define RAID6_TIME_JIFFIES_LG2 4
@@ -239,7 +242,7 @@ static inline const struct raid6_calls *raid6_choose_gen(
/* Try to pick the best algorithm */
/* This code uses the gfmul table as convenient data set to abuse */
-int __init raid6_select_algo(void)
+static int __init raid6_select_algo(void)
{
const int disks = RAID6_TEST_DISKS;
diff --git a/lib/raid6/recov.c b/lib/raid6/recov.c
index b5e47c008b41..8d113196632e 100644
--- a/lib/raid6/recov.c
+++ b/lib/raid6/recov.c
@@ -99,37 +99,3 @@ const struct raid6_recov_calls raid6_recov_intx1 = {
.name = "intx1",
.priority = 0,
};
-
-#ifndef __KERNEL__
-/* Testing only */
-
-/* Recover two failed blocks. */
-void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, void **ptrs)
-{
- if ( faila > failb ) {
- int tmp = faila;
- faila = failb;
- failb = tmp;
- }
-
- if ( failb == disks-1 ) {
- if ( faila == disks-2 ) {
- /* P+Q failure. Just rebuild the syndrome. */
- raid6_call.gen_syndrome(disks, bytes, ptrs);
- } else {
- /* data+Q failure. Reconstruct data from P,
- then rebuild syndrome. */
- /* NOT IMPLEMENTED - equivalent to RAID-5 */
- }
- } else {
- if ( failb == disks-2 ) {
- /* data+P failure. */
- raid6_datap_recov(disks, bytes, faila, ptrs);
- } else {
- /* data+data failure. */
- raid6_2data_recov(disks, bytes, faila, failb, ptrs);
- }
- }
-}
-
-#endif
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
index 09bbe2b14cce..268b085af4d3 100644
--- a/lib/raid6/test/Makefile
+++ b/lib/raid6/test/Makefile
@@ -1,156 +1,3 @@
# SPDX-License-Identifier: GPL-2.0
-#
-# This is a simple Makefile to test some of the RAID-6 code
-# from userspace.
-#
-pound := \#
-
-# Adjust as desired
-CC = gcc
-OPTFLAGS = -O2
-CFLAGS = -I.. -I ../../../include -g $(OPTFLAGS)
-LD = ld
-AWK = awk -f
-AR = ar
-RANLIB = ranlib
-OBJS = int1.o int2.o int4.o int8.o int16.o int32.o recov.o algos.o tables.o
-
-ARCH := $(shell uname -m 2>/dev/null | sed -e /s/i.86/i386/)
-ifeq ($(ARCH),i386)
- CFLAGS += -DCONFIG_X86_32
- IS_X86 = yes
-endif
-ifeq ($(ARCH),x86_64)
- CFLAGS += -DCONFIG_X86_64
- IS_X86 = yes
-endif
-
-ifeq ($(ARCH),arm)
- CFLAGS += -I../../../arch/arm/include -mfpu=neon
- HAS_NEON = yes
-endif
-ifeq ($(ARCH),aarch64)
- CFLAGS += -I../../../arch/arm64/include
- HAS_NEON = yes
-endif
-
-ifeq ($(findstring riscv,$(ARCH)),riscv)
- CFLAGS += -I../../../arch/riscv/include -DCONFIG_RISCV=1
- HAS_RVV = yes
-endif
-
-ifeq ($(findstring ppc,$(ARCH)),ppc)
- CFLAGS += -I../../../arch/powerpc/include
- HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\
- gcc -c -x c - >/dev/null && rm ./-.o && echo yes)
-endif
-
-ifeq ($(ARCH),loongarch64)
- CFLAGS += -I../../../arch/loongarch/include -DCONFIG_LOONGARCH=1
- CFLAGS += $(shell echo 'vld $$vr0, $$zero, 0' | \
- gcc -c -x assembler - >/dev/null 2>&1 && \
- rm ./-.o && echo -DCONFIG_CPU_HAS_LSX=1)
- CFLAGS += $(shell echo 'xvld $$xr0, $$zero, 0' | \
- gcc -c -x assembler - >/dev/null 2>&1 && \
- rm ./-.o && echo -DCONFIG_CPU_HAS_LASX=1)
-endif
-
-ifeq ($(IS_X86),yes)
- OBJS += mmx.o sse1.o sse2.o avx2.o recov_ssse3.o recov_avx2.o avx512.o recov_avx512.o
- CFLAGS += -DCONFIG_X86
-else ifeq ($(HAS_NEON),yes)
- OBJS += neon.o neon1.o neon2.o neon4.o neon8.o recov_neon.o recov_neon_inner.o
- CFLAGS += -DCONFIG_KERNEL_MODE_NEON=1
-else ifeq ($(HAS_ALTIVEC),yes)
- CFLAGS += -DCONFIG_ALTIVEC
- OBJS += altivec1.o altivec2.o altivec4.o altivec8.o \
- vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
-else ifeq ($(ARCH),loongarch64)
- OBJS += loongarch_simd.o recov_loongarch_simd.o
-else ifeq ($(HAS_RVV),yes)
- OBJS += rvv.o recov_rvv.o
- CFLAGS += -DCONFIG_RISCV_ISA_V=1
-endif
-
-.c.o:
- $(CC) $(CFLAGS) -c -o $@ $<
-
-%.c: ../%.c
- cp -f $< $@
-
-%.uc: ../%.uc
- cp -f $< $@
-
-all: raid6.a raid6test
-
-raid6.a: $(OBJS)
- rm -f $@
- $(AR) cq $@ $^
- $(RANLIB) $@
-
-raid6test: test.c raid6.a
- $(CC) $(CFLAGS) -o raid6test $^
-
-neon1.c: neon.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=1 < neon.uc > $@
-
-neon2.c: neon.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=2 < neon.uc > $@
-
-neon4.c: neon.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=4 < neon.uc > $@
-
-neon8.c: neon.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=8 < neon.uc > $@
-
-altivec1.c: altivec.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=1 < altivec.uc > $@
-
-altivec2.c: altivec.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=2 < altivec.uc > $@
-
-altivec4.c: altivec.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=4 < altivec.uc > $@
-
-altivec8.c: altivec.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=8 < altivec.uc > $@
-
-vpermxor1.c: vpermxor.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=1 < vpermxor.uc > $@
-
-vpermxor2.c: vpermxor.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=2 < vpermxor.uc > $@
-
-vpermxor4.c: vpermxor.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=4 < vpermxor.uc > $@
-
-vpermxor8.c: vpermxor.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=8 < vpermxor.uc > $@
-
-int1.c: int.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=1 < int.uc > $@
-
-int2.c: int.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=2 < int.uc > $@
-
-int4.c: int.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=4 < int.uc > $@
-
-int8.c: int.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=8 < int.uc > $@
-
-int16.c: int.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=16 < int.uc > $@
-
-int32.c: int.uc ../unroll.awk
- $(AWK) ../unroll.awk -vN=32 < int.uc > $@
-
-tables.c: mktables
- ./mktables > tables.c
-
-clean:
- rm -f *.o *.a mktables mktables.c *.uc int*.c altivec*.c vpermxor*.c neon*.c tables.c raid6test
-
-spotless: clean
- rm -f *~
+obj-$(CONFIG_RAID6_PQ_KUNIT_TEST) += test.o
diff --git a/lib/raid6/test/test.c b/lib/raid6/test/test.c
index 841a55242aba..97e036b19049 100644
--- a/lib/raid6/test/test.c
+++ b/lib/raid6/test/test.c
@@ -1,43 +1,37 @@
// SPDX-License-Identifier: GPL-2.0-or-later
-/* -*- linux-c -*- ------------------------------------------------------- *
- *
- * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved
- *
- * ----------------------------------------------------------------------- */
-
/*
- * raid6test.c
+ * Copyright 2002-2007 H. Peter Anvin - All Rights Reserved
*
- * Test RAID-6 recovery with various algorithms
+ * Test RAID-6 recovery algorithms.
*/
-#include <stdlib.h>
-#include <stdio.h>
-#include <string.h>
+#include <kunit/test.h>
+#include <linux/prandom.h>
#include <linux/raid/pq.h>
-#define NDISKS 16 /* Including P and Q */
+MODULE_IMPORT_NS("EXPORTED_FOR_KUNIT_TESTING");
+
+#define RAID6_KUNIT_SEED 42
-const char raid6_empty_zero_page[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+#define NDISKS 16 /* Including P and Q */
-char *dataptrs[NDISKS];
-char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
-char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
-char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+static struct rnd_state rng;
+static void *dataptrs[NDISKS];
+static char data[NDISKS][PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+static char recovi[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
+static char recovj[PAGE_SIZE] __attribute__((aligned(PAGE_SIZE)));
static void makedata(int start, int stop)
{
- int i, j;
+ int i;
for (i = start; i <= stop; i++) {
- for (j = 0; j < PAGE_SIZE; j++)
- data[i][j] = rand();
-
+ prandom_bytes_state(&rng, data[i], PAGE_SIZE);
dataptrs[i] = data[i];
}
}
-static char disk_type(int d)
+static char member_type(int d)
{
switch (d) {
case NDISKS-2:
@@ -49,104 +43,121 @@ static char disk_type(int d)
}
}
-static int test_disks(int i, int j)
+static void test_disks(struct kunit *test, const struct raid6_calls *calls,
+ const struct raid6_recov_calls *ra, int faila, int failb)
{
- int erra, errb;
-
memset(recovi, 0xf0, PAGE_SIZE);
memset(recovj, 0xba, PAGE_SIZE);
- dataptrs[i] = recovi;
- dataptrs[j] = recovj;
-
- raid6_dual_recov(NDISKS, PAGE_SIZE, i, j, (void **)&dataptrs);
-
- erra = memcmp(data[i], recovi, PAGE_SIZE);
- errb = memcmp(data[j], recovj, PAGE_SIZE);
-
- if (i < NDISKS-2 && j == NDISKS-1) {
- /* We don't implement the DQ failure scenario, since it's
- equivalent to a RAID-5 failure (XOR, then recompute Q) */
- erra = errb = 0;
+ dataptrs[faila] = recovi;
+ dataptrs[failb] = recovj;
+
+ if (faila > failb)
+ swap(faila, failb);
+
+ if (failb == NDISKS - 1) {
+ /*
+ * We don't implement the data+Q failure scenario, since it
+ * is equivalent to a RAID-5 failure (XOR, then recompute Q).
+ */
+ if (faila != NDISKS - 2)
+ goto skip;
+
+ /* P+Q failure. Just rebuild the syndrome. */
+ calls->gen_syndrome(NDISKS, PAGE_SIZE, dataptrs);
+ } else if (failb == NDISKS - 2) {
+ /* data+P failure. */
+ ra->datap(NDISKS, PAGE_SIZE, faila, dataptrs);
} else {
- printf("algo=%-8s faila=%3d(%c) failb=%3d(%c) %s\n",
- raid6_call.name,
- i, disk_type(i),
- j, disk_type(j),
- (!erra && !errb) ? "OK" :
- !erra ? "ERRB" :
- !errb ? "ERRA" : "ERRAB");
+ /* data+data failure. */
+ ra->data2(NDISKS, PAGE_SIZE, faila, failb, dataptrs);
}
- dataptrs[i] = data[i];
- dataptrs[j] = data[j];
-
- return erra || errb;
+ KUNIT_EXPECT_MEMEQ_MSG(test, data[faila], recovi, PAGE_SIZE,
+ "algo=%-8s/%-8s faila miscompared: %3d[%c] (failb=%3d[%c])\n",
+ calls->name, ra->name,
+ faila, member_type(faila),
+ failb, member_type(failb));
+ KUNIT_EXPECT_MEMEQ_MSG(test, data[failb], recovj, PAGE_SIZE,
+ "algo=%-8s/%-8s failb miscompared: %3d[%c] (faila=%3d[%c])\n",
+ calls->name, ra->name,
+ failb, member_type(failb),
+ faila, member_type(faila));
+
+skip:
+ dataptrs[faila] = data[faila];
+ dataptrs[failb] = data[failb];
}
-int main(int argc, char *argv[])
+static void raid6_test(struct kunit *test)
{
const struct raid6_calls *const *algo;
const struct raid6_recov_calls *const *ra;
int i, j, p1, p2;
- int err = 0;
-
- makedata(0, NDISKS-1);
for (ra = raid6_recov_algos; *ra; ra++) {
if ((*ra)->valid && !(*ra)->valid())
continue;
- raid6_2data_recov = (*ra)->data2;
- raid6_datap_recov = (*ra)->datap;
-
- printf("using recovery %s\n", (*ra)->name);
-
for (algo = raid6_algos; *algo; algo++) {
- if ((*algo)->valid && !(*algo)->valid())
- continue;
+ const struct raid6_calls *calls = *algo;
- raid6_call = **algo;
+ if (calls->valid && !calls->valid())
+ continue;
/* Nuke syndromes */
- memset(data[NDISKS-2], 0xee, 2*PAGE_SIZE);
+ memset(data[NDISKS - 2], 0xee, PAGE_SIZE);
+ memset(data[NDISKS - 1], 0xee, PAGE_SIZE);
/* Generate assumed good syndrome */
- raid6_call.gen_syndrome(NDISKS, PAGE_SIZE,
+ calls->gen_syndrome(NDISKS, PAGE_SIZE,
(void **)&dataptrs);
for (i = 0; i < NDISKS-1; i++)
for (j = i+1; j < NDISKS; j++)
- err += test_disks(i, j);
+ test_disks(test, calls, *ra, i, j);
- if (!raid6_call.xor_syndrome)
+ if (!calls->xor_syndrome)
continue;
for (p1 = 0; p1 < NDISKS-2; p1++)
for (p2 = p1; p2 < NDISKS-2; p2++) {
/* Simulate rmw run */
- raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE,
+ calls->xor_syndrome(NDISKS, p1, p2, PAGE_SIZE,
(void **)&dataptrs);
makedata(p1, p2);
- raid6_call.xor_syndrome(NDISKS, p1, p2, PAGE_SIZE,
+ calls->xor_syndrome(NDISKS, p1, p2, PAGE_SIZE,
(void **)&dataptrs);
for (i = 0; i < NDISKS-1; i++)
for (j = i+1; j < NDISKS; j++)
- err += test_disks(i, j);
+ test_disks(test, calls,
+ *ra, i, j);
}
}
- printf("\n");
}
+}
- printf("\n");
- /* Pick the best algorithm test */
- raid6_select_algo();
-
- if (err)
- printf("\n*** ERRORS FOUND ***\n");
+static struct kunit_case raid6_test_cases[] = {
+ KUNIT_CASE(raid6_test),
+ {},
+};
- return err;
+static int raid6_suite_init(struct kunit_suite *suite)
+{
+ prandom_seed_state(&rng, RAID6_KUNIT_SEED);
+ makedata(0, NDISKS - 1);
+ return 0;
}
+
+static struct kunit_suite raid6_test_suite = {
+ .name = "raid6",
+ .test_cases = raid6_test_cases,
+ .suite_init = raid6_suite_init,
+};
+kunit_test_suite(raid6_test_suite);
+
+MODULE_DESCRIPTION("Unit test for the XOR library functions");
+MODULE_LICENSE("GPL");
--
2.47.3
^ permalink raw reply related
* cleanup the RAID6 P/Q library
From: Christoph Hellwig @ 2026-03-24 6:40 UTC (permalink / raw)
To: Andrew Morton
Cc: Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, Thomas Gleixner, Ingo Molnar, Borislav Petkov,
Dave Hansen, x86, H. Peter Anvin, Herbert Xu, Dan Williams,
Chris Mason, David Sterba, Arnd Bergmann, Song Liu, Yu Kuai,
Li Nan, linux-kernel, linux-arm-kernel, loongarch, linuxppc-dev,
linux-riscv, linux-s390, linux-crypto, linux-btrfs, linux-arch,
linux-raid
Hi all,
this series cleans up the RAID6 P/Q library to match the recent updates
to the RAID 5 XOR library and other CRC/crypto libraries. This includes
providing properly documented external interfaces, hiding the internals,
using static_call instead of indirect calls and turning the user space
test suite into an in-kernel kunit test which is also extended to
improve coverage.
Note that this changes registration so that non-priority algorithms are
not registered, which greatly helps with the benchmark time at boot time.
I'd like to encourage all architecture maintainers to see if they can
further optimized this by registering as few as possible algorithms when
there is a clear benefit in optimized or more unrolled implementations.
This series sits on top of the "cleanup the RAID5 XOR library v3" series.
A git tree is also available here:
git://git.infradead.org/users/hch/misc.git lib-raid6
Gitweb:
https://git.infradead.org/?p=users/hch/misc.git;a=shortlog;h=refs/heads/lib-raid6
Diffstat:
b/Documentation/crypto/async-tx-api.rst | 4
b/crypto/async_tx/async_pq.c | 9
b/crypto/async_tx/async_raid6_recov.c | 9
b/drivers/dma/bcm-sba-raid.c | 1
b/drivers/md/raid5.c | 4
b/fs/btrfs/raid56.c | 8
b/include/linux/raid/pq.h | 216 ------------
b/include/linux/raid/pq_tables.h | 19 +
b/lib/Kconfig | 11
b/lib/Makefile | 1
b/lib/raid/Kconfig | 33 +
b/lib/raid/Makefile | 2
b/lib/raid/raid6/Makefile | 124 +++++++
b/lib/raid/raid6/algos.c | 381 ++++++++++++++++++++++
b/lib/raid/raid6/algos.h | 41 ++
b/lib/raid/raid6/arm/neon.c | 23 -
b/lib/raid/raid6/arm/pq_arch.h | 22 +
b/lib/raid/raid6/arm/recov_neon.c | 25 -
b/lib/raid/raid6/int.uc | 10
b/lib/raid/raid6/loongarch/loongarch_simd.c | 31 -
b/lib/raid/raid6/loongarch/pq_arch.h | 23 +
b/lib/raid/raid6/loongarch/recov_loongarch_simd.c | 39 --
b/lib/raid/raid6/mktables.c | 28 -
b/lib/raid/raid6/powerpc/altivec.uc | 32 -
b/lib/raid/raid6/powerpc/pq_arch.h | 31 +
b/lib/raid/raid6/powerpc/vpermxor.uc | 29 -
b/lib/raid/raid6/recov.c | 62 ---
b/lib/raid/raid6/riscv/pq_arch.h | 21 +
b/lib/raid/raid6/riscv/recov_rvv.c | 14
b/lib/raid/raid6/riscv/rvv.h | 26 -
b/lib/raid/raid6/s390/pq_arch.h | 15
b/lib/raid/raid6/s390/recov_s390xc.c | 14
b/lib/raid/raid6/s390/s390vx.uc | 15
b/lib/raid/raid6/tests/Makefile | 3
b/lib/raid/raid6/tests/raid6_kunit.c | 307 +++++++++++++++++
b/lib/raid/raid6/x86/avx2.c | 47 --
b/lib/raid/raid6/x86/avx512.c | 57 +--
b/lib/raid/raid6/x86/mmx.c | 39 --
b/lib/raid/raid6/x86/pq_arch.h | 96 +++++
b/lib/raid/raid6/x86/recov_avx2.c | 22 -
b/lib/raid/raid6/x86/recov_avx512.c | 26 -
b/lib/raid/raid6/x86/recov_ssse3.c | 23 -
b/lib/raid/raid6/x86/sse1.c | 49 --
b/lib/raid/raid6/x86/sse2.c | 47 --
lib/raid6/Makefile | 83 ----
lib/raid6/algos.c | 291 ----------------
lib/raid6/loongarch.h | 38 --
lib/raid6/test/.gitignore | 3
lib/raid6/test/Makefile | 156 ---------
lib/raid6/test/test.c | 152 --------
lib/raid6/x86.h | 75 ----
51 files changed, 1329 insertions(+), 1508 deletions(-)
^ permalink raw reply
* [PATCH 26/26] xor: add a kunit test case
From: Christoph Hellwig @ 2026-03-24 6:22 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Add a test case for the XOR routines loosely based on the CRC kunit
test.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
lib/raid/.kunitconfig | 3 +
lib/raid/Kconfig | 11 ++
lib/raid/xor/Makefile | 2 +-
lib/raid/xor/tests/Makefile | 3 +
lib/raid/xor/tests/xor_kunit.c | 187 +++++++++++++++++++++++++++++++++
5 files changed, 205 insertions(+), 1 deletion(-)
create mode 100644 lib/raid/.kunitconfig
create mode 100644 lib/raid/xor/tests/Makefile
create mode 100644 lib/raid/xor/tests/xor_kunit.c
diff --git a/lib/raid/.kunitconfig b/lib/raid/.kunitconfig
new file mode 100644
index 000000000000..351d22ed1954
--- /dev/null
+++ b/lib/raid/.kunitconfig
@@ -0,0 +1,3 @@
+CONFIG_KUNIT=y
+CONFIG_BTRFS_FS=y
+CONFIG_XOR_KUNIT_TEST=y
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
index 2ccf0c60de01..1fc4b00e0d71 100644
--- a/lib/raid/Kconfig
+++ b/lib/raid/Kconfig
@@ -17,3 +17,14 @@ config XOR_BLOCKS_ARCH
default y if X86_32
default y if X86_64
bool
+
+config XOR_KUNIT_TEST
+ tristate "KUnit tests for xor_gen" if !KUNIT_ALL_TESTS
+ depends on KUNIT
+ depends on XOR_BLOCKS
+ default KUNIT_ALL_TESTS
+ help
+ Unit tests for the XOR library functions.
+
+ This is intended to help people writing architecture-specific
+ optimized versions. If unsure, say N.
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index df55823c4d82..4d633dfd5b90 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -29,7 +29,7 @@ xor-$(CONFIG_SPARC64) += sparc/xor-sparc64.o sparc/xor-sparc64-glue.o
xor-$(CONFIG_S390) += s390/xor.o
xor-$(CONFIG_X86_32) += x86/xor-avx.o x86/xor-sse.o x86/xor-mmx.o
xor-$(CONFIG_X86_64) += x86/xor-avx.o x86/xor-sse.o
-
+obj-y += tests/
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
CFLAGS_REMOVE_arm/xor-neon.o += $(CC_FLAGS_NO_FPU)
diff --git a/lib/raid/xor/tests/Makefile b/lib/raid/xor/tests/Makefile
new file mode 100644
index 000000000000..661e8f6ffd1f
--- /dev/null
+++ b/lib/raid/xor/tests/Makefile
@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: GPL-2.0-only
+
+obj-$(CONFIG_XOR_KUNIT_TEST) += xor_kunit.o
diff --git a/lib/raid/xor/tests/xor_kunit.c b/lib/raid/xor/tests/xor_kunit.c
new file mode 100644
index 000000000000..01cbdf44f6b0
--- /dev/null
+++ b/lib/raid/xor/tests/xor_kunit.c
@@ -0,0 +1,187 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Unit test the XOR library functions.
+ *
+ * Copyright 2024 Google LLC
+ * Copyright 2026 Christoph Hellwig
+ *
+ * Based on the CRC tests by Eric Biggers <ebiggers@google.com>.
+ */
+#include <kunit/test.h>
+#include <linux/prandom.h>
+#include <linux/string_choices.h>
+#include <linux/vmalloc.h>
+#include <linux/raid/xor.h>
+
+#define XOR_KUNIT_SEED 42
+#define XOR_KUNIT_MAX_BYTES 16384
+#define XOR_KUNIT_MAX_BUFFERS 64
+#define XOR_KUNIT_NUM_TEST_ITERS 1000
+
+static struct rnd_state rng;
+static void *test_buffers[XOR_KUNIT_MAX_BUFFERS];
+static void *test_dest;
+static void *test_ref;
+static size_t test_buflen;
+
+static u32 rand32(void)
+{
+ return prandom_u32_state(&rng);
+}
+
+/* Reference implementation using dumb byte-wise XOR */
+static void xor_ref(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
+{
+ unsigned int off, idx;
+ u8 *d = dest;
+
+ for (off = 0; off < bytes; off++) {
+ for (idx = 0; idx < src_cnt; idx++) {
+ u8 *src = srcs[idx];
+
+ d[off] ^= src[off];
+ }
+ }
+}
+
+/* Generate a random length that is a multiple of 512. */
+static unsigned int random_length(unsigned int max_length)
+{
+ return (rand32() % (max_length + 1)) & ~511;
+}
+
+/* Generate a random alignment that is a multiple of 64. */
+static unsigned int random_alignment(unsigned int max_alignment)
+{
+ return (rand32() % (max_alignment + 1)) & ~63;
+}
+
+static void xor_generate_random_data(void)
+{
+ int i;
+
+ prandom_bytes_state(&rng, test_dest, test_buflen);
+ memcpy(test_ref, test_dest, test_buflen);
+ for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++)
+ prandom_bytes_state(&rng, test_buffers[i], test_buflen);
+}
+
+/* Test that xor_gen gives the same result as a reference implementation. */
+static void xor_test(struct kunit *test)
+{
+ void *aligned_buffers[XOR_KUNIT_MAX_BUFFERS];
+ size_t i;
+
+ for (i = 0; i < XOR_KUNIT_NUM_TEST_ITERS; i++) {
+ unsigned int nr_buffers =
+ (rand32() % XOR_KUNIT_MAX_BUFFERS) + 1;
+ unsigned int len = random_length(XOR_KUNIT_MAX_BYTES);
+ unsigned int max_alignment, align = 0;
+ void *buffers;
+
+ if (rand32() % 8 == 0)
+ /* Refresh the data occasionally. */
+ xor_generate_random_data();
+
+ /*
+ * If we're not using the entire buffer size, inject randomize
+ * alignment into the buffer.
+ */
+ max_alignment = XOR_KUNIT_MAX_BYTES - len;
+ if (max_alignment == 0) {
+ buffers = test_buffers;
+ } else if (rand32() % 2 == 0) {
+ /* Use random alignments mod 64 */
+ int j;
+
+ for (j = 0; j < nr_buffers; j++)
+ aligned_buffers[j] = test_buffers[j] +
+ random_alignment(max_alignment);
+ buffers = aligned_buffers;
+ align = random_alignment(max_alignment);
+ } else {
+ /* Go up to the guard page, to catch buffer overreads */
+ int j;
+
+ align = test_buflen - len;
+ for (j = 0; j < nr_buffers; j++)
+ aligned_buffers[j] = test_buffers[j] + align;
+ buffers = aligned_buffers;
+ }
+
+ /*
+ * Compute the XOR, and verify that it equals the XOR computed
+ * by a simple byte-at-a-time reference implementation.
+ */
+ xor_ref(test_ref + align, buffers, nr_buffers, len);
+ xor_gen(test_dest + align, buffers, nr_buffers, len);
+ KUNIT_EXPECT_MEMEQ_MSG(test, test_ref + align,
+ test_dest + align, len,
+ "Wrong result with buffers=%u, len=%u, unaligned=%s, at_end=%s",
+ nr_buffers, len,
+ str_yes_no(max_alignment),
+ str_yes_no(align + len == test_buflen));
+ }
+}
+
+static struct kunit_case xor_test_cases[] = {
+ KUNIT_CASE(xor_test),
+ {},
+};
+
+static int xor_suite_init(struct kunit_suite *suite)
+{
+ int i;
+
+ /*
+ * Allocate the test buffer using vmalloc() with a page-aligned length
+ * so that it is immediately followed by a guard page. This allows
+ * buffer overreads to be detected, even in assembly code.
+ */
+ test_buflen = round_up(XOR_KUNIT_MAX_BYTES, PAGE_SIZE);
+ test_ref = vmalloc(test_buflen);
+ if (!test_ref)
+ return -ENOMEM;
+ test_dest = vmalloc(test_buflen);
+ if (!test_dest)
+ goto out_free_ref;
+ for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++) {
+ test_buffers[i] = vmalloc(test_buflen);
+ if (!test_buffers[i])
+ goto out_free_buffers;
+ }
+
+ prandom_seed_state(&rng, XOR_KUNIT_SEED);
+ xor_generate_random_data();
+ return 0;
+
+out_free_buffers:
+ while (--i >= 0)
+ vfree(test_buffers[i]);
+ vfree(test_dest);
+out_free_ref:
+ vfree(test_ref);
+ return -ENOMEM;
+}
+
+static void xor_suite_exit(struct kunit_suite *suite)
+{
+ int i;
+
+ vfree(test_ref);
+ vfree(test_dest);
+ for (i = 0; i < XOR_KUNIT_MAX_BUFFERS; i++)
+ vfree(test_buffers[i]);
+}
+
+static struct kunit_suite xor_test_suite = {
+ .name = "xor",
+ .test_cases = xor_test_cases,
+ .suite_init = xor_suite_init,
+ .suite_exit = xor_suite_exit,
+};
+kunit_test_suite(xor_test_suite);
+
+MODULE_DESCRIPTION("Unit test for the XOR library functions");
+MODULE_LICENSE("GPL");
--
2.47.3
^ permalink raw reply related
* [PATCH 25/26] xor: use static_call for xor_gen
From: Christoph Hellwig @ 2026-03-24 6:22 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Avoid the indirect call for xor_generation by using a static_call.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
lib/raid/xor/xor-core.c | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index a9b984d47d8c..67dc8ade7f0b 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -11,10 +11,10 @@
#include <linux/raid/xor.h>
#include <linux/jiffies.h>
#include <linux/preempt.h>
+#include <linux/static_call.h>
#include "xor_impl.h"
-/* The xor routine to use. */
-static struct xor_block_template *active_template;
+DEFINE_STATIC_CALL_NULL(xor_gen_impl, *xor_block_8regs.xor_gen);
/**
* xor_gen - generate RAID-style XOR information
@@ -36,13 +36,13 @@ void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes)
lockdep_assert_preemption_enabled();
WARN_ON_ONCE(bytes & 511);
- active_template->xor_gen(dest, srcs, src_cnt, bytes);
+ static_call(xor_gen_impl)(dest, srcs, src_cnt, bytes);
}
EXPORT_SYMBOL(xor_gen);
/* Set of all registered templates. */
static struct xor_block_template *__initdata template_list;
-static bool __initdata xor_forced = false;
+static struct xor_block_template *forced_template;
/**
* xor_register - register a XOR template
@@ -68,7 +68,7 @@ void __init xor_register(struct xor_block_template *tmpl)
*/
void __init xor_force(struct xor_block_template *tmpl)
{
- active_template = tmpl;
+ forced_template = tmpl;
}
#define BENCH_SIZE 4096
@@ -110,7 +110,7 @@ static int __init calibrate_xor_blocks(void)
void *b1, *b2;
struct xor_block_template *f, *fastest;
- if (xor_forced)
+ if (forced_template)
return 0;
b1 = (void *) __get_free_pages(GFP_KERNEL, 2);
@@ -127,7 +127,7 @@ static int __init calibrate_xor_blocks(void)
if (f->speed > fastest->speed)
fastest = f;
}
- active_template = fastest;
+ static_call_update(xor_gen_impl, fastest->xor_gen);
pr_info("xor: using function: %s (%d MB/sec)\n",
fastest->name, fastest->speed);
@@ -155,10 +155,10 @@ static int __init xor_init(void)
* If this arch/cpu has a short-circuited selection, don't loop through
* all the possible functions, just use the best one.
*/
- if (active_template) {
+ if (forced_template) {
pr_info("xor: automatically using best checksumming function %-10s\n",
- active_template->name);
- xor_forced = true;
+ forced_template->name);
+ static_call_update(xor_gen_impl, forced_template->xor_gen);
return 0;
}
@@ -169,7 +169,7 @@ static int __init xor_init(void)
* Pick the first template as the temporary default until calibration
* happens.
*/
- active_template = template_list;
+ static_call_update(xor_gen_impl, template_list->xor_gen);
return 0;
#endif
}
--
2.47.3
^ permalink raw reply related
* [PATCH 24/26] xor: pass the entire operation to the low-level ops
From: Christoph Hellwig @ 2026-03-24 6:22 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Currently the high-level xor code chunks up all operations into small
units for only up to 1 + 4 vectors, and passes it to four different
methods. This means the FPU/vector context is entered and left a lot
for wide stripes, and a lot of indirect expensive indirect calls are
performed. Switch to passing the entire gen_xor request to the
low-level ops, and provide a macro to dispatch it to the existing
helper.
This reduce the number of indirect calls and FPU/vector context switches
by a factor approaching nr_stripes / 4, and also reduces source and
binary code size.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
include/linux/raid/xor.h | 5 --
lib/raid/xor/alpha/xor.c | 19 ++++----
lib/raid/xor/arm/xor-neon-glue.c | 49 ++------------------
lib/raid/xor/arm/xor-neon.c | 9 +---
lib/raid/xor/arm/xor.c | 10 ++--
lib/raid/xor/arm/xor_arch.h | 3 ++
lib/raid/xor/arm64/xor-neon-glue.c | 44 ++----------------
lib/raid/xor/arm64/xor-neon.c | 20 +++++---
lib/raid/xor/arm64/xor-neon.h | 32 ++-----------
lib/raid/xor/loongarch/xor_simd_glue.c | 62 +++++--------------------
lib/raid/xor/powerpc/xor_vmx.c | 40 ++++++++--------
lib/raid/xor/powerpc/xor_vmx.h | 16 +------
lib/raid/xor/powerpc/xor_vmx_glue.c | 49 ++------------------
lib/raid/xor/riscv/xor-glue.c | 43 +++--------------
lib/raid/xor/s390/xor.c | 9 ++--
lib/raid/xor/sparc/xor-sparc32.c | 9 ++--
lib/raid/xor/sparc/xor-sparc64-glue.c | 19 ++++----
lib/raid/xor/x86/xor-avx.c | 29 ++++--------
lib/raid/xor/x86/xor-mmx.c | 64 ++++++++++----------------
lib/raid/xor/x86/xor-sse.c | 63 +++++++++----------------
lib/raid/xor/xor-32regs-prefetch.c | 10 ++--
lib/raid/xor/xor-32regs.c | 9 ++--
lib/raid/xor/xor-8regs-prefetch.c | 11 +++--
lib/raid/xor/xor-8regs.c | 9 ++--
lib/raid/xor/xor-core.c | 47 ++-----------------
lib/raid/xor/xor_impl.h | 48 +++++++++++++------
26 files changed, 224 insertions(+), 504 deletions(-)
diff --git a/include/linux/raid/xor.h b/include/linux/raid/xor.h
index 6d9a39fd85dd..870558c9d36e 100644
--- a/include/linux/raid/xor.h
+++ b/include/linux/raid/xor.h
@@ -2,11 +2,6 @@
#ifndef _XOR_H
#define _XOR_H
-#define MAX_XOR_BLOCKS 4
-
-extern void xor_blocks(unsigned int count, unsigned int bytes,
- void *dest, void **srcs);
-
void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes);
#endif /* _XOR_H */
diff --git a/lib/raid/xor/alpha/xor.c b/lib/raid/xor/alpha/xor.c
index 90694cc47395..a8f72f2dd3a5 100644
--- a/lib/raid/xor/alpha/xor.c
+++ b/lib/raid/xor/alpha/xor.c
@@ -832,18 +832,17 @@ xor_alpha_prefetch_5: \n\
.end xor_alpha_prefetch_5 \n\
");
+DO_XOR_BLOCKS(alpha, xor_alpha_2, xor_alpha_3, xor_alpha_4, xor_alpha_5);
+
struct xor_block_template xor_block_alpha = {
- .name = "alpha",
- .do_2 = xor_alpha_2,
- .do_3 = xor_alpha_3,
- .do_4 = xor_alpha_4,
- .do_5 = xor_alpha_5,
+ .name = "alpha",
+ .xor_gen = xor_gen_alpha,
};
+DO_XOR_BLOCKS(alpha_prefetch, xor_alpha_prefetch_2, xor_alpha_prefetch_3,
+ xor_alpha_prefetch_4, xor_alpha_prefetch_5);
+
struct xor_block_template xor_block_alpha_prefetch = {
- .name = "alpha prefetch",
- .do_2 = xor_alpha_prefetch_2,
- .do_3 = xor_alpha_prefetch_3,
- .do_4 = xor_alpha_prefetch_4,
- .do_5 = xor_alpha_prefetch_5,
+ .name = "alpha prefetch",
+ .xor_gen = xor_gen_alpha_prefetch,
};
diff --git a/lib/raid/xor/arm/xor-neon-glue.c b/lib/raid/xor/arm/xor-neon-glue.c
index 7afd6294464b..cea39e019904 100644
--- a/lib/raid/xor/arm/xor-neon-glue.c
+++ b/lib/raid/xor/arm/xor-neon-glue.c
@@ -5,54 +5,15 @@
#include "xor_impl.h"
#include "xor_arch.h"
-extern struct xor_block_template const xor_block_neon_inner;
-
-static void
-xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- kernel_neon_begin();
- xor_block_neon_inner.do_2(bytes, p1, p2);
- kernel_neon_end();
-}
-
-static void
-xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- kernel_neon_begin();
- xor_block_neon_inner.do_3(bytes, p1, p2, p3);
- kernel_neon_end();
-}
-
-static void
-xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- kernel_neon_begin();
- xor_block_neon_inner.do_4(bytes, p1, p2, p3, p4);
- kernel_neon_end();
-}
-
-static void
-xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
+static void xor_gen_neon(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
{
kernel_neon_begin();
- xor_block_neon_inner.do_5(bytes, p1, p2, p3, p4, p5);
+ xor_gen_neon_inner(dest, srcs, src_cnt, bytes);
kernel_neon_end();
}
struct xor_block_template xor_block_neon = {
- .name = "neon",
- .do_2 = xor_neon_2,
- .do_3 = xor_neon_3,
- .do_4 = xor_neon_4,
- .do_5 = xor_neon_5
+ .name = "neon",
+ .xor_gen = xor_gen_neon,
};
diff --git a/lib/raid/xor/arm/xor-neon.c b/lib/raid/xor/arm/xor-neon.c
index 806a42c5952c..23147e3a7904 100644
--- a/lib/raid/xor/arm/xor-neon.c
+++ b/lib/raid/xor/arm/xor-neon.c
@@ -4,6 +4,7 @@
*/
#include "xor_impl.h"
+#include "xor_arch.h"
#ifndef __ARM_NEON__
#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon'
@@ -22,10 +23,4 @@
#define NO_TEMPLATE
#include "../xor-8regs.c"
-struct xor_block_template const xor_block_neon_inner = {
- .name = "__inner_neon__",
- .do_2 = xor_8regs_2,
- .do_3 = xor_8regs_3,
- .do_4 = xor_8regs_4,
- .do_5 = xor_8regs_5,
-};
+__DO_XOR_BLOCKS(neon_inner, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5);
diff --git a/lib/raid/xor/arm/xor.c b/lib/raid/xor/arm/xor.c
index 5bd5f048bbe9..45139b6c55ea 100644
--- a/lib/raid/xor/arm/xor.c
+++ b/lib/raid/xor/arm/xor.c
@@ -127,10 +127,10 @@ xor_arm4regs_5(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines);
}
+DO_XOR_BLOCKS(arm4regs, xor_arm4regs_2, xor_arm4regs_3, xor_arm4regs_4,
+ xor_arm4regs_5);
+
struct xor_block_template xor_block_arm4regs = {
- .name = "arm4regs",
- .do_2 = xor_arm4regs_2,
- .do_3 = xor_arm4regs_3,
- .do_4 = xor_arm4regs_4,
- .do_5 = xor_arm4regs_5,
+ .name = "arm4regs",
+ .xor_gen = xor_gen_arm4regs,
};
diff --git a/lib/raid/xor/arm/xor_arch.h b/lib/raid/xor/arm/xor_arch.h
index 5a7eedb48fbb..775ff835df65 100644
--- a/lib/raid/xor/arm/xor_arch.h
+++ b/lib/raid/xor/arm/xor_arch.h
@@ -7,6 +7,9 @@
extern struct xor_block_template xor_block_arm4regs;
extern struct xor_block_template xor_block_neon;
+void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes);
+
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_arm4regs);
diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c
index 3db0a318cf5b..f0284f86feb4 100644
--- a/lib/raid/xor/arm64/xor-neon-glue.c
+++ b/lib/raid/xor/arm64/xor-neon-glue.c
@@ -10,50 +10,16 @@
#include "xor-neon.h"
#define XOR_TEMPLATE(_name) \
-static void \
-xor_##_name##_2(unsigned long bytes, unsigned long * __restrict p1, \
- const unsigned long * __restrict p2) \
+static void xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \
+ unsigned int bytes) \
{ \
scoped_ksimd() \
- __xor_##_name##_2(bytes, p1, p2); \
-} \
- \
-static void \
-xor_##_name##_3(unsigned long bytes, unsigned long * __restrict p1, \
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3) \
-{ \
- scoped_ksimd() \
- __xor_##_name##_3(bytes, p1, p2, p3); \
-} \
- \
-static void \
-xor_##_name##_4(unsigned long bytes, unsigned long * __restrict p1, \
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3, \
- const unsigned long * __restrict p4) \
-{ \
- scoped_ksimd() \
- __xor_##_name##_4(bytes, p1, p2, p3, p4); \
-} \
- \
-static void \
-xor_##_name##_5(unsigned long bytes, unsigned long * __restrict p1, \
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3, \
- const unsigned long * __restrict p4, \
- const unsigned long * __restrict p5) \
-{ \
- scoped_ksimd() \
- __xor_##_name##_5(bytes, p1, p2, p3, p4, p5); \
+ xor_gen_##_name##_inner(dest, srcs, src_cnt, bytes); \
} \
\
struct xor_block_template xor_block_##_name = { \
- .name = __stringify(_name), \
- .do_2 = xor_##_name##_2, \
- .do_3 = xor_##_name##_3, \
- .do_4 = xor_##_name##_4, \
- .do_5 = xor_##_name##_5 \
+ .name = __stringify(_name), \
+ .xor_gen = xor_gen_##_name, \
};
XOR_TEMPLATE(neon);
diff --git a/lib/raid/xor/arm64/xor-neon.c b/lib/raid/xor/arm64/xor-neon.c
index 61f00c4fee49..97ef3cb92496 100644
--- a/lib/raid/xor/arm64/xor-neon.c
+++ b/lib/raid/xor/arm64/xor-neon.c
@@ -10,7 +10,7 @@
#include "xor_arch.h"
#include "xor-neon.h"
-void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
uint64_t *dp1 = (uint64_t *)p1;
@@ -37,7 +37,7 @@ void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+static void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
@@ -73,7 +73,7 @@ void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+static void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
@@ -118,7 +118,7 @@ void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
@@ -172,6 +172,9 @@ void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
+__DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
+ __xor_neon_5);
+
static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
{
uint64x2_t res;
@@ -182,7 +185,7 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
return res;
}
-void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
+static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
@@ -216,7 +219,7 @@ void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
+static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
@@ -259,7 +262,7 @@ void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
+static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
@@ -304,3 +307,6 @@ void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
dp5 += 8;
} while (--lines > 0);
}
+
+__DO_XOR_BLOCKS(eor3_inner, __xor_neon_2, __xor_eor3_3, __xor_eor3_4,
+ __xor_eor3_5);
diff --git a/lib/raid/xor/arm64/xor-neon.h b/lib/raid/xor/arm64/xor-neon.h
index cec0ac846fea..514699ba8f5f 100644
--- a/lib/raid/xor/arm64/xor-neon.h
+++ b/lib/raid/xor/arm64/xor-neon.h
@@ -1,30 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-
-#define __xor_eor3_2 __xor_neon_2
-void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
+void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes);
+void xor_gen_eor3_inner(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes);
diff --git a/lib/raid/xor/loongarch/xor_simd_glue.c b/lib/raid/xor/loongarch/xor_simd_glue.c
index b387aa0213b4..7f324d924f87 100644
--- a/lib/raid/xor/loongarch/xor_simd_glue.c
+++ b/lib/raid/xor/loongarch/xor_simd_glue.c
@@ -11,63 +11,23 @@
#include "xor_arch.h"
#include "xor_simd.h"
-#define MAKE_XOR_GLUE_2(flavor) \
-static void xor_##flavor##_2(unsigned long bytes, unsigned long * __restrict p1,\
- const unsigned long * __restrict p2) \
+#define MAKE_XOR_GLUES(flavor) \
+DO_XOR_BLOCKS(flavor##_inner, __xor_##flavor##_2, __xor_##flavor##_3, \
+ __xor_##flavor##_4, __xor_##flavor##_5); \
+ \
+static void xor_gen_##flavor(void *dest, void **srcs, unsigned int src_cnt, \
+ unsigned int bytes) \
{ \
kernel_fpu_begin(); \
- __xor_##flavor##_2(bytes, p1, p2); \
+ xor_gen_##flavor##_inner(dest, srcs, src_cnt, bytes); \
kernel_fpu_end(); \
} \
-
-#define MAKE_XOR_GLUE_3(flavor) \
-static void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1,\
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3) \
-{ \
- kernel_fpu_begin(); \
- __xor_##flavor##_3(bytes, p1, p2, p3); \
- kernel_fpu_end(); \
-} \
-
-#define MAKE_XOR_GLUE_4(flavor) \
-static void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1,\
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3, \
- const unsigned long * __restrict p4) \
-{ \
- kernel_fpu_begin(); \
- __xor_##flavor##_4(bytes, p1, p2, p3, p4); \
- kernel_fpu_end(); \
-} \
-
-#define MAKE_XOR_GLUE_5(flavor) \
-static void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1,\
- const unsigned long * __restrict p2, \
- const unsigned long * __restrict p3, \
- const unsigned long * __restrict p4, \
- const unsigned long * __restrict p5) \
-{ \
- kernel_fpu_begin(); \
- __xor_##flavor##_5(bytes, p1, p2, p3, p4, p5); \
- kernel_fpu_end(); \
-} \
-
-#define MAKE_XOR_GLUES(flavor) \
- MAKE_XOR_GLUE_2(flavor); \
- MAKE_XOR_GLUE_3(flavor); \
- MAKE_XOR_GLUE_4(flavor); \
- MAKE_XOR_GLUE_5(flavor); \
- \
-struct xor_block_template xor_block_##flavor = { \
- .name = __stringify(flavor), \
- .do_2 = xor_##flavor##_2, \
- .do_3 = xor_##flavor##_3, \
- .do_4 = xor_##flavor##_4, \
- .do_5 = xor_##flavor##_5, \
+ \
+struct xor_block_template xor_block_##flavor = { \
+ .name = __stringify(flavor), \
+ .xor_gen = xor_gen_##flavor \
}
-
#ifdef CONFIG_CPU_HAS_LSX
MAKE_XOR_GLUES(lsx);
#endif /* CONFIG_CPU_HAS_LSX */
diff --git a/lib/raid/xor/powerpc/xor_vmx.c b/lib/raid/xor/powerpc/xor_vmx.c
index aab49d056d18..09bed98c1bc7 100644
--- a/lib/raid/xor/powerpc/xor_vmx.c
+++ b/lib/raid/xor/powerpc/xor_vmx.c
@@ -10,6 +10,7 @@
* Sparse (as at v0.5.0) gets very, very confused by this file.
* Make it a bit simpler for it.
*/
+#include "xor_impl.h"
#if !defined(__CHECKER__)
#include <altivec.h>
#else
@@ -49,9 +50,9 @@ typedef vector signed char unative_t;
V1##_3 = vec_xor(V1##_3, V2##_3); \
} while (0)
-void __xor_altivec_2(unsigned long bytes,
- unsigned long * __restrict v1_in,
- const unsigned long * __restrict v2_in)
+static void __xor_altivec_2(unsigned long bytes,
+ unsigned long * __restrict v1_in,
+ const unsigned long * __restrict v2_in)
{
DEFINE(v1);
DEFINE(v2);
@@ -68,10 +69,10 @@ void __xor_altivec_2(unsigned long bytes,
} while (--lines > 0);
}
-void __xor_altivec_3(unsigned long bytes,
- unsigned long * __restrict v1_in,
- const unsigned long * __restrict v2_in,
- const unsigned long * __restrict v3_in)
+static void __xor_altivec_3(unsigned long bytes,
+ unsigned long * __restrict v1_in,
+ const unsigned long * __restrict v2_in,
+ const unsigned long * __restrict v3_in)
{
DEFINE(v1);
DEFINE(v2);
@@ -92,11 +93,11 @@ void __xor_altivec_3(unsigned long bytes,
} while (--lines > 0);
}
-void __xor_altivec_4(unsigned long bytes,
- unsigned long * __restrict v1_in,
- const unsigned long * __restrict v2_in,
- const unsigned long * __restrict v3_in,
- const unsigned long * __restrict v4_in)
+static void __xor_altivec_4(unsigned long bytes,
+ unsigned long * __restrict v1_in,
+ const unsigned long * __restrict v2_in,
+ const unsigned long * __restrict v3_in,
+ const unsigned long * __restrict v4_in)
{
DEFINE(v1);
DEFINE(v2);
@@ -121,12 +122,12 @@ void __xor_altivec_4(unsigned long bytes,
} while (--lines > 0);
}
-void __xor_altivec_5(unsigned long bytes,
- unsigned long * __restrict v1_in,
- const unsigned long * __restrict v2_in,
- const unsigned long * __restrict v3_in,
- const unsigned long * __restrict v4_in,
- const unsigned long * __restrict v5_in)
+static void __xor_altivec_5(unsigned long bytes,
+ unsigned long * __restrict v1_in,
+ const unsigned long * __restrict v2_in,
+ const unsigned long * __restrict v3_in,
+ const unsigned long * __restrict v4_in,
+ const unsigned long * __restrict v5_in)
{
DEFINE(v1);
DEFINE(v2);
@@ -154,3 +155,6 @@ void __xor_altivec_5(unsigned long bytes,
v5 += 4;
} while (--lines > 0);
}
+
+__DO_XOR_BLOCKS(altivec_inner, __xor_altivec_2, __xor_altivec_3,
+ __xor_altivec_4, __xor_altivec_5);
diff --git a/lib/raid/xor/powerpc/xor_vmx.h b/lib/raid/xor/powerpc/xor_vmx.h
index 573c41d90dac..1d26c1133a86 100644
--- a/lib/raid/xor/powerpc/xor_vmx.h
+++ b/lib/raid/xor/powerpc/xor_vmx.h
@@ -6,17 +6,5 @@
* outside of the enable/disable altivec block.
*/
-void __xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void __xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-void __xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void __xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
+void xor_gen_altivec_inner(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes);
diff --git a/lib/raid/xor/powerpc/xor_vmx_glue.c b/lib/raid/xor/powerpc/xor_vmx_glue.c
index 56e99ddfb64f..dbfbb5cadc36 100644
--- a/lib/raid/xor/powerpc/xor_vmx_glue.c
+++ b/lib/raid/xor/powerpc/xor_vmx_glue.c
@@ -12,56 +12,17 @@
#include "xor_arch.h"
#include "xor_vmx.h"
-static void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
+static void xor_gen_altivec(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
{
preempt_disable();
enable_kernel_altivec();
- __xor_altivec_2(bytes, p1, p2);
- disable_kernel_altivec();
- preempt_enable();
-}
-
-static void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_3(bytes, p1, p2, p3);
- disable_kernel_altivec();
- preempt_enable();
-}
-
-static void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_4(bytes, p1, p2, p3, p4);
- disable_kernel_altivec();
- preempt_enable();
-}
-
-static void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_5(bytes, p1, p2, p3, p4, p5);
+ xor_gen_altivec_inner(dest, srcs, src_cnt, bytes);
disable_kernel_altivec();
preempt_enable();
}
struct xor_block_template xor_block_altivec = {
- .name = "altivec",
- .do_2 = xor_altivec_2,
- .do_3 = xor_altivec_3,
- .do_4 = xor_altivec_4,
- .do_5 = xor_altivec_5,
+ .name = "altivec",
+ .xor_gen = xor_gen_altivec,
};
diff --git a/lib/raid/xor/riscv/xor-glue.c b/lib/raid/xor/riscv/xor-glue.c
index 060e5f22ebcc..2e4c1b05d998 100644
--- a/lib/raid/xor/riscv/xor-glue.c
+++ b/lib/raid/xor/riscv/xor-glue.c
@@ -9,48 +9,17 @@
#include "xor_impl.h"
#include "xor_arch.h"
-static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2)
-{
- kernel_vector_begin();
- xor_regs_2_(bytes, p1, p2);
- kernel_vector_end();
-}
-
-static void xor_vector_3(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3)
-{
- kernel_vector_begin();
- xor_regs_3_(bytes, p1, p2, p3);
- kernel_vector_end();
-}
-
-static void xor_vector_4(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3,
- const unsigned long *__restrict p4)
-{
- kernel_vector_begin();
- xor_regs_4_(bytes, p1, p2, p3, p4);
- kernel_vector_end();
-}
+DO_XOR_BLOCKS(vector_inner, xor_regs_2_, xor_regs_3_, xor_regs_4_, xor_regs_5_);
-static void xor_vector_5(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3,
- const unsigned long *__restrict p4,
- const unsigned long *__restrict p5)
+static void xor_gen_vector(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
{
kernel_vector_begin();
- xor_regs_5_(bytes, p1, p2, p3, p4, p5);
+ xor_gen_vector_inner(dest, srcs, src_cnt, bytes);
kernel_vector_end();
}
struct xor_block_template xor_block_rvv = {
- .name = "rvv",
- .do_2 = xor_vector_2,
- .do_3 = xor_vector_3,
- .do_4 = xor_vector_4,
- .do_5 = xor_vector_5
+ .name = "rvv",
+ .xor_gen = xor_gen_vector,
};
diff --git a/lib/raid/xor/s390/xor.c b/lib/raid/xor/s390/xor.c
index c28cb56fec92..0c478678a129 100644
--- a/lib/raid/xor/s390/xor.c
+++ b/lib/raid/xor/s390/xor.c
@@ -125,10 +125,9 @@ static void xor_xc_5(unsigned long bytes, unsigned long * __restrict p1,
: : "0", "cc", "memory");
}
+DO_XOR_BLOCKS(xc, xor_xc_2, xor_xc_3, xor_xc_4, xor_xc_5);
+
struct xor_block_template xor_block_xc = {
- .name = "xc",
- .do_2 = xor_xc_2,
- .do_3 = xor_xc_3,
- .do_4 = xor_xc_4,
- .do_5 = xor_xc_5,
+ .name = "xc",
+ .xor_gen = xor_gen_xc,
};
diff --git a/lib/raid/xor/sparc/xor-sparc32.c b/lib/raid/xor/sparc/xor-sparc32.c
index 307c4a84f535..fb37631e90e6 100644
--- a/lib/raid/xor/sparc/xor-sparc32.c
+++ b/lib/raid/xor/sparc/xor-sparc32.c
@@ -244,10 +244,9 @@ sparc_5(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
+DO_XOR_BLOCKS(sparc32, sparc_2, sparc_3, sparc_4, sparc_5);
+
struct xor_block_template xor_block_SPARC = {
- .name = "SPARC",
- .do_2 = sparc_2,
- .do_3 = sparc_3,
- .do_4 = sparc_4,
- .do_5 = sparc_5,
+ .name = "SPARC",
+ .xor_gen = xor_gen_sparc32,
};
diff --git a/lib/raid/xor/sparc/xor-sparc64-glue.c b/lib/raid/xor/sparc/xor-sparc64-glue.c
index 5f90c2460b54..a8a686e0d258 100644
--- a/lib/raid/xor/sparc/xor-sparc64-glue.c
+++ b/lib/raid/xor/sparc/xor-sparc64-glue.c
@@ -28,12 +28,11 @@ void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1,
/* XXX Ugh, write cheetah versions... -DaveM */
+DO_XOR_BLOCKS(vis, xor_vis_2, xor_vis_3, xor_vis_4, xor_vis_5);
+
struct xor_block_template xor_block_VIS = {
- .name = "VIS",
- .do_2 = xor_vis_2,
- .do_3 = xor_vis_3,
- .do_4 = xor_vis_4,
- .do_5 = xor_vis_5,
+ .name = "VIS",
+ .xor_gen = xor_gen_vis,
};
void xor_niagara_2(unsigned long bytes, unsigned long * __restrict p1,
@@ -51,10 +50,10 @@ void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5);
+DO_XOR_BLOCKS(niagara, xor_niagara_2, xor_niagara_3, xor_niagara_4,
+ xor_niagara_5);
+
struct xor_block_template xor_block_niagara = {
- .name = "Niagara",
- .do_2 = xor_niagara_2,
- .do_3 = xor_niagara_3,
- .do_4 = xor_niagara_4,
- .do_5 = xor_niagara_5,
+ .name = "Niagara",
+ .xor_gen = xor_gen_niagara,
};
diff --git a/lib/raid/xor/x86/xor-avx.c b/lib/raid/xor/x86/xor-avx.c
index d411efa1ff43..f7777d7aa269 100644
--- a/lib/raid/xor/x86/xor-avx.c
+++ b/lib/raid/xor/x86/xor-avx.c
@@ -29,8 +29,6 @@ static void xor_avx_2(unsigned long bytes, unsigned long * __restrict p0,
{
unsigned long lines = bytes >> 9;
- kernel_fpu_begin();
-
while (lines--) {
#undef BLOCK
#define BLOCK(i, reg) \
@@ -47,8 +45,6 @@ do { \
p0 = (unsigned long *)((uintptr_t)p0 + 512);
p1 = (unsigned long *)((uintptr_t)p1 + 512);
}
-
- kernel_fpu_end();
}
static void xor_avx_3(unsigned long bytes, unsigned long * __restrict p0,
@@ -57,8 +53,6 @@ static void xor_avx_3(unsigned long bytes, unsigned long * __restrict p0,
{
unsigned long lines = bytes >> 9;
- kernel_fpu_begin();
-
while (lines--) {
#undef BLOCK
#define BLOCK(i, reg) \
@@ -78,8 +72,6 @@ do { \
p1 = (unsigned long *)((uintptr_t)p1 + 512);
p2 = (unsigned long *)((uintptr_t)p2 + 512);
}
-
- kernel_fpu_end();
}
static void xor_avx_4(unsigned long bytes, unsigned long * __restrict p0,
@@ -89,8 +81,6 @@ static void xor_avx_4(unsigned long bytes, unsigned long * __restrict p0,
{
unsigned long lines = bytes >> 9;
- kernel_fpu_begin();
-
while (lines--) {
#undef BLOCK
#define BLOCK(i, reg) \
@@ -113,8 +103,6 @@ do { \
p2 = (unsigned long *)((uintptr_t)p2 + 512);
p3 = (unsigned long *)((uintptr_t)p3 + 512);
}
-
- kernel_fpu_end();
}
static void xor_avx_5(unsigned long bytes, unsigned long * __restrict p0,
@@ -125,8 +113,6 @@ static void xor_avx_5(unsigned long bytes, unsigned long * __restrict p0,
{
unsigned long lines = bytes >> 9;
- kernel_fpu_begin();
-
while (lines--) {
#undef BLOCK
#define BLOCK(i, reg) \
@@ -152,14 +138,19 @@ do { \
p3 = (unsigned long *)((uintptr_t)p3 + 512);
p4 = (unsigned long *)((uintptr_t)p4 + 512);
}
+}
+
+DO_XOR_BLOCKS(avx_inner, xor_avx_2, xor_avx_3, xor_avx_4, xor_avx_5);
+static void xor_gen_avx(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
+{
+ kernel_fpu_begin();
+ xor_gen_avx_inner(dest, srcs, src_cnt, bytes);
kernel_fpu_end();
}
struct xor_block_template xor_block_avx = {
- .name = "avx",
- .do_2 = xor_avx_2,
- .do_3 = xor_avx_3,
- .do_4 = xor_avx_4,
- .do_5 = xor_avx_5,
+ .name = "avx",
+ .xor_gen = xor_gen_avx,
};
diff --git a/lib/raid/xor/x86/xor-mmx.c b/lib/raid/xor/x86/xor-mmx.c
index e48c58f92874..63a8b0444fce 100644
--- a/lib/raid/xor/x86/xor-mmx.c
+++ b/lib/raid/xor/x86/xor-mmx.c
@@ -21,8 +21,6 @@ xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 7;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -55,8 +53,6 @@ xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
"+r" (p1), "+r" (p2)
:
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -66,8 +62,6 @@ xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 7;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -105,8 +99,6 @@ xor_pII_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
"+r" (p1), "+r" (p2), "+r" (p3)
:
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -117,8 +109,6 @@ xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 7;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -161,8 +151,6 @@ xor_pII_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
"+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
:
: "memory");
-
- kernel_fpu_end();
}
@@ -175,8 +163,6 @@ xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 7;
- kernel_fpu_begin();
-
/* Make sure GCC forgets anything it knows about p4 or p5,
such that it won't pass to the asm volatile below a
register that is shared with any other variable. That's
@@ -237,8 +223,6 @@ xor_pII_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
Clobber them just to be sure nobody does something stupid
like assuming they have some legal value. */
asm("" : "=r" (p4), "=r" (p5));
-
- kernel_fpu_end();
}
#undef LD
@@ -255,8 +239,6 @@ xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 6;
- kernel_fpu_begin();
-
asm volatile(
" .align 32 ;\n"
" 1: ;\n"
@@ -293,8 +275,6 @@ xor_p5_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
"+r" (p1), "+r" (p2)
:
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -304,8 +284,6 @@ xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 6;
- kernel_fpu_begin();
-
asm volatile(
" .align 32,0x90 ;\n"
" 1: ;\n"
@@ -351,8 +329,6 @@ xor_p5_mmx_3(unsigned long bytes, unsigned long * __restrict p1,
"+r" (p1), "+r" (p2), "+r" (p3)
:
: "memory" );
-
- kernel_fpu_end();
}
static void
@@ -363,8 +339,6 @@ xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 6;
- kernel_fpu_begin();
-
asm volatile(
" .align 32,0x90 ;\n"
" 1: ;\n"
@@ -419,8 +393,6 @@ xor_p5_mmx_4(unsigned long bytes, unsigned long * __restrict p1,
"+r" (p1), "+r" (p2), "+r" (p3), "+r" (p4)
:
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -432,8 +404,6 @@ xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 6;
- kernel_fpu_begin();
-
/* Make sure GCC forgets anything it knows about p4 or p5,
such that it won't pass to the asm volatile below a
register that is shared with any other variable. That's
@@ -510,22 +480,36 @@ xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
Clobber them just to be sure nobody does something stupid
like assuming they have some legal value. */
asm("" : "=r" (p4), "=r" (p5));
+}
+
+DO_XOR_BLOCKS(pII_mmx_inner, xor_pII_mmx_2, xor_pII_mmx_3, xor_pII_mmx_4,
+ xor_pII_mmx_5);
+static void xor_gen_pII_mmx(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
+{
+ kernel_fpu_begin();
+ xor_gen_pII_mmx_inner(dest, srcs, src_cnt, bytes);
kernel_fpu_end();
}
struct xor_block_template xor_block_pII_mmx = {
- .name = "pII_mmx",
- .do_2 = xor_pII_mmx_2,
- .do_3 = xor_pII_mmx_3,
- .do_4 = xor_pII_mmx_4,
- .do_5 = xor_pII_mmx_5,
+ .name = "pII_mmx",
+ .xor_gen = xor_gen_pII_mmx,
};
+DO_XOR_BLOCKS(p5_mmx_inner, xor_p5_mmx_2, xor_p5_mmx_3, xor_p5_mmx_4,
+ xor_p5_mmx_5);
+
+static void xor_gen_p5_mmx(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
+{
+ kernel_fpu_begin();
+ xor_gen_p5_mmx_inner(dest, srcs, src_cnt, bytes);
+ kernel_fpu_end();
+}
+
struct xor_block_template xor_block_p5_mmx = {
- .name = "p5_mmx",
- .do_2 = xor_p5_mmx_2,
- .do_3 = xor_p5_mmx_3,
- .do_4 = xor_p5_mmx_4,
- .do_5 = xor_p5_mmx_5,
+ .name = "p5_mmx",
+ .xor_gen = xor_gen_p5_mmx,
};
diff --git a/lib/raid/xor/x86/xor-sse.c b/lib/raid/xor/x86/xor-sse.c
index 5993ed688c15..c6626ecae6ba 100644
--- a/lib/raid/xor/x86/xor-sse.c
+++ b/lib/raid/xor/x86/xor-sse.c
@@ -51,8 +51,6 @@ xor_sse_2(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 8;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -93,8 +91,6 @@ xor_sse_2(unsigned long bytes, unsigned long * __restrict p1,
[p1] "+r" (p1), [p2] "+r" (p2)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -103,8 +99,6 @@ xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 8;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -128,8 +122,6 @@ xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1,
[p1] "+r" (p1), [p2] "+r" (p2)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -139,8 +131,6 @@ xor_sse_3(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 8;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -188,8 +178,6 @@ xor_sse_3(unsigned long bytes, unsigned long * __restrict p1,
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -199,8 +187,6 @@ xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 8;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -226,8 +212,6 @@ xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1,
[p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -238,8 +222,6 @@ xor_sse_4(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 8;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -294,8 +276,6 @@ xor_sse_4(unsigned long bytes, unsigned long * __restrict p1,
[p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -306,8 +286,6 @@ xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 8;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -335,8 +313,6 @@ xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1,
[p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -348,8 +324,6 @@ xor_sse_5(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 8;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -411,8 +385,6 @@ xor_sse_5(unsigned long bytes, unsigned long * __restrict p1,
[p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
-
- kernel_fpu_end();
}
static void
@@ -424,8 +396,6 @@ xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1,
{
unsigned long lines = bytes >> 8;
- kernel_fpu_begin();
-
asm volatile(
#undef BLOCK
#define BLOCK(i) \
@@ -455,22 +425,35 @@ xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1,
[p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
: [inc] XOR_CONSTANT_CONSTRAINT (256UL)
: "memory");
+}
+
+DO_XOR_BLOCKS(sse_inner, xor_sse_2, xor_sse_3, xor_sse_4, xor_sse_5);
+static void xor_gen_sse(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
+{
+ kernel_fpu_begin();
+ xor_gen_sse_inner(dest, srcs, src_cnt, bytes);
kernel_fpu_end();
}
struct xor_block_template xor_block_sse = {
- .name = "sse",
- .do_2 = xor_sse_2,
- .do_3 = xor_sse_3,
- .do_4 = xor_sse_4,
- .do_5 = xor_sse_5,
+ .name = "sse",
+ .xor_gen = xor_gen_sse,
};
+DO_XOR_BLOCKS(sse_pf64_inner, xor_sse_2_pf64, xor_sse_3_pf64, xor_sse_4_pf64,
+ xor_sse_5_pf64);
+
+static void xor_gen_sse_pf64(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes)
+{
+ kernel_fpu_begin();
+ xor_gen_sse_pf64_inner(dest, srcs, src_cnt, bytes);
+ kernel_fpu_end();
+}
+
struct xor_block_template xor_block_sse_pf64 = {
- .name = "prefetch64-sse",
- .do_2 = xor_sse_2_pf64,
- .do_3 = xor_sse_3_pf64,
- .do_4 = xor_sse_4_pf64,
- .do_5 = xor_sse_5_pf64,
+ .name = "prefetch64-sse",
+ .xor_gen = xor_gen_sse_pf64,
};
diff --git a/lib/raid/xor/xor-32regs-prefetch.c b/lib/raid/xor/xor-32regs-prefetch.c
index 2856a8e50cb8..ade2a7d8cbe2 100644
--- a/lib/raid/xor/xor-32regs-prefetch.c
+++ b/lib/raid/xor/xor-32regs-prefetch.c
@@ -258,10 +258,10 @@ xor_32regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
goto once_more;
}
+DO_XOR_BLOCKS(32regs_p, xor_32regs_p_2, xor_32regs_p_3, xor_32regs_p_4,
+ xor_32regs_p_5);
+
struct xor_block_template xor_block_32regs_p = {
- .name = "32regs_prefetch",
- .do_2 = xor_32regs_p_2,
- .do_3 = xor_32regs_p_3,
- .do_4 = xor_32regs_p_4,
- .do_5 = xor_32regs_p_5,
+ .name = "32regs_prefetch",
+ .xor_gen = xor_gen_32regs_p,
};
diff --git a/lib/raid/xor/xor-32regs.c b/lib/raid/xor/xor-32regs.c
index cc44d64032fa..acb4a10d1e95 100644
--- a/lib/raid/xor/xor-32regs.c
+++ b/lib/raid/xor/xor-32regs.c
@@ -209,10 +209,9 @@ xor_32regs_5(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
+DO_XOR_BLOCKS(32regs, xor_32regs_2, xor_32regs_3, xor_32regs_4, xor_32regs_5);
+
struct xor_block_template xor_block_32regs = {
- .name = "32regs",
- .do_2 = xor_32regs_2,
- .do_3 = xor_32regs_3,
- .do_4 = xor_32regs_4,
- .do_5 = xor_32regs_5,
+ .name = "32regs",
+ .xor_gen = xor_gen_32regs,
};
diff --git a/lib/raid/xor/xor-8regs-prefetch.c b/lib/raid/xor/xor-8regs-prefetch.c
index 1d53aec50d27..451527a951b1 100644
--- a/lib/raid/xor/xor-8regs-prefetch.c
+++ b/lib/raid/xor/xor-8regs-prefetch.c
@@ -136,10 +136,11 @@ xor_8regs_p_5(unsigned long bytes, unsigned long * __restrict p1,
goto once_more;
}
+
+DO_XOR_BLOCKS(8regs_p, xor_8regs_p_2, xor_8regs_p_3, xor_8regs_p_4,
+ xor_8regs_p_5);
+
struct xor_block_template xor_block_8regs_p = {
- .name = "8regs_prefetch",
- .do_2 = xor_8regs_p_2,
- .do_3 = xor_8regs_p_3,
- .do_4 = xor_8regs_p_4,
- .do_5 = xor_8regs_p_5,
+ .name = "8regs_prefetch",
+ .xor_gen = xor_gen_8regs_p,
};
diff --git a/lib/raid/xor/xor-8regs.c b/lib/raid/xor/xor-8regs.c
index 72a44e898c55..1edaed8acffe 100644
--- a/lib/raid/xor/xor-8regs.c
+++ b/lib/raid/xor/xor-8regs.c
@@ -94,11 +94,10 @@ xor_8regs_5(unsigned long bytes, unsigned long * __restrict p1,
}
#ifndef NO_TEMPLATE
+DO_XOR_BLOCKS(8regs, xor_8regs_2, xor_8regs_3, xor_8regs_4, xor_8regs_5);
+
struct xor_block_template xor_block_8regs = {
- .name = "8regs",
- .do_2 = xor_8regs_2,
- .do_3 = xor_8regs_3,
- .do_4 = xor_8regs_4,
- .do_5 = xor_8regs_5,
+ .name = "8regs",
+ .xor_gen = xor_gen_8regs,
};
#endif /* NO_TEMPLATE */
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index fdd0cf5584e7..a9b984d47d8c 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -13,39 +13,9 @@
#include <linux/preempt.h>
#include "xor_impl.h"
-/* The xor routines to use. */
+/* The xor routine to use. */
static struct xor_block_template *active_template;
-void
-xor_blocks(unsigned int src_count, unsigned int bytes, void *dest, void **srcs)
-{
- unsigned long *p1, *p2, *p3, *p4;
-
- lockdep_assert_preemption_enabled();
-
- p1 = (unsigned long *) srcs[0];
- if (src_count == 1) {
- active_template->do_2(bytes, dest, p1);
- return;
- }
-
- p2 = (unsigned long *) srcs[1];
- if (src_count == 2) {
- active_template->do_3(bytes, dest, p1, p2);
- return;
- }
-
- p3 = (unsigned long *) srcs[2];
- if (src_count == 3) {
- active_template->do_4(bytes, dest, p1, p2, p3);
- return;
- }
-
- p4 = (unsigned long *) srcs[3];
- active_template->do_5(bytes, dest, p1, p2, p3, p4);
-}
-EXPORT_SYMBOL(xor_blocks);
-
/**
* xor_gen - generate RAID-style XOR information
* @dest: destination vector
@@ -63,18 +33,10 @@ EXPORT_SYMBOL(xor_blocks);
*/
void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes)
{
- unsigned int src_off = 0;
-
+ lockdep_assert_preemption_enabled();
WARN_ON_ONCE(bytes & 511);
- while (src_cnt > 0) {
- unsigned int this_cnt = min(src_cnt, MAX_XOR_BLOCKS);
-
- xor_blocks(this_cnt, bytes, dest, srcs + src_off);
-
- src_cnt -= this_cnt;
- src_off += this_cnt;
- }
+ active_template->xor_gen(dest, srcs, src_cnt, bytes);
}
EXPORT_SYMBOL(xor_gen);
@@ -118,6 +80,7 @@ do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
int speed;
unsigned long reps;
ktime_t min, start, t0;
+ void *srcs[1] = { b2 };
preempt_disable();
@@ -128,7 +91,7 @@ do_xor_speed(struct xor_block_template *tmpl, void *b1, void *b2)
cpu_relax();
do {
mb(); /* prevent loop optimization */
- tmpl->do_2(BENCH_SIZE, b1, b2);
+ tmpl->xor_gen(b1, srcs, 1, BENCH_SIZE);
mb();
} while (reps++ < REPS || (t0 = ktime_get()) == start);
min = ktime_sub(t0, start);
diff --git a/lib/raid/xor/xor_impl.h b/lib/raid/xor/xor_impl.h
index 44b6c99e2093..09ae2916f71e 100644
--- a/lib/raid/xor/xor_impl.h
+++ b/lib/raid/xor/xor_impl.h
@@ -3,27 +3,47 @@
#define _XOR_IMPL_H
#include <linux/init.h>
+#include <linux/minmax.h>
struct xor_block_template {
struct xor_block_template *next;
const char *name;
int speed;
- void (*do_2)(unsigned long, unsigned long * __restrict,
- const unsigned long * __restrict);
- void (*do_3)(unsigned long, unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict);
- void (*do_4)(unsigned long, unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict);
- void (*do_5)(unsigned long, unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict,
- const unsigned long * __restrict);
+ void (*xor_gen)(void *dest, void **srcs, unsigned int src_cnt,
+ unsigned int bytes);
};
+#define __DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4) \
+void \
+xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \
+ unsigned int bytes) \
+{ \
+ unsigned int src_off = 0; \
+ \
+ while (src_cnt > 0) { \
+ unsigned int this_cnt = min(src_cnt, 4); \
+ \
+ if (this_cnt == 1) \
+ _handle1(bytes, dest, srcs[src_off]); \
+ else if (this_cnt == 2) \
+ _handle2(bytes, dest, srcs[src_off], \
+ srcs[src_off + 1]); \
+ else if (this_cnt == 3) \
+ _handle3(bytes, dest, srcs[src_off], \
+ srcs[src_off + 1], srcs[src_off + 2]); \
+ else \
+ _handle4(bytes, dest, srcs[src_off], \
+ srcs[src_off + 1], srcs[src_off + 2], \
+ srcs[src_off + 3]); \
+ \
+ src_cnt -= this_cnt; \
+ src_off += this_cnt; \
+ } \
+}
+
+#define DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4) \
+ static __DO_XOR_BLOCKS(_name, _handle1, _handle2, _handle3, _handle4)
+
/* generic implementations */
extern struct xor_block_template xor_block_8regs;
extern struct xor_block_template xor_block_32regs;
--
2.47.3
^ permalink raw reply related
* [PATCH 23/26] btrfs: use xor_gen
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Use the new xor_gen helper instead of open coding the loop around
xor_blocks. This helper is very similar to the existing run_xor helper
in btrfs, except that the destination buffer is passed explicitly.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: David Sterba <dsterba@suse.com>
---
fs/btrfs/raid56.c | 27 ++++-----------------------
1 file changed, 4 insertions(+), 23 deletions(-)
diff --git a/fs/btrfs/raid56.c b/fs/btrfs/raid56.c
index 02105d68accb..ad091c56ce77 100644
--- a/fs/btrfs/raid56.c
+++ b/fs/btrfs/raid56.c
@@ -617,26 +617,6 @@ static void cache_rbio(struct btrfs_raid_bio *rbio)
spin_unlock(&table->cache_lock);
}
-/*
- * helper function to run the xor_blocks api. It is only
- * able to do MAX_XOR_BLOCKS at a time, so we need to
- * loop through.
- */
-static void run_xor(void **pages, int src_cnt, ssize_t len)
-{
- int src_off = 0;
- int xor_src_cnt = 0;
- void *dest = pages[src_cnt];
-
- while(src_cnt > 0) {
- xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
- xor_blocks(xor_src_cnt, len, dest, pages + src_off);
-
- src_cnt -= xor_src_cnt;
- src_off += xor_src_cnt;
- }
-}
-
/*
* Returns true if the bio list inside this rbio covers an entire stripe (no
* rmw required).
@@ -1434,7 +1414,8 @@ static void generate_pq_vertical_step(struct btrfs_raid_bio *rbio, unsigned int
} else {
/* raid5 */
memcpy(pointers[rbio->nr_data], pointers[0], step);
- run_xor(pointers + 1, rbio->nr_data - 1, step);
+ xor_gen(pointers[rbio->nr_data], pointers + 1, rbio->nr_data - 1,
+ step);
}
for (stripe = stripe - 1; stripe >= 0; stripe--)
kunmap_local(pointers[stripe]);
@@ -2034,7 +2015,7 @@ static void recover_vertical_step(struct btrfs_raid_bio *rbio,
pointers[rbio->nr_data - 1] = p;
/* Xor in the rest */
- run_xor(pointers, rbio->nr_data - 1, step);
+ xor_gen(p, pointers, rbio->nr_data - 1, step);
}
cleanup:
@@ -2672,7 +2653,7 @@ static bool verify_one_parity_step(struct btrfs_raid_bio *rbio,
} else {
/* RAID5. */
memcpy(pointers[nr_data], pointers[0], step);
- run_xor(pointers + 1, nr_data - 1, step);
+ xor_gen(pointers[nr_data], pointers + 1, nr_data - 1, step);
}
/* Check scrubbing parity and repair it. */
--
2.47.3
^ permalink raw reply related
* [PATCH 22/26] async_xor: use xor_gen
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Replace use of the loop around xor_blocks with the easier to use xor_gen
API.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
crypto/async_tx/async_xor.c | 34 ++++++++++------------------------
1 file changed, 10 insertions(+), 24 deletions(-)
diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c
index 2c499654a36c..84458375b202 100644
--- a/crypto/async_tx/async_xor.c
+++ b/crypto/async_tx/async_xor.c
@@ -103,7 +103,6 @@ do_sync_xor_offs(struct page *dest, unsigned int offset,
{
int i;
int xor_src_cnt = 0;
- int src_off = 0;
void *dest_buf;
void **srcs;
@@ -117,23 +116,12 @@ do_sync_xor_offs(struct page *dest, unsigned int offset,
if (src_list[i])
srcs[xor_src_cnt++] = page_address(src_list[i]) +
(src_offs ? src_offs[i] : offset);
- src_cnt = xor_src_cnt;
+
/* set destination address */
dest_buf = page_address(dest) + offset;
-
if (submit->flags & ASYNC_TX_XOR_ZERO_DST)
memset(dest_buf, 0, len);
-
- while (src_cnt > 0) {
- /* process up to 'MAX_XOR_BLOCKS' sources */
- xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
- xor_blocks(xor_src_cnt, len, dest_buf, &srcs[src_off]);
-
- /* drop completed sources */
- src_cnt -= xor_src_cnt;
- src_off += xor_src_cnt;
- }
-
+ xor_gen(dest_buf, srcs, xor_src_cnt, len);
async_tx_sync_epilog(submit);
}
@@ -168,11 +156,10 @@ dma_xor_aligned_offsets(struct dma_device *device, unsigned int offset,
*
* honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
*
- * xor_blocks always uses the dest as a source so the
- * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
- * the calculation. The assumption with dma engines is that they only
- * use the destination buffer as a source when it is explicitly specified
- * in the source list.
+ * xor_gen always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST flag
+ * must be set to not include dest data in the calculation. The assumption with
+ * dma engines is that they only use the destination buffer as a source when it
+ * is explicitly specified in the source list.
*
* src_list note: if the dest is also a source it must be at index zero.
* The contents of this array will be overwritten if a scribble region
@@ -259,11 +246,10 @@ EXPORT_SYMBOL_GPL(async_xor_offs);
*
* honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST
*
- * xor_blocks always uses the dest as a source so the
- * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in
- * the calculation. The assumption with dma engines is that they only
- * use the destination buffer as a source when it is explicitly specified
- * in the source list.
+ * xor_gen always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST flag
+ * must be set to not include dest data in the calculation. The assumption with
+ * dma engines is that they only use the destination buffer as a source when it
+ * is explicitly specified in the source list.
*
* src_list note: if the dest is also a source it must be at index zero.
* The contents of this array will be overwritten if a scribble region
--
2.47.3
^ permalink raw reply related
* [PATCH 21/26] xor: add a better public API
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
xor_blocks is very annoying to use, because it is limited to 4 + 1
sources / destinations, has an odd argument order and is completely
undocumented.
Lift the code that loops around it from btrfs and async_tx/async_xor into
common code under the name xor_gen and properly document it.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
include/linux/raid/xor.h | 2 ++
lib/raid/xor/xor-core.c | 32 ++++++++++++++++++++++++++++++++
2 files changed, 34 insertions(+)
diff --git a/include/linux/raid/xor.h b/include/linux/raid/xor.h
index 02bda8d99534..6d9a39fd85dd 100644
--- a/include/linux/raid/xor.h
+++ b/include/linux/raid/xor.h
@@ -7,4 +7,6 @@
extern void xor_blocks(unsigned int count, unsigned int bytes,
void *dest, void **srcs);
+void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes);
+
#endif /* _XOR_H */
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index 47e09ae954b2..fdd0cf5584e7 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -46,6 +46,38 @@ xor_blocks(unsigned int src_count, unsigned int bytes, void *dest, void **srcs)
}
EXPORT_SYMBOL(xor_blocks);
+/**
+ * xor_gen - generate RAID-style XOR information
+ * @dest: destination vector
+ * @srcs: source vectors
+ * @src_cnt: number of source vectors
+ * @bytes: length in bytes of each vector
+ *
+ * Performs bit-wise XOR operation into @dest for each of the @src_cnt vectors
+ * in @srcs for a length of @bytes bytes. @src_count must be non-zero, and the
+ * memory pointed to by @dest and each member of @srcs must be at least 64-byte
+ * aligned. @bytes must be non-zero and a multiple of 512.
+ *
+ * Note: for typical RAID uses, @dest either needs to be zeroed, or filled with
+ * the first disk, which then needs to be removed from @srcs.
+ */
+void xor_gen(void *dest, void **srcs, unsigned int src_cnt, unsigned int bytes)
+{
+ unsigned int src_off = 0;
+
+ WARN_ON_ONCE(bytes & 511);
+
+ while (src_cnt > 0) {
+ unsigned int this_cnt = min(src_cnt, MAX_XOR_BLOCKS);
+
+ xor_blocks(this_cnt, bytes, dest, srcs + src_off);
+
+ src_cnt -= this_cnt;
+ src_off += this_cnt;
+ }
+}
+EXPORT_SYMBOL(xor_gen);
+
/* Set of all registered templates. */
static struct xor_block_template *__initdata template_list;
static bool __initdata xor_forced = false;
--
2.47.3
^ permalink raw reply related
* [PATCH 20/26] xor: make xor.ko self-contained in lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the asm/xor.h headers to lib/raid/xor/$(SRCARCH)/xor_arch.h and
include/linux/raid/xor_impl.h to lib/raid/xor/xor_impl.h so that the
xor.ko module implementation is self-contained in lib/raid/.
As this remove the asm-generic mechanism a new kconfig symbol is
added to indicate that a architecture-specific implementations
exists, and xor_arch.h should be included.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/um/include/asm/xor.h | 8 --------
include/asm-generic/Kbuild | 1 -
include/asm-generic/xor.h | 11 -----------
lib/raid/Kconfig | 15 +++++++++++++++
lib/raid/xor/Makefile | 6 ++++++
lib/raid/xor/alpha/xor.c | 4 ++--
.../asm/xor.h => lib/raid/xor/alpha/xor_arch.h | 2 --
lib/raid/xor/arm/xor-neon-glue.c | 4 ++--
lib/raid/xor/arm/xor-neon.c | 2 +-
lib/raid/xor/arm/xor.c | 4 ++--
.../asm/xor.h => lib/raid/xor/arm/xor_arch.h | 2 --
lib/raid/xor/arm64/xor-neon-glue.c | 4 ++--
lib/raid/xor/arm64/xor-neon.c | 4 ++--
.../asm/xor.h => lib/raid/xor/arm64/xor_arch.h | 3 ---
.../xor.h => lib/raid/xor/loongarch/xor_arch.h | 7 -------
lib/raid/xor/loongarch/xor_simd_glue.c | 4 ++--
.../xor.h => lib/raid/xor/powerpc/xor_arch.h | 7 -------
lib/raid/xor/powerpc/xor_vmx_glue.c | 4 ++--
lib/raid/xor/riscv/xor-glue.c | 4 ++--
.../asm/xor.h => lib/raid/xor/riscv/xor_arch.h | 2 --
lib/raid/xor/s390/xor.c | 4 ++--
.../asm/xor.h => lib/raid/xor/s390/xor_arch.h | 6 ------
lib/raid/xor/sparc/xor-sparc32.c | 4 ++--
lib/raid/xor/sparc/xor-sparc64-glue.c | 4 ++--
.../asm/xor.h => lib/raid/xor/sparc/xor_arch.h | 9 ---------
lib/raid/xor/um/xor_arch.h | 2 ++
lib/raid/xor/x86/xor-avx.c | 4 ++--
lib/raid/xor/x86/xor-mmx.c | 4 ++--
lib/raid/xor/x86/xor-sse.c | 4 ++--
.../asm/xor.h => lib/raid/xor/x86/xor_arch.h | 7 -------
lib/raid/xor/xor-32regs-prefetch.c | 3 +--
lib/raid/xor/xor-32regs.c | 3 +--
lib/raid/xor/xor-8regs-prefetch.c | 3 +--
lib/raid/xor/xor-8regs.c | 3 +--
lib/raid/xor/xor-core.c | 18 +++++++++++-------
.../linux/raid => lib/raid/xor}/xor_impl.h | 6 ++++++
36 files changed, 73 insertions(+), 109 deletions(-)
delete mode 100644 arch/um/include/asm/xor.h
delete mode 100644 include/asm-generic/xor.h
rename arch/alpha/include/asm/xor.h => lib/raid/xor/alpha/xor_arch.h (90%)
rename arch/arm/include/asm/xor.h => lib/raid/xor/arm/xor_arch.h (87%)
rename arch/arm64/include/asm/xor.h => lib/raid/xor/arm64/xor_arch.h (89%)
rename arch/loongarch/include/asm/xor.h => lib/raid/xor/loongarch/xor_arch.h (85%)
rename arch/powerpc/include/asm/xor.h => lib/raid/xor/powerpc/xor_arch.h (77%)
rename arch/riscv/include/asm/xor.h => lib/raid/xor/riscv/xor_arch.h (84%)
rename arch/s390/include/asm/xor.h => lib/raid/xor/s390/xor_arch.h (71%)
rename arch/sparc/include/asm/xor.h => lib/raid/xor/sparc/xor_arch.h (81%)
create mode 100644 lib/raid/xor/um/xor_arch.h
rename arch/x86/include/asm/xor.h => lib/raid/xor/x86/xor_arch.h (89%)
rename {include/linux/raid => lib/raid/xor}/xor_impl.h (80%)
diff --git a/arch/um/include/asm/xor.h b/arch/um/include/asm/xor.h
deleted file mode 100644
index 99e5c7e1f475..000000000000
--- a/arch/um/include/asm/xor.h
+++ /dev/null
@@ -1,8 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_UM_XOR_H
-#define _ASM_UM_XOR_H
-
-#include <asm/cpufeature.h>
-#include <../../x86/include/asm/xor.h>
-
-#endif
diff --git a/include/asm-generic/Kbuild b/include/asm-generic/Kbuild
index 9aff61e7b8f2..2c53a1e0b760 100644
--- a/include/asm-generic/Kbuild
+++ b/include/asm-generic/Kbuild
@@ -65,4 +65,3 @@ mandatory-y += vermagic.h
mandatory-y += vga.h
mandatory-y += video.h
mandatory-y += word-at-a-time.h
-mandatory-y += xor.h
diff --git a/include/asm-generic/xor.h b/include/asm-generic/xor.h
deleted file mode 100644
index fc151fdc45ab..000000000000
--- a/include/asm-generic/xor.h
+++ /dev/null
@@ -1,11 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm-generic/xor.h
- *
- * Generic optimized RAID-5 checksumming functions.
- */
-
-extern struct xor_block_template xor_block_8regs;
-extern struct xor_block_template xor_block_32regs;
-extern struct xor_block_template xor_block_8regs_p;
-extern struct xor_block_template xor_block_32regs_p;
diff --git a/lib/raid/Kconfig b/lib/raid/Kconfig
index 01b73a1c303f..2ccf0c60de01 100644
--- a/lib/raid/Kconfig
+++ b/lib/raid/Kconfig
@@ -2,3 +2,18 @@
config XOR_BLOCKS
tristate
+
+# selected by architectures that provide an optimized XOR implementation
+config XOR_BLOCKS_ARCH
+ depends on XOR_BLOCKS
+ default y if ALPHA
+ default y if ARM && KERNEL_MODE_NEON
+ default y if ARM64
+ default y if CPU_HAS_LSX # loongarch
+ default y if ALTIVEC # powerpc
+ default y if RISCV_ISA_V
+ default y if SPARC
+ default y if S390
+ default y if X86_32
+ default y if X86_64
+ bool
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 05aca96041b3..df55823c4d82 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -1,5 +1,7 @@
# SPDX-License-Identifier: GPL-2.0
+ccflags-y += -I $(src)
+
obj-$(CONFIG_XOR_BLOCKS) += xor.o
xor-y += xor-core.o
@@ -8,6 +10,10 @@ xor-y += xor-32regs.o
xor-y += xor-8regs-prefetch.o
xor-y += xor-32regs-prefetch.o
+ifeq ($(CONFIG_XOR_BLOCKS_ARCH),y)
+CFLAGS_xor-core.o += -I$(src)/$(SRCARCH)
+endif
+
xor-$(CONFIG_ALPHA) += alpha/xor.o
xor-$(CONFIG_ARM) += arm/xor.o
ifeq ($(CONFIG_ARM),y)
diff --git a/lib/raid/xor/alpha/xor.c b/lib/raid/xor/alpha/xor.c
index 0964ac420604..90694cc47395 100644
--- a/lib/raid/xor/alpha/xor.c
+++ b/lib/raid/xor/alpha/xor.c
@@ -2,8 +2,8 @@
/*
* Optimized XOR parity functions for alpha EV5 and EV6
*/
-#include <linux/raid/xor_impl.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
extern void
xor_alpha_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/arch/alpha/include/asm/xor.h b/lib/raid/xor/alpha/xor_arch.h
similarity index 90%
rename from arch/alpha/include/asm/xor.h
rename to lib/raid/xor/alpha/xor_arch.h
index e517be577a09..0dcfea578a48 100644
--- a/arch/alpha/include/asm/xor.h
+++ b/lib/raid/xor/alpha/xor_arch.h
@@ -1,7 +1,6 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#include <asm/special_insns.h>
-#include <asm-generic/xor.h>
extern struct xor_block_template xor_block_alpha;
extern struct xor_block_template xor_block_alpha_prefetch;
@@ -10,7 +9,6 @@ extern struct xor_block_template xor_block_alpha_prefetch;
* Force the use of alpha_prefetch if EV6, as it is significantly faster in the
* cold cache case.
*/
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
if (implver() == IMPLVER_EV6) {
diff --git a/lib/raid/xor/arm/xor-neon-glue.c b/lib/raid/xor/arm/xor-neon-glue.c
index c7b162b383a2..7afd6294464b 100644
--- a/lib/raid/xor/arm/xor-neon-glue.c
+++ b/lib/raid/xor/arm/xor-neon-glue.c
@@ -2,8 +2,8 @@
/*
* Copyright (C) 2001 Russell King
*/
-#include <linux/raid/xor_impl.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
extern struct xor_block_template const xor_block_neon_inner;
diff --git a/lib/raid/xor/arm/xor-neon.c b/lib/raid/xor/arm/xor-neon.c
index c9d4378b0f0e..806a42c5952c 100644
--- a/lib/raid/xor/arm/xor-neon.c
+++ b/lib/raid/xor/arm/xor-neon.c
@@ -3,7 +3,7 @@
* Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
*/
-#include <linux/raid/xor_impl.h>
+#include "xor_impl.h"
#ifndef __ARM_NEON__
#error You should compile this file with '-march=armv7-a -mfloat-abi=softfp -mfpu=neon'
diff --git a/lib/raid/xor/arm/xor.c b/lib/raid/xor/arm/xor.c
index 2263341dbbcd..5bd5f048bbe9 100644
--- a/lib/raid/xor/arm/xor.c
+++ b/lib/raid/xor/arm/xor.c
@@ -2,8 +2,8 @@
/*
* Copyright (C) 2001 Russell King
*/
-#include <linux/raid/xor_impl.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
#define __XOR(a1, a2) a1 ^= a2
diff --git a/arch/arm/include/asm/xor.h b/lib/raid/xor/arm/xor_arch.h
similarity index 87%
rename from arch/arm/include/asm/xor.h
rename to lib/raid/xor/arm/xor_arch.h
index 989c55872ef6..5a7eedb48fbb 100644
--- a/arch/arm/include/asm/xor.h
+++ b/lib/raid/xor/arm/xor_arch.h
@@ -2,13 +2,11 @@
/*
* Copyright (C) 2001 Russell King
*/
-#include <asm-generic/xor.h>
#include <asm/neon.h>
extern struct xor_block_template xor_block_arm4regs;
extern struct xor_block_template xor_block_neon;
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_arm4regs);
diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c
index 08c3e3573388..3db0a318cf5b 100644
--- a/lib/raid/xor/arm64/xor-neon-glue.c
+++ b/lib/raid/xor/arm64/xor-neon-glue.c
@@ -4,9 +4,9 @@
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
-#include <linux/raid/xor_impl.h>
#include <asm/simd.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
#include "xor-neon.h"
#define XOR_TEMPLATE(_name) \
diff --git a/lib/raid/xor/arm64/xor-neon.c b/lib/raid/xor/arm64/xor-neon.c
index 61194c292917..61f00c4fee49 100644
--- a/lib/raid/xor/arm64/xor-neon.c
+++ b/lib/raid/xor/arm64/xor-neon.c
@@ -4,10 +4,10 @@
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
-#include <linux/raid/xor_impl.h>
#include <linux/cache.h>
#include <asm/neon-intrinsics.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
#include "xor-neon.h"
void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/arch/arm64/include/asm/xor.h b/lib/raid/xor/arm64/xor_arch.h
similarity index 89%
rename from arch/arm64/include/asm/xor.h
rename to lib/raid/xor/arm64/xor_arch.h
index 4782c760bcac..5dbb40319501 100644
--- a/arch/arm64/include/asm/xor.h
+++ b/lib/raid/xor/arm64/xor_arch.h
@@ -3,14 +3,11 @@
* Authors: Jackie Liu <liuyun01@kylinos.cn>
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
-
-#include <asm-generic/xor.h>
#include <asm/simd.h>
extern struct xor_block_template xor_block_neon;
extern struct xor_block_template xor_block_eor3;
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
diff --git a/arch/loongarch/include/asm/xor.h b/lib/raid/xor/loongarch/xor_arch.h
similarity index 85%
rename from arch/loongarch/include/asm/xor.h
rename to lib/raid/xor/loongarch/xor_arch.h
index 7e32f72f8b03..fe5e8244fd0e 100644
--- a/arch/loongarch/include/asm/xor.h
+++ b/lib/raid/xor/loongarch/xor_arch.h
@@ -2,9 +2,6 @@
/*
* Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
*/
-#ifndef _ASM_LOONGARCH_XOR_H
-#define _ASM_LOONGARCH_XOR_H
-
#include <asm/cpu-features.h>
/*
@@ -15,12 +12,10 @@
* the scalar ones, maybe for errata or micro-op reasons. It may be
* appropriate to revisit this after one or two more uarch generations.
*/
-#include <asm-generic/xor.h>
extern struct xor_block_template xor_block_lsx;
extern struct xor_block_template xor_block_lasx;
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
@@ -36,5 +31,3 @@ static __always_inline void __init arch_xor_init(void)
xor_register(&xor_block_lasx);
#endif
}
-
-#endif /* _ASM_LOONGARCH_XOR_H */
diff --git a/lib/raid/xor/loongarch/xor_simd_glue.c b/lib/raid/xor/loongarch/xor_simd_glue.c
index 11fa3b47ba83..b387aa0213b4 100644
--- a/lib/raid/xor/loongarch/xor_simd_glue.c
+++ b/lib/raid/xor/loongarch/xor_simd_glue.c
@@ -6,9 +6,9 @@
*/
#include <linux/sched.h>
-#include <linux/raid/xor_impl.h>
#include <asm/fpu.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
#include "xor_simd.h"
#define MAKE_XOR_GLUE_2(flavor) \
diff --git a/arch/powerpc/include/asm/xor.h b/lib/raid/xor/powerpc/xor_arch.h
similarity index 77%
rename from arch/powerpc/include/asm/xor.h
rename to lib/raid/xor/powerpc/xor_arch.h
index 3293ac87181c..3b00a4a2fd67 100644
--- a/arch/powerpc/include/asm/xor.h
+++ b/lib/raid/xor/powerpc/xor_arch.h
@@ -5,15 +5,10 @@
*
* Author: Anton Blanchard <anton@au.ibm.com>
*/
-#ifndef _ASM_POWERPC_XOR_H
-#define _ASM_POWERPC_XOR_H
-
#include <asm/cpu_has_feature.h>
-#include <asm-generic/xor.h>
extern struct xor_block_template xor_block_altivec;
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
@@ -25,5 +20,3 @@ static __always_inline void __init arch_xor_init(void)
xor_register(&xor_block_altivec);
#endif
}
-
-#endif /* _ASM_POWERPC_XOR_H */
diff --git a/lib/raid/xor/powerpc/xor_vmx_glue.c b/lib/raid/xor/powerpc/xor_vmx_glue.c
index c41e38340700..56e99ddfb64f 100644
--- a/lib/raid/xor/powerpc/xor_vmx_glue.c
+++ b/lib/raid/xor/powerpc/xor_vmx_glue.c
@@ -7,9 +7,9 @@
#include <linux/preempt.h>
#include <linux/sched.h>
-#include <linux/raid/xor_impl.h>
#include <asm/switch_to.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
#include "xor_vmx.h"
static void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/lib/raid/xor/riscv/xor-glue.c b/lib/raid/xor/riscv/xor-glue.c
index 11666a4b6b68..060e5f22ebcc 100644
--- a/lib/raid/xor/riscv/xor-glue.c
+++ b/lib/raid/xor/riscv/xor-glue.c
@@ -3,11 +3,11 @@
* Copyright (C) 2021 SiFive
*/
-#include <linux/raid/xor_impl.h>
#include <asm/vector.h>
#include <asm/switch_to.h>
#include <asm/asm-prototypes.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1,
const unsigned long *__restrict p2)
diff --git a/arch/riscv/include/asm/xor.h b/lib/raid/xor/riscv/xor_arch.h
similarity index 84%
rename from arch/riscv/include/asm/xor.h
rename to lib/raid/xor/riscv/xor_arch.h
index 614d9209d078..9240857d760b 100644
--- a/arch/riscv/include/asm/xor.h
+++ b/lib/raid/xor/riscv/xor_arch.h
@@ -3,11 +3,9 @@
* Copyright (C) 2021 SiFive
*/
#include <asm/vector.h>
-#include <asm-generic/xor.h>
extern struct xor_block_template xor_block_rvv;
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
diff --git a/lib/raid/xor/s390/xor.c b/lib/raid/xor/s390/xor.c
index acbd268adfc8..c28cb56fec92 100644
--- a/lib/raid/xor/s390/xor.c
+++ b/lib/raid/xor/s390/xor.c
@@ -7,8 +7,8 @@
*/
#include <linux/types.h>
-#include <linux/raid/xor_impl.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
static void xor_xc_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
diff --git a/arch/s390/include/asm/xor.h b/lib/raid/xor/s390/xor_arch.h
similarity index 71%
rename from arch/s390/include/asm/xor.h
rename to lib/raid/xor/s390/xor_arch.h
index 4e2233f64da9..4a233ed2b97a 100644
--- a/arch/s390/include/asm/xor.h
+++ b/lib/raid/xor/s390/xor_arch.h
@@ -5,15 +5,9 @@
* Copyright IBM Corp. 2016
* Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
*/
-#ifndef _ASM_S390_XOR_H
-#define _ASM_S390_XOR_H
-
extern struct xor_block_template xor_block_xc;
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
xor_force(&xor_block_xc);
}
-
-#endif /* _ASM_S390_XOR_H */
diff --git a/lib/raid/xor/sparc/xor-sparc32.c b/lib/raid/xor/sparc/xor-sparc32.c
index b65a75a6e59d..307c4a84f535 100644
--- a/lib/raid/xor/sparc/xor-sparc32.c
+++ b/lib/raid/xor/sparc/xor-sparc32.c
@@ -5,8 +5,8 @@
*
* Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz)
*/
-#include <linux/raid/xor_impl.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
static void
sparc_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/lib/raid/xor/sparc/xor-sparc64-glue.c b/lib/raid/xor/sparc/xor-sparc64-glue.c
index 3c67c8c3a0e8..5f90c2460b54 100644
--- a/lib/raid/xor/sparc/xor-sparc64-glue.c
+++ b/lib/raid/xor/sparc/xor-sparc64-glue.c
@@ -8,8 +8,8 @@
* Copyright (C) 2006 David S. Miller <davem@davemloft.net>
*/
-#include <linux/raid/xor_impl.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
diff --git a/arch/sparc/include/asm/xor.h b/lib/raid/xor/sparc/xor_arch.h
similarity index 81%
rename from arch/sparc/include/asm/xor.h
rename to lib/raid/xor/sparc/xor_arch.h
index f923b009fc24..af288abe4e91 100644
--- a/arch/sparc/include/asm/xor.h
+++ b/lib/raid/xor/sparc/xor_arch.h
@@ -3,16 +3,12 @@
* Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
* Copyright (C) 2006 David S. Miller <davem@davemloft.net>
*/
-#ifndef ___ASM_SPARC_XOR_H
-#define ___ASM_SPARC_XOR_H
-
#if defined(__sparc__) && defined(__arch64__)
#include <asm/spitfire.h>
extern struct xor_block_template xor_block_VIS;
extern struct xor_block_template xor_block_niagara;
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
/* Force VIS for everything except Niagara. */
@@ -28,12 +24,8 @@ static __always_inline void __init arch_xor_init(void)
}
#else /* sparc64 */
-/* For grins, also test the generic routines. */
-#include <asm-generic/xor.h>
-
extern struct xor_block_template xor_block_SPARC;
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
xor_register(&xor_block_8regs);
@@ -41,4 +33,3 @@ static __always_inline void __init arch_xor_init(void)
xor_register(&xor_block_SPARC);
}
#endif /* !sparc64 */
-#endif /* ___ASM_SPARC_XOR_H */
diff --git a/lib/raid/xor/um/xor_arch.h b/lib/raid/xor/um/xor_arch.h
new file mode 100644
index 000000000000..a33e57a26c5e
--- /dev/null
+++ b/lib/raid/xor/um/xor_arch.h
@@ -0,0 +1,2 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#include <../x86/xor_arch.h>
diff --git a/lib/raid/xor/x86/xor-avx.c b/lib/raid/xor/x86/xor-avx.c
index b49cb5199e70..d411efa1ff43 100644
--- a/lib/raid/xor/x86/xor-avx.c
+++ b/lib/raid/xor/x86/xor-avx.c
@@ -8,9 +8,9 @@
* Based on Ingo Molnar and Zach Brown's respective MMX and SSE routines
*/
#include <linux/compiler.h>
-#include <linux/raid/xor_impl.h>
#include <asm/fpu/api.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
#define BLOCK4(i) \
BLOCK(32 * i, 0) \
diff --git a/lib/raid/xor/x86/xor-mmx.c b/lib/raid/xor/x86/xor-mmx.c
index cf0fafea33b7..e48c58f92874 100644
--- a/lib/raid/xor/x86/xor-mmx.c
+++ b/lib/raid/xor/x86/xor-mmx.c
@@ -4,9 +4,9 @@
*
* Copyright (C) 1998 Ingo Molnar.
*/
-#include <linux/raid/xor_impl.h>
#include <asm/fpu/api.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
#define LD(x, y) " movq 8*("#x")(%1), %%mm"#y" ;\n"
#define ST(x, y) " movq %%mm"#y", 8*("#x")(%1) ;\n"
diff --git a/lib/raid/xor/x86/xor-sse.c b/lib/raid/xor/x86/xor-sse.c
index 0e727ced8b00..5993ed688c15 100644
--- a/lib/raid/xor/x86/xor-sse.c
+++ b/lib/raid/xor/x86/xor-sse.c
@@ -12,9 +12,9 @@
* x86-64 changes / gcc fixes from Andi Kleen.
* Copyright 2002 Andi Kleen, SuSE Labs.
*/
-#include <linux/raid/xor_impl.h>
#include <asm/fpu/api.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
+#include "xor_arch.h"
#ifdef CONFIG_X86_32
/* reduce register pressure */
diff --git a/arch/x86/include/asm/xor.h b/lib/raid/xor/x86/xor_arch.h
similarity index 89%
rename from arch/x86/include/asm/xor.h
rename to lib/raid/xor/x86/xor_arch.h
index d1aab8275908..99fe85a213c6 100644
--- a/arch/x86/include/asm/xor.h
+++ b/lib/raid/xor/x86/xor_arch.h
@@ -1,9 +1,5 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _ASM_X86_XOR_H
-#define _ASM_X86_XOR_H
-
#include <asm/cpufeature.h>
-#include <asm-generic/xor.h>
extern struct xor_block_template xor_block_pII_mmx;
extern struct xor_block_template xor_block_p5_mmx;
@@ -20,7 +16,6 @@ extern struct xor_block_template xor_block_avx;
*
* 32-bit without MMX can fall back to the generic routines.
*/
-#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
if (boot_cpu_has(X86_FEATURE_AVX) &&
@@ -39,5 +34,3 @@ static __always_inline void __init arch_xor_init(void)
xor_register(&xor_block_32regs_p);
}
}
-
-#endif /* _ASM_X86_XOR_H */
diff --git a/lib/raid/xor/xor-32regs-prefetch.c b/lib/raid/xor/xor-32regs-prefetch.c
index 8666c287f777..2856a8e50cb8 100644
--- a/lib/raid/xor/xor-32regs-prefetch.c
+++ b/lib/raid/xor/xor-32regs-prefetch.c
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/prefetch.h>
-#include <linux/raid/xor_impl.h>
-#include <asm-generic/xor.h>
+#include "xor_impl.h"
static void
xor_32regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/lib/raid/xor/xor-32regs.c b/lib/raid/xor/xor-32regs.c
index 58d4fac43eb4..cc44d64032fa 100644
--- a/lib/raid/xor/xor-32regs.c
+++ b/lib/raid/xor/xor-32regs.c
@@ -1,6 +1,5 @@
// SPDX-License-Identifier: GPL-2.0-or-later
-#include <linux/raid/xor_impl.h>
-#include <asm-generic/xor.h>
+#include "xor_impl.h"
static void
xor_32regs_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/lib/raid/xor/xor-8regs-prefetch.c b/lib/raid/xor/xor-8regs-prefetch.c
index 67061e35a0a6..1d53aec50d27 100644
--- a/lib/raid/xor/xor-8regs-prefetch.c
+++ b/lib/raid/xor/xor-8regs-prefetch.c
@@ -1,7 +1,6 @@
// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/prefetch.h>
-#include <linux/raid/xor_impl.h>
-#include <asm-generic/xor.h>
+#include "xor_impl.h"
static void
xor_8regs_p_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/lib/raid/xor/xor-8regs.c b/lib/raid/xor/xor-8regs.c
index 769f796ab2cf..72a44e898c55 100644
--- a/lib/raid/xor/xor-8regs.c
+++ b/lib/raid/xor/xor-8regs.c
@@ -1,6 +1,5 @@
// SPDX-License-Identifier: GPL-2.0-or-later
-#include <linux/raid/xor_impl.h>
-#include <asm-generic/xor.h>
+#include "xor_impl.h"
static void
xor_8regs_2(unsigned long bytes, unsigned long * __restrict p1,
diff --git a/lib/raid/xor/xor-core.c b/lib/raid/xor/xor-core.c
index 01a42995b7a5..47e09ae954b2 100644
--- a/lib/raid/xor/xor-core.c
+++ b/lib/raid/xor/xor-core.c
@@ -9,10 +9,9 @@
#include <linux/module.h>
#include <linux/gfp.h>
#include <linux/raid/xor.h>
-#include <linux/raid/xor_impl.h>
#include <linux/jiffies.h>
#include <linux/preempt.h>
-#include <asm/xor.h>
+#include "xor_impl.h"
/* The xor routines to use. */
static struct xor_block_template *active_template;
@@ -141,16 +140,21 @@ static int __init calibrate_xor_blocks(void)
return 0;
}
-static int __init xor_init(void)
-{
-#ifdef arch_xor_init
- arch_xor_init();
+#ifdef CONFIG_XOR_BLOCKS_ARCH
+#include "xor_arch.h" /* $SRCARCH/xor_arch.h */
#else
+static void __init arch_xor_init(void)
+{
xor_register(&xor_block_8regs);
xor_register(&xor_block_8regs_p);
xor_register(&xor_block_32regs);
xor_register(&xor_block_32regs_p);
-#endif
+}
+#endif /* CONFIG_XOR_BLOCKS_ARCH */
+
+static int __init xor_init(void)
+{
+ arch_xor_init();
/*
* If this arch/cpu has a short-circuited selection, don't loop through
diff --git a/include/linux/raid/xor_impl.h b/lib/raid/xor/xor_impl.h
similarity index 80%
rename from include/linux/raid/xor_impl.h
rename to lib/raid/xor/xor_impl.h
index 6ed4c445ab24..44b6c99e2093 100644
--- a/include/linux/raid/xor_impl.h
+++ b/lib/raid/xor/xor_impl.h
@@ -24,6 +24,12 @@ struct xor_block_template {
const unsigned long * __restrict);
};
+/* generic implementations */
+extern struct xor_block_template xor_block_8regs;
+extern struct xor_block_template xor_block_32regs;
+extern struct xor_block_template xor_block_8regs_p;
+extern struct xor_block_template xor_block_32regs_p;
+
void __init xor_register(struct xor_block_template *tmpl);
void __init xor_force(struct xor_block_template *tmpl);
--
2.47.3
^ permalink raw reply related
* [PATCH 19/26] xor: avoid indirect calls for arm64-optimized ops
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Remove the inner xor_block_templates, and instead have two separate
actual template that call into the neon-enabled compilation unit.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/arm64/include/asm/xor.h | 13 ++--
lib/raid/xor/arm64/xor-neon-glue.c | 95 +++++++++++++++---------------
lib/raid/xor/arm64/xor-neon.c | 73 +++++++++--------------
lib/raid/xor/arm64/xor-neon.h | 30 ++++++++++
4 files changed, 114 insertions(+), 97 deletions(-)
create mode 100644 lib/raid/xor/arm64/xor-neon.h
diff --git a/arch/arm64/include/asm/xor.h b/arch/arm64/include/asm/xor.h
index 81718f010761..4782c760bcac 100644
--- a/arch/arm64/include/asm/xor.h
+++ b/arch/arm64/include/asm/xor.h
@@ -7,15 +7,18 @@
#include <asm-generic/xor.h>
#include <asm/simd.h>
-extern struct xor_block_template xor_block_arm64;
-void __init xor_neon_init(void);
+extern struct xor_block_template xor_block_neon;
+extern struct xor_block_template xor_block_eor3;
#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
- xor_neon_init();
xor_register(&xor_block_8regs);
xor_register(&xor_block_32regs);
- if (cpu_has_neon())
- xor_register(&xor_block_arm64);
+ if (cpu_has_neon()) {
+ if (cpu_have_named_feature(SHA3))
+ xor_register(&xor_block_eor3);
+ else
+ xor_register(&xor_block_neon);
+ }
}
diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c
index 067a2095659a..08c3e3573388 100644
--- a/lib/raid/xor/arm64/xor-neon-glue.c
+++ b/lib/raid/xor/arm64/xor-neon-glue.c
@@ -7,51 +7,54 @@
#include <linux/raid/xor_impl.h>
#include <asm/simd.h>
#include <asm/xor.h>
+#include "xor-neon.h"
-extern struct xor_block_template const xor_block_inner_neon;
-
-static void
-xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_2(bytes, p1, p2);
-}
-
-static void
-xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_3(bytes, p1, p2, p3);
-}
-
-static void
-xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_4(bytes, p1, p2, p3, p4);
-}
-
-static void
-xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- scoped_ksimd()
- xor_block_inner_neon.do_5(bytes, p1, p2, p3, p4, p5);
-}
-
-struct xor_block_template xor_block_arm64 = {
- .name = "arm64_neon",
- .do_2 = xor_neon_2,
- .do_3 = xor_neon_3,
- .do_4 = xor_neon_4,
- .do_5 = xor_neon_5
+#define XOR_TEMPLATE(_name) \
+static void \
+xor_##_name##_2(unsigned long bytes, unsigned long * __restrict p1, \
+ const unsigned long * __restrict p2) \
+{ \
+ scoped_ksimd() \
+ __xor_##_name##_2(bytes, p1, p2); \
+} \
+ \
+static void \
+xor_##_name##_3(unsigned long bytes, unsigned long * __restrict p1, \
+ const unsigned long * __restrict p2, \
+ const unsigned long * __restrict p3) \
+{ \
+ scoped_ksimd() \
+ __xor_##_name##_3(bytes, p1, p2, p3); \
+} \
+ \
+static void \
+xor_##_name##_4(unsigned long bytes, unsigned long * __restrict p1, \
+ const unsigned long * __restrict p2, \
+ const unsigned long * __restrict p3, \
+ const unsigned long * __restrict p4) \
+{ \
+ scoped_ksimd() \
+ __xor_##_name##_4(bytes, p1, p2, p3, p4); \
+} \
+ \
+static void \
+xor_##_name##_5(unsigned long bytes, unsigned long * __restrict p1, \
+ const unsigned long * __restrict p2, \
+ const unsigned long * __restrict p3, \
+ const unsigned long * __restrict p4, \
+ const unsigned long * __restrict p5) \
+{ \
+ scoped_ksimd() \
+ __xor_##_name##_5(bytes, p1, p2, p3, p4, p5); \
+} \
+ \
+struct xor_block_template xor_block_##_name = { \
+ .name = __stringify(_name), \
+ .do_2 = xor_##_name##_2, \
+ .do_3 = xor_##_name##_3, \
+ .do_4 = xor_##_name##_4, \
+ .do_5 = xor_##_name##_5 \
};
+
+XOR_TEMPLATE(neon);
+XOR_TEMPLATE(eor3);
diff --git a/lib/raid/xor/arm64/xor-neon.c b/lib/raid/xor/arm64/xor-neon.c
index 8d2d185090db..61194c292917 100644
--- a/lib/raid/xor/arm64/xor-neon.c
+++ b/lib/raid/xor/arm64/xor-neon.c
@@ -8,9 +8,10 @@
#include <linux/cache.h>
#include <asm/neon-intrinsics.h>
#include <asm/xor.h>
+#include "xor-neon.h"
-static void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
+void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
@@ -36,9 +37,9 @@ static void xor_arm64_neon_2(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
+void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
@@ -72,10 +73,10 @@ static void xor_arm64_neon_3(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
+void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
@@ -117,11 +118,11 @@ static void xor_arm64_neon_4(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
+void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
@@ -171,14 +172,6 @@ static void xor_arm64_neon_5(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-struct xor_block_template xor_block_inner_neon __ro_after_init = {
- .name = "__inner_neon__",
- .do_2 = xor_arm64_neon_2,
- .do_3 = xor_arm64_neon_3,
- .do_4 = xor_arm64_neon_4,
- .do_5 = xor_arm64_neon_5,
-};
-
static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
{
uint64x2_t res;
@@ -189,10 +182,9 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
return res;
}
-static void xor_arm64_eor3_3(unsigned long bytes,
- unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
+void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
@@ -224,11 +216,10 @@ static void xor_arm64_eor3_3(unsigned long bytes,
} while (--lines > 0);
}
-static void xor_arm64_eor3_4(unsigned long bytes,
- unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
+void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
@@ -268,12 +259,11 @@ static void xor_arm64_eor3_4(unsigned long bytes,
} while (--lines > 0);
}
-static void xor_arm64_eor3_5(unsigned long bytes,
- unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
+void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
{
uint64_t *dp1 = (uint64_t *)p1;
uint64_t *dp2 = (uint64_t *)p2;
@@ -314,12 +304,3 @@ static void xor_arm64_eor3_5(unsigned long bytes,
dp5 += 8;
} while (--lines > 0);
}
-
-void __init xor_neon_init(void)
-{
- if (cpu_have_named_feature(SHA3)) {
- xor_block_inner_neon.do_3 = xor_arm64_eor3_3;
- xor_block_inner_neon.do_4 = xor_arm64_eor3_4;
- xor_block_inner_neon.do_5 = xor_arm64_eor3_5;
- }
-}
diff --git a/lib/raid/xor/arm64/xor-neon.h b/lib/raid/xor/arm64/xor-neon.h
new file mode 100644
index 000000000000..cec0ac846fea
--- /dev/null
+++ b/lib/raid/xor/arm64/xor-neon.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+
+void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2);
+void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
+
+#define __xor_eor3_2 __xor_neon_2
+void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
--
2.47.3
^ permalink raw reply related
* [PATCH 18/26] x86: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR code out of line into lib/raid.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/x86/include/asm/xor.h | 518 ++----------------
arch/x86/include/asm/xor_64.h | 32 --
lib/raid/xor/Makefile | 2 +
.../xor_avx.h => lib/raid/xor/x86/xor-avx.c | 14 +-
.../xor_32.h => lib/raid/xor/x86/xor-mmx.c | 60 +-
lib/raid/xor/x86/xor-sse.c | 476 ++++++++++++++++
6 files changed, 522 insertions(+), 580 deletions(-)
delete mode 100644 arch/x86/include/asm/xor_64.h
rename arch/x86/include/asm/xor_avx.h => lib/raid/xor/x86/xor-avx.c (95%)
rename arch/x86/include/asm/xor_32.h => lib/raid/xor/x86/xor-mmx.c (90%)
create mode 100644 lib/raid/xor/x86/xor-sse.c
diff --git a/arch/x86/include/asm/xor.h b/arch/x86/include/asm/xor.h
index 33f5620d8d69..d1aab8275908 100644
--- a/arch/x86/include/asm/xor.h
+++ b/arch/x86/include/asm/xor.h
@@ -2,498 +2,42 @@
#ifndef _ASM_X86_XOR_H
#define _ASM_X86_XOR_H
-/*
- * Optimized RAID-5 checksumming functions for SSE.
- */
-
-/*
- * Cache avoiding checksumming functions utilizing KNI instructions
- * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
- */
+#include <asm/cpufeature.h>
+#include <asm-generic/xor.h>
-/*
- * Based on
- * High-speed RAID5 checksumming functions utilizing SSE instructions.
- * Copyright (C) 1998 Ingo Molnar.
- */
+extern struct xor_block_template xor_block_pII_mmx;
+extern struct xor_block_template xor_block_p5_mmx;
+extern struct xor_block_template xor_block_sse;
+extern struct xor_block_template xor_block_sse_pf64;
+extern struct xor_block_template xor_block_avx;
/*
- * x86-64 changes / gcc fixes from Andi Kleen.
- * Copyright 2002 Andi Kleen, SuSE Labs.
+ * When SSE is available, use it as it can write around L2. We may also be able
+ * to load into the L1 only depending on how the cpu deals with a load to a line
+ * that is being prefetched.
+ *
+ * When AVX2 is available, force using it as it is better by all measures.
*
- * This hasn't been optimized for the hammer yet, but there are likely
- * no advantages to be gotten from x86-64 here anyways.
+ * 32-bit without MMX can fall back to the generic routines.
*/
-
-#include <asm/fpu/api.h>
-
-#ifdef CONFIG_X86_32
-/* reduce register pressure */
-# define XOR_CONSTANT_CONSTRAINT "i"
-#else
-# define XOR_CONSTANT_CONSTRAINT "re"
-#endif
-
-#define OFFS(x) "16*("#x")"
-#define PF_OFFS(x) "256+16*("#x")"
-#define PF0(x) " prefetchnta "PF_OFFS(x)"(%[p1]) ;\n"
-#define LD(x, y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n"
-#define ST(x, y) " movaps %%xmm"#y", "OFFS(x)"(%[p1]) ;\n"
-#define PF1(x) " prefetchnta "PF_OFFS(x)"(%[p2]) ;\n"
-#define PF2(x) " prefetchnta "PF_OFFS(x)"(%[p3]) ;\n"
-#define PF3(x) " prefetchnta "PF_OFFS(x)"(%[p4]) ;\n"
-#define PF4(x) " prefetchnta "PF_OFFS(x)"(%[p5]) ;\n"
-#define XO1(x, y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n"
-#define XO2(x, y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n"
-#define XO3(x, y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n"
-#define XO4(x, y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n"
-#define NOP(x)
-
-#define BLK64(pf, op, i) \
- pf(i) \
- op(i, 0) \
- op(i + 1, 1) \
- op(i + 2, 2) \
- op(i + 3, 3)
-
-static void
-xor_sse_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- LD(i, 0) \
- LD(i + 1, 1) \
- PF1(i) \
- PF1(i + 2) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- PF0(i + 4) \
- PF0(i + 6) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- ST(i, 0) \
- ST(i + 1, 1) \
- ST(i + 2, 2) \
- ST(i + 3, 3) \
-
-
- PF0(0)
- PF0(2)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines),
- [p1] "+r" (p1), [p2] "+r" (p2)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- BLK64(PF0, LD, i) \
- BLK64(PF1, XO1, i) \
- BLK64(NOP, ST, i) \
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines),
- [p1] "+r" (p1), [p2] "+r" (p2)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- PF1(i) \
- PF1(i + 2) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- PF2(i) \
- PF2(i + 2) \
- PF0(i + 4) \
- PF0(i + 6) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- XO2(i, 0) \
- XO2(i + 1, 1) \
- XO2(i + 2, 2) \
- XO2(i + 3, 3) \
- ST(i, 0) \
- ST(i + 1, 1) \
- ST(i + 2, 2) \
- ST(i + 3, 3) \
-
-
- PF0(0)
- PF0(2)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines),
- [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ if (boot_cpu_has(X86_FEATURE_AVX) &&
+ boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+ xor_force(&xor_block_avx);
+ } else if (IS_ENABLED(CONFIG_X86_64) || boot_cpu_has(X86_FEATURE_XMM)) {
+ xor_register(&xor_block_sse);
+ xor_register(&xor_block_sse_pf64);
+ } else if (boot_cpu_has(X86_FEATURE_MMX)) {
+ xor_register(&xor_block_pII_mmx);
+ xor_register(&xor_block_p5_mmx);
+ } else {
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_8regs_p);
+ xor_register(&xor_block_32regs);
+ xor_register(&xor_block_32regs_p);
+ }
}
-static void
-xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- BLK64(PF0, LD, i) \
- BLK64(PF1, XO1, i) \
- BLK64(PF2, XO2, i) \
- BLK64(NOP, ST, i) \
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines),
- [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- PF1(i) \
- PF1(i + 2) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- PF2(i) \
- PF2(i + 2) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- PF3(i) \
- PF3(i + 2) \
- PF0(i + 4) \
- PF0(i + 6) \
- XO2(i, 0) \
- XO2(i + 1, 1) \
- XO2(i + 2, 2) \
- XO2(i + 3, 3) \
- XO3(i, 0) \
- XO3(i + 1, 1) \
- XO3(i + 2, 2) \
- XO3(i + 3, 3) \
- ST(i, 0) \
- ST(i + 1, 1) \
- ST(i + 2, 2) \
- ST(i + 3, 3) \
-
-
- PF0(0)
- PF0(2)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " add %[inc], %[p4] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines), [p1] "+r" (p1),
- [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- BLK64(PF0, LD, i) \
- BLK64(PF1, XO1, i) \
- BLK64(PF2, XO2, i) \
- BLK64(PF3, XO3, i) \
- BLK64(NOP, ST, i) \
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " add %[inc], %[p4] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines), [p1] "+r" (p1),
- [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- PF1(i) \
- PF1(i + 2) \
- LD(i, 0) \
- LD(i + 1, 1) \
- LD(i + 2, 2) \
- LD(i + 3, 3) \
- PF2(i) \
- PF2(i + 2) \
- XO1(i, 0) \
- XO1(i + 1, 1) \
- XO1(i + 2, 2) \
- XO1(i + 3, 3) \
- PF3(i) \
- PF3(i + 2) \
- XO2(i, 0) \
- XO2(i + 1, 1) \
- XO2(i + 2, 2) \
- XO2(i + 3, 3) \
- PF4(i) \
- PF4(i + 2) \
- PF0(i + 4) \
- PF0(i + 6) \
- XO3(i, 0) \
- XO3(i + 1, 1) \
- XO3(i + 2, 2) \
- XO3(i + 3, 3) \
- XO4(i, 0) \
- XO4(i + 1, 1) \
- XO4(i + 2, 2) \
- XO4(i + 3, 3) \
- ST(i, 0) \
- ST(i + 1, 1) \
- ST(i + 2, 2) \
- ST(i + 3, 3) \
-
-
- PF0(0)
- PF0(2)
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " add %[inc], %[p4] ;\n"
- " add %[inc], %[p5] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
- [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static void
-xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- unsigned long lines = bytes >> 8;
-
- kernel_fpu_begin();
-
- asm volatile(
-#undef BLOCK
-#define BLOCK(i) \
- BLK64(PF0, LD, i) \
- BLK64(PF1, XO1, i) \
- BLK64(PF2, XO2, i) \
- BLK64(PF3, XO3, i) \
- BLK64(PF4, XO4, i) \
- BLK64(NOP, ST, i) \
-
- " .align 32 ;\n"
- " 1: ;\n"
-
- BLOCK(0)
- BLOCK(4)
- BLOCK(8)
- BLOCK(12)
-
- " add %[inc], %[p1] ;\n"
- " add %[inc], %[p2] ;\n"
- " add %[inc], %[p3] ;\n"
- " add %[inc], %[p4] ;\n"
- " add %[inc], %[p5] ;\n"
- " dec %[cnt] ;\n"
- " jnz 1b ;\n"
- : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
- [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
- : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
- : "memory");
-
- kernel_fpu_end();
-}
-
-static struct xor_block_template xor_block_sse_pf64 = {
- .name = "prefetch64-sse",
- .do_2 = xor_sse_2_pf64,
- .do_3 = xor_sse_3_pf64,
- .do_4 = xor_sse_4_pf64,
- .do_5 = xor_sse_5_pf64,
-};
-
-#undef LD
-#undef XO1
-#undef XO2
-#undef XO3
-#undef XO4
-#undef ST
-#undef NOP
-#undef BLK64
-#undef BLOCK
-
-#undef XOR_CONSTANT_CONSTRAINT
-
-#ifdef CONFIG_X86_32
-# include <asm/xor_32.h>
-#else
-# include <asm/xor_64.h>
-#endif
-
#endif /* _ASM_X86_XOR_H */
diff --git a/arch/x86/include/asm/xor_64.h b/arch/x86/include/asm/xor_64.h
deleted file mode 100644
index 2d2ceb241866..000000000000
--- a/arch/x86/include/asm/xor_64.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_X86_XOR_64_H
-#define _ASM_X86_XOR_64_H
-
-static struct xor_block_template xor_block_sse = {
- .name = "generic_sse",
- .do_2 = xor_sse_2,
- .do_3 = xor_sse_3,
- .do_4 = xor_sse_4,
- .do_5 = xor_sse_5,
-};
-
-
-/* Also try the AVX routines */
-#include <asm/xor_avx.h>
-
-/* We force the use of the SSE xor block because it can write around L2.
- We may also be able to load into the L1 only depending on how the cpu
- deals with a load to a line that is being prefetched. */
-#define arch_xor_init arch_xor_init
-static __always_inline void __init arch_xor_init(void)
-{
- if (boot_cpu_has(X86_FEATURE_AVX) &&
- boot_cpu_has(X86_FEATURE_OSXSAVE)) {
- xor_force(&xor_block_avx);
- } else {
- xor_register(&xor_block_sse_pf64);
- xor_register(&xor_block_sse);
- }
-}
-
-#endif /* _ASM_X86_XOR_64_H */
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 3db6c2b2f26a..05aca96041b3 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -21,6 +21,8 @@ xor-$(CONFIG_RISCV_ISA_V) += riscv/xor.o riscv/xor-glue.o
xor-$(CONFIG_SPARC32) += sparc/xor-sparc32.o
xor-$(CONFIG_SPARC64) += sparc/xor-sparc64.o sparc/xor-sparc64-glue.o
xor-$(CONFIG_S390) += s390/xor.o
+xor-$(CONFIG_X86_32) += x86/xor-avx.o x86/xor-sse.o x86/xor-mmx.o
+xor-$(CONFIG_X86_64) += x86/xor-avx.o x86/xor-sse.o
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
diff --git a/arch/x86/include/asm/xor_avx.h b/lib/raid/xor/x86/xor-avx.c
similarity index 95%
rename from arch/x86/include/asm/xor_avx.h
rename to lib/raid/xor/x86/xor-avx.c
index c600888436bb..b49cb5199e70 100644
--- a/arch/x86/include/asm/xor_avx.h
+++ b/lib/raid/xor/x86/xor-avx.c
@@ -1,18 +1,16 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-#ifndef _ASM_X86_XOR_AVX_H
-#define _ASM_X86_XOR_AVX_H
-
+// SPDX-License-Identifier: GPL-2.0-only
/*
- * Optimized RAID-5 checksumming functions for AVX
+ * Optimized XOR parity functions for AVX
*
* Copyright (C) 2012 Intel Corporation
* Author: Jim Kukunas <james.t.kukunas@linux.intel.com>
*
* Based on Ingo Molnar and Zach Brown's respective MMX and SSE routines
*/
-
#include <linux/compiler.h>
+#include <linux/raid/xor_impl.h>
#include <asm/fpu/api.h>
+#include <asm/xor.h>
#define BLOCK4(i) \
BLOCK(32 * i, 0) \
@@ -158,12 +156,10 @@ do { \
kernel_fpu_end();
}
-static struct xor_block_template xor_block_avx = {
+struct xor_block_template xor_block_avx = {
.name = "avx",
.do_2 = xor_avx_2,
.do_3 = xor_avx_3,
.do_4 = xor_avx_4,
.do_5 = xor_avx_5,
};
-
-#endif
diff --git a/arch/x86/include/asm/xor_32.h b/lib/raid/xor/x86/xor-mmx.c
similarity index 90%
rename from arch/x86/include/asm/xor_32.h
rename to lib/raid/xor/x86/xor-mmx.c
index ee32d08c27bc..cf0fafea33b7 100644
--- a/arch/x86/include/asm/xor_32.h
+++ b/lib/raid/xor/x86/xor-mmx.c
@@ -1,15 +1,12 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-#ifndef _ASM_X86_XOR_32_H
-#define _ASM_X86_XOR_32_H
-
-/*
- * Optimized RAID-5 checksumming functions for MMX.
- */
-
+// SPDX-License-Identifier: GPL-2.0-or-later
/*
- * High-speed RAID5 checksumming functions utilizing MMX instructions.
+ * Optimized XOR parity functions for MMX.
+ *
* Copyright (C) 1998 Ingo Molnar.
*/
+#include <linux/raid/xor_impl.h>
+#include <asm/fpu/api.h>
+#include <asm/xor.h>
#define LD(x, y) " movq 8*("#x")(%1), %%mm"#y" ;\n"
#define ST(x, y) " movq %%mm"#y", 8*("#x")(%1) ;\n"
@@ -18,8 +15,6 @@
#define XO3(x, y) " pxor 8*("#x")(%4), %%mm"#y" ;\n"
#define XO4(x, y) " pxor 8*("#x")(%5), %%mm"#y" ;\n"
-#include <asm/fpu/api.h>
-
static void
xor_pII_mmx_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
@@ -519,7 +514,7 @@ xor_p5_mmx_5(unsigned long bytes, unsigned long * __restrict p1,
kernel_fpu_end();
}
-static struct xor_block_template xor_block_pII_mmx = {
+struct xor_block_template xor_block_pII_mmx = {
.name = "pII_mmx",
.do_2 = xor_pII_mmx_2,
.do_3 = xor_pII_mmx_3,
@@ -527,49 +522,10 @@ static struct xor_block_template xor_block_pII_mmx = {
.do_5 = xor_pII_mmx_5,
};
-static struct xor_block_template xor_block_p5_mmx = {
+struct xor_block_template xor_block_p5_mmx = {
.name = "p5_mmx",
.do_2 = xor_p5_mmx_2,
.do_3 = xor_p5_mmx_3,
.do_4 = xor_p5_mmx_4,
.do_5 = xor_p5_mmx_5,
};
-
-static struct xor_block_template xor_block_pIII_sse = {
- .name = "pIII_sse",
- .do_2 = xor_sse_2,
- .do_3 = xor_sse_3,
- .do_4 = xor_sse_4,
- .do_5 = xor_sse_5,
-};
-
-/* Also try the AVX routines */
-#include <asm/xor_avx.h>
-
-/* Also try the generic routines. */
-#include <asm-generic/xor.h>
-
-/* We force the use of the SSE xor block because it can write around L2.
- We may also be able to load into the L1 only depending on how the cpu
- deals with a load to a line that is being prefetched. */
-#define arch_xor_init arch_xor_init
-static __always_inline void __init arch_xor_init(void)
-{
- if (boot_cpu_has(X86_FEATURE_AVX) &&
- boot_cpu_has(X86_FEATURE_OSXSAVE)) {
- xor_force(&xor_block_avx);
- } else if (boot_cpu_has(X86_FEATURE_XMM)) {
- xor_register(&xor_block_pIII_sse);
- xor_register(&xor_block_sse_pf64);
- } else if (boot_cpu_has(X86_FEATURE_MMX)) {
- xor_register(&xor_block_pII_mmx);
- xor_register(&xor_block_p5_mmx);
- } else {
- xor_register(&xor_block_8regs);
- xor_register(&xor_block_8regs_p);
- xor_register(&xor_block_32regs);
- xor_register(&xor_block_32regs_p);
- }
-}
-
-#endif /* _ASM_X86_XOR_32_H */
diff --git a/lib/raid/xor/x86/xor-sse.c b/lib/raid/xor/x86/xor-sse.c
new file mode 100644
index 000000000000..0e727ced8b00
--- /dev/null
+++ b/lib/raid/xor/x86/xor-sse.c
@@ -0,0 +1,476 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Optimized XOR parity functions for SSE.
+ *
+ * Cache avoiding checksumming functions utilizing KNI instructions
+ * Copyright (C) 1999 Zach Brown (with obvious credit due Ingo)
+ *
+ * Based on
+ * High-speed RAID5 checksumming functions utilizing SSE instructions.
+ * Copyright (C) 1998 Ingo Molnar.
+ *
+ * x86-64 changes / gcc fixes from Andi Kleen.
+ * Copyright 2002 Andi Kleen, SuSE Labs.
+ */
+#include <linux/raid/xor_impl.h>
+#include <asm/fpu/api.h>
+#include <asm/xor.h>
+
+#ifdef CONFIG_X86_32
+/* reduce register pressure */
+# define XOR_CONSTANT_CONSTRAINT "i"
+#else
+# define XOR_CONSTANT_CONSTRAINT "re"
+#endif
+
+#define OFFS(x) "16*("#x")"
+#define PF_OFFS(x) "256+16*("#x")"
+#define PF0(x) " prefetchnta "PF_OFFS(x)"(%[p1]) ;\n"
+#define LD(x, y) " movaps "OFFS(x)"(%[p1]), %%xmm"#y" ;\n"
+#define ST(x, y) " movaps %%xmm"#y", "OFFS(x)"(%[p1]) ;\n"
+#define PF1(x) " prefetchnta "PF_OFFS(x)"(%[p2]) ;\n"
+#define PF2(x) " prefetchnta "PF_OFFS(x)"(%[p3]) ;\n"
+#define PF3(x) " prefetchnta "PF_OFFS(x)"(%[p4]) ;\n"
+#define PF4(x) " prefetchnta "PF_OFFS(x)"(%[p5]) ;\n"
+#define XO1(x, y) " xorps "OFFS(x)"(%[p2]), %%xmm"#y" ;\n"
+#define XO2(x, y) " xorps "OFFS(x)"(%[p3]), %%xmm"#y" ;\n"
+#define XO3(x, y) " xorps "OFFS(x)"(%[p4]), %%xmm"#y" ;\n"
+#define XO4(x, y) " xorps "OFFS(x)"(%[p5]), %%xmm"#y" ;\n"
+#define NOP(x)
+
+#define BLK64(pf, op, i) \
+ pf(i) \
+ op(i, 0) \
+ op(i + 1, 1) \
+ op(i + 2, 2) \
+ op(i + 3, 3)
+
+static void
+xor_sse_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ unsigned long lines = bytes >> 8;
+
+ kernel_fpu_begin();
+
+ asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+ LD(i, 0) \
+ LD(i + 1, 1) \
+ PF1(i) \
+ PF1(i + 2) \
+ LD(i + 2, 2) \
+ LD(i + 3, 3) \
+ PF0(i + 4) \
+ PF0(i + 6) \
+ XO1(i, 0) \
+ XO1(i + 1, 1) \
+ XO1(i + 2, 2) \
+ XO1(i + 3, 3) \
+ ST(i, 0) \
+ ST(i + 1, 1) \
+ ST(i + 2, 2) \
+ ST(i + 3, 3) \
+
+
+ PF0(0)
+ PF0(2)
+
+ " .align 32 ;\n"
+ " 1: ;\n"
+
+ BLOCK(0)
+ BLOCK(4)
+ BLOCK(8)
+ BLOCK(12)
+
+ " add %[inc], %[p1] ;\n"
+ " add %[inc], %[p2] ;\n"
+ " dec %[cnt] ;\n"
+ " jnz 1b ;\n"
+ : [cnt] "+r" (lines),
+ [p1] "+r" (p1), [p2] "+r" (p2)
+ : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+ : "memory");
+
+ kernel_fpu_end();
+}
+
+static void
+xor_sse_2_pf64(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ unsigned long lines = bytes >> 8;
+
+ kernel_fpu_begin();
+
+ asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+ BLK64(PF0, LD, i) \
+ BLK64(PF1, XO1, i) \
+ BLK64(NOP, ST, i) \
+
+ " .align 32 ;\n"
+ " 1: ;\n"
+
+ BLOCK(0)
+ BLOCK(4)
+ BLOCK(8)
+ BLOCK(12)
+
+ " add %[inc], %[p1] ;\n"
+ " add %[inc], %[p2] ;\n"
+ " dec %[cnt] ;\n"
+ " jnz 1b ;\n"
+ : [cnt] "+r" (lines),
+ [p1] "+r" (p1), [p2] "+r" (p2)
+ : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+ : "memory");
+
+ kernel_fpu_end();
+}
+
+static void
+xor_sse_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ unsigned long lines = bytes >> 8;
+
+ kernel_fpu_begin();
+
+ asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+ PF1(i) \
+ PF1(i + 2) \
+ LD(i, 0) \
+ LD(i + 1, 1) \
+ LD(i + 2, 2) \
+ LD(i + 3, 3) \
+ PF2(i) \
+ PF2(i + 2) \
+ PF0(i + 4) \
+ PF0(i + 6) \
+ XO1(i, 0) \
+ XO1(i + 1, 1) \
+ XO1(i + 2, 2) \
+ XO1(i + 3, 3) \
+ XO2(i, 0) \
+ XO2(i + 1, 1) \
+ XO2(i + 2, 2) \
+ XO2(i + 3, 3) \
+ ST(i, 0) \
+ ST(i + 1, 1) \
+ ST(i + 2, 2) \
+ ST(i + 3, 3) \
+
+
+ PF0(0)
+ PF0(2)
+
+ " .align 32 ;\n"
+ " 1: ;\n"
+
+ BLOCK(0)
+ BLOCK(4)
+ BLOCK(8)
+ BLOCK(12)
+
+ " add %[inc], %[p1] ;\n"
+ " add %[inc], %[p2] ;\n"
+ " add %[inc], %[p3] ;\n"
+ " dec %[cnt] ;\n"
+ " jnz 1b ;\n"
+ : [cnt] "+r" (lines),
+ [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
+ : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+ : "memory");
+
+ kernel_fpu_end();
+}
+
+static void
+xor_sse_3_pf64(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ unsigned long lines = bytes >> 8;
+
+ kernel_fpu_begin();
+
+ asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+ BLK64(PF0, LD, i) \
+ BLK64(PF1, XO1, i) \
+ BLK64(PF2, XO2, i) \
+ BLK64(NOP, ST, i) \
+
+ " .align 32 ;\n"
+ " 1: ;\n"
+
+ BLOCK(0)
+ BLOCK(4)
+ BLOCK(8)
+ BLOCK(12)
+
+ " add %[inc], %[p1] ;\n"
+ " add %[inc], %[p2] ;\n"
+ " add %[inc], %[p3] ;\n"
+ " dec %[cnt] ;\n"
+ " jnz 1b ;\n"
+ : [cnt] "+r" (lines),
+ [p1] "+r" (p1), [p2] "+r" (p2), [p3] "+r" (p3)
+ : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+ : "memory");
+
+ kernel_fpu_end();
+}
+
+static void
+xor_sse_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ unsigned long lines = bytes >> 8;
+
+ kernel_fpu_begin();
+
+ asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+ PF1(i) \
+ PF1(i + 2) \
+ LD(i, 0) \
+ LD(i + 1, 1) \
+ LD(i + 2, 2) \
+ LD(i + 3, 3) \
+ PF2(i) \
+ PF2(i + 2) \
+ XO1(i, 0) \
+ XO1(i + 1, 1) \
+ XO1(i + 2, 2) \
+ XO1(i + 3, 3) \
+ PF3(i) \
+ PF3(i + 2) \
+ PF0(i + 4) \
+ PF0(i + 6) \
+ XO2(i, 0) \
+ XO2(i + 1, 1) \
+ XO2(i + 2, 2) \
+ XO2(i + 3, 3) \
+ XO3(i, 0) \
+ XO3(i + 1, 1) \
+ XO3(i + 2, 2) \
+ XO3(i + 3, 3) \
+ ST(i, 0) \
+ ST(i + 1, 1) \
+ ST(i + 2, 2) \
+ ST(i + 3, 3) \
+
+
+ PF0(0)
+ PF0(2)
+
+ " .align 32 ;\n"
+ " 1: ;\n"
+
+ BLOCK(0)
+ BLOCK(4)
+ BLOCK(8)
+ BLOCK(12)
+
+ " add %[inc], %[p1] ;\n"
+ " add %[inc], %[p2] ;\n"
+ " add %[inc], %[p3] ;\n"
+ " add %[inc], %[p4] ;\n"
+ " dec %[cnt] ;\n"
+ " jnz 1b ;\n"
+ : [cnt] "+r" (lines), [p1] "+r" (p1),
+ [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
+ : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+ : "memory");
+
+ kernel_fpu_end();
+}
+
+static void
+xor_sse_4_pf64(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ unsigned long lines = bytes >> 8;
+
+ kernel_fpu_begin();
+
+ asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+ BLK64(PF0, LD, i) \
+ BLK64(PF1, XO1, i) \
+ BLK64(PF2, XO2, i) \
+ BLK64(PF3, XO3, i) \
+ BLK64(NOP, ST, i) \
+
+ " .align 32 ;\n"
+ " 1: ;\n"
+
+ BLOCK(0)
+ BLOCK(4)
+ BLOCK(8)
+ BLOCK(12)
+
+ " add %[inc], %[p1] ;\n"
+ " add %[inc], %[p2] ;\n"
+ " add %[inc], %[p3] ;\n"
+ " add %[inc], %[p4] ;\n"
+ " dec %[cnt] ;\n"
+ " jnz 1b ;\n"
+ : [cnt] "+r" (lines), [p1] "+r" (p1),
+ [p2] "+r" (p2), [p3] "+r" (p3), [p4] "+r" (p4)
+ : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+ : "memory");
+
+ kernel_fpu_end();
+}
+
+static void
+xor_sse_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ unsigned long lines = bytes >> 8;
+
+ kernel_fpu_begin();
+
+ asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+ PF1(i) \
+ PF1(i + 2) \
+ LD(i, 0) \
+ LD(i + 1, 1) \
+ LD(i + 2, 2) \
+ LD(i + 3, 3) \
+ PF2(i) \
+ PF2(i + 2) \
+ XO1(i, 0) \
+ XO1(i + 1, 1) \
+ XO1(i + 2, 2) \
+ XO1(i + 3, 3) \
+ PF3(i) \
+ PF3(i + 2) \
+ XO2(i, 0) \
+ XO2(i + 1, 1) \
+ XO2(i + 2, 2) \
+ XO2(i + 3, 3) \
+ PF4(i) \
+ PF4(i + 2) \
+ PF0(i + 4) \
+ PF0(i + 6) \
+ XO3(i, 0) \
+ XO3(i + 1, 1) \
+ XO3(i + 2, 2) \
+ XO3(i + 3, 3) \
+ XO4(i, 0) \
+ XO4(i + 1, 1) \
+ XO4(i + 2, 2) \
+ XO4(i + 3, 3) \
+ ST(i, 0) \
+ ST(i + 1, 1) \
+ ST(i + 2, 2) \
+ ST(i + 3, 3) \
+
+
+ PF0(0)
+ PF0(2)
+
+ " .align 32 ;\n"
+ " 1: ;\n"
+
+ BLOCK(0)
+ BLOCK(4)
+ BLOCK(8)
+ BLOCK(12)
+
+ " add %[inc], %[p1] ;\n"
+ " add %[inc], %[p2] ;\n"
+ " add %[inc], %[p3] ;\n"
+ " add %[inc], %[p4] ;\n"
+ " add %[inc], %[p5] ;\n"
+ " dec %[cnt] ;\n"
+ " jnz 1b ;\n"
+ : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
+ [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
+ : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+ : "memory");
+
+ kernel_fpu_end();
+}
+
+static void
+xor_sse_5_pf64(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ unsigned long lines = bytes >> 8;
+
+ kernel_fpu_begin();
+
+ asm volatile(
+#undef BLOCK
+#define BLOCK(i) \
+ BLK64(PF0, LD, i) \
+ BLK64(PF1, XO1, i) \
+ BLK64(PF2, XO2, i) \
+ BLK64(PF3, XO3, i) \
+ BLK64(PF4, XO4, i) \
+ BLK64(NOP, ST, i) \
+
+ " .align 32 ;\n"
+ " 1: ;\n"
+
+ BLOCK(0)
+ BLOCK(4)
+ BLOCK(8)
+ BLOCK(12)
+
+ " add %[inc], %[p1] ;\n"
+ " add %[inc], %[p2] ;\n"
+ " add %[inc], %[p3] ;\n"
+ " add %[inc], %[p4] ;\n"
+ " add %[inc], %[p5] ;\n"
+ " dec %[cnt] ;\n"
+ " jnz 1b ;\n"
+ : [cnt] "+r" (lines), [p1] "+r" (p1), [p2] "+r" (p2),
+ [p3] "+r" (p3), [p4] "+r" (p4), [p5] "+r" (p5)
+ : [inc] XOR_CONSTANT_CONSTRAINT (256UL)
+ : "memory");
+
+ kernel_fpu_end();
+}
+
+struct xor_block_template xor_block_sse = {
+ .name = "sse",
+ .do_2 = xor_sse_2,
+ .do_3 = xor_sse_3,
+ .do_4 = xor_sse_4,
+ .do_5 = xor_sse_5,
+};
+
+struct xor_block_template xor_block_sse_pf64 = {
+ .name = "prefetch64-sse",
+ .do_2 = xor_sse_2_pf64,
+ .do_3 = xor_sse_3_pf64,
+ .do_4 = xor_sse_4_pf64,
+ .do_5 = xor_sse_5_pf64,
+};
--
2.47.3
^ permalink raw reply related
* [PATCH 17/26] s390: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR into lib/raid and include it it in xor.ko
instead of unconditionally building it into the main kernel image.
Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Heiko Carstens <hca@linux.ibm.com>
---
arch/s390/lib/Makefile | 2 +-
lib/raid/xor/Makefile | 1 +
{arch/s390/lib => lib/raid/xor/s390}/xor.c | 2 --
3 files changed, 2 insertions(+), 3 deletions(-)
rename {arch/s390/lib => lib/raid/xor/s390}/xor.c (98%)
diff --git a/arch/s390/lib/Makefile b/arch/s390/lib/Makefile
index f43f897d3fc0..2bf47204f6ab 100644
--- a/arch/s390/lib/Makefile
+++ b/arch/s390/lib/Makefile
@@ -5,7 +5,7 @@
lib-y += delay.o string.o uaccess.o find.o spinlock.o tishift.o
lib-y += csum-partial.o
-obj-y += mem.o xor.o
+obj-y += mem.o
lib-$(CONFIG_KPROBES) += probes.o
lib-$(CONFIG_UPROBES) += probes.o
obj-$(CONFIG_S390_KPROBES_SANITY_TEST) += test_kprobes_s390.o
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 3a7c887d08ee..3db6c2b2f26a 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -20,6 +20,7 @@ xor-$(CONFIG_ALTIVEC) += powerpc/xor_vmx.o powerpc/xor_vmx_glue.o
xor-$(CONFIG_RISCV_ISA_V) += riscv/xor.o riscv/xor-glue.o
xor-$(CONFIG_SPARC32) += sparc/xor-sparc32.o
xor-$(CONFIG_SPARC64) += sparc/xor-sparc64.o sparc/xor-sparc64-glue.o
+xor-$(CONFIG_S390) += s390/xor.o
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
diff --git a/arch/s390/lib/xor.c b/lib/raid/xor/s390/xor.c
similarity index 98%
rename from arch/s390/lib/xor.c
rename to lib/raid/xor/s390/xor.c
index 3bbe21b40e66..acbd268adfc8 100644
--- a/arch/s390/lib/xor.c
+++ b/lib/raid/xor/s390/xor.c
@@ -7,7 +7,6 @@
*/
#include <linux/types.h>
-#include <linux/export.h>
#include <linux/raid/xor_impl.h>
#include <asm/xor.h>
@@ -133,4 +132,3 @@ struct xor_block_template xor_block_xc = {
.do_4 = xor_xc_4,
.do_5 = xor_xc_5,
};
-EXPORT_SYMBOL(xor_block_xc);
--
2.47.3
^ permalink raw reply related
* [PATCH 16/26] sparc: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR into lib/raid and include it it in xor.ko
instead of always building it into the main kernel image.
The code should probably be split into separate files for the two
implementations, but for now this just does the trivial move.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/sparc/include/asm/asm-prototypes.h | 1 -
arch/sparc/include/asm/xor.h | 45 ++++++++++++++++---
arch/sparc/lib/Makefile | 2 +-
lib/raid/xor/Makefile | 2 +
.../raid/xor/sparc/xor-sparc32.c | 23 ++--------
.../raid/xor/sparc/xor-sparc64-glue.c | 26 +++--------
.../xor.S => lib/raid/xor/sparc/xor-sparc64.S | 10 -----
7 files changed, 52 insertions(+), 57 deletions(-)
rename arch/sparc/include/asm/xor_32.h => lib/raid/xor/sparc/xor-sparc32.c (93%)
rename arch/sparc/include/asm/xor_64.h => lib/raid/xor/sparc/xor-sparc64-glue.c (74%)
rename arch/sparc/lib/xor.S => lib/raid/xor/sparc/xor-sparc64.S (98%)
diff --git a/arch/sparc/include/asm/asm-prototypes.h b/arch/sparc/include/asm/asm-prototypes.h
index 08810808ca6d..bbd1a8afaabf 100644
--- a/arch/sparc/include/asm/asm-prototypes.h
+++ b/arch/sparc/include/asm/asm-prototypes.h
@@ -14,7 +14,6 @@
#include <asm/oplib.h>
#include <asm/pgtable.h>
#include <asm/trap_block.h>
-#include <asm/xor.h>
void *__memscan_zero(void *, size_t);
void *__memscan_generic(void *, int, size_t);
diff --git a/arch/sparc/include/asm/xor.h b/arch/sparc/include/asm/xor.h
index f4c651e203c4..f923b009fc24 100644
--- a/arch/sparc/include/asm/xor.h
+++ b/arch/sparc/include/asm/xor.h
@@ -1,9 +1,44 @@
/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 1997, 1999 Jakub Jelinek (jj@ultra.linux.cz)
+ * Copyright (C) 2006 David S. Miller <davem@davemloft.net>
+ */
#ifndef ___ASM_SPARC_XOR_H
#define ___ASM_SPARC_XOR_H
+
#if defined(__sparc__) && defined(__arch64__)
-#include <asm/xor_64.h>
-#else
-#include <asm/xor_32.h>
-#endif
-#endif
+#include <asm/spitfire.h>
+
+extern struct xor_block_template xor_block_VIS;
+extern struct xor_block_template xor_block_niagara;
+
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ /* Force VIS for everything except Niagara. */
+ if (tlb_type == hypervisor &&
+ (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 ||
+ sun4v_chip_type == SUN4V_CHIP_NIAGARA2 ||
+ sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
+ sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
+ sun4v_chip_type == SUN4V_CHIP_NIAGARA5))
+ xor_force(&xor_block_niagara);
+ else
+ xor_force(&xor_block_VIS);
+}
+#else /* sparc64 */
+
+/* For grins, also test the generic routines. */
+#include <asm-generic/xor.h>
+
+extern struct xor_block_template xor_block_SPARC;
+
+#define arch_xor_init arch_xor_init
+static __always_inline void __init arch_xor_init(void)
+{
+ xor_register(&xor_block_8regs);
+ xor_register(&xor_block_32regs);
+ xor_register(&xor_block_SPARC);
+}
+#endif /* !sparc64 */
+#endif /* ___ASM_SPARC_XOR_H */
diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile
index 783bdec0d7be..dd10cdd6f062 100644
--- a/arch/sparc/lib/Makefile
+++ b/arch/sparc/lib/Makefile
@@ -48,7 +48,7 @@ lib-$(CONFIG_SPARC64) += GENmemcpy.o GENcopy_from_user.o GENcopy_to_user.o
lib-$(CONFIG_SPARC64) += GENpatch.o GENpage.o GENbzero.o
lib-$(CONFIG_SPARC64) += copy_in_user.o memmove.o
-lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o
+lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o hweight.o ffs.o
obj-$(CONFIG_SPARC64) += iomap.o
obj-$(CONFIG_SPARC32) += atomic32.o
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 9e729b50e775..3a7c887d08ee 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -18,6 +18,8 @@ xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o
xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o
xor-$(CONFIG_ALTIVEC) += powerpc/xor_vmx.o powerpc/xor_vmx_glue.o
xor-$(CONFIG_RISCV_ISA_V) += riscv/xor.o riscv/xor-glue.o
+xor-$(CONFIG_SPARC32) += sparc/xor-sparc32.o
+xor-$(CONFIG_SPARC64) += sparc/xor-sparc64.o sparc/xor-sparc64-glue.o
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
diff --git a/arch/sparc/include/asm/xor_32.h b/lib/raid/xor/sparc/xor-sparc32.c
similarity index 93%
rename from arch/sparc/include/asm/xor_32.h
rename to lib/raid/xor/sparc/xor-sparc32.c
index 8fbf0c07ec28..b65a75a6e59d 100644
--- a/arch/sparc/include/asm/xor_32.h
+++ b/lib/raid/xor/sparc/xor-sparc32.c
@@ -1,16 +1,12 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * include/asm/xor.h
- *
- * Optimized RAID-5 checksumming functions for 32-bit Sparc.
- */
-
+// SPDX-License-Identifier: GPL-2.0-or-later
/*
* High speed xor_block operation for RAID4/5 utilizing the
* ldd/std SPARC instructions.
*
* Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz)
*/
+#include <linux/raid/xor_impl.h>
+#include <asm/xor.h>
static void
sparc_2(unsigned long bytes, unsigned long * __restrict p1,
@@ -248,21 +244,10 @@ sparc_5(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static struct xor_block_template xor_block_SPARC = {
+struct xor_block_template xor_block_SPARC = {
.name = "SPARC",
.do_2 = sparc_2,
.do_3 = sparc_3,
.do_4 = sparc_4,
.do_5 = sparc_5,
};
-
-/* For grins, also test the generic routines. */
-#include <asm-generic/xor.h>
-
-#define arch_xor_init arch_xor_init
-static __always_inline void __init arch_xor_init(void)
-{
- xor_register(&xor_block_8regs);
- xor_register(&xor_block_32regs);
- xor_register(&xor_block_SPARC);
-}
diff --git a/arch/sparc/include/asm/xor_64.h b/lib/raid/xor/sparc/xor-sparc64-glue.c
similarity index 74%
rename from arch/sparc/include/asm/xor_64.h
rename to lib/raid/xor/sparc/xor-sparc64-glue.c
index e0482ecc0a68..3c67c8c3a0e8 100644
--- a/arch/sparc/include/asm/xor_64.h
+++ b/lib/raid/xor/sparc/xor-sparc64-glue.c
@@ -1,7 +1,5 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
+// SPDX-License-Identifier: GPL-2.0-or-later
/*
- * include/asm/xor.h
- *
* High speed xor_block operation for RAID4/5 utilizing the
* UltraSparc Visual Instruction Set and Niagara block-init
* twin-load instructions.
@@ -10,7 +8,8 @@
* Copyright (C) 2006 David S. Miller <davem@davemloft.net>
*/
-#include <asm/spitfire.h>
+#include <linux/raid/xor_impl.h>
+#include <asm/xor.h>
void xor_vis_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2);
@@ -29,7 +28,7 @@ void xor_vis_5(unsigned long bytes, unsigned long * __restrict p1,
/* XXX Ugh, write cheetah versions... -DaveM */
-static struct xor_block_template xor_block_VIS = {
+struct xor_block_template xor_block_VIS = {
.name = "VIS",
.do_2 = xor_vis_2,
.do_3 = xor_vis_3,
@@ -52,25 +51,10 @@ void xor_niagara_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p4,
const unsigned long * __restrict p5);
-static struct xor_block_template xor_block_niagara = {
+struct xor_block_template xor_block_niagara = {
.name = "Niagara",
.do_2 = xor_niagara_2,
.do_3 = xor_niagara_3,
.do_4 = xor_niagara_4,
.do_5 = xor_niagara_5,
};
-
-#define arch_xor_init arch_xor_init
-static __always_inline void __init arch_xor_init(void)
-{
- /* Force VIS for everything except Niagara. */
- if (tlb_type == hypervisor &&
- (sun4v_chip_type == SUN4V_CHIP_NIAGARA1 ||
- sun4v_chip_type == SUN4V_CHIP_NIAGARA2 ||
- sun4v_chip_type == SUN4V_CHIP_NIAGARA3 ||
- sun4v_chip_type == SUN4V_CHIP_NIAGARA4 ||
- sun4v_chip_type == SUN4V_CHIP_NIAGARA5))
- xor_force(&xor_block_niagara);
- else
- xor_force(&xor_block_VIS);
-}
diff --git a/arch/sparc/lib/xor.S b/lib/raid/xor/sparc/xor-sparc64.S
similarity index 98%
rename from arch/sparc/lib/xor.S
rename to lib/raid/xor/sparc/xor-sparc64.S
index 35461e3b2a9b..a7b74d473bd4 100644
--- a/arch/sparc/lib/xor.S
+++ b/lib/raid/xor/sparc/xor-sparc64.S
@@ -1,7 +1,5 @@
/* SPDX-License-Identifier: GPL-2.0 */
/*
- * arch/sparc64/lib/xor.S
- *
* High speed xor_block operation for RAID4/5 utilizing the
* UltraSparc Visual Instruction Set and Niagara store-init/twin-load.
*
@@ -92,7 +90,6 @@ ENTRY(xor_vis_2)
retl
wr %g0, 0, %fprs
ENDPROC(xor_vis_2)
-EXPORT_SYMBOL(xor_vis_2)
ENTRY(xor_vis_3)
rd %fprs, %o5
@@ -159,7 +156,6 @@ ENTRY(xor_vis_3)
retl
wr %g0, 0, %fprs
ENDPROC(xor_vis_3)
-EXPORT_SYMBOL(xor_vis_3)
ENTRY(xor_vis_4)
rd %fprs, %o5
@@ -245,7 +241,6 @@ ENTRY(xor_vis_4)
retl
wr %g0, 0, %fprs
ENDPROC(xor_vis_4)
-EXPORT_SYMBOL(xor_vis_4)
ENTRY(xor_vis_5)
save %sp, -192, %sp
@@ -352,7 +347,6 @@ ENTRY(xor_vis_5)
ret
restore
ENDPROC(xor_vis_5)
-EXPORT_SYMBOL(xor_vis_5)
/* Niagara versions. */
ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */
@@ -399,7 +393,6 @@ ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */
ret
restore
ENDPROC(xor_niagara_2)
-EXPORT_SYMBOL(xor_niagara_2)
ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */
save %sp, -192, %sp
@@ -461,7 +454,6 @@ ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */
ret
restore
ENDPROC(xor_niagara_3)
-EXPORT_SYMBOL(xor_niagara_3)
ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */
save %sp, -192, %sp
@@ -544,7 +536,6 @@ ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */
ret
restore
ENDPROC(xor_niagara_4)
-EXPORT_SYMBOL(xor_niagara_4)
ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */
save %sp, -192, %sp
@@ -643,4 +634,3 @@ ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=s
ret
restore
ENDPROC(xor_niagara_5)
-EXPORT_SYMBOL(xor_niagara_5)
--
2.47.3
^ permalink raw reply related
* [PATCH 15/26] riscv: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR into lib/raid and include it it in xor.ko
instead of always building it into the main kernel image.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/riscv/include/asm/xor.h | 54 +------------------
arch/riscv/lib/Makefile | 1 -
lib/raid/xor/Makefile | 1 +
lib/raid/xor/riscv/xor-glue.c | 56 ++++++++++++++++++++
{arch/riscv/lib => lib/raid/xor/riscv}/xor.S | 4 --
5 files changed, 59 insertions(+), 57 deletions(-)
create mode 100644 lib/raid/xor/riscv/xor-glue.c
rename {arch/riscv/lib => lib/raid/xor/riscv}/xor.S (92%)
diff --git a/arch/riscv/include/asm/xor.h b/arch/riscv/include/asm/xor.h
index ed5f27903efc..614d9209d078 100644
--- a/arch/riscv/include/asm/xor.h
+++ b/arch/riscv/include/asm/xor.h
@@ -2,60 +2,10 @@
/*
* Copyright (C) 2021 SiFive
*/
-
-#include <linux/hardirq.h>
-#include <asm-generic/xor.h>
-#ifdef CONFIG_RISCV_ISA_V
#include <asm/vector.h>
-#include <asm/switch_to.h>
-#include <asm/asm-prototypes.h>
-
-static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2)
-{
- kernel_vector_begin();
- xor_regs_2_(bytes, p1, p2);
- kernel_vector_end();
-}
-
-static void xor_vector_3(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3)
-{
- kernel_vector_begin();
- xor_regs_3_(bytes, p1, p2, p3);
- kernel_vector_end();
-}
-
-static void xor_vector_4(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3,
- const unsigned long *__restrict p4)
-{
- kernel_vector_begin();
- xor_regs_4_(bytes, p1, p2, p3, p4);
- kernel_vector_end();
-}
-
-static void xor_vector_5(unsigned long bytes, unsigned long *__restrict p1,
- const unsigned long *__restrict p2,
- const unsigned long *__restrict p3,
- const unsigned long *__restrict p4,
- const unsigned long *__restrict p5)
-{
- kernel_vector_begin();
- xor_regs_5_(bytes, p1, p2, p3, p4, p5);
- kernel_vector_end();
-}
+#include <asm-generic/xor.h>
-static struct xor_block_template xor_block_rvv = {
- .name = "rvv",
- .do_2 = xor_vector_2,
- .do_3 = xor_vector_3,
- .do_4 = xor_vector_4,
- .do_5 = xor_vector_5
-};
-#endif /* CONFIG_RISCV_ISA_V */
+extern struct xor_block_template xor_block_rvv;
#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
diff --git a/arch/riscv/lib/Makefile b/arch/riscv/lib/Makefile
index bbc031124974..e220c35764eb 100644
--- a/arch/riscv/lib/Makefile
+++ b/arch/riscv/lib/Makefile
@@ -16,5 +16,4 @@ lib-$(CONFIG_MMU) += uaccess.o
lib-$(CONFIG_64BIT) += tishift.o
lib-$(CONFIG_RISCV_ISA_ZICBOZ) += clear_page.o
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
-lib-$(CONFIG_RISCV_ISA_V) += xor.o
lib-$(CONFIG_RISCV_ISA_V) += riscv_v_helpers.o
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 006b44ce46bf..9e729b50e775 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -17,6 +17,7 @@ xor-$(CONFIG_ARM64) += arm64/xor-neon.o arm64/xor-neon-glue.o
xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o
xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o
xor-$(CONFIG_ALTIVEC) += powerpc/xor_vmx.o powerpc/xor_vmx_glue.o
+xor-$(CONFIG_RISCV_ISA_V) += riscv/xor.o riscv/xor-glue.o
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
diff --git a/lib/raid/xor/riscv/xor-glue.c b/lib/raid/xor/riscv/xor-glue.c
new file mode 100644
index 000000000000..11666a4b6b68
--- /dev/null
+++ b/lib/raid/xor/riscv/xor-glue.c
@@ -0,0 +1,56 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Copyright (C) 2021 SiFive
+ */
+
+#include <linux/raid/xor_impl.h>
+#include <asm/vector.h>
+#include <asm/switch_to.h>
+#include <asm/asm-prototypes.h>
+#include <asm/xor.h>
+
+static void xor_vector_2(unsigned long bytes, unsigned long *__restrict p1,
+ const unsigned long *__restrict p2)
+{
+ kernel_vector_begin();
+ xor_regs_2_(bytes, p1, p2);
+ kernel_vector_end();
+}
+
+static void xor_vector_3(unsigned long bytes, unsigned long *__restrict p1,
+ const unsigned long *__restrict p2,
+ const unsigned long *__restrict p3)
+{
+ kernel_vector_begin();
+ xor_regs_3_(bytes, p1, p2, p3);
+ kernel_vector_end();
+}
+
+static void xor_vector_4(unsigned long bytes, unsigned long *__restrict p1,
+ const unsigned long *__restrict p2,
+ const unsigned long *__restrict p3,
+ const unsigned long *__restrict p4)
+{
+ kernel_vector_begin();
+ xor_regs_4_(bytes, p1, p2, p3, p4);
+ kernel_vector_end();
+}
+
+static void xor_vector_5(unsigned long bytes, unsigned long *__restrict p1,
+ const unsigned long *__restrict p2,
+ const unsigned long *__restrict p3,
+ const unsigned long *__restrict p4,
+ const unsigned long *__restrict p5)
+{
+ kernel_vector_begin();
+ xor_regs_5_(bytes, p1, p2, p3, p4, p5);
+ kernel_vector_end();
+}
+
+struct xor_block_template xor_block_rvv = {
+ .name = "rvv",
+ .do_2 = xor_vector_2,
+ .do_3 = xor_vector_3,
+ .do_4 = xor_vector_4,
+ .do_5 = xor_vector_5
+};
diff --git a/arch/riscv/lib/xor.S b/lib/raid/xor/riscv/xor.S
similarity index 92%
rename from arch/riscv/lib/xor.S
rename to lib/raid/xor/riscv/xor.S
index b28f2430e52f..56fb7fc1e2cd 100644
--- a/arch/riscv/lib/xor.S
+++ b/lib/raid/xor/riscv/xor.S
@@ -18,7 +18,6 @@ SYM_FUNC_START(xor_regs_2_)
bnez a0, xor_regs_2_
ret
SYM_FUNC_END(xor_regs_2_)
-EXPORT_SYMBOL(xor_regs_2_)
SYM_FUNC_START(xor_regs_3_)
vsetvli a4, a0, e8, m8, ta, ma
@@ -35,7 +34,6 @@ SYM_FUNC_START(xor_regs_3_)
bnez a0, xor_regs_3_
ret
SYM_FUNC_END(xor_regs_3_)
-EXPORT_SYMBOL(xor_regs_3_)
SYM_FUNC_START(xor_regs_4_)
vsetvli a5, a0, e8, m8, ta, ma
@@ -55,7 +53,6 @@ SYM_FUNC_START(xor_regs_4_)
bnez a0, xor_regs_4_
ret
SYM_FUNC_END(xor_regs_4_)
-EXPORT_SYMBOL(xor_regs_4_)
SYM_FUNC_START(xor_regs_5_)
vsetvli a6, a0, e8, m8, ta, ma
@@ -78,4 +75,3 @@ SYM_FUNC_START(xor_regs_5_)
bnez a0, xor_regs_5_
ret
SYM_FUNC_END(xor_regs_5_)
-EXPORT_SYMBOL(xor_regs_5_)
--
2.47.3
^ permalink raw reply related
* [PATCH 14/26] powerpc: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR into lib/raid and include it it in xor.ko
instead of always building it into the main kernel image.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/powerpc/include/asm/xor.h | 17 +----
arch/powerpc/include/asm/xor_altivec.h | 22 ------
arch/powerpc/lib/Makefile | 5 --
arch/powerpc/lib/xor_vmx_glue.c | 63 -----------------
lib/raid/xor/Makefile | 5 ++
.../lib => lib/raid/xor/powerpc}/xor_vmx.c | 0
.../lib => lib/raid/xor/powerpc}/xor_vmx.h | 0
lib/raid/xor/powerpc/xor_vmx_glue.c | 67 +++++++++++++++++++
8 files changed, 74 insertions(+), 105 deletions(-)
delete mode 100644 arch/powerpc/include/asm/xor_altivec.h
delete mode 100644 arch/powerpc/lib/xor_vmx_glue.c
rename {arch/powerpc/lib => lib/raid/xor/powerpc}/xor_vmx.c (100%)
rename {arch/powerpc/lib => lib/raid/xor/powerpc}/xor_vmx.h (100%)
create mode 100644 lib/raid/xor/powerpc/xor_vmx_glue.c
diff --git a/arch/powerpc/include/asm/xor.h b/arch/powerpc/include/asm/xor.h
index 30224c5279c4..3293ac87181c 100644
--- a/arch/powerpc/include/asm/xor.h
+++ b/arch/powerpc/include/asm/xor.h
@@ -8,24 +8,11 @@
#ifndef _ASM_POWERPC_XOR_H
#define _ASM_POWERPC_XOR_H
-#ifdef CONFIG_ALTIVEC
-
-#include <asm/cputable.h>
#include <asm/cpu_has_feature.h>
-#include <asm/xor_altivec.h>
-
-static struct xor_block_template xor_block_altivec = {
- .name = "altivec",
- .do_2 = xor_altivec_2,
- .do_3 = xor_altivec_3,
- .do_4 = xor_altivec_4,
- .do_5 = xor_altivec_5,
-};
-#endif /* CONFIG_ALTIVEC */
-
-/* Also try the generic routines. */
#include <asm-generic/xor.h>
+extern struct xor_block_template xor_block_altivec;
+
#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
diff --git a/arch/powerpc/include/asm/xor_altivec.h b/arch/powerpc/include/asm/xor_altivec.h
deleted file mode 100644
index 294620a25f80..000000000000
--- a/arch/powerpc/include/asm/xor_altivec.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef _ASM_POWERPC_XOR_ALTIVEC_H
-#define _ASM_POWERPC_XOR_ALTIVEC_H
-
-#ifdef CONFIG_ALTIVEC
-void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3);
-void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5);
-
-#endif
-#endif /* _ASM_POWERPC_XOR_ALTIVEC_H */
diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile
index f14ecab674a3..002edc3f01d5 100644
--- a/arch/powerpc/lib/Makefile
+++ b/arch/powerpc/lib/Makefile
@@ -73,9 +73,4 @@ obj-$(CONFIG_PPC_LIB_RHEAP) += rheap.o
obj-$(CONFIG_FTR_FIXUP_SELFTEST) += feature-fixups-test.o
-obj-$(CONFIG_ALTIVEC) += xor_vmx.o xor_vmx_glue.o
-CFLAGS_xor_vmx.o += -mhard-float -maltivec $(call cc-option,-mabi=altivec)
-# Enable <altivec.h>
-CFLAGS_xor_vmx.o += -isystem $(shell $(CC) -print-file-name=include)
-
obj-$(CONFIG_PPC64) += $(obj64-y)
diff --git a/arch/powerpc/lib/xor_vmx_glue.c b/arch/powerpc/lib/xor_vmx_glue.c
deleted file mode 100644
index 35d917ece4d1..000000000000
--- a/arch/powerpc/lib/xor_vmx_glue.c
+++ /dev/null
@@ -1,63 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0-or-later
-/*
- * Altivec XOR operations
- *
- * Copyright 2017 IBM Corp.
- */
-
-#include <linux/preempt.h>
-#include <linux/export.h>
-#include <linux/sched.h>
-#include <asm/switch_to.h>
-#include <asm/xor_altivec.h>
-#include "xor_vmx.h"
-
-void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_2(bytes, p1, p2);
- disable_kernel_altivec();
- preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_2);
-
-void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_3(bytes, p1, p2, p3);
- disable_kernel_altivec();
- preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_3);
-
-void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_4(bytes, p1, p2, p3, p4);
- disable_kernel_altivec();
- preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_4);
-
-void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2,
- const unsigned long * __restrict p3,
- const unsigned long * __restrict p4,
- const unsigned long * __restrict p5)
-{
- preempt_disable();
- enable_kernel_altivec();
- __xor_altivec_5(bytes, p1, p2, p3, p4, p5);
- disable_kernel_altivec();
- preempt_enable();
-}
-EXPORT_SYMBOL(xor_altivec_5);
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index e8868f5fc396..006b44ce46bf 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -16,6 +16,7 @@ endif
xor-$(CONFIG_ARM64) += arm64/xor-neon.o arm64/xor-neon-glue.o
xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o
xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o
+xor-$(CONFIG_ALTIVEC) += powerpc/xor_vmx.o powerpc/xor_vmx_glue.o
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
@@ -23,3 +24,7 @@ CFLAGS_REMOVE_arm/xor-neon.o += $(CC_FLAGS_NO_FPU)
CFLAGS_arm64/xor-neon.o += $(CC_FLAGS_FPU)
CFLAGS_REMOVE_arm64/xor-neon.o += $(CC_FLAGS_NO_FPU)
+
+CFLAGS_powerpc/xor_vmx.o += -mhard-float -maltivec \
+ $(call cc-option,-mabi=altivec) \
+ -isystem $(shell $(CC) -print-file-name=include)
diff --git a/arch/powerpc/lib/xor_vmx.c b/lib/raid/xor/powerpc/xor_vmx.c
similarity index 100%
rename from arch/powerpc/lib/xor_vmx.c
rename to lib/raid/xor/powerpc/xor_vmx.c
diff --git a/arch/powerpc/lib/xor_vmx.h b/lib/raid/xor/powerpc/xor_vmx.h
similarity index 100%
rename from arch/powerpc/lib/xor_vmx.h
rename to lib/raid/xor/powerpc/xor_vmx.h
diff --git a/lib/raid/xor/powerpc/xor_vmx_glue.c b/lib/raid/xor/powerpc/xor_vmx_glue.c
new file mode 100644
index 000000000000..c41e38340700
--- /dev/null
+++ b/lib/raid/xor/powerpc/xor_vmx_glue.c
@@ -0,0 +1,67 @@
+// SPDX-License-Identifier: GPL-2.0-or-later
+/*
+ * Altivec XOR operations
+ *
+ * Copyright 2017 IBM Corp.
+ */
+
+#include <linux/preempt.h>
+#include <linux/sched.h>
+#include <linux/raid/xor_impl.h>
+#include <asm/switch_to.h>
+#include <asm/xor.h>
+#include "xor_vmx.h"
+
+static void xor_altivec_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2)
+{
+ preempt_disable();
+ enable_kernel_altivec();
+ __xor_altivec_2(bytes, p1, p2);
+ disable_kernel_altivec();
+ preempt_enable();
+}
+
+static void xor_altivec_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3)
+{
+ preempt_disable();
+ enable_kernel_altivec();
+ __xor_altivec_3(bytes, p1, p2, p3);
+ disable_kernel_altivec();
+ preempt_enable();
+}
+
+static void xor_altivec_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4)
+{
+ preempt_disable();
+ enable_kernel_altivec();
+ __xor_altivec_4(bytes, p1, p2, p3, p4);
+ disable_kernel_altivec();
+ preempt_enable();
+}
+
+static void xor_altivec_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5)
+{
+ preempt_disable();
+ enable_kernel_altivec();
+ __xor_altivec_5(bytes, p1, p2, p3, p4, p5);
+ disable_kernel_altivec();
+ preempt_enable();
+}
+
+struct xor_block_template xor_block_altivec = {
+ .name = "altivec",
+ .do_2 = xor_altivec_2,
+ .do_3 = xor_altivec_3,
+ .do_4 = xor_altivec_4,
+ .do_5 = xor_altivec_5,
+};
--
2.47.3
^ permalink raw reply related
* [PATCH 13/26] loongarch: move the XOR code to lib/raid/
From: Christoph Hellwig @ 2026-03-24 6:21 UTC (permalink / raw)
To: Andrew Morton
Cc: Richard Henderson, Matt Turner, Magnus Lindholm, Russell King,
Catalin Marinas, Will Deacon, Ard Biesheuvel, Huacai Chen,
WANG Xuerui, Madhavan Srinivasan, Michael Ellerman,
Nicholas Piggin, Christophe Leroy (CS GROUP), Paul Walmsley,
Palmer Dabbelt, Albert Ou, Alexandre Ghiti, Heiko Carstens,
Vasily Gorbik, Alexander Gordeev, Christian Borntraeger,
Sven Schnelle, David S. Miller, Andreas Larsson,
Richard Weinberger, Anton Ivanov, Johannes Berg, Thomas Gleixner,
Ingo Molnar, Borislav Petkov, Dave Hansen, x86, H. Peter Anvin,
Herbert Xu, Dan Williams, Chris Mason, David Sterba,
Arnd Bergmann, Song Liu, Yu Kuai, Li Nan, Theodore Ts'o,
Jason A. Donenfeld, linux-alpha, linux-kernel, linux-arm-kernel,
loongarch, linuxppc-dev, linux-riscv, linux-s390, sparclinux,
linux-um, linux-crypto, linux-btrfs, linux-arch, linux-raid
In-Reply-To: <20260324062211.3216301-1-hch@lst.de>
Move the optimized XOR into lib/raid and include it it in xor.ko
instead of always building it into the main kernel image.
Signed-off-by: Christoph Hellwig <hch@lst.de>
---
arch/loongarch/include/asm/xor.h | 24 ++----------
arch/loongarch/include/asm/xor_simd.h | 34 ----------------
arch/loongarch/lib/Makefile | 2 -
lib/raid/xor/Makefile | 2 +
.../lib => lib/raid/xor/loongarch}/xor_simd.c | 0
.../lib => lib/raid/xor/loongarch}/xor_simd.h | 0
.../raid/xor/loongarch}/xor_simd_glue.c | 39 +++++++++++--------
.../raid/xor/loongarch}/xor_template.c | 0
8 files changed, 27 insertions(+), 74 deletions(-)
delete mode 100644 arch/loongarch/include/asm/xor_simd.h
rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_simd.c (100%)
rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_simd.h (100%)
rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_simd_glue.c (64%)
rename {arch/loongarch/lib => lib/raid/xor/loongarch}/xor_template.c (100%)
diff --git a/arch/loongarch/include/asm/xor.h b/arch/loongarch/include/asm/xor.h
index d17c0e3b047f..7e32f72f8b03 100644
--- a/arch/loongarch/include/asm/xor.h
+++ b/arch/loongarch/include/asm/xor.h
@@ -6,27 +6,6 @@
#define _ASM_LOONGARCH_XOR_H
#include <asm/cpu-features.h>
-#include <asm/xor_simd.h>
-
-#ifdef CONFIG_CPU_HAS_LSX
-static struct xor_block_template xor_block_lsx = {
- .name = "lsx",
- .do_2 = xor_lsx_2,
- .do_3 = xor_lsx_3,
- .do_4 = xor_lsx_4,
- .do_5 = xor_lsx_5,
-};
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-static struct xor_block_template xor_block_lasx = {
- .name = "lasx",
- .do_2 = xor_lasx_2,
- .do_3 = xor_lasx_3,
- .do_4 = xor_lasx_4,
- .do_5 = xor_lasx_5,
-};
-#endif /* CONFIG_CPU_HAS_LASX */
/*
* For grins, also test the generic routines.
@@ -38,6 +17,9 @@ static struct xor_block_template xor_block_lasx = {
*/
#include <asm-generic/xor.h>
+extern struct xor_block_template xor_block_lsx;
+extern struct xor_block_template xor_block_lasx;
+
#define arch_xor_init arch_xor_init
static __always_inline void __init arch_xor_init(void)
{
diff --git a/arch/loongarch/include/asm/xor_simd.h b/arch/loongarch/include/asm/xor_simd.h
deleted file mode 100644
index 471b96332f38..000000000000
--- a/arch/loongarch/include/asm/xor_simd.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-or-later */
-/*
- * Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
- */
-#ifndef _ASM_LOONGARCH_XOR_SIMD_H
-#define _ASM_LOONGARCH_XOR_SIMD_H
-
-#ifdef CONFIG_CPU_HAS_LSX
-void xor_lsx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_lsx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void xor_lsx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_lsx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LSX */
-
-#ifdef CONFIG_CPU_HAS_LASX
-void xor_lasx_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-void xor_lasx_3(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3);
-void xor_lasx_4(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4);
-void xor_lasx_5(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2, const unsigned long * __restrict p3,
- const unsigned long * __restrict p4, const unsigned long * __restrict p5);
-#endif /* CONFIG_CPU_HAS_LASX */
-
-#endif /* _ASM_LOONGARCH_XOR_SIMD_H */
diff --git a/arch/loongarch/lib/Makefile b/arch/loongarch/lib/Makefile
index ccea3bbd4353..827a88529a42 100644
--- a/arch/loongarch/lib/Makefile
+++ b/arch/loongarch/lib/Makefile
@@ -8,6 +8,4 @@ lib-y += delay.o memset.o memcpy.o memmove.o \
obj-$(CONFIG_ARCH_SUPPORTS_INT128) += tishift.o
-obj-$(CONFIG_CPU_HAS_LSX) += xor_simd.o xor_simd_glue.o
-
obj-$(CONFIG_FUNCTION_ERROR_INJECTION) += error-inject.o
diff --git a/lib/raid/xor/Makefile b/lib/raid/xor/Makefile
index 4ab0e7411ff7..e8868f5fc396 100644
--- a/lib/raid/xor/Makefile
+++ b/lib/raid/xor/Makefile
@@ -14,6 +14,8 @@ ifeq ($(CONFIG_ARM),y)
xor-$(CONFIG_KERNEL_MODE_NEON) += arm/xor-neon.o arm/xor-neon-glue.o
endif
xor-$(CONFIG_ARM64) += arm64/xor-neon.o arm64/xor-neon-glue.o
+xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd.o
+xor-$(CONFIG_CPU_HAS_LSX) += loongarch/xor_simd_glue.o
CFLAGS_arm/xor-neon.o += $(CC_FLAGS_FPU)
diff --git a/arch/loongarch/lib/xor_simd.c b/lib/raid/xor/loongarch/xor_simd.c
similarity index 100%
rename from arch/loongarch/lib/xor_simd.c
rename to lib/raid/xor/loongarch/xor_simd.c
diff --git a/arch/loongarch/lib/xor_simd.h b/lib/raid/xor/loongarch/xor_simd.h
similarity index 100%
rename from arch/loongarch/lib/xor_simd.h
rename to lib/raid/xor/loongarch/xor_simd.h
diff --git a/arch/loongarch/lib/xor_simd_glue.c b/lib/raid/xor/loongarch/xor_simd_glue.c
similarity index 64%
rename from arch/loongarch/lib/xor_simd_glue.c
rename to lib/raid/xor/loongarch/xor_simd_glue.c
index 393f689dbcf6..11fa3b47ba83 100644
--- a/arch/loongarch/lib/xor_simd_glue.c
+++ b/lib/raid/xor/loongarch/xor_simd_glue.c
@@ -5,24 +5,23 @@
* Copyright (C) 2023 WANG Xuerui <git@xen0n.name>
*/
-#include <linux/export.h>
#include <linux/sched.h>
+#include <linux/raid/xor_impl.h>
#include <asm/fpu.h>
-#include <asm/xor_simd.h>
+#include <asm/xor.h>
#include "xor_simd.h"
#define MAKE_XOR_GLUE_2(flavor) \
-void xor_##flavor##_2(unsigned long bytes, unsigned long * __restrict p1, \
+static void xor_##flavor##_2(unsigned long bytes, unsigned long * __restrict p1,\
const unsigned long * __restrict p2) \
{ \
kernel_fpu_begin(); \
__xor_##flavor##_2(bytes, p1, p2); \
kernel_fpu_end(); \
} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_2)
#define MAKE_XOR_GLUE_3(flavor) \
-void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1, \
+static void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1,\
const unsigned long * __restrict p2, \
const unsigned long * __restrict p3) \
{ \
@@ -30,10 +29,9 @@ void xor_##flavor##_3(unsigned long bytes, unsigned long * __restrict p1, \
__xor_##flavor##_3(bytes, p1, p2, p3); \
kernel_fpu_end(); \
} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_3)
#define MAKE_XOR_GLUE_4(flavor) \
-void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1, \
+static void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1,\
const unsigned long * __restrict p2, \
const unsigned long * __restrict p3, \
const unsigned long * __restrict p4) \
@@ -42,10 +40,9 @@ void xor_##flavor##_4(unsigned long bytes, unsigned long * __restrict p1, \
__xor_##flavor##_4(bytes, p1, p2, p3, p4); \
kernel_fpu_end(); \
} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_4)
#define MAKE_XOR_GLUE_5(flavor) \
-void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1, \
+static void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1,\
const unsigned long * __restrict p2, \
const unsigned long * __restrict p3, \
const unsigned long * __restrict p4, \
@@ -55,18 +52,26 @@ void xor_##flavor##_5(unsigned long bytes, unsigned long * __restrict p1, \
__xor_##flavor##_5(bytes, p1, p2, p3, p4, p5); \
kernel_fpu_end(); \
} \
-EXPORT_SYMBOL_GPL(xor_##flavor##_5)
-#define MAKE_XOR_GLUES(flavor) \
- MAKE_XOR_GLUE_2(flavor); \
- MAKE_XOR_GLUE_3(flavor); \
- MAKE_XOR_GLUE_4(flavor); \
- MAKE_XOR_GLUE_5(flavor)
+#define MAKE_XOR_GLUES(flavor) \
+ MAKE_XOR_GLUE_2(flavor); \
+ MAKE_XOR_GLUE_3(flavor); \
+ MAKE_XOR_GLUE_4(flavor); \
+ MAKE_XOR_GLUE_5(flavor); \
+ \
+struct xor_block_template xor_block_##flavor = { \
+ .name = __stringify(flavor), \
+ .do_2 = xor_##flavor##_2, \
+ .do_3 = xor_##flavor##_3, \
+ .do_4 = xor_##flavor##_4, \
+ .do_5 = xor_##flavor##_5, \
+}
+
#ifdef CONFIG_CPU_HAS_LSX
MAKE_XOR_GLUES(lsx);
-#endif
+#endif /* CONFIG_CPU_HAS_LSX */
#ifdef CONFIG_CPU_HAS_LASX
MAKE_XOR_GLUES(lasx);
-#endif
+#endif /* CONFIG_CPU_HAS_LASX */
diff --git a/arch/loongarch/lib/xor_template.c b/lib/raid/xor/loongarch/xor_template.c
similarity index 100%
rename from arch/loongarch/lib/xor_template.c
rename to lib/raid/xor/loongarch/xor_template.c
--
2.47.3
^ permalink raw reply related
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox