From: Ard Biesheuvel <ardb+git@google.com>
To: linux-kbuild@vger.kernel.org,
linux-arm-kernel@lists.infradead.org,
linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>, Christoph Hellwig <hch@lst.de>,
Eric Biggers <ebiggers@kernel.org>,
Nathan Chancellor <nathan@kernel.org>
Subject: [PATCH 2/4] xor: arm/arm64: Move NEON xor_gen() dispatch into the glue code
Date: Thu, 29 Oct 2026 14:50:22 +0100 [thread overview]
Message-ID: <20261029135020.272082-8-ardb+git@google.com> (raw)
In-Reply-To: <20261029135020.272082-6-ardb+git@google.com>
From: Ard Biesheuvel <ardb@kernel.org>
The NEON intrinsics code in xor-neon.c and arm64/xor-eor3.c includes xor_impl.h
for the definition of __DO_XOR_BLOCKS(), which pulls in <linux/types.h> via
<linux/init.h> and <linux/minmax.h>. This only works because
asm/neon-intrinsics.h overrides the compiler's predefined macros that
<stdint.h> uses to define the C99 types, so that they match the kernel's
definitions in <linux/types.h>.
So instead, follow the approach taken by LoongArch, and instantiate the
dispatch routines using DO_XOR_BLOCKS() in the glue code, which already
includes the kernel headers, and does not use NEON intrinsics itself. This
requires the NEON routines that handle a fixed number of sources to be exposed
to the glue code, which means that the arm64 glue code can use __xor_neon_2()
directly for the EOR3 flavor as well, and so the __xor_eor3_2() alias can be
dropped.
This makes the NEON intrinsics code independent of the kernel's own headers,
which will allow the macro overrides in asm/neon-intrinsics.h to be removed.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
lib/raid/xor/arm/xor-neon-glue.c | 3 ++
lib/raid/xor/arm/xor-neon.h | 16 ++++++++--
lib/raid/xor/arm64/xor-eor3.c | 15 ++--------
lib/raid/xor/arm64/xor-neon-glue.c | 5 ++++
lib/raid/xor/arm64/xor-neon.h | 31 +++++++++++++++++---
lib/raid/xor/xor-neon.c | 16 +++-------
6 files changed, 56 insertions(+), 30 deletions(-)
diff --git a/lib/raid/xor/arm/xor-neon-glue.c b/lib/raid/xor/arm/xor-neon-glue.c
index cea39e019904..783474bfe81e 100644
--- a/lib/raid/xor/arm/xor-neon-glue.c
+++ b/lib/raid/xor/arm/xor-neon-glue.c
@@ -5,6 +5,9 @@
#include "xor_impl.h"
#include "xor_arch.h"
+DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
+ __xor_neon_5);
+
static void xor_gen_neon(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
diff --git a/lib/raid/xor/arm/xor-neon.h b/lib/raid/xor/arm/xor-neon.h
index 406e0356f05b..5592dec4de50 100644
--- a/lib/raid/xor/arm/xor-neon.h
+++ b/lib/raid/xor/arm/xor-neon.h
@@ -3,5 +3,17 @@
extern struct xor_block_template xor_block_arm4regs;
extern struct xor_block_template xor_block_neon;
-void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
- unsigned int bytes);
+void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2);
+void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
diff --git a/lib/raid/xor/arm64/xor-eor3.c b/lib/raid/xor/arm64/xor-eor3.c
index e44016c363f1..41bb577b63eb 100644
--- a/lib/raid/xor/arm64/xor-eor3.c
+++ b/lib/raid/xor/arm64/xor-eor3.c
@@ -1,14 +1,8 @@
// SPDX-License-Identifier: GPL-2.0-only
-#include <linux/cache.h>
#include <asm/neon-intrinsics.h>
-#include "xor_impl.h"
-#include "xor_arch.h"
#include "xor-neon.h"
-extern void __xor_eor3_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-
static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
{
uint64x2_t res;
@@ -19,7 +13,7 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
return res;
}
-static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
@@ -53,7 +47,7 @@ static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
@@ -96,7 +90,7 @@ static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
@@ -141,6 +135,3 @@ static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
dp5 += 8;
} while (--lines > 0);
}
-
-__DO_XOR_BLOCKS(eor3_inner, __xor_eor3_2, __xor_eor3_3, __xor_eor3_4,
- __xor_eor3_5);
diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c
index f0284f86feb4..ae133d699649 100644
--- a/lib/raid/xor/arm64/xor-neon-glue.c
+++ b/lib/raid/xor/arm64/xor-neon-glue.c
@@ -9,6 +9,11 @@
#include "xor_arch.h"
#include "xor-neon.h"
+DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
+ __xor_neon_5);
+DO_XOR_BLOCKS(eor3_inner, __xor_neon_2, __xor_eor3_3, __xor_eor3_4,
+ __xor_eor3_5);
+
#define XOR_TEMPLATE(_name) \
static void xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \
unsigned int bytes) \
diff --git a/lib/raid/xor/arm64/xor-neon.h b/lib/raid/xor/arm64/xor-neon.h
index 514699ba8f5f..7d7fb96343df 100644
--- a/lib/raid/xor/arm64/xor-neon.h
+++ b/lib/raid/xor/arm64/xor-neon.h
@@ -1,6 +1,29 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
- unsigned int bytes);
-void xor_gen_eor3_inner(void *dest, void **srcs, unsigned int src_cnt,
- unsigned int bytes);
+void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2);
+void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
+
+void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
diff --git a/lib/raid/xor/xor-neon.c b/lib/raid/xor/xor-neon.c
index c7c3cf634e23..6b2fd14f5041 100644
--- a/lib/raid/xor/xor-neon.c
+++ b/lib/raid/xor/xor-neon.c
@@ -4,12 +4,11 @@
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
-#include "xor_impl.h"
#include "xor-neon.h"
#include <asm/neon-intrinsics.h>
-static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
uint64_t *dp1 = (uint64_t *)p1;
@@ -36,7 +35,7 @@ static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
@@ -72,7 +71,7 @@ static void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
@@ -117,7 +116,7 @@ static void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
@@ -170,10 +169,3 @@ static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
dp5 += 8;
} while (--lines > 0);
}
-
-__DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
- __xor_neon_5);
-
-#ifdef CONFIG_ARM64
-extern typeof(__xor_neon_2) __xor_eor3_2 __alias(__xor_neon_2);
-#endif
--
2.56.0.313.g2479322e68-goog
next prev parent reply other threads:[~2026-10-29 13:50 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-29 13:50 [PATCH 0/4] Get rid of all NEON intrinsics stdint.h type hacks Ard Biesheuvel
2026-10-29 13:50 ` [PATCH 1/4] lib/crc: Don't include <linux/types.h> in NEON intrinsics code Ard Biesheuvel
2026-10-29 13:50 ` Ard Biesheuvel [this message]
2026-10-29 13:50 ` [PATCH 3/4] arm64: Drop stdint.h type overrides from asm/neon-intrinsics.h Ard Biesheuvel
2026-10-29 13:50 ` [PATCH 4/4] ARM: " Ard Biesheuvel
2026-10-29 21:41 ` [PATCH 0/4] Get rid of all NEON intrinsics stdint.h type hacks Nathan Chancellor
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261029135020.272082-8-ardb+git@google.com \
--to=ardb+git@google.com \
--cc=ardb@kernel.org \
--cc=ebiggers@kernel.org \
--cc=hch@lst.de \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kbuild@vger.kernel.org \
--cc=nathan@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox