From: Ard Biesheuvel <ardb+git@google.com>
To: linux-kbuild@vger.kernel.org,
linux-arm-kernel@lists.infradead.org,
linux-crypto@vger.kernel.org
Cc: Ard Biesheuvel <ardb@kernel.org>, Christoph Hellwig <hch@lst.de>,
Eric Biggers <ebiggers@kernel.org>,
Nathan Chancellor <nathan@kernel.org>
Subject: [PATCH 2/4] xor: arm/arm64: Move NEON xor_gen() dispatch into the glue code
Date: Thu, 29 Oct 2026 14:50:22 +0100 [thread overview]
Message-ID: <20261029135020.272082-8-ardb+git@google.com> (raw)
In-Reply-To: <20261029135020.272082-6-ardb+git@google.com>
From: Ard Biesheuvel <ardb@kernel.org>
The NEON intrinsics code in xor-neon.c and arm64/xor-eor3.c includes xor_impl.h
for the definition of __DO_XOR_BLOCKS(), which pulls in <linux/types.h> via
<linux/init.h> and <linux/minmax.h>. This only works because
asm/neon-intrinsics.h overrides the compiler's predefined macros that
<stdint.h> uses to define the C99 types, so that they match the kernel's
definitions in <linux/types.h>.
So instead, follow the approach taken by LoongArch, and instantiate the
dispatch routines using DO_XOR_BLOCKS() in the glue code, which already
includes the kernel headers, and does not use NEON intrinsics itself. This
requires the NEON routines that handle a fixed number of sources to be exposed
to the glue code, which means that the arm64 glue code can use __xor_neon_2()
directly for the EOR3 flavor as well, and so the __xor_eor3_2() alias can be
dropped.
This makes the NEON intrinsics code independent of the kernel's own headers,
which will allow the macro overrides in asm/neon-intrinsics.h to be removed.
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
---
lib/raid/xor/arm/xor-neon-glue.c | 3 ++
lib/raid/xor/arm/xor-neon.h | 16 ++++++++--
lib/raid/xor/arm64/xor-eor3.c | 15 ++--------
lib/raid/xor/arm64/xor-neon-glue.c | 5 ++++
lib/raid/xor/arm64/xor-neon.h | 31 +++++++++++++++++---
lib/raid/xor/xor-neon.c | 16 +++-------
6 files changed, 56 insertions(+), 30 deletions(-)
diff --git a/lib/raid/xor/arm/xor-neon-glue.c b/lib/raid/xor/arm/xor-neon-glue.c
index cea39e019904..783474bfe81e 100644
--- a/lib/raid/xor/arm/xor-neon-glue.c
+++ b/lib/raid/xor/arm/xor-neon-glue.c
@@ -5,6 +5,9 @@
#include "xor_impl.h"
#include "xor_arch.h"
+DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
+ __xor_neon_5);
+
static void xor_gen_neon(void *dest, void **srcs, unsigned int src_cnt,
unsigned int bytes)
{
diff --git a/lib/raid/xor/arm/xor-neon.h b/lib/raid/xor/arm/xor-neon.h
index 406e0356f05b..5592dec4de50 100644
--- a/lib/raid/xor/arm/xor-neon.h
+++ b/lib/raid/xor/arm/xor-neon.h
@@ -3,5 +3,17 @@
extern struct xor_block_template xor_block_arm4regs;
extern struct xor_block_template xor_block_neon;
-void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
- unsigned int bytes);
+void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2);
+void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
diff --git a/lib/raid/xor/arm64/xor-eor3.c b/lib/raid/xor/arm64/xor-eor3.c
index e44016c363f1..41bb577b63eb 100644
--- a/lib/raid/xor/arm64/xor-eor3.c
+++ b/lib/raid/xor/arm64/xor-eor3.c
@@ -1,14 +1,8 @@
// SPDX-License-Identifier: GPL-2.0-only
-#include <linux/cache.h>
#include <asm/neon-intrinsics.h>
-#include "xor_impl.h"
-#include "xor_arch.h"
#include "xor-neon.h"
-extern void __xor_eor3_2(unsigned long bytes, unsigned long * __restrict p1,
- const unsigned long * __restrict p2);
-
static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
{
uint64x2_t res;
@@ -19,7 +13,7 @@ static inline uint64x2_t eor3(uint64x2_t p, uint64x2_t q, uint64x2_t r)
return res;
}
-static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
@@ -53,7 +47,7 @@ static void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
@@ -96,7 +90,7 @@ static void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
@@ -141,6 +135,3 @@ static void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
dp5 += 8;
} while (--lines > 0);
}
-
-__DO_XOR_BLOCKS(eor3_inner, __xor_eor3_2, __xor_eor3_3, __xor_eor3_4,
- __xor_eor3_5);
diff --git a/lib/raid/xor/arm64/xor-neon-glue.c b/lib/raid/xor/arm64/xor-neon-glue.c
index f0284f86feb4..ae133d699649 100644
--- a/lib/raid/xor/arm64/xor-neon-glue.c
+++ b/lib/raid/xor/arm64/xor-neon-glue.c
@@ -9,6 +9,11 @@
#include "xor_arch.h"
#include "xor-neon.h"
+DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
+ __xor_neon_5);
+DO_XOR_BLOCKS(eor3_inner, __xor_neon_2, __xor_eor3_3, __xor_eor3_4,
+ __xor_eor3_5);
+
#define XOR_TEMPLATE(_name) \
static void xor_gen_##_name(void *dest, void **srcs, unsigned int src_cnt, \
unsigned int bytes) \
diff --git a/lib/raid/xor/arm64/xor-neon.h b/lib/raid/xor/arm64/xor-neon.h
index 514699ba8f5f..7d7fb96343df 100644
--- a/lib/raid/xor/arm64/xor-neon.h
+++ b/lib/raid/xor/arm64/xor-neon.h
@@ -1,6 +1,29 @@
/* SPDX-License-Identifier: GPL-2.0-only */
-void xor_gen_neon_inner(void *dest, void **srcs, unsigned int src_cnt,
- unsigned int bytes);
-void xor_gen_eor3_inner(void *dest, void **srcs, unsigned int src_cnt,
- unsigned int bytes);
+void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2);
+void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
+
+void __xor_eor3_3(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3);
+void __xor_eor3_4(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4);
+void __xor_eor3_5(unsigned long bytes, unsigned long * __restrict p1,
+ const unsigned long * __restrict p2,
+ const unsigned long * __restrict p3,
+ const unsigned long * __restrict p4,
+ const unsigned long * __restrict p5);
diff --git a/lib/raid/xor/xor-neon.c b/lib/raid/xor/xor-neon.c
index c7c3cf634e23..6b2fd14f5041 100644
--- a/lib/raid/xor/xor-neon.c
+++ b/lib/raid/xor/xor-neon.c
@@ -4,12 +4,11 @@
* Copyright (C) 2018,Tianjin KYLIN Information Technology Co., Ltd.
*/
-#include "xor_impl.h"
#include "xor-neon.h"
#include <asm/neon-intrinsics.h>
-static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2)
{
uint64_t *dp1 = (uint64_t *)p1;
@@ -36,7 +35,7 @@ static void __xor_neon_2(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3)
{
@@ -72,7 +71,7 @@ static void __xor_neon_3(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4)
@@ -117,7 +116,7 @@ static void __xor_neon_4(unsigned long bytes, unsigned long * __restrict p1,
} while (--lines > 0);
}
-static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
+void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
const unsigned long * __restrict p2,
const unsigned long * __restrict p3,
const unsigned long * __restrict p4,
@@ -170,10 +169,3 @@ static void __xor_neon_5(unsigned long bytes, unsigned long * __restrict p1,
dp5 += 8;
} while (--lines > 0);
}
-
-__DO_XOR_BLOCKS(neon_inner, __xor_neon_2, __xor_neon_3, __xor_neon_4,
- __xor_neon_5);
-
-#ifdef CONFIG_ARM64
-extern typeof(__xor_neon_2) __xor_eor3_2 __alias(__xor_neon_2);
-#endif
--
2.56.0.313.g2479322e68-goog
next prev parent reply other threads:[~2026-10-29 13:50 UTC|newest]
Thread overview: 6+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-29 13:50 [PATCH 0/4] Get rid of all NEON intrinsics stdint.h type hacks Ard Biesheuvel
2026-10-29 13:50 ` [PATCH 1/4] lib/crc: Don't include <linux/types.h> in NEON intrinsics code Ard Biesheuvel
2026-10-29 13:50 ` Ard Biesheuvel [this message]
2026-10-29 13:50 ` [PATCH 3/4] arm64: Drop stdint.h type overrides from asm/neon-intrinsics.h Ard Biesheuvel
2026-10-29 13:50 ` [PATCH 4/4] ARM: " Ard Biesheuvel
2026-10-29 21:41 ` [PATCH 0/4] Get rid of all NEON intrinsics stdint.h type hacks Nathan Chancellor
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261029135020.272082-8-ardb+git@google.com \
--to=ardb+git@google.com \
--cc=ardb@kernel.org \
--cc=ebiggers@kernel.org \
--cc=hch@lst.de \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-crypto@vger.kernel.org \
--cc=linux-kbuild@vger.kernel.org \
--cc=nathan@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.