* [PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk
@ 2026-10-23 18:15 Joshua Machauer
2026-10-25 10:48 ` Marc Zyngier
0 siblings, 1 reply; 3+ messages in thread
From: Joshua Machauer @ 2026-10-23 18:15 UTC (permalink / raw)
To: Marc Zyngier, Thomas Gleixner
Cc: Radu Rendec, linux-arm-kernel, linux-kernel, nh-open-source,
Joshua Machauer
From: Joshua Machauer <machaujo@amazon.com>
On systems that give every CPU its own redistributor frame, a CPU coming
online finds its frame by walking the frames from the first one until
GICR_TYPER matches its affinity. CPU n therefore performs O(n) MMIO
reads, and bringing up N CPUs costs O(N^2). Secondary CPUs boot
serially, so all of it adds to boot time.
gic_update_rdist_properties() already walks every frame once at boot.
Record each possible CPU's frame there and have gic_populate_rdist() try
it first, so bringing a CPU online costs one MMIO read. A CPU without a
recorded frame, such as an ACPI online-capable CPU absent at init, still
scans.
Recording compares each frame's affinity against that of every possible
CPU. That is still O(N^2) work at init, but it reads cpu_logical_map[],
which stays in the cache, instead of device memory.
The frame is kept apart from rd_base, which
inherit_vpe_l1_table_from_rd() reads as "this CPU has booted".
NVIDIA Vera, 352 CPUs in two redistributor regions, 69 ms faster.
Without this patch:
[ 0.007006] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
[ 0.463026] GICv3: CPU351: found redistributor 100610001 region 1:0x0000fe4043140000
With it:
[ 0.007794] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
[ 0.394395] GICv3: CPU351: found redistributor 100610001 region 1:0x0000fe4043140000
KVM guest with 352 vCPUs on that Vera system, where every GICR_TYPER
read traps to the hypervisor, 139 ms faster. Without this patch:
[ 0.023231] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
[ 0.432895] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000
With it:
[ 0.023456] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
[ 0.293807] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000
Graviton4, 192 CPUs with one redistributor region each, 5.6 ms faster.
Without this patch:
[ 0.009300] GICv3: CPU1: found redistributor 10000 region 1:0x000000b8fc1c0000
[ 0.091853] GICv3: CPU191: found redistributor 1005f0000 region 191:0x000004b8fd940000
With it:
[ 0.009343] GICv3: CPU1: found redistributor 10000 region 1:0x000000b8fc1c0000
[ 0.086299] GICv3: CPU191: found redistributor 1005f0000 region 191:0x000004b8fd940000
KVM guest with 192 vCPUs on that Graviton4 system, 33 ms faster.
Without this patch:
[ 0.008751] GICv3: CPU1: found redistributor 1 region 0:0x0000000010220000
[ 0.115862] GICv3: CPU191: found redistributor 1050f region 0:0x00000000119e0000
With it:
[ 0.009036] GICv3: CPU1: found redistributor 1 region 0:0x0000000010220000
[ 0.082898] GICv3: CPU191: found redistributor 1050f region 0:0x00000000119e0000
Assisted-by: LLM
Signed-off-by: Joshua Machauer <machaujo@amazon.de>
---
drivers/irqchip/irq-gic-v3.c | 66 ++++++++++++++++++++++++++++++------
1 file changed, 55 insertions(+), 11 deletions(-)
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 6e1fa5b247fc..8f1007facda9 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -1017,25 +1017,43 @@ static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
return ret ? -ENODEV : 0;
}
+/*
+ * Convert a CPU's affinity to a 32bit value that can be matched to
+ * GICR_TYPER bits [63:32].
+ */
+static u32 gic_cpu_to_rdist_aff(int cpu)
+{
+ u64 mpidr = gic_cpu_to_affinity(cpu);
+
+ return (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
+ MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+ MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
+ MPIDR_AFFINITY_LEVEL(mpidr, 0));
+}
+
+/*
+ * The redistributor frame of each possible CPU, as found by the walk in
+ * gic_update_rdist_properties(). A CPU coming up checks this frame first
+ * instead of scanning every frame from the start, which on a system with
+ * one redistributor per CPU makes bringing up N CPUs cost O(N^2) MMIO
+ * reads.
+ */
+struct gic_rdist_frame {
+ struct redist_region *region;
+ void __iomem *ptr;
+};
+
+static DEFINE_PER_CPU(struct gic_rdist_frame, gic_rdist_frame);
+
static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
{
unsigned long mpidr;
u64 typer;
- u32 aff;
- /*
- * Convert affinity to a 32bit value that can be matched to
- * GICR_TYPER bits [63:32].
- */
mpidr = gic_cpu_to_affinity(smp_processor_id());
- aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
- MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
- MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
- MPIDR_AFFINITY_LEVEL(mpidr, 0));
-
typer = gic_read_typer(ptr + GICR_TYPER);
- if ((typer >> 32) == aff) {
+ if ((typer >> 32) == gic_cpu_to_rdist_aff(smp_processor_id())) {
u64 offset = ptr - region->redist_base;
raw_spin_lock_init(&gic_data_rdist()->rd_lock);
gic_data_rdist_rd_base() = ptr;
@@ -1054,6 +1072,16 @@ static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
static int gic_populate_rdist(void)
{
+ struct gic_rdist_frame *frame = this_cpu_ptr(&gic_rdist_frame);
+
+ /*
+ * A CPU that was not present when the frames were walked, such as an
+ * ACPI online-capable CPU added later, has no recorded frame and falls
+ * back to the scan.
+ */
+ if (frame->ptr && !__gic_populate_rdist(frame->region, frame->ptr))
+ return 0;
+
if (gic_iterate_rdists(__gic_populate_rdist) == 0)
return 0;
@@ -1064,12 +1092,28 @@ static int gic_populate_rdist(void)
return -ENODEV;
}
+static void gic_record_rdist_frame(struct redist_region *region,
+ void __iomem *ptr, u64 typer)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ if (gic_cpu_to_rdist_aff(cpu) == (typer >> 32)) {
+ per_cpu(gic_rdist_frame, cpu).region = region;
+ per_cpu(gic_rdist_frame, cpu).ptr = ptr;
+ return;
+ }
+ }
+}
+
static int __gic_update_rdist_properties(struct redist_region *region,
void __iomem *ptr)
{
u64 typer = gic_read_typer(ptr + GICR_TYPER);
u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
+ gic_record_rdist_frame(region, ptr, typer);
+
/* Boot-time cleanup */
if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
u64 val;
--
2.47.3
^ permalink raw reply related [flat|nested] 3+ messages in thread
* Re: [PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk
2026-10-23 18:15 [PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk Joshua Machauer
@ 2026-10-25 10:48 ` Marc Zyngier
2026-11-19 18:42 ` [PATCH v2] irqchip/gic-v3: Fill in " Joshua Machauer
0 siblings, 1 reply; 3+ messages in thread
From: Marc Zyngier @ 2026-10-25 10:48 UTC (permalink / raw)
To: Joshua Machauer
Cc: Thomas Gleixner, Radu Rendec, linux-arm-kernel, linux-kernel,
nh-open-source, Joshua Machauer
On Fri, 23 Oct 2026 19:15:10 +0100,
Joshua Machauer <machaujo@amazon.de> wrote:
>
> From: Joshua Machauer <machaujo@amazon.com>
>
> On systems that give every CPU its own redistributor frame, a CPU coming
Is there *any* sort of GICv3-based systems that do not match this
description?
> online finds its frame by walking the frames from the first one until
> GICR_TYPER matches its affinity. CPU n therefore performs O(n) MMIO
> reads, and bringing up N CPUs costs O(N^2). Secondary CPUs boot
> serially, so all of it adds to boot time.
>
> gic_update_rdist_properties() already walks every frame once at boot.
> Record each possible CPU's frame there and have gic_populate_rdist() try
> it first, so bringing a CPU online costs one MMIO read. A CPU without a
> recorded frame, such as an ACPI online-capable CPU absent at init, still
> scans.
>
> Recording compares each frame's affinity against that of every possible
> CPU. That is still O(N^2) work at init, but it reads cpu_logical_map[],
> which stays in the cache, instead of device memory.
>
> The frame is kept apart from rd_base, which
> inherit_vpe_l1_table_from_rd() reads as "this CPU has booted".
But this is a pointless duplication of the existing tracking, which
could equally be solved with a flag.
I'm not opposed to this, even if the gains are vanishingly small. But
it can't come at the cost of duplicated state.
Thanks,
M.
--
Without deviation from the norm, progress is not possible.
^ permalink raw reply [flat|nested] 3+ messages in thread
* [PATCH v2] irqchip/gic-v3: Fill in each CPU's redistributor from the init walk
2026-10-25 10:48 ` Marc Zyngier
@ 2026-11-19 18:42 ` Joshua Machauer
0 siblings, 0 replies; 3+ messages in thread
From: Joshua Machauer @ 2026-11-19 18:42 UTC (permalink / raw)
To: Marc Zyngier, Thomas Gleixner
Cc: Radu Rendec, linux-arm-kernel, linux-kernel, nh-open-source
A CPU coming online finds its redistributor by walking the frames from
the first one until GICR_TYPER matches its affinity. CPU n therefore
performs O(n) MMIO reads, and bringing up N CPUs costs O(N^2).
Secondary CPUs boot serially, so all of it adds to boot time.
gic_update_rdist_properties() already walks every frame once at boot.
Give each frame to the possible CPU it belongs to there, by setting that
CPU's rd_base and phys_base and initialising its rd_lock. A CPU coming
online then finds its frame assigned and does no MMIO at all. A CPU
absent at init, such as an ACPI online-capable CPU, still scans.
Finding the CPU compares each frame's affinity against that of every
possible CPU. That is still O(N^2) work at init, but it reads
cpu_logical_map[], which stays in the cache, instead of device memory.
inherit_vpe_l1_table_from_rd() and allocate_vpe_l2_table() take a
non-NULL rd_base to mean that the CPU has come up. Mark that with a new
RD_LOCAL_POPULATED flag instead, set in gic_populate_rdist(), and move
the RD_LOCAL_* flags to irq-gic-common.h so that both drivers see them.
NVIDIA Vera, 352 CPUs in two redistributor regions, 73 ms faster.
Without this patch:
[ 0.007441] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
[ 0.873847] GICv3: CPU351: found redistributor 100600001 region 1:0x0000fe40430c0000
With it:
[ 0.006401] GICv3: CPU1: found redistributor 20000 region 0:0x0000fe0040180000
[ 0.800722] GICv3: CPU351: found redistributor 100600001 region 1:0x0000fe40430c0000
KVM guest with 352 vCPUs on that Vera system, where every GICR_TYPER
read traps to the hypervisor, 139 ms faster. Without this patch:
[ 0.023483] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
[ 0.433820] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000
With it:
[ 0.023817] GICv3: CPU1: found redistributor 100 region 0:0x0000000010240000
[ 0.295205] GICv3: CPU351: found redistributor 15701 region 0:0x0000000012de0000
Signed-off-by: Joshua Machauer <machaujo@amazon.de>
---
An LLM coding assistant drafted the code and this changelog and ran the
boot measurements. I reviewed and edited the result.
v2:
- Fill in rd_base and phys_base instead of keeping a separate per-CPU
copy of the frame (Marc)
- Add RD_LOCAL_POPULATED for the two users that took a non-NULL
rd_base to mean the CPU has come up
- Initialise rd_lock once, where the frame is assigned, instead of
each time the CPU comes online
- Drop the qualifier about systems with one redistributor per CPU,
which is every GICv3 system (Marc)
- Rebase onto tip irq/core (v7.4-rc1)
- Trim the test results to the NVIDIA Vera runs, remeasured on v2
---
drivers/irqchip/irq-gic-common.h | 7 ++
drivers/irqchip/irq-gic-v3-its.c | 11 ++-
drivers/irqchip/irq-gic-v3.c | 113 +++++++++++++++++++++----------
3 files changed, 90 insertions(+), 41 deletions(-)
diff --git a/drivers/irqchip/irq-gic-common.h b/drivers/irqchip/irq-gic-common.h
index 710cab61d9195..ffe5546d6189b 100644
--- a/drivers/irqchip/irq-gic-common.h
+++ b/drivers/irqchip/irq-gic-common.h
@@ -33,4 +33,11 @@ void gic_enable_of_quirks(const struct device_node *np,
#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1)
#define RDIST_FLAGS_FORCE_NON_SHAREABLE (1 << 2)
+/* Per-CPU redistributor flags, in struct rdists::rdist->flags */
+#define RD_LOCAL_LPI_ENABLED BIT(0)
+#define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1)
+#define RD_LOCAL_MEMRESERVE_DONE BIT(2)
+/* The CPU has come up */
+#define RD_LOCAL_POPULATED BIT(3)
+
#endif /* _IRQ_GIC_COMMON_H */
diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c
index 282e4bafb084f..ba6a6edf34230 100644
--- a/drivers/irqchip/irq-gic-v3-its.c
+++ b/drivers/irqchip/irq-gic-v3-its.c
@@ -50,10 +50,6 @@
#define ITS_FLAGS_FORCE_NON_SHAREABLE (1ULL << 3)
#define ITS_FLAGS_WORKAROUND_HISILICON_162100801 (1ULL << 4)
-#define RD_LOCAL_LPI_ENABLED BIT(0)
-#define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1)
-#define RD_LOCAL_MEMRESERVE_DONE BIT(2)
-
static u32 lpi_id_bits;
/*
@@ -2802,7 +2798,8 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask)
for_each_possible_cpu(cpu) {
void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base;
- if (!base || cpu == smp_processor_id())
+ if (!(gic_data_rdist_cpu(cpu)->flags & RD_LOCAL_POPULATED) ||
+ cpu == smp_processor_id())
continue;
val = gic_read_typer(base + GICR_TYPER);
@@ -2839,8 +2836,8 @@ static bool allocate_vpe_l2_table(int cpu, u32 id)
if (!gic_rdists->has_rvpeid)
return true;
- /* Skip non-present CPUs */
- if (!base)
+ /* Skip CPUs that have not come up yet */
+ if (!(gic_data_rdist_cpu(cpu)->flags & RD_LOCAL_POPULATED))
return true;
val = gicr_read_vpropbaser(base + SZ_128K + GICR_VPROPBASER);
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index 94873646d46fb..a507d84bf8dbc 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -1019,51 +1019,94 @@ static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *))
return ret ? -ENODEV : 0;
}
-static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr)
+/*
+ * Convert a CPU's affinity to a 32bit value that can be matched to
+ * GICR_TYPER bits [63:32].
+ */
+static u32 gic_cpu_to_rdist_aff(int cpu)
{
- unsigned long mpidr;
- u64 typer;
- u32 aff;
+ u64 mpidr = gic_cpu_to_affinity(cpu);
- /*
- * Convert affinity to a 32bit value that can be matched to
- * GICR_TYPER bits [63:32].
- */
- mpidr = gic_cpu_to_affinity(smp_processor_id());
+ return (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
+ MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+ MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
+ MPIDR_AFFINITY_LEVEL(mpidr, 0));
+}
- aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
- MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
- MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
- MPIDR_AFFINITY_LEVEL(mpidr, 0));
+/* Give @cpu the redistributor frame at @ptr */
+static void gic_assign_rdist(int cpu, struct redist_region *region,
+ void __iomem *ptr)
+{
+ raw_spin_lock_init(&per_cpu_ptr(gic_data.rdists.rdist, cpu)->rd_lock);
+ per_cpu_ptr(gic_data.rdists.rdist, cpu)->rd_base = ptr;
+ per_cpu_ptr(gic_data.rdists.rdist, cpu)->phys_base =
+ region->phys_base + (ptr - region->redist_base);
+}
- typer = gic_read_typer(ptr + GICR_TYPER);
- if ((typer >> 32) == aff) {
- u64 offset = ptr - region->redist_base;
- raw_spin_lock_init(&gic_data_rdist()->rd_lock);
- gic_data_rdist_rd_base() = ptr;
- gic_data_rdist()->phys_base = region->phys_base + offset;
-
- pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
- smp_processor_id(), mpidr,
- (int)(region - gic_data.redist_regions),
- &gic_data_rdist()->phys_base);
- return 0;
+static int __gic_find_rdist(struct redist_region *region, void __iomem *ptr)
+{
+ u64 typer = gic_read_typer(ptr + GICR_TYPER);
+
+ if ((typer >> 32) != gic_cpu_to_rdist_aff(smp_processor_id()))
+ return 1; /* Try next one */
+
+ gic_assign_rdist(smp_processor_id(), region, ptr);
+ return 0;
+}
+
+/* The region holding @ptr: the one with the highest base not above it */
+static int gic_rdist_region_index(void __iomem *ptr)
+{
+ int i, idx = 0;
+
+ for (i = 1; i < gic_data.nr_redist_regions; i++) {
+ if (gic_data.redist_regions[i].redist_base <= ptr &&
+ gic_data.redist_regions[i].redist_base >
+ gic_data.redist_regions[idx].redist_base)
+ idx = i;
}
- /* Try next one */
- return 1;
+ return idx;
}
static int gic_populate_rdist(void)
{
- if (gic_iterate_rdists(__gic_populate_rdist) == 0)
- return 0;
+ /*
+ * The frame is normally assigned at init by
+ * gic_update_rdist_properties(). A CPU that was absent then, such as
+ * an ACPI online-capable CPU, scans for its frame instead.
+ */
+ if (!gic_data_rdist_rd_base() &&
+ gic_iterate_rdists(__gic_find_rdist)) {
+ /* We couldn't even deal with ourselves... */
+ WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
+ smp_processor_id(),
+ (unsigned long)cpu_logical_map(smp_processor_id()));
+ return -ENODEV;
+ }
- /* We couldn't even deal with ourselves... */
- WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n",
- smp_processor_id(),
- (unsigned long)cpu_logical_map(smp_processor_id()));
- return -ENODEV;
+ gic_data_rdist()->flags |= RD_LOCAL_POPULATED;
+
+ pr_info("CPU%d: found redistributor %lx region %d:%pa\n",
+ smp_processor_id(),
+ (unsigned long)gic_cpu_to_affinity(smp_processor_id()),
+ gic_rdist_region_index(gic_data_rdist_rd_base()),
+ &gic_data_rdist()->phys_base);
+ return 0;
+}
+
+/* Give the frame at @ptr to the possible CPU whose affinity it reports */
+static void gic_find_rdist_cpu(struct redist_region *region, void __iomem *ptr,
+ u64 typer)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ if (gic_cpu_to_rdist_aff(cpu) == (typer >> 32)) {
+ gic_assign_rdist(cpu, region, ptr);
+ return;
+ }
+ }
}
static int __gic_update_rdist_properties(struct redist_region *region,
@@ -1072,6 +1115,8 @@ static int __gic_update_rdist_properties(struct redist_region *region,
u64 typer = gic_read_typer(ptr + GICR_TYPER);
u32 ctlr = readl_relaxed(ptr + GICR_CTLR);
+ gic_find_rdist_cpu(region, ptr, typer);
+
/* Boot-time cleanup */
if ((typer & GICR_TYPER_VLPIS) && (typer & GICR_TYPER_RVPEID)) {
u64 val;
--
2.47.3
^ permalink raw reply related [flat|nested] 3+ messages in thread
end of thread, other threads:[~2026-11-19 18:42 UTC | newest]
Thread overview: 3+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-10-23 18:15 [PATCH] irqchip/gic-v3: Remember each CPU's redistributor from the init walk Joshua Machauer
2026-10-25 10:48 ` Marc Zyngier
2026-11-19 18:42 ` [PATCH v2] irqchip/gic-v3: Fill in " Joshua Machauer
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox