Re: [RFC] PCI_IRQ_AFFINITY limits MSI-X allocation on 384 CPU / 1000+ NVMe system

[Keith Busch <kbusch@kernel.org>]

On Tue, Aug 04, 2026 at 11:27:04PM +0200, Thomas Gleixner wrote:
> 
> We could come up with some less restrictive mechanism, but that would at
> the end run into the vector limitation on hotplug/hibernate because you
> can't fit more than ~200 vectors into the last online CPU.
> 
> OTOH. With 1000 devices which consume also one non-managed interrupt for
> their management queues, i.e. a total of 1000, that's not going to work
> anyway.

I think we'd have to at some point declare that extremely mismatched
setups just can't be reasonably supported. :)

My understanding is you've got the managed IRQ's allocating a vector up
front for every possible CPU they could be migrated to primarily to
ensure CPU unplug can always move the effective affinity to an active
CPU. I totally get that not having this guarantee can cause a device to
appear unresponsive, but maybe this is unlikely enough to accept? Or
maybe not, I am not sure; it just seems like a fringe case that trying
to mitigate causes a more likely problem.

Anyway, if we can tolerate dynamic vector allocation that only happens
on effective CPU assigment, here's a PoC I did some basic sanity testing
with:

---
diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c
index bddc544653999..314c764cb5456 100644
--- a/arch/x86/kernel/apic/vector.c
+++ b/arch/x86/kernel/apic/vector.c
@@ -199,19 +199,15 @@ static void vector_assign_managed_shutdown(struct irq_data *irqd)
 	apic_update_irq_cfg(irqd, MANAGED_IRQ_SHUTDOWN_VECTOR, cpu);
 }
 
-static int reserve_managed_vector(struct irq_data *irqd)
+static void mark_managed_vector(struct irq_data *irqd)
 {
-	const struct cpumask *affmsk = irq_data_get_affinity_mask(irqd);
 	struct apic_chip_data *apicd = apic_chip_data(irqd);
 	unsigned long flags;
-	int ret;
 
 	raw_spin_lock_irqsave(&vector_lock, flags);
 	apicd->is_managed = true;
-	ret = irq_matrix_reserve_managed(vector_matrix, affmsk);
 	raw_spin_unlock_irqrestore(&vector_lock, flags);
-	trace_vector_reserve_managed(irqd->irq, ret);
-	return ret;
+	trace_vector_reserve_managed(irqd->irq, 0);
 }
 
 static void reserve_irq_vector_locked(struct irq_data *irqd)
@@ -315,8 +311,10 @@ static int assign_irq_vector_any_locked(struct irq_data *irqd)
 static int
 assign_irq_vector_policy(struct irq_data *irqd, struct irq_alloc_info *info)
 {
-	if (irqd_affinity_is_managed(irqd))
-		return reserve_managed_vector(irqd);
+	if (irqd_affinity_is_managed(irqd)) {
+		mark_managed_vector(irqd);
+		return 0;
+	}
 	if (info->mask)
 		return assign_irq_vector(irqd, info->mask);
 	/*
@@ -483,7 +481,6 @@ static int x86_vector_activate(struct irq_domain *dom, struct irq_data *irqd,
 
 static void vector_free_reserved_and_managed(struct irq_data *irqd)
 {
-	const struct cpumask *dest = irq_data_get_affinity_mask(irqd);
 	struct apic_chip_data *apicd = apic_chip_data(irqd);
 
 	trace_vector_teardown(irqd->irq, apicd->is_managed,
@@ -491,8 +488,6 @@ static void vector_free_reserved_and_managed(struct irq_data *irqd)
 
 	if (apicd->has_reserved)
 		irq_matrix_remove_reserved(vector_matrix);
-	if (apicd->is_managed)
-		irq_matrix_remove_managed(vector_matrix, dest);
 }
 
 static void x86_vector_free_irqs(struct irq_domain *domain,
diff --git a/include/linux/irq.h b/include/linux/irq.h
index f485369b1b4f7..60e7573606c44 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -1251,8 +1251,6 @@ struct irq_matrix *irq_alloc_matrix(unsigned int matrix_bits,
 void irq_matrix_online(struct irq_matrix *m);
 void irq_matrix_offline(struct irq_matrix *m);
 void irq_matrix_assign_system(struct irq_matrix *m, unsigned int bit, bool replace);
-int irq_matrix_reserve_managed(struct irq_matrix *m, const struct cpumask *msk);
-void irq_matrix_remove_managed(struct irq_matrix *m, const struct cpumask *msk);
 int irq_matrix_alloc_managed(struct irq_matrix *m, const struct cpumask *msk,
 				unsigned int *mapped_cpu);
 void irq_matrix_reserve(struct irq_matrix *m);
diff --git a/kernel/irq/matrix.c b/kernel/irq/matrix.c
index faafb43a4e611..fab44ce35732f 100644
--- a/kernel/irq/matrix.c
+++ b/kernel/irq/matrix.c
@@ -164,7 +164,9 @@ static unsigned int matrix_find_best_cpu_managed(struct irq_matrix *m,
 	for_each_cpu(cpu, msk) {
 		cm = per_cpu_ptr(m->maps, cpu);
 
-		if (!cm->online || cm->managed_allocated > allocated)
+		if (!cm->online || !cm->available)
+			continue;
+		if (cm->managed_allocated > allocated)
 			continue;
 
 		best_cpu = cpu;
@@ -204,95 +206,28 @@ void irq_matrix_assign_system(struct irq_matrix *m, unsigned int bit,
 	trace_irq_matrix_assign_system(bit, m);
 }
 
-/**
- * irq_matrix_reserve_managed - Reserve a managed interrupt in a CPU map
- * @m:		Matrix pointer
- * @msk:	On which CPUs the bits should be reserved.
- *
- * Can be called for offline CPUs. Note, this will only reserve one bit
- * on all CPUs in @msk, but it's not guaranteed that the bits are at the
- * same offset on all CPUs
- */
-int irq_matrix_reserve_managed(struct irq_matrix *m, const struct cpumask *msk)
-{
-	unsigned int cpu, failed_cpu;
-
-	for_each_cpu(cpu, msk) {
-		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);
-		unsigned int bit;
-
-		bit = matrix_alloc_area(m, cm, 1, true);
-		if (bit >= m->alloc_end)
-			goto cleanup;
-		cm->managed++;
-		if (cm->online) {
-			cm->available--;
-			m->global_available--;
-		}
-		trace_irq_matrix_reserve_managed(bit, cpu, m, cm);
-	}
-	return 0;
-cleanup:
-	failed_cpu = cpu;
-	for_each_cpu(cpu, msk) {
-		if (cpu == failed_cpu)
-			break;
-		irq_matrix_remove_managed(m, cpumask_of(cpu));
-	}
-	return -ENOSPC;
-}
-
-/**
- * irq_matrix_remove_managed - Remove managed interrupts in a CPU map
- * @m:		Matrix pointer
- * @msk:	On which CPUs the bits should be removed
- *
- * Can be called for offline CPUs
- *
- * This removes not allocated managed interrupts from the map. It does
- * not matter which one because the managed interrupts free their
- * allocation when they shut down. If not, the accounting is screwed,
- * but all what can be done at this point is warn about it.
- */
-void irq_matrix_remove_managed(struct irq_matrix *m, const struct cpumask *msk)
-{
-	unsigned int cpu;
-
-	for_each_cpu(cpu, msk) {
-		struct cpumap *cm = per_cpu_ptr(m->maps, cpu);
-		unsigned int bit, end = m->alloc_end;
-
-		if (WARN_ON_ONCE(!cm->managed))
-			continue;
-
-		/* Get managed bit which are not allocated */
-		bitmap_andnot(m->scratch_map, cm->managed_map, cm->alloc_map, end);
-
-		bit = find_first_bit(m->scratch_map, end);
-		if (WARN_ON_ONCE(bit >= end))
-			continue;
-
-		clear_bit(bit, cm->managed_map);
-
-		cm->managed--;
-		if (cm->online) {
-			cm->available++;
-			m->global_available++;
-		}
-		trace_irq_matrix_remove_managed(bit, cpu, m, cm);
-	}
-}
-
 /**
  * irq_matrix_alloc_managed - Allocate a managed interrupt in a CPU map
  * @m:		Matrix pointer
  * @msk:	Which CPUs to search in
  * @mapped_cpu:	Pointer to store the CPU for which the irq was allocated
+ *
+ * Managed interrupts used to reserve a vector on every CPU of their affinity
+ * mask before any of them was used, so that migrating one on CPU offline could
+ * never fail. That costs one vector on every CPU of the mask for each
+ * interrupt, which does not scale with the number of devices: a mask spread
+ * over all CPUs consumes a vector on every CPU no matter how few interrupts
+ * the device asked for.
+ *
+ * The vector is therefore taken only on the CPU the interrupt is assigned to,
+ * and released again when that vector is freed. Migration has to allocate a
+ * new vector and can fail, which is why lapic_can_unplug_cpu() has to account
+ * for managed interrupts before letting a CPU go down.
  */
 int irq_matrix_alloc_managed(struct irq_matrix *m, const struct cpumask *msk,
 			     unsigned int *mapped_cpu)
 {
-	unsigned int bit, cpu, end;
+	unsigned int bit, cpu;
 	struct cpumap *cm;
 
 	if (cpumask_empty(msk))
@@ -303,16 +238,17 @@ int irq_matrix_alloc_managed(struct irq_matrix *m, const struct cpumask *msk,
 		return -ENOSPC;
 
 	cm = per_cpu_ptr(m->maps, cpu);
-	end = m->alloc_end;
-	/* Get managed bit which are not allocated */
-	bitmap_andnot(m->scratch_map, cm->managed_map, cm->alloc_map, end);
-	bit = find_first_bit(m->scratch_map, end);
-	if (bit >= end)
+	bit = matrix_alloc_area(m, cm, 1, true);
+	if (bit >= m->alloc_end)
 		return -ENOSPC;
+
+	cm->managed++;
 	set_bit(bit, cm->alloc_map);
 	cm->allocated++;
 	cm->managed_allocated++;
+	cm->available--;
 	m->total_allocated++;
+	m->global_available--;
 	*mapped_cpu = cpu;
 	trace_irq_matrix_alloc_managed(bit, cpu, m, cm);
 	return bit;
@@ -433,17 +369,20 @@ void irq_matrix_free(struct irq_matrix *m, unsigned int cpu,
 		return;
 
 	cm->allocated--;
-	if(managed)
+	if (managed) {
+		/* The vector reservation was taken along with the vector */
+		clear_bit(bit, cm->managed_map);
+		cm->managed--;
 		cm->managed_allocated--;
+	}
 
 	if (cm->online)
 		m->total_allocated--;
 
-	if (!managed) {
-		cm->available++;
-		if (cm->online)
-			m->global_available++;
-	}
+	cm->available++;
+	if (cm->online)
+		m->global_available++;
+
 	trace_irq_matrix_free(bit, cpu, m, cm);
 }
 
@@ -472,16 +411,18 @@ unsigned int irq_matrix_reserved(struct irq_matrix *m)
 }
 
 /**
- * irq_matrix_allocated - Get the number of allocated non-managed irqs on the local CPU
+ * irq_matrix_allocated - Get the number of allocated irqs on the local CPU
  * @m:		Pointer to the matrix to search
  *
- * This returns number of allocated non-managed interrupts.
+ * This returns the number of allocated interrupts, managed ones included.
+ * Managed interrupts hold a vector only on the CPU they are assigned to, so
+ * they have to be relocated on CPU offline just like the regular ones.
  */
 unsigned int irq_matrix_allocated(struct irq_matrix *m)
 {
 	struct cpumap *cm = this_cpu_ptr(m->maps);
 
-	return cm->allocated - cm->managed_allocated;
+	return cm->allocated;
 }
 
 #ifdef CONFIG_GENERIC_IRQ_DEBUGFS
--