[PATCH 0/3] improve group_cpus initialization routines

[PATCH 0/3] improve group_cpus initialization routines

[Yury Norov (NVIDIA)]

Add missing core API and improve cpumasks usage in the initialization
code.

Yury Norov (NVIDIA) (3):
  bitmap: cpumask: introduce and_andnot search helper and iterator
  group_cpus: don't call cpumask_weight() prematurely
  group_cpus: simplify inner loop in grp_spread_init_one()

 include/linux/cpumask.h | 22 +++++++++++++++++++++
 include/linux/find.h    | 38 +++++++++++++++++++++++++++++++++++++
 lib/find_bit.c          |  9 +++++++++
 lib/group_cpus.c        | 42 ++++++++++++-----------------------------
 4 files changed, 81 insertions(+), 30 deletions(-)

-- 
2.43.0

[PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator

[Yury Norov (NVIDIA)]

Like other similar iterators, *_and_andnot helps to get rid of temporary
on-stack bitmaps and associate housekeeping code.

Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
 include/linux/cpumask.h | 22 ++++++++++++++++++++++
 include/linux/find.h    | 38 ++++++++++++++++++++++++++++++++++++++
 lib/find_bit.c          |  9 +++++++++
 3 files changed, 69 insertions(+)

diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index ff8f41ab7ce6..6de16a0e6e7b 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -406,6 +406,28 @@ unsigned int cpumask_random(const struct cpumask *src)
 #define for_each_cpu_and(cpu, mask1, mask2)				\
 	for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
 
+/**
+ * for_each_cpu_and_andnot_from - iterate over every cpu in all masks
+ * @cpu: the (optionally unsigned) integer iterator
+ * @mask1: the first cpumask pointer
+ * @mask2: the second cpumask pointer
+ * @mask3: the third cpumask pointer
+ *
+ * This saves a temporary CPU mask in many places.  It is equivalent to:
+ *	struct cpumask tmp;
+ *	cpumask_and(&tmp, &mask1, &mask2);
+ *	cpumask_andnot(&tmp, &tmp, &mask3);
+ *	for_each_cpu_from(cpu, &tmp)
+ *		...
+ *
+ * After the loop, cpu is >= nr_cpu_ids.
+ */
+#define for_each_cpu_and_andnot_from(cpu, mask1, mask2, mask3)				\
+	for_each_and_andnot_bit_from(cpu, cpumask_bits(mask1),				\
+					  cpumask_bits(mask2),				\
+					  cpumask_bits(mask3),				\
+					  small_cpumask_bits)
+
 /**
  * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
  *			 those present in another.
diff --git a/include/linux/find.h b/include/linux/find.h
index 9d720ad92bc1..daf72078c25e 100644
--- a/include/linux/find.h
+++ b/include/linux/find.h
@@ -14,6 +14,9 @@ unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long
 					unsigned long nbits, unsigned long start);
 unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
 					unsigned long nbits, unsigned long start);
+unsigned long _find_next_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
+					 const unsigned long *addr3, unsigned long size,
+					 unsigned long n);
 unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
 					unsigned long nbits, unsigned long start);
 unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
@@ -135,6 +138,36 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1,
 }
 #endif
 
+/**
+ * find_next_and_andnot_bit - find the next set bit in *addr1 and *addr2
+ *			      excluding all the bits in *addr3
+ * @addr1: The first address to base the search on
+ * @addr2: The second address to base the search on
+ * @addr3: The second address to base the search on
+ * @size: The bitmap size in bits
+ * @offset: The bitnumber to start searching at
+ *
+ * Returns the bit number for the next set bit
+ * If no bits found, returns >= @size.
+ */
+static __always_inline
+unsigned long find_next_and_andnot_bit(const unsigned long *addr1,
+		const unsigned long *addr2, const unsigned long *addr3,
+		unsigned long size, unsigned long offset)
+{
+	if (small_const_nbits(size)) {
+		unsigned long val;
+
+		if (unlikely(offset >= size))
+			return size;
+
+		val = *addr1 & *addr2 & ~*addr3;
+		return val ? __ffs(val) : size;
+	}
+
+	return _find_next_and_andnot_bit(addr1, addr2, addr3, size, offset);
+}
+
 #ifndef find_next_or_bit
 /**
  * find_next_or_bit - find the next set bit in either memory regions
@@ -595,6 +628,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
 	     (bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
 	     (bit)++)
 
+#define for_each_and_andnot_bit_from(bit, addr1, addr2, addr3, size)			\
+	for (; (bit) = find_next_and_andnot_bit((addr1), (addr2), (addr3),		\
+						 (size), (bit)), (bit) < (size);	\
+	       (bit)++)
+
 #define for_each_or_bit(bit, addr1, addr2, size) \
 	for ((bit) = 0;									\
 	     (bit) = find_next_or_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
diff --git a/lib/find_bit.c b/lib/find_bit.c
index d4b5a29e3e72..aec79207c566 100644
--- a/lib/find_bit.c
+++ b/lib/find_bit.c
@@ -206,6 +206,15 @@ unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned l
 EXPORT_SYMBOL(_find_next_andnot_bit);
 #endif
 
+unsigned long _find_next_and_andnot_bit(const unsigned long *addr1,
+					const unsigned long *addr2,
+					const unsigned long *addr3,
+					unsigned long nbits, unsigned long start)
+{
+	return FIND_NEXT_BIT(addr1[idx] & addr2[idx] & ~addr3[idx], /* nop */, nbits, start);
+}
+EXPORT_SYMBOL(_find_next_and_andnot_bit);
+
 #ifndef find_next_or_bit
 unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
 					unsigned long nbits, unsigned long start)
-- 
2.43.0

[PATCH 2/3] group_cpus: don't call cpumask_weight() prematurely

[Yury Norov (NVIDIA)]

alloc_nodes_groups() and __group_cpus_evenly() call cpumask_weight()
unconditionally in the for_each() loops. cpumask_weight() is O(N), so
the complexity of the functions become O(MAX_NUMNODES * nr_cpu_ids).

This call may be avoided if the nmsk is empty.

Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
 lib/group_cpus.c | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/lib/group_cpus.c b/lib/group_cpus.c
index 6d08ac05f371..6aae1560b796 100644
--- a/lib/group_cpus.c
+++ b/lib/group_cpus.c
@@ -142,15 +142,11 @@ static void alloc_nodes_groups(unsigned int numgrps,
 	}
 
 	for_each_node_mask(n, nodemsk) {
-		unsigned ncpus;
-
-		cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]);
-		ncpus = cpumask_weight(nmsk);
-
-		if (!ncpus)
+		if (!cpumask_and(nmsk, cpu_mask, node_to_cpumask[n]))
 			continue;
-		remaining_ncpus += ncpus;
-		node_groups[n].ncpus = ncpus;
+
+		node_groups[n].ncpus = cpumask_weight(nmsk);
+		remaining_ncpus += node_groups[n].ncpus;
 	}
 
 	numgrps = min_t(unsigned, remaining_ncpus, numgrps);
@@ -294,11 +290,10 @@ static int __group_cpus_evenly(unsigned int startgrp, unsigned int numgrps,
 			continue;
 
 		/* Get the cpus on this node which are in the mask */
-		cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id]);
-		ncpus = cpumask_weight(nmsk);
-		if (!ncpus)
+		if (!cpumask_and(nmsk, cpu_mask, node_to_cpumask[nv->id]))
 			continue;
 
+		ncpus = cpumask_weight(nmsk);
 		WARN_ON_ONCE(nv->ngroups > ncpus);
 
 		/* Account for rounding errors */
-- 
2.43.0

[PATCH 3/3] group_cpus: simplify inner loop in grp_spread_init_one()

[Yury Norov (NVIDIA)]

Three optimizations for grp_spread_init_one().

1. Drop most of housekeeping code in grp_spread_init_one() with
   for_each_cpu_and_andnot_from().

2. Fix Shlemiel the Painter's algorithm by adding 'sibl = cpu' line. This
   improves the grp_spread_init_one() complexity from quadratic to linear.

3. Don't clear the nmsk because it's rewritten in the caller code anyways,
   and switch to non-atomic bit setter for irqmsk as the mask is local
   and implies no concurrency.

Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
 lib/group_cpus.c | 25 ++++++-------------------
 1 file changed, 6 insertions(+), 19 deletions(-)

diff --git a/lib/group_cpus.c b/lib/group_cpus.c
index 6aae1560b796..35aba99d8cd0 100644
--- a/lib/group_cpus.c
+++ b/lib/group_cpus.c
@@ -17,27 +17,14 @@ static void grp_spread_init_one(struct cpumask *irqmsk, struct cpumask *nmsk,
 	const struct cpumask *siblmsk;
 	int cpu, sibl;
 
-	for ( ; cpus_per_grp > 0; ) {
-		cpu = cpumask_first(nmsk);
-
-		/* Should not happen, but I'm too lazy to think about it */
-		if (cpu >= nr_cpu_ids)
-			return;
-
-		cpumask_clear_cpu(cpu, nmsk);
-		cpumask_set_cpu(cpu, irqmsk);
-		cpus_per_grp--;
-
+	for_each_cpu(cpu, nmsk) {
 		/* If the cpu has siblings, use them first */
 		siblmsk = topology_sibling_cpumask(cpu);
-		for (sibl = -1; cpus_per_grp > 0; ) {
-			sibl = cpumask_next(sibl, siblmsk);
-			if (sibl >= nr_cpu_ids)
-				break;
-			if (!cpumask_test_and_clear_cpu(sibl, nmsk))
-				continue;
-			cpumask_set_cpu(sibl, irqmsk);
-			cpus_per_grp--;
+		sibl = cpu;
+		for_each_cpu_and_andnot_from(sibl, nmsk, siblmsk, irqmsk) {
+			__cpumask_set_cpu(sibl, irqmsk);
+			if (--cpus_per_grp)
+				return;
 		}
 	}
 }
-- 
2.43.0

Re: [PATCH 0/3] improve group_cpus initialization routines

[Yury Norov]

Ping?

On Tue, Nov 18, 2025 at 10:13:02PM -0500, Yury Norov (NVIDIA) wrote:
> Add missing core API and improve cpumasks usage in the initialization
> code.
> 
> Yury Norov (NVIDIA) (3):
>   bitmap: cpumask: introduce and_andnot search helper and iterator
>   group_cpus: don't call cpumask_weight() prematurely
>   group_cpus: simplify inner loop in grp_spread_init_one()
> 
>  include/linux/cpumask.h | 22 +++++++++++++++++++++
>  include/linux/find.h    | 38 +++++++++++++++++++++++++++++++++++++
>  lib/find_bit.c          |  9 +++++++++
>  lib/group_cpus.c        | 42 ++++++++++++-----------------------------
>  4 files changed, 81 insertions(+), 30 deletions(-)
> 
> -- 
> 2.43.0

[PATCH 0/3] improve group_cpus initialization routines

[Yury Norov]

From: Yury Norov (NVIDIA) <yury.norov@gmail.com>

Add missing core API and improve cpumasks usage in the initialization
code.

Yury Norov (NVIDIA) (3):
  bitmap: cpumask: introduce and_andnot search helper and iterator
  group_cpus: don't call cpumask_weight() prematurely
  group_cpus: simplify inner loop in grp_spread_init_one()

 include/linux/cpumask.h | 22 +++++++++++++++++++++
 include/linux/find.h    | 38 +++++++++++++++++++++++++++++++++++++
 lib/find_bit.c          |  9 +++++++++
 lib/group_cpus.c        | 42 ++++++++++++-----------------------------
 4 files changed, 81 insertions(+), 30 deletions(-)

-- 
2.43.0

[PATCH 1/3] bitmap: cpumask: introduce and_andnot search helper and iterator

[Yury Norov]

From: "Yury Norov (NVIDIA)" <yury.norov@gmail.com>

Like other similar iterators, it helps to get rid of temporary
on-stack bitmaps and associate housekeeping code.

Signed-off-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
 include/linux/cpumask.h | 22 ++++++++++++++++++++++
 include/linux/find.h    | 38 ++++++++++++++++++++++++++++++++++++++
 lib/find_bit.c          |  9 +++++++++
 3 files changed, 69 insertions(+)

diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index ff8f41ab7ce6..8f8cb29d56d9 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -406,6 +406,28 @@ unsigned int cpumask_random(const struct cpumask *src)
 #define for_each_cpu_and(cpu, mask1, mask2)				\
 	for_each_and_bit(cpu, cpumask_bits(mask1), cpumask_bits(mask2), small_cpumask_bits)
 
+/**
+ * for_each_cpu_and_andnot_from - iterate over every cpu in all masks
+ * @cpu: the (optionally unsigned) integer iterator
+ * @mask1: the first cpumask pointer
+ * @mask2: the second cpumask pointer
+ * @mask3: the third cpumask pointer
+ *
+ * This saves a temporary CPU mask in many places.  It is equivalent to:
+ *	struct cpumask tmp;
+ *	cpumask_and(&tmp, &mask1, &mask2);
+ *	cpumask_andnot(&tmp, &tmp, &mask3);
+ *	for_each_cpu_from(cpu, &tmp)
+ *		...
+ *
+ * After the loop, cpu is >= nr_cpu_ids.
+ */
+#define for_each_cpu_and_andnot_from(cpu, mask1, mask2, mask3)				\
+	for_each_and_andnot_bit_from(cpu, cpumask_bits(mask1),				\
+					  cpumask_bits(mask2),				\
+					  cpumask_bits(mask3),				\
+					  small_cpumask_bits)
+
 /**
  * for_each_cpu_andnot - iterate over every cpu present in one mask, excluding
  *			 those present in another.
diff --git a/include/linux/find.h b/include/linux/find.h
index 9d720ad92bc1..d43c811aef5c 100644
--- a/include/linux/find.h
+++ b/include/linux/find.h
@@ -14,6 +14,9 @@ unsigned long _find_next_and_bit(const unsigned long *addr1, const unsigned long
 					unsigned long nbits, unsigned long start);
 unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
 					unsigned long nbits, unsigned long start);
+unsigned long _find_next_and_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
+					 const unsigned long *addr3, unsigned long size,
+					 unsigned long n);
 unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
 					unsigned long nbits, unsigned long start);
 unsigned long _find_next_zero_bit(const unsigned long *addr, unsigned long nbits,
@@ -135,6 +138,36 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1,
 }
 #endif
 
+/**
+ * find_next_and_andnot_bit - find the next set bit in *addr1 and *addr2
+ *			       excluding all the bits in *addr3
+ * @addr1: The first address to base the search on
+ * @addr2: The second address to base the search on
+ * @addr3: The third address to base the search on
+ * @size: The bitmap size in bits
+ * @offset: The bitnumber to start searching at
+ *
+ * Returns the bit number for the next such bit
+ * If no bits found, returns >= @size.
+ */
+static __always_inline
+unsigned long find_next_and_andnot_bit(const unsigned long *addr1,
+		const unsigned long *addr2, const unsigned long *addr3,
+		unsigned long size, unsigned long offset)
+{
+	if (small_const_nbits(size)) {
+		unsigned long val;
+
+		if (unlikely(offset >= size))
+			return size;
+
+		val = *addr1 & *addr2 & ~*addr3 & GENMASK(size - 1, offset);
+		return val ? __ffs(val) : size;
+	}
+
+	return _find_next_and_andnot_bit(addr1, addr2, addr3, size, offset);
+}
+
 #ifndef find_next_or_bit
 /**
  * find_next_or_bit - find the next set bit in either memory regions
@@ -595,6 +628,11 @@ unsigned long find_next_bit_le(const void *addr, unsigned
 	     (bit) = find_next_andnot_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
 	     (bit)++)
 
+#define for_each_and_andnot_bit_from(bit, addr1, addr2, addr3, size)			\
+	for (; (bit) = find_next_and_andnot_bit((addr1), (addr2), (addr3),		\
+						 (size), (bit)), (bit) < (size);	\
+	       (bit)++)
+
 #define for_each_or_bit(bit, addr1, addr2, size) \
 	for ((bit) = 0;									\
 	     (bit) = find_next_or_bit((addr1), (addr2), (size), (bit)), (bit) < (size);\
diff --git a/lib/find_bit.c b/lib/find_bit.c
index d4b5a29e3e72..aec79207c566 100644
--- a/lib/find_bit.c
+++ b/lib/find_bit.c
@@ -206,6 +206,15 @@ unsigned long _find_next_andnot_bit(const unsigned long *addr1, const unsigned l
 EXPORT_SYMBOL(_find_next_andnot_bit);
 #endif
 
+unsigned long _find_next_and_andnot_bit(const unsigned long *addr1,
+					const unsigned long *addr2,
+					const unsigned long *addr3,
+					unsigned long nbits, unsigned long start)
+{
+	return FIND_NEXT_BIT(addr1[idx] & addr2[idx] & ~addr3[idx], /* nop */, nbits, start);
+}
+EXPORT_SYMBOL(_find_next_and_andnot_bit);
+
 #ifndef find_next_or_bit
 unsigned long _find_next_or_bit(const unsigned long *addr1, const unsigned long *addr2,
 					unsigned long nbits, unsigned long start)
-- 
2.43.0