[PATCH 4/6] sched: serialize preferred cpumask update better

[Yury Norov <ynorov@nvidia.com>]

This series adds more flexibility to control the 'preferred' status of
CPUs, so that multiple modules may access the mask concurrently. Thus,
add the cpu_preferred_mutex.

With that, the scheme would look like this:

 - if your code takes cpus_write_lock(), it's enough to write-access
   the preferred cpumask;
 - if your code takes cpus_read_lock(), you'd also take
   cpu_preferred_mutex in order to modify the preferred mask.

This approach also fixes the existing race between steal governor
sampling and hotplug/unplug.

Fixes: 27d47ebce4d6 ("virt/steal_governor: Implement steal_governor policy loop")
Signed-off-by: Yury Norov <ynorov@nvidia.com>
---
 drivers/virt/steal_governor.c | 21 ++++++++++++++++-----
 include/linux/cpumask.h       | 15 ++++++++++++---
 kernel/cpu.c                  |  4 ++++
 3 files changed, 32 insertions(+), 8 deletions(-)

diff --git a/drivers/virt/steal_governor.c b/drivers/virt/steal_governor.c
index 4cd9971fec45..bc3936e69eed 100644
--- a/drivers/virt/steal_governor.c
+++ b/drivers/virt/steal_governor.c
@@ -23,6 +23,7 @@
 #include <linux/ktime.h>
 #include <linux/math64.h>
 #include <linux/module.h>
+#include <linux/mutex.h>
 #include <linux/sched/isolation.h>
 #include <linux/topology.h>
 #include <linux/types.h>
@@ -134,7 +135,8 @@ static void decrease_preferred_cpus(void)
 	int target_cpu = nr_cpu_ids;
 	int cpu;
 
-	guard(cpus_read_lock)();
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&cpu_preferred_mutex);
 	cpu = cpumask_first_and(housekeeping_cpumask(HK_TYPE_KERNEL_NOISE),
 				cpu_preferred_mask);
 	if (cpu >= nr_cpu_ids)
@@ -174,7 +176,8 @@ static void increase_preferred_cpus(void)
 {
 	int first_cpu, cpu;
 
-	guard(cpus_read_lock)();
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&cpu_preferred_mutex);
 	first_cpu = cpumask_first_andnot(cpu_active_mask, cpu_preferred_mask);
 
 	/* All CPUs are preferred. Nothing to increase further */
@@ -188,6 +191,9 @@ static void increase_preferred_cpus(void)
 
 static bool preferred_cpus_valid(void)
 {
+	lockdep_assert_cpus_held();
+	lockdep_assert_held(&cpu_preferred_mutex);
+
 	if (cpumask_empty(cpu_preferred_mask)) {
 		pr_err("empty preferred mask. stopping\n");
 		return false;
@@ -208,6 +214,9 @@ static void steal_governor_loop(struct work_struct *work)
 	u64 curr_steal, delta_steal, delta_ns, steal_ratio;
 	ktime_t now;
 
+	guard(cpus_read_lock)();
+	guard(mutex)(&cpu_preferred_mutex);
+
 	now = ktime_get();
 	delta_ns = ktime_to_ns(ktime_sub(now, sg_ctx.time));
 
@@ -238,8 +247,7 @@ static void steal_governor_loop(struct work_struct *work)
 	 * module restores to active if CPU hotplug breaks those assumptions.
 	 */
 	if (!preferred_cpus_valid()) {
-		scoped_guard(cpus_read_lock)
-			reset_preferred_cpumask();
+		reset_preferred_cpumask();
 		return;
 	}
 
@@ -276,8 +284,11 @@ static int __init steal_governor_init(void)
 static void __exit steal_governor_exit(void)
 {
 	disable_delayed_work_sync(&sg_ctx.work);
-	scoped_guard(cpus_read_lock)
+	scoped_guard(cpus_read_lock) {
+		guard(mutex)(&cpu_preferred_mutex);
+
 		reset_preferred_cpumask();
+	}
 	pr_info("disabled\n");
 }
 
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index c65abfc8a97f..fc911a153b44 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -123,7 +123,14 @@ extern struct cpumask __cpu_active_mask;
 extern struct cpumask __cpu_dying_mask;
 
 #ifdef CONFIG_PREFERRED_CPU
+/*
+ * Drivers must take cpus_read_lock() before cpu_preferred_mutex and
+ * hold both across mask validation and updates. If cpus_write_lock()
+ * is held, it guarantees the exclusive access to the mask, thus
+ * holding cpu_preferred_mutex isn't necessary.
+ */
 extern struct cpumask __cpu_preferred_mask;
+extern struct mutex cpu_preferred_mutex;
 #else
 #define __cpu_preferred_mask __cpu_active_mask
 #endif
@@ -1199,12 +1206,14 @@ void init_cpu_possible(const struct cpumask *src);
  * reset_preferred_cpumask - reset preferred CPUs to the active mask.
  *
  * Callers must serialize against CPU hotplug and other preferred cpumask
- * writers.
+ * writers. Requires cpus_read_lock() and cpu_preferred_mutex; or
+ * cpus_write_lock().
  */
 static inline void reset_preferred_cpumask(void)
 {
-	if (IS_ENABLED(CONFIG_PREFERRED_CPU))
-		cpumask_copy(&__cpu_preferred_mask, cpu_active_mask);
+#ifdef CONFIG_PREFERRED_CPU
+	cpumask_copy(&__cpu_preferred_mask, cpu_active_mask);
+#endif
 }
 
 void set_cpu_online(unsigned int cpu, bool online);
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 376d297a6292..dec08a90ea77 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3106,6 +3106,10 @@ EXPORT_SYMBOL(__num_online_cpus);
 #ifdef CONFIG_PREFERRED_CPU
 struct cpumask __cpu_preferred_mask __read_mostly;
 EXPORT_SYMBOL_GPL(__cpu_preferred_mask);
+
+/* Serialize policy drivers; acquire inside cpus_read_lock(). */
+DEFINE_MUTEX(cpu_preferred_mutex);
+EXPORT_SYMBOL_GPL(cpu_preferred_mutex);
 #endif
 
 void init_cpu_present(const struct cpumask *src)
-- 
2.53.0