[PATCH 7/7] cpu: make preferred CPU policy writers mutually exclusive

[Yury Norov <ynorov@nvidia.com>]

Serializing preferred mask updates still lets competing policy drivers
repeatedly override each other's selections.

Disallow multiple policy drivers running at the same time with the
exclusive ownership rule.

Suggested-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Yury Norov <ynorov@nvidia.com>
---
 Documentation/scheduler/sched-paravirt.rst |  2 +-
 drivers/virt/steal_governor.c              | 13 +++--
 include/linux/cpumask.h                    |  8 +++
 lib/cpumask.c                              | 62 ++++++++++++++++++++++
 4 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/Documentation/scheduler/sched-paravirt.rst b/Documentation/scheduler/sched-paravirt.rst
index 311cdbf2722b..d8f446c3ab6e 100644
--- a/Documentation/scheduler/sched-paravirt.rst
+++ b/Documentation/scheduler/sched-paravirt.rst
@@ -48,7 +48,7 @@ Notes:
    makes decisions should enable it. For example, steal_governor driver
    (``CONFIG_STEAL_GOVERNOR``). On enabling the driver, CPU preferred state
    can change based on steal time. Without the driver, preferred CPUs is
-   same as active CPUs.
+   same as active CPUs. Only one governor may be enabled at a time.
 
 2. This feature works for the FAIR class only.
 
diff --git a/drivers/virt/steal_governor.c b/drivers/virt/steal_governor.c
index f71dbf27e4a9..b456b03714e4 100644
--- a/drivers/virt/steal_governor.c
+++ b/drivers/virt/steal_governor.c
@@ -291,6 +291,10 @@ static int __init steal_governor_init(void)
 	}
 
 	INIT_DELAYED_WORK(&sg_ctx.work, steal_governor_loop);
+	ret = claim_preferred_cpumask(&sg_ctx);
+	if (ret)
+		return ret;
+
 	/* Lock to prevent unobserved hotplug event during setup. */
 	scoped_guard(cpus_read_lock) {
 		sg_ctx.steal = get_system_steal_time();
@@ -301,8 +305,10 @@ static int __init steal_governor_init(void)
 							 hotplug_callback,
 							 hotplug_callback);
 	}
-	if (ret < 0)
+	if (ret < 0) {
+		release_preferred_cpumask(&sg_ctx);
 		return ret;
+	}
 	sg_ctx.hp_state = ret;
 
 	schedule_delayed_work(&sg_ctx.work, sg_ctx.delay);
@@ -316,10 +322,7 @@ static void __exit steal_governor_exit(void)
 {
 	disable_delayed_work_sync(&sg_ctx.work);
 	cpuhp_remove_state_nocalls(sg_ctx.hp_state);
-	guard(cpus_read_lock)();
-	guard(mutex)(&cpu_preferred_mutex);
-
-	reset_preferred_cpumask();
+	release_preferred_cpumask(&sg_ctx);
 	pr_info("disabled\n");
 }
 
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index 3716213d0730..1f4d93adb917 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -132,6 +132,14 @@ extern struct cpumask __cpu_preferred_mask;
  * holding cpu_preferred_mutex isn't necessary.
  */
 extern struct mutex cpu_preferred_mutex;
+
+/*
+ * Drivers must claim exclusive ownership of cpu_preferred_mask
+ * for the whole period of operation.
+ */
+int claim_preferred_cpumask(const void *owner);
+void release_preferred_cpumask(const void *owner);
+bool preferred_cpumask_owned_by(const void *owner);
 #else
 #define __cpu_preferred_mask __cpu_active_mask
 #endif
diff --git a/lib/cpumask.c b/lib/cpumask.c
index 2e16e19a2bdb..ee710c3c8920 100644
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -3,8 +3,10 @@
 #include <linux/kernel.h>
 #include <linux/bitops.h>
 #include <linux/cpumask.h>
+#include <linux/cpuhplock.h>
 #include <linux/export.h>
 #include <linux/memblock.h>
+#include <linux/mutex.h>
 #include <linux/numa.h>
 
 /* These are not inline because of header tangles. */
@@ -170,4 +172,64 @@ EXPORT_SYMBOL(cpumask_any_distribute);
 #ifdef CONFIG_PREFERRED_CPU
 DEFINE_MUTEX(cpu_preferred_mutex);
 EXPORT_SYMBOL_GPL(cpu_preferred_mutex);
+
+static const void *preferred_cpumask_owner;
+
+/**
+ * preferred_cpumask_owned_by - test preferred CPU policy ownership
+ * @owner: policy identity
+ *
+ * Return: true if @owner holds the policy, false otherwise.
+ */
+bool preferred_cpumask_owned_by(const void *owner)
+{
+	lockdep_assert_held(&cpu_preferred_mutex);
+
+	return preferred_cpumask_owner == owner;
+}
+EXPORT_SYMBOL_GPL(preferred_cpumask_owned_by);
+
+/**
+ * claim_preferred_cpumask - acquire exclusive preferred CPU policy ownership
+ * @owner: non-NULL identity that remains valid until ownership is released
+ *
+ * Call before starting policy work, normally on module init.
+ *
+ * Return: 0 on success, -EBUSY if claimed, or -EINVAL for a NULL identity.
+ */
+int claim_preferred_cpumask(const void *owner)
+{
+	if (!owner)
+		return -EINVAL;
+
+	guard(cpus_read_lock)();
+	guard(mutex)(&cpu_preferred_mutex);
+
+	if (preferred_cpumask_owner)
+		return -EBUSY;
+
+	preferred_cpumask_owner = owner;
+	reset_preferred_cpumask();
+	return 0;
+}
+EXPORT_SYMBOL_GPL(claim_preferred_cpumask);
+
+/**
+ * release_preferred_cpumask - release exclusive preferred CPU policy ownership
+ * @owner: identity used to claim ownership
+ *
+ * Call after stopping policy work, normally on module exit.
+ */
+void release_preferred_cpumask(const void *owner)
+{
+	guard(cpus_read_lock)();
+	guard(mutex)(&cpu_preferred_mutex);
+
+	if (WARN_ON_ONCE(!owner || !preferred_cpumask_owned_by(owner)))
+		return;
+
+	reset_preferred_cpumask();
+	preferred_cpumask_owner = NULL;
+}
+EXPORT_SYMBOL_GPL(release_preferred_cpumask);
 #endif
-- 
2.53.0