[PATCH v4 01/11] sched/isolation: Add runtime housekeeping mask updates with boot snapshots

[Jing Wu <realwujing@gmail.com>]

The housekeeping cpumasks for kernel noise (HK_TYPE_KERNEL_NOISE) and
managed interrupts (HK_TYPE_MANAGED_IRQ) are computed once at boot from
the nohz_full= and isolcpus=managed_irq arguments.  Dynamic CPU
isolation driven by cpuset isolated partitions needs to update these
masks after boot.

Add housekeeping_update_types() to recompute one or more housekeeping
masks as (boot snapshot & ~isolated), publish them via RCU and free the
old masks after a grace period.

Introduce HK_TYPE_KERNEL_NOISE_BOOT and HK_TYPE_MANAGED_IRQ_BOOT to
record the immutable boot configuration.  Anchoring every update on the
boot snapshot keeps the runtime mask a subset of the boot set and lets
de-isolation restore the boot configuration exactly.

When neither nohz_full= nor isolcpus=nohz was given at boot no boot
snapshot exists, so cpu_possible_mask is the implicit boot set and
HK_TYPE_KERNEL_NOISE is enabled on the first runtime call.  In that
case housekeeping_update_types() also calls sched_tick_offload_init()
to allocate the tick offload percpu data that would otherwise only be
allocated by housekeeping_init() when nohz_full= is present at boot.
Remove __init from sched_tick_offload_init() and guard it with a
tick_work_cpu check so it is safe to call at runtime.

Co-developed-by: Qiliang Yuan <yuanql9@chinatelecom.cn>
Signed-off-by: Qiliang Yuan <yuanql9@chinatelecom.cn>
Signed-off-by: Jing Wu <realwujing@gmail.com>
---
 include/linux/sched/isolation.h |  32 ++++++-
 kernel/sched/core.c             |   4 +-
 kernel/sched/isolation.c        | 197 +++++++++++++++++++++++++++++++++++-----
 kernel/sched/sched.h            |   2 +-
 4 files changed, 209 insertions(+), 26 deletions(-)

diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h
index cf0fd03dd7a24..70602a74c1410 100644
--- a/include/linux/sched/isolation.h
+++ b/include/linux/sched/isolation.h
@@ -14,10 +14,26 @@ enum hk_type {
 	 * is always a subset of HK_TYPE_DOMAIN_BOOT.
 	 */
 	HK_TYPE_DOMAIN,
-	/* Inverse of boot-time isolcpus=managed_irq argument */
-	HK_TYPE_MANAGED_IRQ,
-	/* Inverse of boot-time nohz_full= or isolcpus=nohz arguments */
+
+	/*
+	 * Inverse of the boot-time nohz_full= or isolcpus=nohz arguments.
+	 * When neither is given, DHM still records cpu_possible_mask here so
+	 * that kernel-noise isolation can be enabled purely at runtime.
+	 */
+	HK_TYPE_KERNEL_NOISE_BOOT,
+	/*
+	 * A subset of HK_TYPE_KERNEL_NOISE_BOOT: it may exclude additional
+	 * CPUs isolated at runtime via cpuset isolated partitions.
+	 */
 	HK_TYPE_KERNEL_NOISE,
+
+	/* Inverse of the boot-time isolcpus=managed_irq argument */
+	HK_TYPE_MANAGED_IRQ_BOOT,
+	/*
+	 * A subset of HK_TYPE_MANAGED_IRQ_BOOT: it may exclude additional
+	 * CPUs isolated at runtime via cpuset isolated partitions.
+	 */
+	HK_TYPE_MANAGED_IRQ,
 	HK_TYPE_MAX,
 
 	/*
@@ -40,10 +56,13 @@ enum hk_type {
 DECLARE_STATIC_KEY_FALSE(housekeeping_overridden);
 extern int housekeeping_any_cpu(enum hk_type type);
 extern const struct cpumask *housekeeping_cpumask(enum hk_type type);
+extern const struct cpumask *housekeeping_cpumask_rcu(enum hk_type type);
 extern bool housekeeping_enabled(enum hk_type type);
 extern void housekeeping_affine(struct task_struct *t, enum hk_type type);
 extern bool housekeeping_test_cpu(int cpu, enum hk_type type);
 extern int housekeeping_update(struct cpumask *isol_mask);
+extern int housekeeping_update_types(unsigned long type_mask,
+				     struct cpumask *isol_mask);
 extern void __init housekeeping_init(void);
 
 #else
@@ -58,6 +77,11 @@ static inline const struct cpumask *housekeeping_cpumask(enum hk_type type)
 	return cpu_possible_mask;
 }
 
+static inline const struct cpumask *housekeeping_cpumask_rcu(enum hk_type type)
+{
+	return cpu_possible_mask;
+}
+
 static inline bool housekeeping_enabled(enum hk_type type)
 {
 	return false;
@@ -72,6 +96,8 @@ static inline bool housekeeping_test_cpu(int cpu, enum hk_type type)
 }
 
 static inline int housekeeping_update(struct cpumask *isol_mask) { return 0; }
+static inline int housekeeping_update_types(unsigned long type_mask,
+					    struct cpumask *isol_mask) { return 0; }
 static inline void housekeeping_init(void) { }
 #endif /* CONFIG_CPU_ISOLATION */
 
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b8871449d3c69..79c3349f65bb4 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5813,8 +5813,10 @@ static void sched_tick_stop(int cpu)
 }
 #endif /* CONFIG_HOTPLUG_CPU */
 
-int __init sched_tick_offload_init(void)
+int sched_tick_offload_init(void)
 {
+	if (tick_work_cpu)
+		return 0;
 	tick_work_cpu = alloc_percpu(struct tick_work);
 	BUG_ON(!tick_work_cpu);
 	return 0;
diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c
index ef152d401fe20..4602c8d0108e4 100644
--- a/kernel/sched/isolation.c
+++ b/kernel/sched/isolation.c
@@ -12,10 +12,12 @@
 #include "sched.h"
 
 enum hk_flags {
-	HK_FLAG_DOMAIN_BOOT	= BIT(HK_TYPE_DOMAIN_BOOT),
-	HK_FLAG_DOMAIN		= BIT(HK_TYPE_DOMAIN),
-	HK_FLAG_MANAGED_IRQ	= BIT(HK_TYPE_MANAGED_IRQ),
-	HK_FLAG_KERNEL_NOISE	= BIT(HK_TYPE_KERNEL_NOISE),
+	HK_FLAG_DOMAIN_BOOT	  = BIT(HK_TYPE_DOMAIN_BOOT),
+	HK_FLAG_DOMAIN		  = BIT(HK_TYPE_DOMAIN),
+	HK_FLAG_KERNEL_NOISE_BOOT = BIT(HK_TYPE_KERNEL_NOISE_BOOT),
+	HK_FLAG_KERNEL_NOISE	  = BIT(HK_TYPE_KERNEL_NOISE),
+	HK_FLAG_MANAGED_IRQ_BOOT  = BIT(HK_TYPE_MANAGED_IRQ_BOOT),
+	HK_FLAG_MANAGED_IRQ	  = BIT(HK_TYPE_MANAGED_IRQ),
 };
 
 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
@@ -34,25 +36,40 @@ bool housekeeping_enabled(enum hk_type type)
 }
 EXPORT_SYMBOL_GPL(housekeeping_enabled);
 
+/*
+ * Types that can change at runtime via cpuset isolated partitions.
+ * Boot-only types (DOMAIN_BOOT) are always safe to read without lockdep.
+ */
+static bool housekeeping_type_can_change(enum hk_type type)
+{
+	switch (type) {
+	case HK_TYPE_DOMAIN:
+	case HK_TYPE_KERNEL_NOISE:
+	case HK_TYPE_MANAGED_IRQ:
+		return true;
+	default:
+		return false;
+	}
+}
+
 static bool housekeeping_dereference_check(enum hk_type type)
 {
-	if (IS_ENABLED(CONFIG_LOCKDEP) && type == HK_TYPE_DOMAIN) {
-		/* Cpuset isn't even writable yet? */
-		if (system_state <= SYSTEM_SCHEDULING)
-			return true;
+	if (!IS_ENABLED(CONFIG_LOCKDEP) || !housekeeping_type_can_change(type))
+		return true;
 
-		/* CPU hotplug write locked, so cpuset partition can't be overwritten */
-		if (IS_ENABLED(CONFIG_HOTPLUG_CPU) && lockdep_is_cpus_write_held())
-			return true;
+	/* Cpuset isn't even writable yet? */
+	if (system_state <= SYSTEM_SCHEDULING)
+		return true;
 
-		/* Cpuset lock held, partitions not writable */
-		if (IS_ENABLED(CONFIG_CPUSETS) && lockdep_is_cpuset_held())
-			return true;
+	/* CPU hotplug write locked, so cpuset partition can't be overwritten */
+	if (IS_ENABLED(CONFIG_HOTPLUG_CPU) && lockdep_is_cpus_write_held())
+		return true;
 
-		return false;
-	}
+	/* Cpuset lock held, partitions not writable */
+	if (IS_ENABLED(CONFIG_CPUSETS) && lockdep_is_cpuset_held())
+		return true;
 
-	return true;
+	return false;
 }
 
 static inline struct cpumask *housekeeping_cpumask_dereference(enum hk_type type)
@@ -75,12 +92,26 @@ const struct cpumask *housekeeping_cpumask(enum hk_type type)
 }
 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
 
+const struct cpumask *housekeeping_cpumask_rcu(enum hk_type type)
+{
+	const struct cpumask *mask = NULL;
+
+	if (static_branch_unlikely(&housekeeping_overridden)) {
+		if (READ_ONCE(housekeeping.flags) & BIT(type))
+			mask = rcu_dereference(housekeeping.cpumasks[type]);
+	}
+	if (!mask)
+		mask = cpu_possible_mask;
+	return mask;
+}
+EXPORT_SYMBOL_GPL(housekeeping_cpumask_rcu);
+
 int housekeeping_any_cpu(enum hk_type type)
 {
 	int cpu;
 
 	if (static_branch_unlikely(&housekeeping_overridden)) {
-		if (housekeeping.flags & BIT(type)) {
+		if (READ_ONCE(housekeeping.flags) & BIT(type)) {
 			cpu = sched_numa_find_closest(housekeeping_cpumask(type), smp_processor_id());
 			if (cpu < nr_cpu_ids)
 				return cpu;
@@ -162,6 +193,130 @@ int housekeeping_update(struct cpumask *isol_mask)
 	return 0;
 }
 
+/**
+ * housekeeping_update_types - Update housekeeping masks for specified types
+ * @type_mask: Bitmask of housekeeping types to update
+ * @isol_mask: CPUs being added to the isolation set
+ *
+ * For each type in @type_mask, compute the trial mask as
+ * (boot snapshot & ~@isol_mask), validate it against @cpu_online_mask,
+ * then swap the RCU mask pointer and free the old mask after
+ * synchronize_rcu().  Anchoring on the immutable boot snapshot
+ * (HK_TYPE_*_BOOT) keeps the runtime mask a subset of the boot set and
+ * lets de-isolation restore the boot configuration exactly.
+ *
+ * The updated mask only takes effect for subsystems as CPUs cycle
+ * through hotplug; callers isolate CPUs via the CPU hotplug machinery so
+ * that tick, RCU and interrupt state is reconfigured by the existing
+ * online/offline callbacks rather than reconfigured in place.
+ *
+ * HK_TYPE_KERNEL_NOISE also supports runtime first-enable: when neither
+ * nohz_full= nor isolcpus=nohz was given at boot, no boot snapshot
+ * exists, so cpu_possible_mask is the implicit boot set and the type
+ * flag is set in housekeeping.flags on the first call.
+ *
+ * Return: 0 on success, -ENOMEM on allocation failure, -EINVAL if
+ * a trial mask has no online CPUs.
+ */
+int housekeeping_update_types(unsigned long type_mask,
+			      struct cpumask *isol_mask)
+{
+	struct cpumask *trials[HK_TYPE_MAX] = {};
+	struct cpumask *old_masks[HK_TYPE_MAX] = {};
+	enum hk_type type;
+	int ret = 0;
+
+	for_each_set_bit(type, &type_mask, HK_TYPE_MAX) {
+		const struct cpumask *base;
+
+		if (type == HK_TYPE_DOMAIN_BOOT)
+			continue;
+		if (!housekeeping_enabled(type)) {
+			/*
+			 * HK_TYPE_KERNEL_NOISE supports runtime first-enable
+			 * for DHM isolated partitions created without nohz_full=
+			 * at boot.  All other types must be boot-enabled.
+			 */
+			if (type != HK_TYPE_KERNEL_NOISE)
+				continue;
+		}
+
+		/*
+		 * Compute the trial mask relative to the immutable boot
+		 * snapshot, never relative to the current (already shrunk)
+		 * mask.  Using the current mask would let it shrink
+		 * monotonically across isolation/de-isolation cycles and would
+		 * never recover CPUs once de-isolated.  Anchoring on the boot
+		 * snapshot keeps the runtime mask a subset of the boot set and
+		 * lets de-isolation restore exactly the boot configuration.
+		 *
+		 * HK_TYPE_KERNEL_NOISE additionally supports runtime
+		 * first-enable: when no nohz_full=/isolcpus=nohz was given at
+		 * boot, no boot snapshot exists, so cpu_possible_mask is the
+		 * implicit boot set.
+		 */
+		if (type == HK_TYPE_KERNEL_NOISE &&
+		    !(housekeeping.flags & HK_FLAG_KERNEL_NOISE_BOOT))
+			base = cpu_possible_mask;
+		else if (type == HK_TYPE_KERNEL_NOISE)
+			base = housekeeping_cpumask(HK_TYPE_KERNEL_NOISE_BOOT);
+		else if (type == HK_TYPE_MANAGED_IRQ)
+			base = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ_BOOT);
+		else
+			base = housekeeping_cpumask(type);
+		trials[type] = kmalloc(cpumask_size(), GFP_KERNEL);
+		if (!trials[type]) {
+			ret = -ENOMEM;
+			goto err_free;
+		}
+		cpumask_andnot(trials[type], base, isol_mask);
+		if (!cpumask_intersects(trials[type], cpu_online_mask)) {
+			ret = -EINVAL;
+			goto err_free;
+		}
+	}
+
+	if (!housekeeping.flags) {
+		ret = -EINVAL;
+		goto err_free;
+	}
+
+	for_each_set_bit(type, &type_mask, HK_TYPE_MAX) {
+		if (!trials[type])
+			continue;
+		old_masks[type] = housekeeping_cpumask_dereference(type);
+		/* First-time runtime enable: register the type now. */
+		if (!housekeeping_enabled(type)) {
+			WRITE_ONCE(housekeeping.flags,
+				   housekeeping.flags | BIT(type));
+			/*
+			 * HK_TYPE_KERNEL_NOISE first-enable at runtime
+			 * (zero-boot-param path): tick offload percpu data
+			 * was never allocated at boot since nohz_full= was
+			 * absent.  Allocate it now before CPUs cycle through
+			 * hotplug and sched_tick_stop() dereferences
+			 * tick_work_cpu.
+			 */
+			if (type == HK_TYPE_KERNEL_NOISE)
+				WARN_ON_ONCE(sched_tick_offload_init());
+		}
+		rcu_assign_pointer(housekeeping.cpumasks[type], trials[type]);
+		trials[type] = NULL;
+	}
+
+	synchronize_rcu();
+
+	for_each_set_bit(type, &type_mask, HK_TYPE_MAX)
+		kfree(old_masks[type]);
+
+	return 0;
+
+err_free:
+	for_each_set_bit(type, &type_mask, HK_TYPE_MAX)
+		kfree(trials[type]);
+	return ret;
+}
+
 void __init housekeeping_init(void)
 {
 	enum hk_type type;
@@ -305,7 +460,7 @@ static int __init housekeeping_nohz_full_setup(char *str)
 {
 	unsigned long flags;
 
-	flags = HK_FLAG_KERNEL_NOISE;
+	flags = HK_FLAG_KERNEL_NOISE | HK_FLAG_KERNEL_NOISE_BOOT;
 
 	return housekeeping_setup(str, flags);
 }
@@ -324,7 +479,7 @@ static int __init housekeeping_isolcpus_setup(char *str)
 		 */
 		if (!strncmp(str, "nohz,", 5)) {
 			str += 5;
-			flags |= HK_FLAG_KERNEL_NOISE;
+			flags |= HK_FLAG_KERNEL_NOISE | HK_FLAG_KERNEL_NOISE_BOOT;
 			continue;
 		}
 
@@ -336,7 +491,7 @@ static int __init housekeeping_isolcpus_setup(char *str)
 
 		if (!strncmp(str, "managed_irq,", 12)) {
 			str += 12;
-			flags |= HK_FLAG_MANAGED_IRQ;
+			flags |= HK_FLAG_MANAGED_IRQ | HK_FLAG_MANAGED_IRQ_BOOT;
 			continue;
 		}
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 9f63b15d309d1..9d54ac3f2251d 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2925,7 +2925,7 @@ extern void post_init_entity_util_avg(struct task_struct *p);
 
 #ifdef CONFIG_NO_HZ_FULL
 extern bool sched_can_stop_tick(struct rq *rq);
-extern int __init sched_tick_offload_init(void);
+extern int sched_tick_offload_init(void);
 
 /*
  * Tick may be needed by tasks in the runqueue depending on their policy and

-- 
2.43.0