[PATCH 04/23] tick/nohz: Allow runtime changes in full dynticks CPUs

[Waiman Long <longman@redhat.com>]

Full dynticks can only be enabled if "nohz_full" boot option has been
been specified with or without parameter. Any change in the list of
nohz_full CPUs have to be reflected in tick_nohz_full_mask. Introduce
a new tick_nohz_full_update_cpus() helper that can be called to update
the tick_nohz_full_mask at run time. The housekeeping_update() function
is modified to call the new helper when the HK_TYPE_KERNEL_NOSIE cpumask
is going to be changed.

We also need to enable CPU context tracking for those CPUs that
are in tick_nohz_full_mask. So remove __init from tick_nohz_init()
and ct_cpu_track_user() so that they be called later when an isolated
cpuset partition is being created. The __ro_after_init attribute is
taken away from context_tracking_key as well.

Also add a new ct_cpu_untrack_user() function to reverse the action of
ct_cpu_track_user() in case we need to disable the nohz_full mode of
a CPU.

With nohz_full enabled, the boot CPU (typically CPU 0) will be the
tick CPU which cannot be shut down easily. So the boot CPU should not
be used in an isolated cpuset partition.

With runtime modification of nohz_full CPUs, tick_do_timer_cpu can become
TICK_DO_TIMER_NONE. So remove the two TICK_DO_TIMER_NONE WARN_ON_ONCE()
checks in tick-sched.c to avoid unnecessary warnings.

Signed-off-by: Waiman Long <longman@redhat.com>
---
 include/linux/context_tracking.h |  1 +
 include/linux/tick.h             |  2 ++
 kernel/context_tracking.c        | 15 ++++++++++---
 kernel/sched/isolation.c         |  3 +++
 kernel/time/tick-sched.c         | 37 ++++++++++++++++++++++++++------
 5 files changed, 48 insertions(+), 10 deletions(-)

diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h
index a3fea7f9fef6..1a6b816f1ad6 100644
--- a/include/linux/context_tracking.h
+++ b/include/linux/context_tracking.h
@@ -17,6 +17,7 @@
 #define CONTEXT_TRACKING_FORCE_ENABLE	(-1)
 
 extern void ct_cpu_track_user(int cpu);
+extern void ct_cpu_untrack_user(int cpu);
 
 /* Called with interrupts disabled.  */
 extern void __ct_user_enter(enum ctx_state state);
diff --git a/include/linux/tick.h b/include/linux/tick.h
index 738007d6f577..05586f14461c 100644
--- a/include/linux/tick.h
+++ b/include/linux/tick.h
@@ -274,6 +274,7 @@ static inline void tick_dep_clear_signal(struct signal_struct *signal,
 extern void tick_nohz_full_kick_cpu(int cpu);
 extern void __tick_nohz_task_switch(void);
 extern void __init tick_nohz_full_setup(cpumask_var_t cpumask);
+extern void tick_nohz_full_update_cpus(struct cpumask *cpumask);
 #else
 static inline bool tick_nohz_full_enabled(void) { return false; }
 static inline bool tick_nohz_full_cpu(int cpu) { return false; }
@@ -299,6 +300,7 @@ static inline void tick_dep_clear_signal(struct signal_struct *signal,
 static inline void tick_nohz_full_kick_cpu(int cpu) { }
 static inline void __tick_nohz_task_switch(void) { }
 static inline void tick_nohz_full_setup(cpumask_var_t cpumask) { }
+static inline void tick_nohz_full_update_cpus(struct cpumask *cpumask) { }
 #endif
 
 static inline void tick_nohz_task_switch(void)
diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c
index 925999de1a28..394e432630a3 100644
--- a/kernel/context_tracking.c
+++ b/kernel/context_tracking.c
@@ -411,7 +411,7 @@ static __always_inline void ct_kernel_enter(bool user, int offset) { }
 #define CREATE_TRACE_POINTS
 #include <trace/events/context_tracking.h>
 
-DEFINE_STATIC_KEY_FALSE_RO(context_tracking_key);
+DEFINE_STATIC_KEY_FALSE(context_tracking_key);
 EXPORT_SYMBOL_GPL(context_tracking_key);
 
 static noinstr bool context_tracking_recursion_enter(void)
@@ -674,9 +674,9 @@ void user_exit_callable(void)
 }
 NOKPROBE_SYMBOL(user_exit_callable);
 
-void __init ct_cpu_track_user(int cpu)
+void ct_cpu_track_user(int cpu)
 {
-	static __initdata bool initialized = false;
+	static bool initialized;
 
 	if (cpu == CONTEXT_TRACKING_FORCE_ENABLE) {
 		static_branch_inc(&context_tracking_key);
@@ -700,6 +700,15 @@ void __init ct_cpu_track_user(int cpu)
 	initialized = true;
 }
 
+void ct_cpu_untrack_user(int cpu)
+{
+	if (!per_cpu(context_tracking.active, cpu))
+		return;
+
+	per_cpu(context_tracking.active, cpu) = false;
+	static_branch_dec(&context_tracking_key);
+}
+
 #ifdef CONFIG_CONTEXT_TRACKING_USER_FORCE
 void __init context_tracking_init(void)
 {
diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c
index c233d55a1e95..48b155e0b290 100644
--- a/kernel/sched/isolation.c
+++ b/kernel/sched/isolation.c
@@ -181,6 +181,9 @@ int housekeeping_update(struct cpumask *isol_mask, unsigned long flags)
 	if ((housekeeping.flags & flags) != flags)
 		WRITE_ONCE(housekeeping.flags, housekeeping.flags | flags);
 
+	if (flags & HK_FLAG_KERNEL_NOISE)
+		tick_nohz_full_update_cpus(isol_mask);
+
 	synchronize_rcu();
 
 	if (flags & HK_FLAG_DOMAIN) {
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index ed877b2c9040..7baa757ca45f 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -241,9 +241,6 @@ static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
 	tick_cpu = READ_ONCE(tick_do_timer_cpu);
 
 	if (IS_ENABLED(CONFIG_NO_HZ_COMMON) && unlikely(tick_cpu == TICK_DO_TIMER_NONE)) {
-#ifdef CONFIG_NO_HZ_FULL
-		WARN_ON_ONCE(tick_nohz_full_running);
-#endif
 		WRITE_ONCE(tick_do_timer_cpu, cpu);
 		tick_cpu = cpu;
 	}
@@ -629,6 +626,36 @@ void __init tick_nohz_full_setup(cpumask_var_t cpumask)
 	tick_nohz_full_running = true;
 }
 
+/* Get the new set of run-time nohz CPU list & update accordingly */
+void tick_nohz_full_update_cpus(struct cpumask *cpumask)
+{
+	int cpu;
+
+	if (!tick_nohz_full_running) {
+		pr_warn_once("Full dynticks cannot be enabled without the nohz_full kernel boot parameter!\n");
+		return;
+	}
+
+	/*
+	 * To properly enable/disable nohz_full dynticks for the affected CPUs,
+	 * the new nohz_full CPUs have to be copied to tick_nohz_full_mask and
+	 * ct_cpu_track_user/ct_cpu_untrack_user() will have to be called
+	 * for those CPUs that have their states changed. Those CPUs should be
+	 * in an offline state.
+	 */
+	for_each_cpu_andnot(cpu, cpumask, tick_nohz_full_mask) {
+		WARN_ON_ONCE(cpu_online(cpu));
+		ct_cpu_track_user(cpu);
+		cpumask_set_cpu(cpu, tick_nohz_full_mask);
+	}
+
+	for_each_cpu_andnot(cpu, tick_nohz_full_mask, cpumask) {
+		WARN_ON_ONCE(cpu_online(cpu));
+		ct_cpu_untrack_user(cpu);
+		cpumask_clear_cpu(cpu, tick_nohz_full_mask);
+	}
+}
+
 bool tick_nohz_cpu_hotpluggable(unsigned int cpu)
 {
 	/*
@@ -1238,10 +1265,6 @@ static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
 		 */
 		if (tick_cpu == cpu)
 			return false;
-
-		/* Should not happen for nohz-full */
-		if (WARN_ON_ONCE(tick_cpu == TICK_DO_TIMER_NONE))
-			return false;
 	}
 
 	return true;
-- 
2.53.0