Re: [PATCH v6 00/29] context_tracking,x86: Defer some IPIs until a user->kernel transition

[Frederic Weisbecker <frederic@kernel.org>]

Le Wed, Oct 29, 2025 at 11:32:58AM +0100, Valentin Schneider a écrit :
> I need to have a think about that one; one pain point I see is the context
> tracking work has to be NMI safe since e.g. an NMI can take us out of
> userspace. Another is that NOHZ-full CPUs need to be special cased in the
> stop machine queueing / completion.
> 
> /me goes fetch a new notebook

Something like the below (untested) ?

diff --git a/arch/x86/include/asm/context_tracking_work.h b/arch/x86/include/asm/context_tracking_work.h
index 485b32881fde..2940e28ecea6 100644
--- a/arch/x86/include/asm/context_tracking_work.h
+++ b/arch/x86/include/asm/context_tracking_work.h
@@ -3,6 +3,7 @@
 #define _ASM_X86_CONTEXT_TRACKING_WORK_H
 
 #include <asm/sync_core.h>
+#include <linux/stop_machine.h>
 
 static __always_inline void arch_context_tracking_work(enum ct_work work)
 {
@@ -10,6 +11,9 @@ static __always_inline void arch_context_tracking_work(enum ct_work work)
 	case CT_WORK_SYNC:
 		sync_core();
 		break;
+	case CT_WORK_STOP_MACHINE:
+		stop_machine_poll_wait();
+		break;
 	case CT_WORK_MAX:
 		WARN_ON_ONCE(true);
 	}
diff --git a/include/linux/context_tracking_work.h b/include/linux/context_tracking_work.h
index 2facc621be06..b63200bd73d6 100644
--- a/include/linux/context_tracking_work.h
+++ b/include/linux/context_tracking_work.h
@@ -6,12 +6,14 @@
 
 enum {
 	CT_WORK_SYNC_OFFSET,
+	CT_WORK_STOP_MACHINE_OFFSET,
 	CT_WORK_MAX_OFFSET
 };
 
 enum ct_work {
-	CT_WORK_SYNC     = BIT(CT_WORK_SYNC_OFFSET),
-	CT_WORK_MAX      = BIT(CT_WORK_MAX_OFFSET)
+	CT_WORK_SYNC         = BIT(CT_WORK_SYNC_OFFSET),
+	CT_WORK_STOP_MACHINE = BIT(CT_WORK_STOP_MACHINE_OFFSET),
+	CT_WORK_MAX          = BIT(CT_WORK_MAX_OFFSET)
 };
 
 #include <asm/context_tracking_work.h>
diff --git a/include/linux/stop_machine.h b/include/linux/stop_machine.h
index 72820503514c..0efe88e84b8a 100644
--- a/include/linux/stop_machine.h
+++ b/include/linux/stop_machine.h
@@ -36,6 +36,7 @@ bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
 void stop_machine_park(int cpu);
 void stop_machine_unpark(int cpu);
 void stop_machine_yield(const struct cpumask *cpumask);
+void stop_machine_poll_wait(void);
 
 extern void print_stop_info(const char *log_lvl, struct task_struct *task);
 
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c
index 3fe6b0c99f3d..8f0281b0db64 100644
--- a/kernel/stop_machine.c
+++ b/kernel/stop_machine.c
@@ -22,6 +22,7 @@
 #include <linux/atomic.h>
 #include <linux/nmi.h>
 #include <linux/sched/wake_q.h>
+#include <linux/sched/isolation.h>
 
 /*
  * Structure to determine completion condition and record errors.  May
@@ -176,6 +177,68 @@ struct multi_stop_data {
 	atomic_t		thread_ack;
 };
 
+static DEFINE_PER_CPU(int, stop_machine_poll);
+
+void stop_machine_poll_wait(void)
+{
+	int *poll = this_cpu_ptr(&stop_machine_poll);
+
+	while (*poll)
+		cpu_relax();
+	/* Enforce the work in stop machine to be visible */
+	smp_mb();
+}
+
+static void stop_machine_poll_start(struct multi_stop_data *msdata)
+{
+	int cpu;
+
+	if (housekeeping_enabled(HK_TYPE_KERNEL_NOISE))
+		return;
+
+	/* Random target can't be known in advance */
+	if (!msdata->active_cpus)
+		return;
+		
+	for_each_cpu_andnot(cpu, cpu_online_mask, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE)) {
+		int *poll = per_cpu_ptr(&stop_machine_poll, cpu);
+
+		if (cpumask_test_cpu(cpu, msdata->active_cpus))
+			continue;
+
+		*poll = 1;
+
+		/*
+		 * Act as a full barrier so that if the work is queued, polling is
+		 * visible.
+		 */
+		if (ct_set_cpu_work(cpu, CT_WORK_STOP_MACHINE))
+			msdata->num_threads--;
+		else
+			*poll = 0;
+	}
+}
+
+static void stop_machine_poll_complete(struct multi_stop_data *msdata)
+{
+	int cpu;
+
+	if (housekeeping_enabled(HK_TYPE_KERNEL_NOISE))
+		return;
+
+	for_each_cpu_andnot(cpu, cpu_online_mask, housekeeping_cpumask(HK_TYPE_KERNEL_NOISE)) {
+		int *poll = per_cpu_ptr(&stop_machine_poll, cpu);
+
+		if (cpumask_test_cpu(cpu, msdata->active_cpus))
+			continue;
+		/*
+		 * The RmW in ack_state() fully orders the work performed in stop_machine()
+		 * with polling.
+		 */
+		*poll = 0;
+	}
+}
+
 static void set_state(struct multi_stop_data *msdata,
 		      enum multi_stop_state newstate)
 {
@@ -186,10 +249,13 @@ static void set_state(struct multi_stop_data *msdata,
 }
 
 /* Last one to ack a state moves to the next state. */
-static void ack_state(struct multi_stop_data *msdata)
+static bool ack_state(struct multi_stop_data *msdata)
 {
-	if (atomic_dec_and_test(&msdata->thread_ack))
+	if (atomic_dec_and_test(&msdata->thread_ack)) {
 		set_state(msdata, msdata->state + 1);
+		return true;
+	}
+	return false;
 }
 
 notrace void __weak stop_machine_yield(const struct cpumask *cpumask)
@@ -240,7 +306,8 @@ static int multi_cpu_stop(void *data)
 			default:
 				break;
 			}
-			ack_state(msdata);
+			if (ack_state(msdata) && msdata->state == MULTI_STOP_EXIT)
+				stop_machine_poll_complete(msdata);
 		} else if (curstate > MULTI_STOP_PREPARE) {
 			/*
 			 * At this stage all other CPUs we depend on must spin
@@ -615,6 +682,8 @@ int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
 		return ret;
 	}
 
+	stop_machine_poll_start(&msdata);
+
 	/* Set the initial state and stop all online cpus. */
 	set_state(&msdata, MULTI_STOP_PREPARE);
 	return stop_cpus(cpu_online_mask, multi_cpu_stop, &msdata);