[PATCH v2 10/17] sched/core: Push current task from non preferred CPU

Shrikanth Hegde
2026-04-07T19:21:48Z

Actively push out RT/CFS running on a non-preferred CPU. Since the task is
running on the CPU, need to stop the cpu and push the task out.
However, if the task in pinned only to non-preferred CPUs, it will continue
running there. This will help in maintaining the userspace affinities
unlike CPU hotplug or isolated cpusets.

Though code is almost same as __balance_push_cpu_stop and quite close to
push_cpu_stop, it is being kept separate as it provides a cleaner
implementation w.r.t to PARAVIRT config.

Add push_task_work_done flag to protect work buffer.
This currently works only fair and rt class.

Signed-off-by: Shrikanth Hegde <sshegde@linux.ibm.com>
---
kernel/sched/core.c | 84 ++++++++++++++++++++++++++++++++++++++++++++
kernel/sched/sched.h | 8 +++++
2 files changed, 92 insertions(+)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index c7f046443dc5..b375c500d49e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5652,6 +5652,10 @@ void sched_tick(void)
unsigned long hw_pressure;
u64 resched_latency;

+ /* push the current CFS/RT task out if its on non-preferred CPU */
+ if (!cpu_preferred(cpu))
+ sched_push_current_non_preferred_cpu(rq);
+
if (housekeeping_cpu(cpu, HK_TYPE_KERNEL_NOISE))
arch_scale_freq_tick();

@@ -11247,4 +11251,84 @@ void sched_change_end(struct sched_change_ctx *ctx)
#ifdef CONFIG_PARAVIRT
struct cpumask __cpu_preferred_mask __read_mostly;
EXPORT_SYMBOL(__cpu_preferred_mask);
+
+/* npc - non preferred CPU */
+static DEFINE_PER_CPU(struct cpu_stop_work, npc_push_task_work);
+
+static int sched_non_preferred_cpu_push_stop(void *arg)
+{
+ struct task_struct *p = arg;
+ struct rq *rq = this_rq();
+ struct rq_flags rf;
+ int cpu;
+
+ raw_spin_lock_irq(&p->pi_lock);
+ rq_lock(rq, &rf);
+ rq->push_task_work_done = 0;
+
+ update_rq_clock(rq);
+
+ if (task_rq(p) == rq && task_on_rq_queued(p)) {
+ cpu = select_fallback_rq(rq->cpu, p);
+ rq = __migrate_task(rq, &rf, p, cpu);
+ }
+
+ rq_unlock(rq, &rf);
+ raw_spin_unlock_irq(&p->pi_lock);
+ put_task_struct(p);
+
+ return 0;
+}
+
+/* Using this CPU will lead to more hypervisor preemptions.
+ * It is better not to use this CPU.
+ *
+ * In case any task is scheduled on such CPU, move it out. In
+ * select_fallback_rq a preferred CPU will be chosen and henceforth
+ * task shouldn't come back to this CPU
+ */
+void sched_push_current_non_preferred_cpu(struct rq *rq)
+{
+ struct task_struct *push_task = rq->curr;
+ unsigned long flags;
+ struct rq_flags rf;
+
+ /* sanity check */
+ if (cpu_preferred(rq->cpu))
+ return;
+
+ /* Idle task can't be pused out */
+ if (rq->curr == rq->idle)
+ return;
+
+ /* Do for only SCHED_NORMAL AND RT for now */
+ if (push_task->sched_class != &fair_sched_class &&
+ push_task->sched_class != &rt_sched_class)
+ return;
+
+ if (kthread_is_per_cpu(push_task) ||
+ is_migration_disabled(push_task))
+ return;
+
+ /* Is there any preferred CPU in the affinity list */
+ if (!task_can_run_on_preferred_cpu(push_task))
+ return;
+
+ /* There is already a stopper thread for this. Dont race with it */
+ if (rq->push_task_work_done == 1)
+ return;
+
+ local_irq_save(flags);
+
+ get_task_struct(push_task);
+
+ rq_lock(rq, &rf);
+ rq->push_task_work_done = 1;
+ rq_unlock(rq, &rf);
+
+ stop_one_cpu_nowait(rq->cpu, sched_non_preferred_cpu_push_stop,
+ push_task, this_cpu_ptr(&npc_push_task_work));
+ local_irq_restore(flags);
+}
+
#endif
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4c45092b2fce..c1d037f11c62 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1239,6 +1239,10 @@ struct rq {
unsigned char nohz_idle_balance;
unsigned char idle_balance;

+#ifdef CONFIG_PARAVIRT
+ bool push_task_work_done;
+#endif
+
unsigned long misfit_task_load;

/* For active balancing */
@@ -4138,11 +4142,15 @@ static inline bool task_can_run_on_preferred_cpu(struct task_struct *p)
{
return cpumask_intersects(p->cpus_ptr, cpu_preferred_mask);
}
+
+void sched_push_current_non_preferred_cpu(struct rq *rq);
#else
static inline bool task_can_run_on_preferred_cpu(struct task_struct *p)
{
return true;
}
+
+static inline void sched_push_current_non_preferred_cpu(struct rq *rq) { }
#endif

#endif /* _KERNEL_SCHED_SCHED_H */
--
2.47.3