Re: [PATCH 07/18] sched/fair: Add push task mechanism for fair

[Chen Yu <yu.c.chen@intel.com>]

On Fri, Oct 02, 2026 at 05:44:04PM +0200, Vincent Guittot wrote:
> EAS is based on wakeup events to efficiently place tasks on the system, but
> there are cases where a task doesn't have wakeup events anymore or at a far
> too low pace. For such situation, we can take advantage of the task being
> put back in the enqueued list to check if it should be pushed on another
> CPU.
> 
> Add a push task mechanism that enables fair scheduler to push runnable
> tasks. EAS will be one user but other feature like filling idle CPUs or
> short slice tasks can also take advantage of it.
>

A hackbench was launched on a Xeon server with 192 cores, using the default
settings to see if there was any regression. According to the test results,
there is no obvious difference, so I think it is good now. Later, I'll also
test with a mix of different time slice workloads:

=========================================
Hackbench comparison
BASE: baseline
TEST: push
=========================================
MODE     G  FD |         BASE(s) |         TEST(s) |   DIFF% | RESULT
------- -- ----+-----------------+-----------------+---------+----------
process 16  20 |     76.438/0.2% |     76.310/0.1% |   0.17% | IMPROVED
process  1  20 |     52.532/2.3% |     51.217/1.1% |   2.50% | IMPROVED
process  4  20 |     61.754/0.6% |     61.758/1.8% |  -0.01% | REGRESSED
process  8  20 |     76.817/0.5% |     79.049/0.2% |  -2.91% | REGRESSED
threads 16  20 |     66.290/1.5% |     67.120/3.5% |  -1.25% | REGRESSED
threads  1  20 |     62.311/2.3% |     61.518/1.7% |   1.27% | IMPROVED
threads  4  20 |     56.376/0.8% |     57.455/0.8% |  -1.91% | REGRESSED
threads  8  20 |     63.871/3.8% |     68.611/0.3% |  -7.42% | REGRESSED

Besides, as suggested by Qais, cache-aware scheduling could also be a user
of the push mechanism. This is an evaluation patch that uses task push for
cache-aware scheduling by allowing the cache-aware sensitive task to leverage
the push path to be pushed to its preferred LLC.

In my opinion, task pushing is triggered less frequently than task wakeup,
because it has a high bar to be launched(preempted runnable task)
This could reduce the risk of contention between the task pushing and the
cache-aware load balancer, and achieve faster task aggregation on its
preferred LLC.

We have launched a simple migration test based on the following change, and
will update later to see what the result is.

---
 kernel/sched/fair.c  | 88 ++++++++++++++++++++++++++++++++++++++++----
 kernel/sched/sched.h |  1 +
 2 files changed, 82 insertions(+), 7 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 7eea0414fd46..d5c6bbf04b47 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1391,7 +1391,8 @@ static bool update_deadline(struct cfs_rq *cfs_rq, struct sched_entity *se)
 
 #include "pelt.h"
 
-static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu);
+static int select_idle_sibling(struct task_struct *p, int prev_cpu, int cpu,
+			       int push_cpu);
 static unsigned long task_h_load(struct task_struct *p);
 static unsigned long capacity_of(int cpu);
 
@@ -9325,7 +9326,8 @@ static inline bool asym_fits_cpu(unsigned long util,
 /*
  * Try and locate an idle core/thread in the LLC cache domain.
  */
-static int select_idle_sibling(struct task_struct *p, int prev, int target)
+static int select_idle_sibling(struct task_struct *p, int prev, int target,
+			       int push_cpu)
 {
 	bool has_idle_core = false;
 	struct sched_domain *sd;
@@ -9348,6 +9350,19 @@ static int select_idle_sibling(struct task_struct *p, int prev, int target)
 	 */
 	lockdep_assert_irqs_disabled();
 
+	/*
+	 * push_cpu must be checked before the target: when
+	 * a task is pushed from the tick, it is rq->curr, so
+	 * idle_cpu_without() would consider the target as
+	 * idle and keep the task on its current (non-preferred) LLC.
+	 */
+	if (push_cpu != -1 && push_cpu != target && !cpus_share_cache(push_cpu, target)) {
+		target = push_cpu;
+		if (choose_idle_cpu(push_cpu, p) &&
+		    asym_fits_cpu(task_util, util_min, util_max, push_cpu))
+			goto select_smt_priority;
+	}
+
 	if (choose_idle_cpu(target, p) &&
 	    asym_fits_cpu(task_util, util_min, util_max, target))
 		goto select_smt_priority;
@@ -10156,6 +10171,15 @@ static bool check_pushable_short_task(struct rq *rq, struct task_struct *p)
 	return false;
 }
 
+#ifdef CONFIG_SCHED_CACHE
+static int check_pushable_cache_task(int this_cpu, struct task_struct *p);
+#else
+static inline int check_pushable_cache_task(int this_cpu, struct task_struct *p)
+{
+	return -1;
+}
+#endif
+
 static bool fair_check_pushable_task(struct rq *rq, struct task_struct *p, struct task_struct *next)
 {
 	if (!__check_pushable_fair_task(rq, p))
@@ -10167,6 +10191,9 @@ static bool fair_check_pushable_task(struct rq *rq, struct task_struct *p, struc
 	if (check_pushable_short_task(rq, p))
 		return true;
 
+	if (check_pushable_cache_task(rq->cpu, p) != -1)
+		return true;
+
 	return false;
 }
 
@@ -10231,7 +10258,7 @@ static bool fair_push_task(struct rq *rq)
 	if (!raw_spin_trylock(&next_task->pi_lock))
 		return true;
 
-	new_cpu = select_task_rq_fair(next_task, prev_cpu, 0);
+	new_cpu = select_task_rq_fair(next_task, prev_cpu, WF_RQ_PUSH);
 
 	/* Task doesn't need to migrate */
 	if (new_cpu == prev_cpu)
@@ -10335,7 +10362,7 @@ static inline bool tick_pushable_task(struct task_struct *p, struct rq *rq, stru
 	if (!raw_spin_trylock(&p->pi_lock))
 		return false;
 
-	new_cpu = select_task_rq_fair(p, cpu, 0);
+	new_cpu = select_task_rq_fair(p, cpu, WF_RQ_PUSH);
 
 	raw_spin_unlock(&p->pi_lock);
 
@@ -10382,7 +10409,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int select_flags)
 {
 	int sync = (select_flags & WF_SYNC) && !(current->flags & PF_EXITING);
 	int want_sibling = !(select_flags & (WF_EXEC | WF_FORK));
-	int new_cpu, cpu = smp_processor_id();
+	int new_cpu, cpu = smp_processor_id(), push_cpu;
 	struct sched_domain *tmp, *sd = NULL;
 	/* SD_flags and WF_flags share the first nibble */
 	int sd_flag = select_flags & 0xF;
@@ -10414,6 +10441,15 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int select_flags)
 	if (select_flags & WF_TTWU)
 		want_affine = !wake_wide(p) && cpumask_test_cpu(cpu, p->cpus_ptr);
 
+	/*
+	 * Only a push triggered by cache aware scheduling will push the
+	 * task towards its preferred LLC. Pushes for other reasons, like short
+	 * slice task, uses the default push strategy.
+	 */
+	push_cpu = -1;
+	if (select_flags & WF_RQ_PUSH)
+		push_cpu = check_pushable_cache_task(prev_cpu, p);
+
 	new_cpu = prev_cpu;
 
 	for_each_domain(cpu, tmp) {
@@ -10441,7 +10477,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int select_flags)
 			break;
 	}
 
-	/* Slow path */
+	/* Slow path,  push task will not go inside due to flags = 0 */
 	if (unlikely(sd)) {
 		new_cpu = sched_balance_find_dst_cpu(sd, p, cpu, prev_cpu, sd_flag);
 		return select_idle_smt_cpu(p, new_cpu);
@@ -10449,7 +10485,7 @@ select_task_rq_fair(struct task_struct *p, int prev_cpu, int select_flags)
 
 	/* Fast path */
 	if (want_sibling)
-		new_cpu = select_idle_sibling(p, prev_cpu, new_cpu);
+		new_cpu = select_idle_sibling(p, prev_cpu, new_cpu, push_cpu);
 
 	return new_cpu;
 }
@@ -11572,6 +11608,44 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
 	return true;
 }
 
+/*
+ * Return the preferred CPU that task p running on this_cpu should be
+ * pushed to for better LLC locality, or -1 if no such push is wanted.
+ */
+static int check_pushable_cache_task(int this_cpu, struct task_struct *p)
+{
+	struct sched_cache_group *grp;
+	int pref_cpu;
+
+	if (!sched_cache_enabled())
+		return -1;
+
+	/* preemption already disabled */
+	grp = rcu_dereference_all(p->sched_cache_grp);
+	if (!grp)
+		return -1;
+
+	pref_cpu = READ_ONCE(grp->cpu);
+	/* If the task's pref_cpu is on the wrong(non-preferred) LLC, move it there */
+	if (pref_cpu == -1 || cpus_share_cache(pref_cpu, this_cpu))
+		return -1;
+
+	if (!cpumask_test_cpu(pref_cpu, p->cpus_ptr) ||
+	    !cpumask_test_cpu(pref_cpu, cpu_active_mask))
+		return -1;
+
+#ifdef CONFIG_NUMA_BALANCING
+	if (static_branch_likely(&sched_numa_balancing) &&
+	    p->numa_preferred_nid != NUMA_NO_NODE &&
+	    p->numa_preferred_nid != cpu_to_node(pref_cpu))
+		return -1;
+#endif
+	if (can_migrate_llc(this_cpu, pref_cpu, task_util(p), true) == mig_forbid)
+		return -1;
+
+	return pref_cpu;
+}
+
 #else
 static inline bool get_llc_stats(int cpu, unsigned long *util,
 				 unsigned long *cap)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 95a18c0d21c6..85527034572d 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2559,6 +2559,7 @@ static inline int task_on_rq_migrating(struct task_struct *p)
 #define WF_CURRENT_CPU		0x40 /* Prefer to move the wakee to the current CPU. */
 #define WF_RQ_SELECTED		0x80 /* ->select_task_rq() was called */
 #define WF_TTWU_RQ		0x100 /* Wakeup completed through ttwu_runnable() */
+#define WF_RQ_PUSH		0x200 /* Push the task  */
 
 static_assert(WF_EXEC == SD_BALANCE_EXEC);
 static_assert(WF_FORK == SD_BALANCE_FORK);
-- 
2.43.0