[PATCH 3/4] sched/fair: Add cookie checks on wake idle path

Fernand Sieber
2025-09-22T12:41:11Z

The wake_affine_idle() function determines whether the previous CPU or the
waking CPU are suitable for running a waking task. Currently it does not
consider core scheduling constraints.

Add cookie compatibility checks to prevent considering a CPU idle when
placing the task there would immediately cause force idle due to an
incompatible sibling task. This reduces unnecessary force idle scenarios
in the wake-up path.

Signed-off-by: Fernand Sieber <sieberf@amazon.com>
---
kernel/sched/fair.c | 19 +++++++++++++------
kernel/sched/sched.h | 33 ++++++++++++++++++++++++++-------
2 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 67746899809e..78b36225a039 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7235,7 +7235,7 @@ static int wake_wide(struct task_struct *p)
* for the overloaded case.
*/
static int
-wake_affine_idle(int this_cpu, int prev_cpu, int sync)
+wake_affine_idle(struct task_struct *p, int this_cpu, int prev_cpu, int sync)
{
/*
* If this_cpu is idle, it implies the wakeup is from interrupt
@@ -7249,17 +7249,24 @@ wake_affine_idle(int this_cpu, int prev_cpu, int sync)
* a cpufreq perspective, it's better to have higher utilisation
* on one CPU.
*/
- if (available_idle_cpu(this_cpu) && cpus_share_cache(this_cpu, prev_cpu))
- return available_idle_cpu(prev_cpu) ? prev_cpu : this_cpu;
+ if (available_idle_cpu(this_cpu) &&
+ cpus_share_cache(this_cpu, prev_cpu) &&
+ sched_core_cookie_match(cpu_rq(this_cpu), p)) {
+ return available_idle_cpu(prev_cpu) &&
+ sched_core_cookie_match(cpu_rq(prev_cpu), p) ?
+ prev_cpu : this_cpu;
+ }

if (sync) {
struct rq *rq = cpu_rq(this_cpu);

- if ((rq->nr_running - cfs_h_nr_delayed(rq)) == 1)
+ if (((rq->nr_running - cfs_h_nr_delayed(rq)) == 1) &&
+ sched_core_cookie_match_sync(rq, p))
return this_cpu;
}

- if (available_idle_cpu(prev_cpu))
+ if (available_idle_cpu(prev_cpu) &&
+ sched_core_cookie_match(cpu_rq(prev_cpu), p))
return prev_cpu;

return nr_cpumask_bits;
@@ -7314,7 +7321,7 @@ static int wake_affine(struct sched_domain *sd, struct task_struct *p,
int target = nr_cpumask_bits;

if (sched_feat(WA_IDLE))
- target = wake_affine_idle(this_cpu, prev_cpu, sync);
+ target = wake_affine_idle(p, this_cpu, prev_cpu, sync);

if (sched_feat(WA_WEIGHT) && target == nr_cpumask_bits)
target = wake_affine_weight(sd, p, this_cpu, prev_cpu, sync);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4e7080123a4c..97cc8c66519e 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1386,27 +1386,41 @@ extern void task_vruntime_update(struct rq *rq, struct task_struct *p, bool in_f
* A special case is that the task's cookie always matches with CPU's core
* cookie if the CPU is in an idle core.
*/
-static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
+static inline bool __sched_core_cookie_match(struct rq *rq,
+ struct task_struct *p,
+ bool sync)
{
- bool idle_core = true;
int cpu;

/* Ignore cookie match if core scheduler is not enabled on the CPU. */
if (!sched_core_enabled(rq))
return true;

+ if (rq->core->core_cookie == p->core_cookie)
+ return true;
+
for_each_cpu(cpu, cpu_smt_mask(cpu_of(rq))) {
- if (!available_idle_cpu(cpu)) {
- idle_core = false;
- break;
- }
+ if (sync && cpu_of(rq) == cpu)
+ continue;
+ if (!available_idle_cpu(cpu))
+ return false;
}

/*
* A CPU in an idle core is always the best choice for tasks with
* cookies.
*/
- return idle_core || rq->core->core_cookie == p->core_cookie;
+ return true;
+}
+
+static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
+{
+ return __sched_core_cookie_match(rq, p, false);
+}
+
+static inline bool sched_core_cookie_match_sync(struct rq *rq, struct task_struct *p)
+{
+ return __sched_core_cookie_match(rq, p, true);
}

static inline bool sched_group_cookie_match(struct rq *rq,
@@ -1464,6 +1478,11 @@ static inline bool sched_core_cookie_match(struct rq *rq, struct task_struct *p)
return true;
}

+static inline bool sched_core_cookie_match_sync(struct rq *rq, struct task_struct *p)
+{
+ return true;
+}
+
static inline bool sched_group_cookie_match(struct rq *rq,
struct task_struct *p,
struct sched_group *group)
--
2.43.0