[PATCH 1/2] sched/fair: reuse the ENQUEUE_DELAYED calculation in enqueue_task_fair()

[PATCH 1/2] sched/fair: reuse the ENQUEUE_DELAYED calculation in enqueue_task_fair()

[Kayra Cizmeci]

Currently flags & ENQUEUE_DELAYED checks spread throughout the
enqueue_task_fair(), connect these checks to a bool 'delayed' that
gets calculated at the start of the function.

No functional change intended.

Signed-off-by: Kayra Cizmeci <kayracizmeci@gmail.com>
---
 kernel/sched/fair.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6d881e530f89..b411384125ec 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7985,7 +7985,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	struct sched_entity *se = &p->se;
 	struct cfs_rq *cfs_rq = &rq->cfs;
 	unsigned long weight;
-	bool curr;
+	bool curr, delayed = (flags & ENQUEUE_DELAYED);
 
 	if (task_is_throttled(p) && enqueue_throttled_task(p))
 		return;
@@ -7996,12 +7996,12 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	 * Let's add the task's estimated utilization to the cfs_rq's
 	 * estimated utilization, before we update schedutil.
 	 */
-	if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED))
+	if (!p->se.sched_delayed || delayed)
 		util_est_enqueue(cfs_rq, p);
 
 	update_curr_eevdf(cfs_rq);
 
-	if (flags & ENQUEUE_DELAYED) {
+	if (delayed) {
 		requeue_delayed_entity(cfs_rq, se);
 		return;
 	}
-- 
2.53.0

[PATCH 2/2] sched/fair: avoid recalculating curr status in place_entity() and requeue_delayed_entity()

[Kayra Cizmeci]

In enqueue_task_fair() a bool is calculated by cfs_rq->curr == se.
But this information gets recalculated on requeue_delayed_entity() and
requeue_delayed_entity() only gets called in enqueue_task_fair().
And on place_entity() if se == curr we call the avg_vruntime_weight()
twice with the same input. place_entity() only gets called in
enqueue_task_fair() and requeue_delayed_entity().

Use the information on enqueue_task_fair() in requeue_delayed_entity().
And place_entity() to avoid calling avg_vruntime_weight() twice.

Signed-off-by: Kayra Cizmeci <kayracizmeci@gmail.com>
---
The patch needs curr to be invariant so I added the same 
(cfs_rq->curr == se) to places in requeue_delayed_entity() and
and enqueue_task_fair(). Like after the place_entity() call in
requeue_delayed_entity(), or like before if (curr) the old place
of the calculation. And added WARN_ON_ONCE(curr != new_calc_curr)
or something like that I don't know how to say it normally.

Then I booted these changes on x86 (Zen 3)
called perf bench sched messaging with 200 groups and 5000 loops.
And then I make sure if the requeue_delayed_entity() was really working
with ftrace. The results were good but I left the computer on 2 more
hours and then checked the results again. It was still good, but
considering that I booted the kernel with busybox I don't think
much happened on that 2 hour window. 

 kernel/sched/fair.c | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index b411384125ec..304ef70685d2 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6175,7 +6175,7 @@ void __setparam_fair(struct task_struct *p, const struct sched_attr *attr)
 }
 
 static void
-place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
+place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags, bool is_curr)
 {
 	u64 vslice, vruntime = avg_vruntime(cfs_rq);
 	unsigned int nr_queued = cfs_rq->h_nr_queued;
@@ -6199,7 +6199,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	 */
 	if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) {
 		struct sched_entity *curr = cfs_rq->curr;
-		long load, weight;
+		long load, weight, curr_weight;
 
 		lag = se->vlag;
 
@@ -6256,10 +6256,17 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 		 *   vl_i = (W + w_i)*vl'_i / W
 		 */
 		load = cfs_rq->sum_weight;
-		if (curr && curr->on_rq)
-			load += avg_vruntime_weight(cfs_rq, curr->h_load.weight);
+		if (curr) {
+			curr_weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight);
+			if (curr->on_rq)
+				load += curr_weight;
+		}
+
+		if (is_curr)
+			weight = curr_weight;
+		else
+			weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
 
-		weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
 		lag *= load + weight;
 		if (WARN_ON_ONCE(!load))
 			load = 1;
@@ -7900,7 +7907,7 @@ static int choose_idle_cpu(int cpu, struct task_struct *p)
 }
 
 static void
-requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
+requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool curr)
 {
 	/*
 	 * se->sched_delayed should imply: se->on_rq == 1.
@@ -7912,10 +7919,10 @@ requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 
 	if (update_entity_lag(cfs_rq, se)) {
 		cfs_rq->h_nr_queued--;
-		if (se != cfs_rq->curr)
+		if (!curr)
 			__dequeue_entity(cfs_rq, se);
-		place_entity(cfs_rq, se, 0);
-		if (se != cfs_rq->curr)
+		place_entity(cfs_rq, se, 0, curr);
+		if (!curr)
 			__enqueue_entity(cfs_rq, se);
 		cfs_rq->h_nr_queued++;
 	}
@@ -8000,9 +8007,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 		util_est_enqueue(cfs_rq, p);
 
 	update_curr_eevdf(cfs_rq);
+	curr = (cfs_rq->curr == se);
 
 	if (delayed) {
-		requeue_delayed_entity(cfs_rq, se);
+		requeue_delayed_entity(cfs_rq, se, curr);
 		return;
 	}
 
@@ -8017,18 +8025,18 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	/*
 	 * XXX comment on the curr thing
 	 */
-	curr = (cfs_rq->curr == se);
+
 	if (curr)
-		place_entity(cfs_rq, se, flags);
+		place_entity(cfs_rq, se, flags, curr);
 
 	if (se->on_rq && se->sched_delayed)
-		requeue_delayed_entity(cfs_rq, se);
+		requeue_delayed_entity(cfs_rq, se, curr);
 
 	weight = enqueue_hierarchy(p, flags);
 
 	if (!curr) {
 		reweight_eevdf(cfs_rq, se, weight, false);
-		place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED);
+		place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED, curr);
 		__enqueue_entity(cfs_rq, se);
 	}
 
-- 
2.53.0

[PATCH v2 0/2] sched/fair: reduce repeated work in enqueue path

[Kayra Cizmeci]

1/2: Gather the flags & DELAYED_QUEUE controls in one place.

2/2: give the calculated curr == se onto requeue_delayed_entity()
and place_entity()

Changelog:

1/2: No changes.

2/2: add (is_curr || curr->on_rq) condition to the 
check for calculating curr_weight so when both of them are false the 
curr_weight won't go to waste.

v1: https://lore.kernel.org/lkml/20260824125223.508178-1-kayracizmeci@gmail.com/

Kayra Cizmeci (2):
  sched/fair: reuse the ENQUEUE_DELAYED calculation in
    enqueue_task_fair()
  sched/fair: avoid recalculating curr status in place_entity() and
    requeue_delayed_entity()

 kernel/sched/fair.c | 42 +++++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 17 deletions(-)

-- 
2.53.0

[PATCH v2 1/2] sched/fair: reuse the ENQUEUE_DELAYED calculation in enqueue_task_fair()

[Kayra Cizmeci]

Currently flags & ENQUEUE_DELAYED checks spread throughout the
enqueue_task_fair(), connect these checks to a bool 'delayed' that
gets calculated at the start of the function.

No functional change intended.

Signed-off-by: Kayra Cizmeci <kayracizmeci@gmail.com>
---
 kernel/sched/fair.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 6d881e530f89..b411384125ec 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7985,7 +7985,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	struct sched_entity *se = &p->se;
 	struct cfs_rq *cfs_rq = &rq->cfs;
 	unsigned long weight;
-	bool curr;
+	bool curr, delayed = (flags & ENQUEUE_DELAYED);
 
 	if (task_is_throttled(p) && enqueue_throttled_task(p))
 		return;
@@ -7996,12 +7996,12 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	 * Let's add the task's estimated utilization to the cfs_rq's
 	 * estimated utilization, before we update schedutil.
 	 */
-	if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED))
+	if (!p->se.sched_delayed || delayed)
 		util_est_enqueue(cfs_rq, p);
 
 	update_curr_eevdf(cfs_rq);
 
-	if (flags & ENQUEUE_DELAYED) {
+	if (delayed) {
 		requeue_delayed_entity(cfs_rq, se);
 		return;
 	}
-- 
2.53.0

[PATCH v2 2/2] sched/fair: avoid recalculating curr status in place_entity() and requeue_delayed_entity()

[Kayra Cizmeci]

In enqueue_task_fair() a bool is calculated by cfs_rq->curr == se.
But this information gets recalculated on requeue_delayed_entity() and
requeue_delayed_entity() only gets called in enqueue_task_fair().
And on place_entity() if se == curr we call the avg_vruntime_weight()
twice with the same input. place_entity() only gets called in
enqueue_task_fair() and requeue_delayed_entity().

Use the information on enqueue_task_fair() in requeue_delayed_entity().
And place_entity() to avoid calling avg_vruntime_weight() twice.

Signed-off-by: Kayra Cizmeci <kayracizmeci@gmail.com>
---
The patch needs curr to be invariant so I added the same 
(cfs_rq->curr == se) to places in requeue_delayed_entity() and
and enqueue_task_fair(). Like after the place_entity() call in
requeue_delayed_entity(), or like before if (curr) the old place
of the calculation. And added WARN_ON_ONCE(curr != new_calc_curr)
or something like that I don't know how to say it normally.

Then I booted these changes on x86 (Zen 3)
called perf bench sched messaging with 200 groups and 5000 loops.
And then I make sure if the requeue_delayed_entity() was really working
with ftrace. The results were good but I left the computer on 2 more
hours and then checked the results again. It was still good, but
considering that I booted the kernel with busybox I don't think
much happened on that 2 hour window.

 kernel/sched/fair.c | 36 ++++++++++++++++++++++--------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index b411384125ec..f1b46fe26d12 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -6175,7 +6175,7 @@ void __setparam_fair(struct task_struct *p, const struct sched_attr *attr)
 }
 
 static void
-place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
+place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags, bool is_curr)
 {
 	u64 vslice, vruntime = avg_vruntime(cfs_rq);
 	unsigned int nr_queued = cfs_rq->h_nr_queued;
@@ -6199,7 +6199,7 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 	 */
 	if (sched_feat(PLACE_LAG) && nr_queued && se->vlag) {
 		struct sched_entity *curr = cfs_rq->curr;
-		long load, weight;
+		long load, weight, curr_weight;
 
 		lag = se->vlag;
 
@@ -6256,10 +6256,17 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags)
 		 *   vl_i = (W + w_i)*vl'_i / W
 		 */
 		load = cfs_rq->sum_weight;
-		if (curr && curr->on_rq)
-			load += avg_vruntime_weight(cfs_rq, curr->h_load.weight);
+		if (curr && (curr->on_rq || is_curr)) {
+			curr_weight = avg_vruntime_weight(cfs_rq, curr->h_load.weight);
+			if (curr->on_rq)
+				load += curr_weight;
+		}
+
+		if (is_curr)
+			weight = curr_weight;
+		else
+			weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
 
-		weight = avg_vruntime_weight(cfs_rq, se->h_load.weight);
 		lag *= load + weight;
 		if (WARN_ON_ONCE(!load))
 			load = 1;
@@ -7900,7 +7907,7 @@ static int choose_idle_cpu(int cpu, struct task_struct *p)
 }
 
 static void
-requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
+requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, bool curr)
 {
 	/*
 	 * se->sched_delayed should imply: se->on_rq == 1.
@@ -7912,10 +7919,10 @@ requeue_delayed_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
 
 	if (update_entity_lag(cfs_rq, se)) {
 		cfs_rq->h_nr_queued--;
-		if (se != cfs_rq->curr)
+		if (!curr)
 			__dequeue_entity(cfs_rq, se);
-		place_entity(cfs_rq, se, 0);
-		if (se != cfs_rq->curr)
+		place_entity(cfs_rq, se, 0, curr);
+		if (!curr)
 			__enqueue_entity(cfs_rq, se);
 		cfs_rq->h_nr_queued++;
 	}
@@ -8000,9 +8007,10 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 		util_est_enqueue(cfs_rq, p);
 
 	update_curr_eevdf(cfs_rq);
+	curr = (cfs_rq->curr == se);
 
 	if (delayed) {
-		requeue_delayed_entity(cfs_rq, se);
+		requeue_delayed_entity(cfs_rq, se, curr);
 		return;
 	}
 
@@ -8017,18 +8025,18 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
 	/*
 	 * XXX comment on the curr thing
 	 */
-	curr = (cfs_rq->curr == se);
+
 	if (curr)
-		place_entity(cfs_rq, se, flags);
+		place_entity(cfs_rq, se, flags, curr);
 
 	if (se->on_rq && se->sched_delayed)
-		requeue_delayed_entity(cfs_rq, se);
+		requeue_delayed_entity(cfs_rq, se, curr);
 
 	weight = enqueue_hierarchy(p, flags);
 
 	if (!curr) {
 		reweight_eevdf(cfs_rq, se, weight, false);
-		place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED);
+		place_entity(cfs_rq, se, flags | ENQUEUE_QUEUED, curr);
 		__enqueue_entity(cfs_rq, se);
 	}
 
-- 
2.53.0

Re: [PATCH v2 1/2] sched/fair: reuse the ENQUEUE_DELAYED calculation in enqueue_task_fair()

[K Prateek Nayak]

On 8/26/2026 3:45 PM, Kayra Cizmeci wrote:
> @@ -7996,12 +7996,12 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags)
>  	 * Let's add the task's estimated utilization to the cfs_rq's
>  	 * estimated utilization, before we update schedutil.
>  	 */
> -	if (!p->se.sched_delayed || (flags & ENQUEUE_DELAYED))
> +	if (!p->se.sched_delayed || delayed)

nit. This reads funny now - not delayed or delayed?

Maybe wakeup_delayed but all of this should be optimized by compiler
at the end and a big ENQUEUE_DELAYED is better for humans who are
reading the code no?

>  		util_est_enqueue(cfs_rq, p);
>  
>  	update_curr_eevdf(cfs_rq);
>  
> -	if (flags & ENQUEUE_DELAYED) {
> +	if (delayed) {
>  		requeue_delayed_entity(cfs_rq, se);
>  		return;
>  	}

-- 
Thanks and Regards,
Prateek