[PATCH v3 04/10] sched/fair: Credit nominated next-buddy in yield_to_task_fair()

[Wanpeng Li <kernellwp@gmail.com>]

From: Wanpeng Li <wanpengli@tencent.com>

After set_next_buddy() nominates the yield_to() target at every level of
its sched_entity hierarchy, walk that same hierarchy and credit bounded
EEVDF lag to each not-yet-eligible entity. This allows pick_eevdf()'s
PICK_BUDDY path to select the nominated target instead of dropping the
hint at the first ineligible group entity.

Gate the walk with YIELD_TO_LAG_CREDIT. With the feature disabled,
yield_to_task_fair() keeps the existing forfeit-based behavior.
yield_to() holds both rq locks via double_rq_lock(), so touching the
target task's cfs_rqs, including remote cfs_rqs, is safe. Stop the walk
where set_next_buddy() stopped, and skip delayed or throttled entities.

Refresh the target rq clock when it differs from the local rq so the
per-level update_curr() calls observe current rq_clock values. The local
rq still uses the existing yield_task_fair() path in this change.

Signed-off-by: Wanpeng Li <wanpengli@tencent.com>
---
 kernel/sched/fair.c | 44 +++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 41 insertions(+), 3 deletions(-)

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index c28682fedb36..48f65a4f1923 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -9346,8 +9346,8 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev, struct t
  * depth so it stays eligible across several picks. The caller clamps it to
  * entity_lag()'s legal bound, so EEVDF fairness is preserved.
  */
-static u64 __maybe_unused
-eevdf_persistent_margin(struct cfs_rq *cfs_rq, struct sched_entity *se)
+static u64 eevdf_persistent_margin(struct cfs_rq *cfs_rq,
+				   struct sched_entity *se)
 {
 	u64 base = sysctl_sched_base_slice;
 	unsigned int n = cfs_rq->h_nr_queued;
@@ -9379,7 +9379,7 @@ eevdf_persistent_margin(struct cfs_rq *cfs_rq, struct sched_entity *se)
  * Idempotent once @se holds the margin. Caller must hold
  * rq_of(cfs_rq)->lock with rq_clock up to date.
  */
-static void __maybe_unused
+static void
 eevdf_credit_entity_vlag(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
 	u64 avruntime, credit, want, margin, max_slice, lag_limit;
@@ -9488,6 +9488,7 @@ static void yield_task_fair(struct rq *rq)
 static bool yield_to_task_fair(struct rq *rq, struct task_struct *p)
 {
 	struct sched_entity *se = &p->se;
+	struct rq *p_rq = task_rq(p);
 
 	/* !se->on_rq also covers throttled task */
 	if (!se->on_rq)
@@ -9496,6 +9497,43 @@ static bool yield_to_task_fair(struct rq *rq, struct task_struct *p)
 	/* Tell the scheduler that we'd really like se to run next. */
 	set_next_buddy(se);
 
+	/* Without lag credit, keep the existing forfeit-based yield. */
+	if (!sched_feat(YIELD_TO_LAG_CREDIT)) {
+		yield_task_fair(rq);
+		return true;
+	}
+
+	/*
+	 * Walk the ancestor chain set_next_buddy() just nominated and credit
+	 * bounded lag to each not-yet-eligible level so pick_eevdf() returns
+	 * it. yield_to() holds both rq locks via double_rq_lock(), so touching
+	 * p's cfs_rqs (possibly on another CPU) is safe; the primitive is
+	 * idempotent, so no rate limiting is needed.
+	 *
+	 * Only refresh p_rq's clock when it differs from the local rq. A
+	 * remote p_rq must be refreshed so the per-level update_curr() is
+	 * accurate. In the same-rq case we skip it: the credit is a
+	 * best-effort hint and the rq clock is recent enough, while the
+	 * trailing yield_task_fair() would otherwise make this a second
+	 * update_rq_clock() on the same rq and trip
+	 * SCHED_WARN_ON(WARN_DOUBLE_CLOCK).
+	 */
+	if (rq != p_rq)
+		update_rq_clock(p_rq);
+
+	for_each_sched_entity(se) {
+		struct cfs_rq *cfs_rq = cfs_rq_of(se);
+
+		if (cfs_rq->next != se)
+			break;
+		if (se->sched_delayed)
+			break;
+		if (throttled_hierarchy(cfs_rq))
+			break;
+
+		eevdf_credit_entity_vlag(cfs_rq, se);
+	}
+
 	yield_task_fair(rq);
 
 	return true;
-- 
2.43.0