From: Qais Yousef <qyousef@layalina.io>
To: Ingo Molnar <mingo@kernel.org>,
Peter Zijlstra <peterz@infradead.org>,
Vincent Guittot <vincent.guittot@linaro.org>
Cc: Christian Loehle <christian.loehle@arm.com>,
Dietmar Eggemann <dietmar.eggemann@arm.com>,
Steven Rostedt <rostedt@goodmis.org>,
John Stultz <jstultz@google.com>,
linux-kernel@vger.kernel.org,
Kayra Cizmeci <kayracizmeci@gmail.com>
Subject: Re: [PATCH v6 0/3] Fix uclamp race against non atomic static key enable
Date: Sun, 22 Nov 2026 17:22:19 +0000 [thread overview]
Message-ID: <awMi01DePXqbdZ9h@aerith> (raw)
In-Reply-To: <20261121041507.39582-1-qyousef@layalina.io>
On 11/21/26 04:15, Qais Yousef wrote:
> So in summary RT throttling is not fixed and being ignored.
For the sake of completeness, the below patch (only compile tested) should fix
it. What I was tripping at before is that tasks don't get dequeued on
throttling, but only the group. This requires a bit of rework to walk the tasks
that are being dequeued and perform full dequeue instead. As stated in the
cover letter, given this is already being discussed for removal and being
marked as broken, not worth the effort. But in case we need to come back to it,
hope the below can remind us and give us head start.
--->8---
From 411ce77117253e5eac301b0acf67e266363c7630 Mon Sep 17 00:00:00 2001
From: Qais Yousef <qyousef@layalina.io>
Date: Sun, 1 Nov 2026 00:10:12 +0000
Subject: [RFC PATCH] sched/rt: Rework how rq->nr_running is manged
This partially reverts commit f4ebcbc0d7e009783256c9daf76bc4b90e645c14.
While trying to address uclamp static key race condition the throttling
logic in RT and fair has highlighted that both perform incomplete
dequeue since they don't go through core. RT logic seemed to diverge
from fair pattern where rq->nr_running was management oddly.
This is due to the fact that tasks are not really dequeued on
throttling, but only the group rt_se; leading to the oddity of how
nr_running is managed.
Bring the logic more inline with the rest of the classes by
adding/subtracting nr_running on inc/dec_tasks() if it is a task. And
add extra logic to walk the group and do rq accounting for nr_running
and uclamp in case of throttling. Less efficient, but more obvious and
robust.
Replace usage of rt_queued with rt_rq_throttled().
Signed-off-by: Qais Yousef <qyousef@layalina.io>
---
kernel/sched/rt.c | 141 ++++++++++++++++++++-----------------------
kernel/sched/sched.h | 20 ++++--
2 files changed, 80 insertions(+), 81 deletions(-)
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index fd116139ac9bf..b12f5502a4c54 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -82,12 +82,11 @@ void init_rt_rq(struct rt_rq *rt_rq)
rt_rq->highest_prio.next = MAX_RT_PRIO-1;
rt_rq->overloaded = 0;
plist_head_init(&rt_rq->pushable_tasks);
- /* We start is dequeued state, because no RT tasks are queued */
- rt_rq->rt_queued = 0;
#ifdef CONFIG_RT_GROUP_SCHED
rt_rq->rt_time = 0;
rt_rq->rt_throttled = 0;
+ rt_rq->tasks_throttled = false;
rt_rq->rt_runtime = 0;
raw_spin_lock_init(&rt_rq->rt_runtime_lock);
rt_rq->tg = &root_task_group;
@@ -429,8 +428,6 @@ static void dequeue_pushable_task(struct rq *rq, struct task_struct *p)
}
}
-static void enqueue_top_rt_rq(struct rt_rq *rt_rq);
-static void dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count);
static inline int on_rt_rq(struct sched_rt_entity *rt_se)
{
@@ -524,6 +521,37 @@ static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags);
+static void walk_rt_tasks(struct rt_rq *rt_rq, bool throttle)
+{
+ struct rq *rq = rq_of_rt_rq(rt_rq);
+ struct rt_prio_array *array = &rt_rq->active;
+ struct sched_rt_entity *rt_se;
+ int idx;
+
+ for_each_set_bit(idx, array->bitmap, MAX_RT_PRIO) {
+ struct list_head *queue = array->queue + idx;
+
+ list_for_each_entry(rt_se, queue, run_list) {
+ struct rt_rq *child_rt_rq = group_rt_rq(rt_se);
+
+ if (child_rt_rq) {
+ if (!child_rt_rq->tasks_throttled)
+ walk_rt_tasks(child_rt_rq, throttle);
+ } else {
+ struct task_struct *p = rt_task_of(rt_se);
+
+ if (throttle) {
+ sub_nr_running(rq, 1);
+ uclamp_rq_dec(rq, p);
+ } else {
+ add_nr_running(rq, 1);
+ uclamp_rq_inc(rq, p);
+ }
+ }
+ }
+ }
+}
+
static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
{
struct task_struct *donor = rq_of_rt_rq(rt_rq)->donor;
@@ -535,9 +563,12 @@ static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
rt_se = rt_rq->tg->rt_se[cpu];
if (rt_rq->rt_nr_running) {
- if (!rt_se)
- enqueue_top_rt_rq(rt_rq);
- else if (!on_rt_rq(rt_se))
+ if (rt_rq->tasks_throttled) {
+ rt_rq->tasks_throttled = false;
+ walk_rt_tasks(rt_rq, false);
+ }
+
+ if (rt_se && !on_rt_rq(rt_se))
enqueue_rt_entity(rt_se, 0);
if (rt_rq->highest_prio.curr < donor->prio)
@@ -552,8 +583,12 @@ static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
rt_se = rt_rq->tg->rt_se[cpu];
+ if (!rt_rq->tasks_throttled) {
+ walk_rt_tasks(rt_rq, true);
+ rt_rq->tasks_throttled = true;
+ }
+
if (!rt_se) {
- dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
/* Kick cpufreq (see the comment in kernel/sched/sched.h). */
cpufreq_update_util(rq_of_rt_rq(rt_rq), 0);
}
@@ -561,11 +596,6 @@ static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
dequeue_rt_entity(rt_se, 0);
}
-static inline int rt_rq_throttled(struct rt_rq *rt_rq)
-{
- return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
-}
-
static int rt_se_boosted(struct sched_rt_entity *rt_se)
{
struct rt_rq *rt_rq = group_rt_rq(rt_se);
@@ -920,23 +950,12 @@ static inline struct rt_rq *group_rt_rq(struct sched_rt_entity *rt_se)
static inline void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
{
- struct rq *rq = rq_of_rt_rq(rt_rq);
-
- if (!rt_rq->rt_nr_running)
- return;
-
- enqueue_top_rt_rq(rt_rq);
- resched_curr(rq);
+ if (rt_rq->rt_nr_running)
+ resched_curr(rq_of_rt_rq(rt_rq));
}
static inline void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
{
- dequeue_top_rt_rq(rt_rq, rt_rq->rt_nr_running);
-}
-
-static inline int rt_rq_throttled(struct rt_rq *rt_rq)
-{
- return false;
}
static inline const struct cpumask *sched_rt_period_mask(void)
@@ -1007,44 +1026,6 @@ static void update_curr_rt(struct rq *rq)
#endif /* CONFIG_RT_GROUP_SCHED */
}
-static void
-dequeue_top_rt_rq(struct rt_rq *rt_rq, unsigned int count)
-{
- struct rq *rq = rq_of_rt_rq(rt_rq);
-
- BUG_ON(&rq->rt != rt_rq);
-
- if (!rt_rq->rt_queued)
- return;
-
- BUG_ON(!rq->nr_running);
-
- sub_nr_running(rq, count);
- rt_rq->rt_queued = 0;
-
-}
-
-static void
-enqueue_top_rt_rq(struct rt_rq *rt_rq)
-{
- struct rq *rq = rq_of_rt_rq(rt_rq);
-
- BUG_ON(&rq->rt != rt_rq);
-
- if (rt_rq->rt_queued)
- return;
-
- if (rt_rq_throttled(rt_rq))
- return;
-
- if (rt_rq->rt_nr_running) {
- add_nr_running(rq, rt_rq->rt_nr_running);
- rt_rq->rt_queued = 1;
- }
-
- /* Kick cpufreq (see the comment in kernel/sched/sched.h). */
- cpufreq_update_util(rq, 0);
-}
static void
inc_rt_prio_smp(struct rt_rq *rt_rq, int prio, int prev_prio)
@@ -1171,6 +1152,19 @@ unsigned int rt_se_rr_nr_running(struct sched_rt_entity *rt_se)
return (tsk->policy == SCHED_RR) ? 1 : 0;
}
+static inline bool rt_se_throttled(struct sched_rt_entity *rt_se)
+{
+#ifdef CONFIG_RT_GROUP_SCHED
+ for_each_sched_rt_entity(rt_se) {
+ struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
+
+ if (rt_rq->tasks_throttled)
+ return true;
+ }
+#endif
+ return false;
+}
+
static inline
void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
{
@@ -1183,8 +1177,10 @@ void inc_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
inc_rt_prio(rt_rq, prio);
inc_rt_group(rt_se, rt_rq);
- if (rt_entity_is_task(rt_se))
+ if (rt_entity_is_task(rt_se)) {
+ add_nr_running(rq_of_rt_rq(rt_rq), 1);
uclamp_rq_inc(rq_of_rt_rq(rt_rq), rt_task_of(rt_se));
+ }
}
static inline
@@ -1198,8 +1194,10 @@ void dec_rt_tasks(struct sched_rt_entity *rt_se, struct rt_rq *rt_rq)
dec_rt_prio(rt_rq, rt_se_prio(rt_se));
dec_rt_group(rt_se, rt_rq);
- if (rt_entity_is_task(rt_se))
+ if (rt_entity_is_task(rt_se)) {
uclamp_rq_dec(rq_of_rt_rq(rt_rq), rt_task_of(rt_se));
+ sub_nr_running(rq_of_rt_rq(rt_rq), 1);
+ }
}
/*
@@ -1389,39 +1387,29 @@ static void __dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flag
static void dequeue_rt_stack(struct sched_rt_entity *rt_se, unsigned int flags)
{
struct sched_rt_entity *back = NULL;
- unsigned int rt_nr_running;
for_each_sched_rt_entity(rt_se) {
rt_se->back = back;
back = rt_se;
}
- rt_nr_running = rt_rq_of_se(back)->rt_nr_running;
-
for (rt_se = back; rt_se; rt_se = rt_se->back) {
if (on_rt_rq(rt_se))
__dequeue_rt_entity(rt_se, flags);
}
-
- dequeue_top_rt_rq(rt_rq_of_se(back), rt_nr_running);
}
static void enqueue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
{
- struct rq *rq = rq_of_rt_se(rt_se);
-
update_stats_enqueue_rt(rt_rq_of_se(rt_se), rt_se, flags);
dequeue_rt_stack(rt_se, flags);
for_each_sched_rt_entity(rt_se)
__enqueue_rt_entity(rt_se, flags);
- enqueue_top_rt_rq(&rq->rt);
}
static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
{
- struct rq *rq = rq_of_rt_se(rt_se);
-
update_stats_dequeue_rt(rt_rq_of_se(rt_se), rt_se, flags);
dequeue_rt_stack(rt_se, flags);
@@ -1432,7 +1420,6 @@ static void dequeue_rt_entity(struct sched_rt_entity *rt_se, unsigned int flags)
if (rt_rq && rt_rq->rt_nr_running)
__enqueue_rt_entity(rt_se, flags);
}
- enqueue_top_rt_rq(&rq->rt);
}
/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 5bbed5ff2d957..f35ec835c6d68 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -872,8 +872,6 @@ struct rt_rq {
bool overloaded;
struct plist_head pushable_tasks;
- int rt_queued;
-
#ifdef CONFIG_RT_GROUP_SCHED
int rt_throttled;
u64 rt_time; /* consumed RT time, goes up in update_curr_rt */
@@ -883,6 +881,8 @@ struct rt_rq {
unsigned int rt_nr_boosted;
+ bool tasks_throttled;
+
struct rq *rq; /* this is always top-level rq, cache? */
#endif
#ifdef CONFIG_CGROUP_SCHED
@@ -890,9 +890,21 @@ struct rt_rq {
#endif
};
+#ifdef CONFIG_RT_GROUP_SCHED
+static inline bool rt_rq_throttled(struct rt_rq *rt_rq)
+{
+ return rt_rq->rt_throttled && !rt_rq->rt_nr_boosted;
+}
+#else
+static inline bool rt_rq_throttled(struct rt_rq *rt_rq)
+{
+ return false;
+}
+#endif
+
static inline bool rt_rq_is_runnable(struct rt_rq *rt_rq)
{
- return rt_rq->rt_queued && rt_rq->rt_nr_running;
+ return !rt_rq_throttled(rt_rq) && rt_rq->rt_nr_running;
}
/* Deadline class' related fields in a runqueue */
@@ -2914,7 +2926,7 @@ static inline bool sched_dl_runnable(struct rq *rq)
static inline bool sched_rt_runnable(struct rq *rq)
{
- return rq->rt.rt_queued > 0;
+ return rt_rq_is_runnable(&rq->rt);
}
static inline bool sched_fair_runnable(struct rq *rq)
--
2.53.0
prev parent reply other threads:[~2026-11-22 17:22 UTC|newest]
Thread overview: 5+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-21 4:15 [PATCH v6 0/3] Fix uclamp race against non atomic static key enable Qais Yousef
2026-11-21 4:15 ` [PATCH v6 1/3] sched/uclamp: Fix race condition leading to imbalanced uclamp_rq Qais Yousef
2026-11-21 4:15 ` [PATCH v6 2/3] sched/uclamp: Use wrappers instead of open coding UCLAMP_FLAG_IDLE Qais Yousef
2026-11-21 4:15 ` [PATCH v6 3/3] sched/uclamp: Move uclamp_rq_{inc, dec}() from core.c Qais Yousef
2026-11-22 17:22 ` Qais Yousef [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=awMi01DePXqbdZ9h@aerith \
--to=qyousef@layalina.io \
--cc=christian.loehle@arm.com \
--cc=dietmar.eggemann@arm.com \
--cc=jstultz@google.com \
--cc=kayracizmeci@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@kernel.org \
--cc=peterz@infradead.org \
--cc=rostedt@goodmis.org \
--cc=vincent.guittot@linaro.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox