* [PATCH 1/4] sched_ext: Remove queued ecaps syncs directly on sched teardown
2026-07-14 23:09 [PATCHSET sched_ext/for-7.3] sched_ext: Sub-scheduler follow-ups Tejun Heo
@ 2026-07-14 23:09 ` Tejun Heo
2026-07-14 23:09 ` [PATCH 2/4] sched_ext: Move scx_dispatch_sched() from sub.h to internal.h Tejun Heo
` (2 subsequent siblings)
3 siblings, 0 replies; 6+ messages in thread
From: Tejun Heo @ 2026-07-14 23:09 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: Emil Tsalapatis, sched-ext, linux-kernel, Tejun Heo
scx_discard_ecaps_to_sync() waited for balance_one() to consume a dying
sched's queued ecaps sync, polling with resched_cpu() + msleep(). The wait
is unbounded - the ext dl_server forces picks through sustained fair or RT
load only while ext tasks are queued, so an ext-idle cpu monopolized by a
higher class can stall the teardown indefinitely.
Remove the node directly instead: take all queued nodes, drop the dying
sched's and resplice the rest. Consumption runs under the rq lock and batch
nodes read as on-list throughout, so the producer-side dedup stays correct.
A node that an in-flight scx_process_sync_ecaps() batch holds across a
dispatch-induced rq unlock still needs a wait, but one bounded by that batch
completing rather than by a future balance.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 2 +-
kernel/sched/ext/sub.c | 67 ++++++++++++++++++++++++++----------------
2 files changed, 43 insertions(+), 26 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index f8f8597b6f78..428f2bfb2cda 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -4877,7 +4877,7 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
*/
WARN_ON_ONCE(!list_empty(&pcpu->deferred_reenq_local.node));
- /* retire the queued ecaps syncs so the pcpu can be freed */
+ /* remove the queued ecaps sync so the pcpu can be freed */
scx_discard_ecaps_to_sync(cpu, pcpu);
/*
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 68ab3675337f..198063a78a3a 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -13,7 +13,6 @@
* Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
* Copyright (c) 2026 Tejun Heo <tj@kernel.org>
*/
-#include <linux/delay.h>
#include <linux/rhashtable.h>
#include "internal.h"
#include "cid.h"
@@ -537,7 +536,12 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
gained = ecaps & ~old;
lost_all |= lost;
- /* tell the sched its effective caps on this cid changed */
+ /*
+ * Tell the sched its effective caps on this cid changed. The
+ * invocation is equivalent to the dispatch path and may drop
+ * and re-acquire the rq lock temporarily while the rest of
+ * @batch is held privately, see scx_discard_ecaps_to_sync().
+ */
if (ecaps != pcpu->reported_ecaps &&
SCX_HAS_OP(pcpu->sch, sub_ecaps_updated) &&
!scx_bypassing(pcpu->sch, cpu)) {
@@ -661,38 +665,51 @@ void scx_offline_ecaps(struct rq *rq)
}
/*
- * @pcpu's sched was unhashed before the grace period, so nothing new queues.
- * Flush its pending sync so the pcpu can be freed. If the cpu is online and
- * scx is enabled, drain via balance_one(). Otherwise, discard under the rq
- * lock.
+ * @pcpu's sched was unhashed before the grace period, so nothing re-queues its
+ * sync node. Remove the node from @rq's pending list so the pcpu can be freed.
*/
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu)
{
struct rq *rq = cpu_rq(cpu);
+ struct llist_node *head = NULL, *tail = NULL;
+ struct llist_node *pos, *tmp;
+
+ /*
+ * llist can't unlink a single node. Take all queued nodes, drop @pcpu's
+ * and resplice the rest. Nodes in the taken batch read as on-list
+ * throughout, so queue_sync_ecaps() stays correct.
+ */
+ if (llist_on_list(&pcpu->ecaps_to_sync_node)) {
+ scoped_guard (rq_lock_irqsave, rq) {
+ llist_for_each_safe(pos, tmp, llist_del_all(&rq->scx.ecaps_to_sync)) {
+ if (pos == &pcpu->ecaps_to_sync_node) {
+ init_llist_node(pos);
+ } else {
+ pos->next = head;
+ head = pos;
+ if (!tail)
+ tail = pos;
+ }
+ }
+ if (head)
+ llist_add_batch(head, tail, &rq->scx.ecaps_to_sync);
+ }
+ }
+ /*
+ * An in-flight scx_process_sync_ecaps() batch may still hold the node
+ * privately across dispatch-induced rq unlocks, reading as on-list.
+ *
+ * Because a bypassing sched gets no op call, init_llist_node() and all
+ * @pcpu accesses share one contiguous lock hold, off-list under the rq
+ * lock means @pcpu won't be accessed again.
+ */
while (true) {
scoped_guard (rq_lock_irqsave, rq) {
- /*
- * scx_process_sync_ecaps() takes the node off the list
- * before it is done accessing @pcpu but does all of it
- * under the rq lock. Off-list observed under the rq
- * lock guarantees that the sync is complete.
- */
if (!llist_on_list(&pcpu->ecaps_to_sync_node))
return;
- /*
- * Discard only when the cpu is truly down. cpu_active()
- * is already set when scx_online_ecaps() queues an online
- * resync while SCX_RQ_ONLINE is not - so test cpu_active(),
- * or that resync would be dropped.
- */
- if (!scx_enabled() || !cpu_active(cpu)) {
- discard_queued_syncs(rq);
- return;
- }
}
- resched_cpu(cpu);
- msleep(1);
+ cpu_relax();
}
}
@@ -704,7 +721,7 @@ void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu)
* still-open link fd defers it) and can leave queued ecaps syncs behind.
* Processing them would decode the dead sched's pshards with the current cid
* layout. Discard them instead. The backing scx_sched_pcpu's are still
- * allocated as the free path drains ecaps_to_sync_node before freeing.
+ * allocated as the free path removes ecaps_to_sync_node before freeing.
*/
void scx_discard_stale_ecaps_syncs(void)
{
--
2.55.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 2/4] sched_ext: Move scx_dispatch_sched() from sub.h to internal.h
2026-07-14 23:09 [PATCHSET sched_ext/for-7.3] sched_ext: Sub-scheduler follow-ups Tejun Heo
2026-07-14 23:09 ` [PATCH 1/4] sched_ext: Remove queued ecaps syncs directly on sched teardown Tejun Heo
@ 2026-07-14 23:09 ` Tejun Heo
2026-07-14 23:35 ` sashiko-bot
2026-07-14 23:09 ` [PATCH 3/4] sched_ext: Gate sub_dispatch_prev with CONFIG_EXT_SUB_SCHED Tejun Heo
2026-07-14 23:09 ` [PATCH 4/4] sched_ext: Add the scx_has_subs static key and gate sub-sched hot paths Tejun Heo
3 siblings, 1 reply; 6+ messages in thread
From: Tejun Heo @ 2026-07-14 23:09 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: Emil Tsalapatis, sched-ext, linux-kernel, Tejun Heo
scx_dispatch_sched() is common dispatch machinery, not sub-scheduler support
code, and looks out of place in sub.h. Move it to internal.h. cid.h is
included near the tail for scx_cpu_arg() as it depends on internal.h
declarations. Pure code move, no functional changes.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/internal.h | 111 ++++++++++++++++++++++++++++++++++++
kernel/sched/ext/sub.h | 109 -----------------------------------
2 files changed, 111 insertions(+), 109 deletions(-)
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index d7a1d6a14ebf..88b2a9b5d3ab 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -2289,4 +2289,115 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
#endif /* CONFIG_EXT_SUB_SCHED */
+#include "cid.h"
+
+/*
+ * One user of this function is scx_bpf_dispatch() which can be called
+ * recursively as sub-sched dispatches nest. Always inline to reduce stack usage
+ * from the call frame.
+ */
+static __always_inline bool
+scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
+ struct task_struct *prev, bool nested)
+{
+ struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
+ int nr_loops = SCX_DSP_MAX_LOOPS;
+ s32 cpu = cpu_of(rq);
+ bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
+ scx_task_on_sched(sch, prev);
+
+ if (scx_consume_global_dsq(sch, rq))
+ return true;
+
+ if (scx_bypass_dsp_enabled(sch)) {
+ /* if @sch is bypassing, only the bypass DSQs are active */
+ if (scx_bypassing(sch, cpu))
+ return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0);
+
+#ifdef CONFIG_EXT_SUB_SCHED
+ /*
+ * If @sch isn't bypassing but its children are, @sch is
+ * responsible for making forward progress for both its own
+ * tasks that aren't bypassing and the bypassing descendants'
+ * tasks. The following implements a simple built-in behavior -
+ * let each CPU try to run the bypass DSQ every Nth time.
+ *
+ * Later, if necessary, we can add an ops flag to suppress the
+ * auto-consumption and a kfunc to consume the bypass DSQ and,
+ * so that the BPF scheduler can fully control scheduling of
+ * bypassed tasks.
+ */
+ struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
+
+ if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
+ scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
+ __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
+ return true;
+ }
+#endif /* CONFIG_EXT_SUB_SCHED */
+ }
+
+ if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
+ return false;
+
+ dspc->rq = rq;
+
+ /*
+ * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq
+ * lock, the local DSQ might still end up empty after a successful
+ * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
+ * produced some tasks, retry. The BPF scheduler may depend on this
+ * looping behavior to simplify its implementation.
+ */
+ do {
+ dspc->nr_tasks = 0;
+
+ if (nested) {
+ SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
+ prev_on_sch ? prev : NULL);
+ } else {
+ /* stash @prev so that nested invocations can access it */
+ rq->scx.sub_dispatch_prev = prev;
+ SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
+ prev_on_sch ? prev : NULL);
+ rq->scx.sub_dispatch_prev = NULL;
+ }
+
+ scx_flush_dispatch_buf(sch, rq);
+
+ if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) {
+ rq->scx.flags |= SCX_RQ_BAL_KEEP;
+ return true;
+ }
+ if (rq->scx.local_dsq.nr)
+ return true;
+ if (scx_consume_global_dsq(sch, rq))
+ return true;
+
+ /*
+ * ops.dispatch() can trap us in this loop by repeatedly
+ * dispatching ineligible tasks. Break out once in a while to
+ * allow the watchdog to run. As IRQ can't be enabled in
+ * balance(), we want to complete this scheduling cycle and then
+ * start a new one. IOW, we want to call resched_curr() on the
+ * next, most likely idle, task, not the current one. Use
+ * __scx_bpf_kick_cpu() for deferred kicking.
+ */
+ if (unlikely(!--nr_loops)) {
+ scx_kick_cpu(sch, cpu, 0);
+ break;
+ }
+ } while (dspc->nr_tasks);
+
+ /*
+ * Prevent the CPU from going idle while bypassed descendants have tasks
+ * queued. Without this fallback, bypassed tasks could stall if the host
+ * scheduler's ops.dispatch() doesn't yield any tasks.
+ */
+ if (scx_bypass_dsp_enabled(sch))
+ return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0);
+
+ return false;
+}
+
#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index f72c18a5972a..d06bf24aee83 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -172,113 +172,4 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p
#endif /* CONFIG_EXT_SUB_SCHED */
-/*
- * One user of this function is scx_bpf_dispatch() which can be called
- * recursively as sub-sched dispatches nest. Always inline to reduce stack usage
- * from the call frame.
- */
-static __always_inline bool
-scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
- struct task_struct *prev, bool nested)
-{
- struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
- int nr_loops = SCX_DSP_MAX_LOOPS;
- s32 cpu = cpu_of(rq);
- bool prev_on_sch = (prev->sched_class == &ext_sched_class) &&
- scx_task_on_sched(sch, prev);
-
- if (scx_consume_global_dsq(sch, rq))
- return true;
-
- if (scx_bypass_dsp_enabled(sch)) {
- /* if @sch is bypassing, only the bypass DSQs are active */
- if (scx_bypassing(sch, cpu))
- return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0);
-
-#ifdef CONFIG_EXT_SUB_SCHED
- /*
- * If @sch isn't bypassing but its children are, @sch is
- * responsible for making forward progress for both its own
- * tasks that aren't bypassing and the bypassing descendants'
- * tasks. The following implements a simple built-in behavior -
- * let each CPU try to run the bypass DSQ every Nth time.
- *
- * Later, if necessary, we can add an ops flag to suppress the
- * auto-consumption and a kfunc to consume the bypass DSQ and,
- * so that the BPF scheduler can fully control scheduling of
- * bypassed tasks.
- */
- struct scx_sched_pcpu *pcpu = per_cpu_ptr(sch->pcpu, cpu);
-
- if (!(pcpu->bypass_host_seq++ % SCX_BYPASS_HOST_NTH) &&
- scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0)) {
- __scx_add_event(sch, SCX_EV_SUB_BYPASS_DISPATCH, 1);
- return true;
- }
-#endif /* CONFIG_EXT_SUB_SCHED */
- }
-
- if (unlikely(!SCX_HAS_OP(sch, dispatch)) || !scx_rq_online(rq))
- return false;
-
- dspc->rq = rq;
-
- /*
- * The dispatch loop. Because scx_flush_dispatch_buf() may drop the rq
- * lock, the local DSQ might still end up empty after a successful
- * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
- * produced some tasks, retry. The BPF scheduler may depend on this
- * looping behavior to simplify its implementation.
- */
- do {
- dspc->nr_tasks = 0;
-
- if (nested) {
- SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
- prev_on_sch ? prev : NULL);
- } else {
- /* stash @prev so that nested invocations can access it */
- rq->scx.sub_dispatch_prev = prev;
- SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
- prev_on_sch ? prev : NULL);
- rq->scx.sub_dispatch_prev = NULL;
- }
-
- scx_flush_dispatch_buf(sch, rq);
-
- if ((prev->scx.flags & SCX_TASK_QUEUED) && prev->scx.slice) {
- rq->scx.flags |= SCX_RQ_BAL_KEEP;
- return true;
- }
- if (rq->scx.local_dsq.nr)
- return true;
- if (scx_consume_global_dsq(sch, rq))
- return true;
-
- /*
- * ops.dispatch() can trap us in this loop by repeatedly
- * dispatching ineligible tasks. Break out once in a while to
- * allow the watchdog to run. As IRQ can't be enabled in
- * balance(), we want to complete this scheduling cycle and then
- * start a new one. IOW, we want to call resched_curr() on the
- * next, most likely idle, task, not the current one. Use
- * __scx_bpf_kick_cpu() for deferred kicking.
- */
- if (unlikely(!--nr_loops)) {
- scx_kick_cpu(sch, cpu, 0);
- break;
- }
- } while (dspc->nr_tasks);
-
- /*
- * Prevent the CPU from going idle while bypassed descendants have tasks
- * queued. Without this fallback, bypassed tasks could stall if the host
- * scheduler's ops.dispatch() doesn't yield any tasks.
- */
- if (scx_bypass_dsp_enabled(sch))
- return scx_consume_dispatch_q(sch, rq, scx_bypass_dsq(sch, cpu), 0);
-
- return false;
-}
-
#endif /* _KERNEL_SCHED_EXT_SUB_H */
--
2.55.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH 2/4] sched_ext: Move scx_dispatch_sched() from sub.h to internal.h
2026-07-14 23:09 ` [PATCH 2/4] sched_ext: Move scx_dispatch_sched() from sub.h to internal.h Tejun Heo
@ 2026-07-14 23:35 ` sashiko-bot
0 siblings, 0 replies; 6+ messages in thread
From: sashiko-bot @ 2026-07-14 23:35 UTC (permalink / raw)
To: Tejun Heo; +Cc: sched-ext
Thank you for your contribution! Sashiko AI review found 1 potential issue(s) to consider:
- [Low] Moving `scx_dispatch_sched()` to `internal.h` and including `cid.h` at the end creates a circular header dependency. `cid.h` includes `internal.h` at the top. When `cid.h` is included first by a C file, it triggers the inclusion of `internal.h`. The `#include "cid.h"` at the end of `internal.h` is skipped by the include guard. Consequently, `scx_dispatch_sched()` is parsed before the rest of `cid.h` has been processed, resulting in an 'implicit declaration of function' compiler error for `scx_cpu_arg()`. This violates the kernel coding standard requiring headers to be self-contained and independently compilable.
--
--- Patch [2]: [PATCH 2/4] sched_ext: Move scx_dispatch_sched() from sub.h to internal.h ---
commit 6f0aa181637de48d619823a3e1db41e281070615
Author: Tejun Heo <tj@kernel.org>
sched_ext: Move scx_dispatch_sched() from sub.h to internal.h
This commit moves scx_dispatch_sched() from sub.h to internal.h,
and includes cid.h at the end of internal.h to support scx_cpu_arg().
> diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
> index d7a1d6a14ebf3..88b2a9b5d3ab2 100644
> --- a/kernel/sched/ext/internal.h
> +++ b/kernel/sched/ext/internal.h
> @@ -2289,4 +2289,115 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
> static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
> #endif /* CONFIG_EXT_SUB_SCHED */
>
> +#include "cid.h"
[Severity: Low]
Does this create a circular header dependency between internal.h and cid.h?
If a C file includes cid.h before internal.h, cid.h includes internal.h
at the top. When processing internal.h, the #include "cid.h" here is
skipped by the include guard.
This causes scx_dispatch_sched() to be parsed before the rest of cid.h is
processed, which would result in an implicit declaration error for
scx_cpu_arg() further down in this function.
Could this cause build failures if headers are included in a different order
in the future?
> +
> +/*
> + * One user of this function is scx_bpf_dispatch() which can be called
> + * recursively as sub-sched dispatches nest. Always inline to reduce stack usage
> + * from the call frame.
> + */
> +static __always_inline bool
> +scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
[ ... ]
--
Sashiko AI review · https://sashiko.dev/#/patchset/20260714230917.84158-1-tj@kernel.org?part=2
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH 3/4] sched_ext: Gate sub_dispatch_prev with CONFIG_EXT_SUB_SCHED
2026-07-14 23:09 [PATCHSET sched_ext/for-7.3] sched_ext: Sub-scheduler follow-ups Tejun Heo
2026-07-14 23:09 ` [PATCH 1/4] sched_ext: Remove queued ecaps syncs directly on sched teardown Tejun Heo
2026-07-14 23:09 ` [PATCH 2/4] sched_ext: Move scx_dispatch_sched() from sub.h to internal.h Tejun Heo
@ 2026-07-14 23:09 ` Tejun Heo
2026-07-14 23:09 ` [PATCH 4/4] sched_ext: Add the scx_has_subs static key and gate sub-sched hot paths Tejun Heo
3 siblings, 0 replies; 6+ messages in thread
From: Tejun Heo @ 2026-07-14 23:09 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: Emil Tsalapatis, sched-ext, linux-kernel, Tejun Heo
rq->scx.sub_dispatch_prev is sub-sched-only but was left unconditional. Move
it into the CONFIG_EXT_SUB_SCHED block next to ecaps_to_sync and gate its
updates.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/internal.h | 19 +++++++++++--------
kernel/sched/sched.h | 3 +--
2 files changed, 12 insertions(+), 10 deletions(-)
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 88b2a9b5d3ab..8c940be9278d 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -2352,16 +2352,19 @@ scx_dispatch_sched(struct scx_sched *sch, struct rq *rq,
do {
dspc->nr_tasks = 0;
- if (nested) {
- SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
- prev_on_sch ? prev : NULL);
- } else {
- /* stash @prev so that nested invocations can access it */
+#ifdef CONFIG_EXT_SUB_SCHED
+ /* stash @prev so that nested invocations can access it */
+ if (!nested)
rq->scx.sub_dispatch_prev = prev;
- SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
- prev_on_sch ? prev : NULL);
+#endif
+
+ SCX_CALL_OP(sch, dispatch, rq, scx_cpu_arg(cpu),
+ prev_on_sch ? prev : NULL);
+
+#ifdef CONFIG_EXT_SUB_SCHED
+ if (!nested)
rq->scx.sub_dispatch_prev = NULL;
- }
+#endif
scx_flush_dispatch_buf(sch, rq);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 54cff94556c0..64d79e9efc3d 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -814,6 +814,7 @@ struct scx_rq {
u64 clock; /* current per-rq clock -- see scx_bpf_now() */
#ifdef CONFIG_EXT_SUB_SCHED
struct llist_head ecaps_to_sync; /* pending ecaps syncs */
+ struct task_struct *sub_dispatch_prev;
#endif
cpumask_var_t cpus_to_sync;
bool kick_sync_pending;
@@ -821,8 +822,6 @@ struct scx_rq {
struct list_head sched_pcpus_to_kick; /* see kick_cpus_irq_workfn() */
- struct task_struct *sub_dispatch_prev;
-
raw_spinlock_t deferred_reenq_lock;
u64 deferred_reenq_locals_seq;
struct list_head deferred_reenq_locals; /* scheds requesting reenq of local DSQ */
--
2.55.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH 4/4] sched_ext: Add the scx_has_subs static key and gate sub-sched hot paths
2026-07-14 23:09 [PATCHSET sched_ext/for-7.3] sched_ext: Sub-scheduler follow-ups Tejun Heo
` (2 preceding siblings ...)
2026-07-14 23:09 ` [PATCH 3/4] sched_ext: Gate sub_dispatch_prev with CONFIG_EXT_SUB_SCHED Tejun Heo
@ 2026-07-14 23:09 ` Tejun Heo
3 siblings, 0 replies; 6+ messages in thread
From: Tejun Heo @ 2026-07-14 23:09 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: Emil Tsalapatis, sched-ext, linux-kernel, Tejun Heo
With CONFIG_EXT_SUB_SCHED=y but no sub-scheduler attached - the common case
- hot paths still pay for sub-sched bookkeeping. Gate it behind
__scx_has_subs, a static key counting live sub-schedulers, so that a
root-only system stops paying.
Most conversions are simple skip-if-no-sub tests. scx_idle_notify() is
special - it's a hierarchy walk, so give it a fast path which notifies the
root directly using the same tests as the walk. A pending
SCX_RQ_SUB_IDLE_RENOTIFY can be ignored as no sub can be owed one and the
caller clears the flag either way.
Suggested-by: Andrea Righi <arighi@nvidia.com>
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 4 ++++
kernel/sched/ext/idle.c | 10 ++++++++++
kernel/sched/ext/internal.h | 19 +++++++++++++++++--
kernel/sched/ext/sub.c | 21 +++++++++++++++++++--
kernel/sched/ext/sub.h | 15 +++++++++++++++
5 files changed, 65 insertions(+), 4 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 428f2bfb2cda..baf5a3daf7db 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -4919,6 +4919,10 @@ static void scx_sched_free_rcu_work(struct work_struct *work)
scx_arena_pool_destroy(sch);
if (sch->arena_map)
bpf_map_put(sch->arena_map);
+
+ /* @sch is completely inactive by now */
+ scx_dec_has_subs(sch);
+
kfree(sch);
}
diff --git a/kernel/sched/ext/idle.c b/kernel/sched/ext/idle.c
index 16ebe3ab8647..d08166de03d8 100644
--- a/kernel/sched/ext/idle.c
+++ b/kernel/sched/ext/idle.c
@@ -746,6 +746,16 @@ static void scx_idle_notify(struct rq *rq, bool idle, bool do_notify, bool root_
lockdep_assert_rq_held(rq);
+ /* with no sub-sched, only the root can be owed a notification */
+ if (!scx_has_subs()) {
+ struct scx_sched *sch = scx_root;
+
+ if ((do_notify || root_renotify) &&
+ SCX_HAS_OP(sch, update_idle) && !scx_bypassing(sch, cpu))
+ SCX_CALL_OP(sch, update_idle, rq, cid, idle);
+ return;
+ }
+
pos = scx_next_descendant_pre(NULL, scx_root);
while (pos) {
bool forced = false;
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 8c940be9278d..e69aa04c6739 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -2110,7 +2110,7 @@ do { \
*/
#define SCX_CALL_OP_TASK(sch, op, locked_rq, task, args...) \
do { \
- WARN_ON_ONCE((sch) != scx_task_sched_rcu(task)); \
+ WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task)); \
__SCX_CALL_OP_TASK((sch), ops, op, locked_rq, task, ##args); \
} while (0)
@@ -2125,7 +2125,7 @@ do { \
#define SCX_CALL_OP_TASK_RET(sch, op, locked_rq, task, args...) \
({ \
__typeof__((sch)->ops.op(task, ##args)) __ret; \
- WARN_ON_ONCE((sch) != scx_task_sched_rcu(task)); \
+ WARN_ON_ONCE(scx_has_subs() && (sch) != scx_task_sched_rcu(task)); \
WARN_ON_ONCE(current->scx.kf_tasks[0]); \
current->scx.kf_tasks[0] = task; \
__ret = SCX_CALL_OP_RET((sch), op, locked_rq, task, ##args); \
@@ -2165,6 +2165,19 @@ static inline bool scx_bypassing(struct scx_sched *sch, s32 cpu)
}
#ifdef CONFIG_EXT_SUB_SCHED
+DECLARE_STATIC_KEY_FALSE(__scx_has_subs);
+
+/**
+ * scx_has_subs - Whether any sub-scheduler exists
+ *
+ * Gates the sub-sched portions of hot paths so that a root-only system doesn't
+ * pay for them. See scx_sub_enable_workfn() and scx_sched_free_rcu_work().
+ */
+static inline bool scx_has_subs(void)
+{
+ return static_branch_unlikely(&__scx_has_subs);
+}
+
/**
* scx_task_sched - Find scx_sched scheduling a task
* @p: task of interest
@@ -2259,6 +2272,8 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch)
return NULL;
}
#else /* CONFIG_EXT_SUB_SCHED */
+static inline bool scx_has_subs(void) { return false; }
+
static inline struct scx_sched *scx_task_sched(const struct task_struct *p)
{
return rcu_dereference_protected(scx_root,
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 198063a78a3a..d92fae5888ac 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -21,6 +21,12 @@
#ifdef CONFIG_EXT_SUB_SCHED
+/*
+ * On while any sub-scheduler exists so that a root-only system doesn't pay for
+ * the sub-sched portions of hot paths. See scx_has_subs().
+ */
+DEFINE_STATIC_KEY_FALSE(__scx_has_subs);
+
/**
* scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
* @pos: current position
@@ -235,6 +241,9 @@ void scx_init_root_caps(struct scx_sched *sch)
struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq,
struct task_struct *p, u64 *enq_flags)
{
+ if (!scx_has_subs())
+ return &rq->scx.local_dsq;
+
s32 cid = __scx_cpu_to_cid(cpu_of(rq));
struct scx_sched *asch = rq->scx.remote_activate_sch ?: sch;
u64 needed = scx_caps_for_enq(*enq_flags);
@@ -313,7 +322,7 @@ void scx_reenq_reject(struct rq *rq)
lockdep_assert_rq_held(rq);
- if (list_empty(&rq->scx.reject_dsq.list))
+ if (!scx_has_subs() || list_empty(&rq->scx.reject_dsq.list))
return;
/*
@@ -496,7 +505,7 @@ void scx_process_sync_ecaps(struct rq *rq, struct task_struct *prev)
lockdep_assert_rq_held(rq);
- if (likely(llist_empty(&rq->scx.ecaps_to_sync)))
+ if (!scx_has_subs() || likely(llist_empty(&rq->scx.ecaps_to_sync)))
return;
/*
@@ -1015,10 +1024,18 @@ void scx_sub_enable_workfn(struct kthread_work *work)
kobject_get(&parent->kobj);
raw_spin_unlock_irq(&scx_sched_lock);
+ /*
+ * Flip the hot-path gates before ops->priv is published - the sub's
+ * programs can e.g. kick cpus from that point on. The matching dec is
+ * at the end of scx_sched_free_rcu_work().
+ */
+ static_branch_inc(&__scx_has_subs);
+
/* scx_alloc_and_add_sched() consumes @cgrp whether it succeeds or not */
sch = scx_alloc_and_add_sched(cmd, cgrp, parent);
kobject_put(&parent->kobj);
if (IS_ERR(sch)) {
+ static_branch_dec(&__scx_has_subs);
ret = PTR_ERR(sch);
goto out_unlock;
}
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index d06bf24aee83..d5f377d35352 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -45,6 +45,13 @@ static inline const char *sch_cgrp_path(struct scx_sched *sch)
return sch->cgrp_path;
}
+/* a dying sub's hot-path influence ends in scx_sched_free_rcu_work() */
+static inline void scx_dec_has_subs(struct scx_sched *sch)
+{
+ if (sch->level)
+ static_branch_dec(&__scx_has_subs);
+}
+
#else /* CONFIG_EXT_SUB_SCHED */
static inline struct scx_sched *scx_next_descendant_pre(struct scx_sched *pos, struct scx_sched *root) { return pos ? NULL : root; }
@@ -67,6 +74,7 @@ static inline void scx_discard_stale_ecaps_syncs(void) {}
static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
static inline void scx_reenq_reject(struct rq *rq) {}
+static inline void scx_dec_has_subs(struct scx_sched *sch) {}
#endif /* CONFIG_EXT_SUB_SCHED */
@@ -97,6 +105,10 @@ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed)
{
u64 ecaps;
+ /* no sub-scheds, no missing caps */
+ if (!scx_has_subs())
+ return 0;
+
/* root holds every cap on every cpu */
if (!sch->level)
return 0;
@@ -157,6 +169,9 @@ static inline u64 scx_caps_implied(u64 cap)
/* may @p keep running on @rq's cpu? requires baseline cpu access */
static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p)
{
+ if (!scx_has_subs())
+ return true;
+
/* a migration-disabled task is let in without caps, keep it likewise */
if (unlikely(is_migration_disabled(p)))
return true;
--
2.55.0
^ permalink raw reply related [flat|nested] 6+ messages in thread