* [PATCH 01/12] sched_ext: Rename scx_local_or_reject_dsq() to scx_resolve_local_dsq()
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 02/12] sched_ext: Make several ext.c helpers available outside ext.c Tejun Heo
` (10 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
The following rescue execution addition gives the function a third possible
destination, making a name that enumerates the outcomes a poor fit. Rename
to the destination-neutral scx_resolve_local_dsq(). No functional changes.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 4 ++--
kernel/sched/ext/sub.c | 6 +++---
kernel/sched/ext/sub.h | 6 +++---
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 312938179e52..595b45746245 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -1507,7 +1507,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
bool is_rq_owned = false;
if (dsq->id == SCX_DSQ_LOCAL) {
- dsq = scx_local_or_reject_dsq(sch, rq, p, &enq_flags);
+ dsq = scx_resolve_local_dsq(sch, rq, p, &enq_flags);
is_rq_owned = true;
}
@@ -2279,7 +2279,7 @@ static void move_local_task_to_local_dsq(struct scx_sched *sch,
struct scx_dispatch_q *src_dsq,
struct rq *dst_rq)
{
- struct scx_dispatch_q *dst_dsq = scx_local_or_reject_dsq(sch, dst_rq, p, &enq_flags);
+ struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
/* @dsq is locked and @p is on @dst_rq */
lockdep_assert_held(&src_dsq->lock);
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index daf7fbd3d0c3..2ea9a690e986 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -230,7 +230,7 @@ void scx_init_root_caps(struct scx_sched *sch)
}
/**
- * scx_local_or_reject_dsq - Pick the local or reject DSQ for an insert
+ * scx_resolve_local_dsq - Pick the local or reject DSQ for an insert
* @sch: enqueuing sub-sched
* @rq: rq whose local DSQ @p targets
* @p: task being inserted
@@ -246,8 +246,8 @@ void scx_init_root_caps(struct scx_sched *sch)
* to and run by its nearest non-bypassing ancestor. If root is bypassing, it
* always holds all caps.
*/
-struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq,
- struct task_struct *p, u64 *enq_flags)
+struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
+ struct task_struct *p, u64 *enq_flags)
{
if (!scx_has_subs())
return &rq->scx.local_dsq;
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index 0019b75a2560..db449559bbe8 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -34,8 +34,8 @@ void scx_online_ecaps(struct rq *rq);
void scx_offline_ecaps(struct rq *rq);
void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu);
void scx_discard_stale_ecaps_syncs(void);
-struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq,
- struct task_struct *p, u64 *enq_flags);
+struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq,
+ struct task_struct *p, u64 *enq_flags);
bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
void scx_reenq_reject(struct rq *rq);
@@ -84,7 +84,7 @@ static inline void scx_online_ecaps(struct rq *rq) {}
static inline void scx_offline_ecaps(struct rq *rq) {}
static inline void scx_discard_ecaps_to_sync(s32 cpu, struct scx_sched_pcpu *pcpu) {}
static inline void scx_discard_stale_ecaps_syncs(void) {}
-static inline struct scx_dispatch_q *scx_local_or_reject_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
+static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
static inline void scx_reenq_reject(struct rq *rq) {}
static inline void scx_dec_has_subs(struct scx_sched *sch) {}
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 02/12] sched_ext: Make several ext.c helpers available outside ext.c
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
2026-08-01 8:51 ` [PATCH 01/12] sched_ext: Rename scx_local_or_reject_dsq() to scx_resolve_local_dsq() Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 03/12] sched_ext: Factor out __scx_bpf_now() Tejun Heo
` (9 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
set_task_slice(), task_unlink_from_dsq(), move_local_task_to_local_dsq(),
init_dsq() and dump_line() will be used outside ext.c. Add the scx_ prefix
and declare them in internal.h. The scx_sched_all list will also be used
outside ext.c, drop its static. No functional changes.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 172 +++++++++++++++++-------------------
kernel/sched/ext/internal.h | 8 ++
2 files changed, 90 insertions(+), 90 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 595b45746245..59c9c1f2dc04 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -38,7 +38,7 @@ struct scx_sched __rcu *scx_root;
* All scheds, writers must hold both scx_enable_mutex and scx_sched_lock.
* Readers can hold either or rcu_read_lock().
*/
-static LIST_HEAD(scx_sched_all);
+LIST_HEAD(scx_sched_all);
#ifdef CONFIG_EXT_SUB_SCHED
const struct rhashtable_params scx_sched_hash_params = {
@@ -900,7 +900,7 @@ struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter)
* @kind: a kind of event to dump
*/
#define scx_dump_event(s, events, kind) do { \
- dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \
+ scx_dump_line(&(s), "%40s: %16lld", #kind, (events)->kind); \
} while (0)
@@ -1248,7 +1248,7 @@ static void set_task_slice_keep_oob(struct task_struct *p, u64 slice)
}
/* set @p's slice, superseding any pending out-of-band request */
-static void set_task_slice(struct task_struct *p, u64 slice)
+void scx_set_task_slice(struct task_struct *p, u64 slice)
{
set_task_slice_keep_oob(p, slice);
clear_task_slice_oob(p);
@@ -1495,7 +1495,7 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
rq->curr->sched_class == &ext_sched_class) {
- set_task_slice(rq->curr, 0);
+ scx_set_task_slice(rq->curr, 0);
resched_curr(rq);
}
}
@@ -1637,8 +1637,7 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
}
-static void task_unlink_from_dsq(struct task_struct *p,
- struct scx_dispatch_q *dsq)
+void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq)
{
WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node));
@@ -1695,7 +1694,7 @@ void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p)
*/
if (p->scx.holding_cpu < 0) {
/* @p must still be on @dsq, dequeue */
- task_unlink_from_dsq(p, dsq);
+ scx_task_unlink_from_dsq(p, dsq);
} else {
/*
* We're racing against dispatch_to_local_dsq() which already
@@ -1722,7 +1721,7 @@ static void dispatch_dequeue_locked(struct task_struct *p,
lockdep_assert_rq_held(task_rq(p));
lockdep_assert_held(&dsq->lock);
- task_unlink_from_dsq(p, dsq);
+ scx_task_unlink_from_dsq(p, dsq);
p->scx.dsq = NULL;
}
@@ -2236,7 +2235,7 @@ static void yield_task_scx(struct rq *rq)
if (SCX_HAS_OP(sch, yield))
SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL);
else
- set_task_slice(p, 0);
+ scx_set_task_slice(p, 0);
}
static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
@@ -2274,10 +2273,9 @@ static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p, int wake_fl
scx_schedule_reenq_local(rq, 0);
}
-static void move_local_task_to_local_dsq(struct scx_sched *sch,
- struct task_struct *p, u64 enq_flags,
- struct scx_dispatch_q *src_dsq,
- struct rq *dst_rq)
+void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p,
+ u64 enq_flags, struct scx_dispatch_q *src_dsq,
+ struct rq *dst_rq)
{
struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
@@ -2458,7 +2456,7 @@ static bool unlink_dsq_and_switch_rq_lock(struct task_struct *p,
lockdep_assert_rq_held(locked_rq);
WARN_ON_ONCE(p->scx.holding_cpu >= 0);
- task_unlink_from_dsq(p, dsq);
+ scx_task_unlink_from_dsq(p, dsq);
p->scx.holding_cpu = cpu;
raw_spin_unlock(&dsq->lock);
@@ -2529,9 +2527,8 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
if (dst_dsq->id == SCX_DSQ_LOCAL) {
/* @p is going from a non-local DSQ to a local DSQ */
if (src_rq == dst_rq) {
- task_unlink_from_dsq(p, src_dsq);
- move_local_task_to_local_dsq(sch, p, enq_flags,
- src_dsq, dst_rq);
+ scx_task_unlink_from_dsq(p, src_dsq);
+ scx_move_local_task_to_local_dsq(sch, p, enq_flags, src_dsq, dst_rq);
raw_spin_unlock(&src_dsq->lock);
} else {
raw_spin_unlock(&src_dsq->lock);
@@ -2581,8 +2578,8 @@ bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
break;
if (rq == task_rq) {
- task_unlink_from_dsq(p, dsq);
- move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq);
+ scx_task_unlink_from_dsq(p, dsq);
+ scx_move_local_task_to_local_dsq(sch, p, enq_flags, dsq, rq);
raw_spin_unlock(&dsq->lock);
return true;
}
@@ -3530,7 +3527,7 @@ static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
* we can't trust the slice management or ops.core_sched_before().
*/
if (scx_bypassing(sch, cpu_of(rq))) {
- set_task_slice(curr, 0);
+ scx_set_task_slice(curr, 0);
touch_core_sched(rq, curr);
} else if (SCX_HAS_OP(sch, tick)) {
SCX_CALL_OP_TASK(sch, tick, rq, curr);
@@ -3685,7 +3682,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
* control, after ops.disable() has observed their final values.
*/
p->scx.dsq_vtime = 0;
- set_task_slice(p, 0);
+ scx_set_task_slice(p, 0);
p->scx.reenq_cnt = 0;
/*
@@ -4184,7 +4181,7 @@ static u32 reenq_local(struct scx_sched *sch, struct rq *rq, u64 reenq_flags)
if ((reenq_flags & SCX_REENQ_CAP_REVOKE) &&
rq->curr->sched_class == &ext_sched_class &&
scx_task_reenq_on_cap_revoke(rq, rq->curr)) {
- set_task_slice(rq->curr, 0);
+ scx_set_task_slice(rq->curr, 0);
resched_curr(rq);
}
@@ -4748,8 +4745,7 @@ DEFINE_SCHED_CLASS(ext) = {
#endif
};
-static s32 init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id,
- struct scx_sched *sch)
+s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch)
{
s32 cpu;
@@ -6375,7 +6371,7 @@ static void dump_newline(struct seq_buf *s)
seq_buf_putc(s, '\n');
}
-static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...)
+__printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...)
{
va_list args;
@@ -6407,7 +6403,7 @@ static void dump_stack_trace(struct seq_buf *s, const char *prefix,
unsigned int i;
for (i = 0; i < len; i++)
- dump_line(s, "%s%pS", prefix, (void *)bt[i]);
+ scx_dump_line(s, "%s%pS", prefix, (void *)bt[i]);
}
static void ops_dump_init(struct seq_buf *s, const char *prefix)
@@ -6457,7 +6453,7 @@ static void ops_dump_flush(void)
*/
c = *end;
*end = '\0';
- dump_line(dd->s, "%s%s", dd->prefix, line);
+ scx_dump_line(dd->s, "%s%s", dd->prefix, line);
if (c == '\0')
break;
@@ -6501,21 +6497,19 @@ static void scx_dump_task(struct scx_sched *sch, struct seq_buf *s, struct scx_d
(unsigned long long)p->scx.dsq->id);
dump_newline(s);
- dump_line(s, " %c%c %s[%d] %s%s %+ldms",
- marker, task_state_to_char(p), p->comm, p->pid,
- own_marker, sch_id_buf,
- jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies));
- dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu",
- scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT,
- p->scx.flags & ~SCX_TASK_STATE_MASK,
- p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK,
- ops_state >> SCX_OPSS_QSEQ_SHIFT);
- dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s",
- p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf);
- dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u",
- p->scx.dsq_vtime, p->scx.slice, p->scx.weight);
- dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr),
- p->migration_disabled);
+ scx_dump_line(s, " %c%c %s[%d] %s%s %+ldms",
+ marker, task_state_to_char(p), p->comm, p->pid, own_marker, sch_id_buf,
+ jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies));
+ scx_dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu",
+ scx_get_task_state(p) >> SCX_TASK_STATE_SHIFT,
+ p->scx.flags & ~SCX_TASK_STATE_MASK, p->scx.dsq_flags,
+ ops_state & SCX_OPSS_STATE_MASK, ops_state >> SCX_OPSS_QSEQ_SHIFT);
+ scx_dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s",
+ p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf);
+ scx_dump_line(s, " dsq_vtime=%llu slice=%llu weight=%u",
+ p->scx.dsq_vtime, p->scx.slice, p->scx.weight);
+ scx_dump_line(s, " cpus=%*pb no_mig=%u", cpumask_pr_args(p->cpus_ptr),
+ p->migration_disabled);
if (SCX_HAS_OP(sch, dump_task)) {
ops_dump_init(s, " ");
@@ -6563,28 +6557,26 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
seq_buf_init(&ns, buf, avail);
dump_newline(&ns);
- dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu",
- cpu, rq->scx.nr_running, rq->scx.flags,
- rq->scx.cpu_released, rq->scx.ops_qseq,
- rq->scx.kick_sync);
- dump_line(&ns, " curr=%s[%d] class=%ps",
- rq->curr->comm, rq->curr->pid,
- rq->curr->sched_class);
+ scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu",
+ cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released,
+ rq->scx.ops_qseq, rq->scx.kick_sync);
+ scx_dump_line(&ns, " curr=%s[%d] class=%ps",
+ rq->curr->comm, rq->curr->pid, rq->curr->sched_class);
if (!cpumask_empty(pcpu->cpus_to_kick))
- dump_line(&ns, " cpus_to_kick : %*pb",
- cpumask_pr_args(pcpu->cpus_to_kick));
+ scx_dump_line(&ns, " cpus_to_kick : %*pb",
+ cpumask_pr_args(pcpu->cpus_to_kick));
if (!cpumask_empty(pcpu->cpus_to_kick_if_idle))
- dump_line(&ns, " idle_to_kick : %*pb",
- cpumask_pr_args(pcpu->cpus_to_kick_if_idle));
+ scx_dump_line(&ns, " idle_to_kick : %*pb",
+ cpumask_pr_args(pcpu->cpus_to_kick_if_idle));
if (!cpumask_empty(pcpu->cpus_to_preempt))
- dump_line(&ns, " cpus_to_preempt: %*pb",
- cpumask_pr_args(pcpu->cpus_to_preempt));
+ scx_dump_line(&ns, " cpus_to_preempt: %*pb",
+ cpumask_pr_args(pcpu->cpus_to_preempt));
if (!cpumask_empty(pcpu->cpus_to_wait))
- dump_line(&ns, " cpus_to_wait : %*pb",
- cpumask_pr_args(pcpu->cpus_to_wait));
+ scx_dump_line(&ns, " cpus_to_wait : %*pb",
+ cpumask_pr_args(pcpu->cpus_to_wait));
if (!cpumask_empty(rq->scx.cpus_to_sync))
- dump_line(&ns, " cpus_to_sync : %*pb",
- cpumask_pr_args(rq->scx.cpus_to_sync));
+ scx_dump_line(&ns, " cpus_to_sync : %*pb",
+ cpumask_pr_args(rq->scx.cpus_to_sync));
used = seq_buf_used(&ns);
if (SCX_HAS_OP(sch, dump_cpu)) {
@@ -6652,25 +6644,25 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
#ifdef CONFIG_EXT_SUB_SCHED
if (sch->level == 0)
- dump_line(&s, "%s: root", sch->ops.name);
+ scx_dump_line(&s, "%s: root", sch->ops.name);
else
- dump_line(&s, "%s: sub%d-%llu %s",
- sch->ops.name, sch->level, sch->ops.sub_cgroup_id,
- sch->cgrp_path);
+ scx_dump_line(&s, "%s: sub%d-%llu %s",
+ sch->ops.name, sch->level, sch->ops.sub_cgroup_id,
+ sch->cgrp_path);
#endif
if (ei->kind == SCX_EXIT_NONE) {
- dump_line(&s, "Debug dump triggered by %s", ei->reason);
+ scx_dump_line(&s, "Debug dump triggered by %s", ei->reason);
} else {
if (ei->exit_cpu >= 0)
- dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:",
- current->comm, current->pid, ei->kind,
- ei->exit_cpu);
+ scx_dump_line(&s, "%s[%d] triggered exit kind %d on CPU %d:",
+ current->comm, current->pid, ei->kind,
+ ei->exit_cpu);
else
- dump_line(&s, "%s[%d] triggered exit kind %d:",
- current->comm, current->pid, ei->kind);
- dump_line(&s, " %s (%s)", ei->reason, ei->msg);
+ scx_dump_line(&s, "%s[%d] triggered exit kind %d:",
+ current->comm, current->pid, ei->kind);
+ scx_dump_line(&s, " %s (%s)", ei->reason, ei->msg);
dump_newline(&s);
- dump_line(&s, "Backtrace:");
+ scx_dump_line(&s, "Backtrace:");
dump_stack_trace(&s, " ", ei->bt, ei->bt_len);
}
@@ -6681,8 +6673,8 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
}
dump_newline(&s);
- dump_line(&s, "CPU states");
- dump_line(&s, "----------");
+ scx_dump_line(&s, "CPU states");
+ scx_dump_line(&s, "----------");
/*
* Dump stalled CPUs first so they aren't lost to dump truncation, then
@@ -6705,8 +6697,8 @@ static void scx_dump_state(struct scx_sched *sch, struct scx_exit_info *ei,
}
dump_newline(&s);
- dump_line(&s, "Event counters");
- dump_line(&s, "--------------");
+ scx_dump_line(&s, "Event counters");
+ scx_dump_line(&s, "--------------");
scx_read_events(sch, &events);
#define SCX_EVENT(name) scx_dump_event(s, &events, name)
@@ -6815,7 +6807,7 @@ static struct scx_sched_pnode *alloc_pnode(struct scx_sched *sch, int node)
if (!pnode)
return NULL;
- if (init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) {
+ if (scx_init_dsq(&pnode->global_dsq, SCX_DSQ_GLOBAL, sch)) {
kfree(pnode);
return NULL;
}
@@ -6876,7 +6868,7 @@ struct scx_sched *scx_alloc_and_add_sched(struct scx_enable_cmd *cmd,
}
for_each_possible_cpu(cpu) {
- ret = init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch);
+ ret = scx_init_dsq(scx_bypass_dsq(sch, cpu), SCX_DSQ_BYPASS, sch);
if (ret) {
bypass_fail_cpu = cpu;
goto err_free_pcpu;
@@ -7509,7 +7501,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
queue_flags |= DEQUEUE_CLASS;
scoped_guard (sched_change, p, queue_flags) {
- set_task_slice(p, READ_ONCE(sch->slice_dfl));
+ scx_set_task_slice(p, READ_ONCE(sch->slice_dfl));
p->sched_class = new_class;
}
}
@@ -8159,7 +8151,7 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
if (cur_class == &ext_sched_class) {
if (likely(!scx_missing_caps(pcpu->sch, cpu,
scx_caps_for_preempt(pcpu->sch, rq))))
- set_task_slice(rq->curr, 0);
+ scx_set_task_slice(rq->curr, 0);
else
__scx_add_event(pcpu->sch,
SCX_EV_SUB_PREEMPT_DENIED, 1);
@@ -8363,9 +8355,9 @@ void __init init_sched_ext_class(void)
int n = cpu_to_node(cpu);
/* local_dsq's sch will be set during scx_root_enable() */
- BUG_ON(init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
+ BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
#ifdef CONFIG_EXT_SUB_SCHED
- BUG_ON(init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
+ BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
#endif
INIT_LIST_HEAD(&rq->scx.runnable_list);
@@ -8522,7 +8514,7 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,
return false;
if (slice)
- set_task_slice(p, slice);
+ scx_set_task_slice(p, slice);
else
set_task_slice_keep_oob(p, p->scx.slice ?: 1);
@@ -8548,7 +8540,7 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,
return false;
if (slice)
- set_task_slice(p, slice);
+ scx_set_task_slice(p, slice);
else
set_task_slice_keep_oob(p, p->scx.slice ?: 1);
@@ -8732,7 +8724,7 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)
p->scx.dsq_vtime = kit->vtime;
if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)
- set_task_slice(p, kit->slice);
+ scx_set_task_slice(p, kit->slice);
/* execute move */
locked_rq = move_task_between_dsqs(sch, p, enq_flags, src_dsq, dst_dsq);
@@ -9055,10 +9047,10 @@ __bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node, const struct bpf_prog_a
return -ENOMEM;
/*
- * init_dsq() must be called in GFP_KERNEL context. Init it with NULL
- * @sch and update afterwards.
+ * scx_init_dsq() must be called in GFP_KERNEL context. Init it with
+ * NULL @sch and update afterwards.
*/
- ret = init_dsq(dsq, dsq_id, NULL);
+ ret = scx_init_dsq(dsq, dsq_id, NULL);
if (ret) {
kfree(dsq);
return ret;
@@ -9152,7 +9144,7 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE)))
__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
else
- set_task_slice(p, slice);
+ scx_set_task_slice(p, slice);
return true;
}
@@ -9701,8 +9693,8 @@ __bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data,
ret = __bstr_format(sch, buf->data, buf->line + dd->cursor,
sizeof(buf->line) - dd->cursor, fmt, data, data__sz);
if (ret < 0) {
- dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)",
- dd->prefix, fmt, data, data__sz, ret);
+ scx_dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)",
+ dd->prefix, fmt, data, data__sz, ret);
return;
}
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 7501ec28958b..e234a51cf82d 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -1968,14 +1968,21 @@ void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp);
void scx_task_iter_unlock(struct scx_task_iter *iter);
void scx_task_iter_stop(struct scx_task_iter *iter);
struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter);
+void scx_set_task_slice(struct task_struct *p, u64 slice);
+void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq);
void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p);
void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
int sticky_cpu);
+void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct *p,
+ u64 enq_flags, struct scx_dispatch_q *src_dsq,
+ struct rq *dst_rq);
bool scx_consume_dispatch_q(struct scx_sched *sch, struct rq *rq,
struct scx_dispatch_q *dsq, u64 enq_flags);
bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq);
bool scx_rq_online(struct rq *rq);
void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq);
+s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch);
+__printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...);
void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
u64 reenq_flags, struct rq *locked_rq);
@@ -2014,6 +2021,7 @@ extern bool scx_cgroup_enabled;
#ifdef CONFIG_EXT_SUB_SCHED
extern const struct rhashtable_params scx_sched_hash_params;
extern struct rhashtable scx_sched_hash;
+extern struct list_head scx_sched_all;
extern struct scx_sched *scx_enabling_sub_sched;
#endif
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 03/12] sched_ext: Factor out __scx_bpf_now()
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
2026-08-01 8:51 ` [PATCH 01/12] sched_ext: Rename scx_local_or_reject_dsq() to scx_resolve_local_dsq() Tejun Heo
2026-08-01 8:51 ` [PATCH 02/12] sched_ext: Make several ext.c helpers available outside ext.c Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 04/12] sched_ext: Reject internal enq_flags in the dsq move kfuncs Tejun Heo
` (8 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
scx_bpf_now() couples the valid-or-fresh rq clock read to the current rq.
The read is useful for kernel-internal timing against a specific rq,
including a remotely locked one. Factor it out into __scx_bpf_now().
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 59 ++++++++++++++++++-------------------
kernel/sched/ext/internal.h | 1 +
2 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 59c9c1f2dc04..9ce7e50c13d3 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -10159,6 +10159,27 @@ __bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid)
return container_of(scx, struct task_struct, scx);
}
+u64 __scx_bpf_now(struct rq *rq)
+{
+ /* the caller must be on @rq's cpu or hold its lock */
+ lockdep_assert((rq == this_rq() && !preemptible()) ||
+ lockdep_is_held(__rq_lockp(rq)));
+
+ if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) {
+ /* if the rq clock is valid, use the cached rq clock */
+ return READ_ONCE(rq->scx.clock);
+ } else {
+ /*
+ * Otherwise, return a fresh rq clock.
+ *
+ * The rq clock is updated outside of the rq lock.
+ * In this case, keep the updated rq clock invalid so the next
+ * read outside the rq lock gets a fresh rq clock.
+ */
+ return sched_clock_cpu(cpu_of(rq));
+ }
+}
+
/**
* scx_bpf_now - Returns a high-performance monotonically non-decreasing
* clock for the current CPU. The clock returned is in nanoseconds.
@@ -10189,36 +10210,14 @@ __bpf_kfunc struct task_struct *scx_bpf_tid_to_task(u64 tid)
*/
__bpf_kfunc u64 scx_bpf_now(void)
{
- struct rq *rq;
- u64 clock;
-
- preempt_disable();
-
- rq = this_rq();
- if (smp_load_acquire(&rq->scx.flags) & SCX_RQ_CLK_VALID) {
- /*
- * If the rq clock is valid, use the cached rq clock.
- *
- * Note that scx_bpf_now() is re-entrant between a process
- * context and an interrupt context (e.g., timer interrupt).
- * However, we don't need to consider the race between them
- * because such race is not observable from a caller.
- */
- clock = READ_ONCE(rq->scx.clock);
- } else {
- /*
- * Otherwise, return a fresh rq clock.
- *
- * The rq clock is updated outside of the rq lock.
- * In this case, keep the updated rq clock invalid so the next
- * kfunc call outside the rq lock gets a fresh rq clock.
- */
- clock = sched_clock_cpu(cpu_of(rq));
- }
-
- preempt_enable();
-
- return clock;
+ /*
+ * Note that scx_bpf_now() is re-entrant between a process context and
+ * an interrupt context (e.g., timer interrupt). However, we don't need
+ * to consider the race between them because such race is not observable
+ * from a caller.
+ */
+ guard(preempt)();
+ return __scx_bpf_now(this_rq());
}
static void scx_read_events(struct scx_sched *sch, struct scx_event_stats *events)
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index e234a51cf82d..9e500d58908e 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -1984,6 +1984,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq);
s32 scx_init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id, struct scx_sched *sch);
__printf(2, 3) void scx_dump_line(struct seq_buf *s, const char *fmt, ...);
void scx_kick_cpu(struct scx_sched *sch, s32 cpu, u64 flags);
+u64 __scx_bpf_now(struct rq *rq);
void schedule_dsq_reenq(struct scx_sched *sch, struct scx_dispatch_q *dsq,
u64 reenq_flags, struct rq *locked_rq);
int __scx_init_task(struct scx_sched *sch, struct task_struct *p,
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 04/12] sched_ext: Reject internal enq_flags in the dsq move kfuncs
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (2 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 03/12] sched_ext: Factor out __scx_bpf_now() Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 05/12] sched_ext: Make SCX_ENQ_IGNORE_CAPS waive the preemption cap too Tejun Heo
` (7 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
The dsq insert kfuncs reject __SCX_ENQ_INTERNAL_MASK bits in
scx_dsq_insert_preamble() instead of scx_vet_enq_flags(). A scheduler can
smuggle internal flags such as SCX_ENQ_CLEAR_OPSS through the dsq move
kfuncs and corrupt the dispatch protocol. Move the rejection into
scx_vet_enq_flags(). The vtime move wrapper OR'd the internal
SCX_ENQ_DSQ_PRIQ bit into enq_flags before the vet; the bit now goes in
inside scx_dsq_move() after the vet.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 21 +++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 9ce7e50c13d3..028cfbcd1314 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -8393,6 +8393,11 @@ static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags)
bool is_local = dsq_id == SCX_DSQ_LOCAL ||
(dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON;
+ if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
+ scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags);
+ return false;
+ }
+
if (*enq_flags & SCX_ENQ_IMMED) {
if (unlikely(!is_local)) {
scx_error(sch, "SCX_ENQ_IMMED on a non-local DSQ 0x%llx", dsq_id);
@@ -8415,11 +8420,6 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p
return false;
}
- if (unlikely(*enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
- scx_error(sch, "invalid enq_flags 0x%llx", *enq_flags);
- return false;
- }
-
/* see SCX_EV_INSERT_NOT_OWNED definition */
if (unlikely(!scx_task_on_sched(sch, p))) {
__scx_add_event(sch, SCX_EV_INSERT_NOT_OWNED, 1);
@@ -8651,7 +8651,8 @@ static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = {
};
static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
- struct task_struct *p, u64 dsq_id, u64 enq_flags)
+ struct task_struct *p, u64 dsq_id, u64 enq_flags,
+ bool priq)
{
struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;
struct scx_sched *sch;
@@ -8673,6 +8674,10 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
if (!scx_vet_enq_flags(sch, dsq_id, &enq_flags))
return false;
+ /* internal bit, can only go in after @enq_flags is vetted */
+ if (priq)
+ enq_flags |= SCX_ENQ_DSQ_PRIQ;
+
/*
* If the BPF scheduler keeps calling this function repeatedly, it can
* cause similar live-lock conditions as scx_consume_dispatch_q().
@@ -8929,7 +8934,7 @@ __bpf_kfunc bool scx_bpf_dsq_move(struct bpf_iter_scx_dsq *it__iter,
u64 enq_flags)
{
return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter,
- p, dsq_id, enq_flags);
+ p, dsq_id, enq_flags, false);
}
/**
@@ -8954,7 +8959,7 @@ __bpf_kfunc bool scx_bpf_dsq_move_vtime(struct bpf_iter_scx_dsq *it__iter,
u64 enq_flags)
{
return scx_dsq_move((struct bpf_iter_scx_dsq_kern *)it__iter,
- p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
+ p, dsq_id, enq_flags, true);
}
__bpf_kfunc_end_defs();
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 05/12] sched_ext: Make SCX_ENQ_IGNORE_CAPS waive the preemption cap too
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (3 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 04/12] sched_ext: Reject internal enq_flags in the dsq move kfuncs Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 06/12] sched_ext: Synchronize slice and dsq_vtime writes Tejun Heo
` (6 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
SCX_ENQ_IGNORE_CAPS is kernel-internal and marks a placement the kernel
forces. scx_caps_for_enq() waives the enqueue cap for it, but a PREEMPT
insert still picks up the preemption cap requirement from
scx_caps_for_preempt(). Update scx_caps_for_preempt() to take enq_flags and
require nothing when SCX_ENQ_IGNORE_CAPS is set.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
kernel/sched/ext/ext.c | 5 +++--
kernel/sched/ext/sub.c | 2 +-
kernel/sched/ext/sub.h | 9 ++++++---
3 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 028cfbcd1314..bf1f9f4d0847 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -8149,8 +8149,9 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
if (kickable && !scx_missing_caps(pcpu->sch, cpu, SCX_CAP_BASE)) {
if (cpumask_test_cpu(cpu, pcpu->cpus_to_preempt)) {
if (cur_class == &ext_sched_class) {
- if (likely(!scx_missing_caps(pcpu->sch, cpu,
- scx_caps_for_preempt(pcpu->sch, rq))))
+ u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0);
+
+ if (likely(!scx_missing_caps(pcpu->sch, cpu, caps)))
scx_set_task_slice(rq->curr, 0);
else
__scx_add_event(pcpu->sch,
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 2ea9a690e986..c30f48ee07f9 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -262,7 +262,7 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
* @p's owner (@sch). Check caps against the scheduling sched.
*/
if (*enq_flags & SCX_ENQ_PREEMPT)
- needed |= scx_caps_for_preempt(asch, rq);
+ needed |= scx_caps_for_preempt(asch, rq, *enq_flags);
missing = scx_missing_caps(asch, cpu_of(rq), needed);
/* requirements met */
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index db449559bbe8..fe1d82e6c1d5 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -140,7 +140,7 @@ static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed)
static inline u64 scx_caps_for_enq(u64 enq_flags)
{
/* a restored task must be put into the local DSQ regardless of caps */
- if (enq_flags & SCX_ENQ_IGNORE_CAPS)
+ if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
return 0;
if (enq_flags & SCX_ENQ_IMMED)
return SCX_CAP_ENQ_IMMED;
@@ -156,10 +156,13 @@ static inline u64 scx_caps_for_task(struct task_struct *p)
}
/* the cap @sch needs to preempt @rq's current task, 0 if none */
-static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq)
+static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags)
{
struct task_struct *curr = rq->curr;
+ /* a kernel-forced placement preempts regardless of caps */
+ if (unlikely(enq_flags & SCX_ENQ_IGNORE_CAPS))
+ return 0;
/* a non-ext task can't be preempted by ext, own-subtree needs no cap */
if (curr->sched_class != &ext_sched_class ||
scx_is_descendant(scx_task_sched(curr), sch))
@@ -195,7 +198,7 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p
#else /* CONFIG_EXT_SUB_SCHED */
static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; }
-static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq) { return 0; }
+static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; }
static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; }
#endif /* CONFIG_EXT_SUB_SCHED */
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 06/12] sched_ext: Synchronize slice and dsq_vtime writes
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (4 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 05/12] sched_ext: Make SCX_ENQ_IGNORE_CAPS waive the preemption cap too Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 07/12] sched_ext: Add SCX_TASK_PROTECTED Tejun Heo
` (5 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
p->scx.slice and p->scx.dsq_vtime writes have no synchronization rules. The
dsq insert kfuncs write both fields synchronously from whatever context
they're called in - a direct dispatch from ops.select_cpu() writes with only
pi_lock held - and, as the kfuncs are safe to call spuriously with the
invalid dispatch discarded later, a scheduler can modify any task's slice by
spuriously calling them. The latter stands in the way of an upcoming patch
which adds kernel-granted slices that the schedulers must not be able to
modify.
Give both fields explicit rules. While the task is running, sleeping or
queued on an rq-owned DSQ, the rq lock protects them - these are the states
where the kernel consumes the slice. While queued on a user DSQ or on the
BPF side, the kernel neither consumes nor decides on the fields and every
writer acts for the BPF scheduler - synchronizing the writers is the
scheduler's responsibility and whichever write lands last wins.
To conform, an insert kfunc no longer writes the fields when called. The
values travel with the dispatch and take effect when the task is inserted. A
discarded dispatch has no side effects. The rq lock rule is asserted at the
slice store.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
include/linux/sched/ext.h | 2 +
kernel/sched/ext/ext.c | 184 ++++++++++++++++++++++--------------
kernel/sched/ext/internal.h | 4 +
3 files changed, 120 insertions(+), 70 deletions(-)
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index bd9c4059e8fc..3166a0c3d892 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -192,6 +192,8 @@ struct sched_ext_entity {
atomic_long_t ops_state;
u64 ddsp_dsq_id;
u64 ddsp_enq_flags;
+ u64 ddsp_slice;
+ u64 ddsp_vtime;
struct scx_dsq_list_node dsq_list; /* dispatch order */
struct rb_node dsq_priq; /* p->scx.dsq_vtime order */
u32 dsq_seq;
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index bf1f9f4d0847..4503655e7b6b 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -1204,34 +1204,29 @@ enum scx_slice_oob_consts {
};
/*
- * Slice write rules
+ * Slice and dsq_vtime write rules
*
- * A task's slice - how long it may hold its cpu - is an occupancy grant owned
- * by the task's scheduler. How it may be written depends on whether the task is
- * running.
+ * While @p is running, sleeping or queued on an rq-owned DSQ, both fields are
+ * protected by the rq lock. While running, the rq lock is required because
+ * update_curr_scx() RMWs the slice and the cap check for slice extension is
+ * only reliable under the rq lock.
*
- * Queued, not running: the slice grants no occupancy yet and nothing consumes
- * it, so the owner writes it directly - via scx_bpf_dsq_insert(), the dsq move
- * kfuncs, or scx_bpf_task_set_slice(). Serializing its own writers is then the
- * scheduler's job, not the kernel's.
+ * While @p is queued on a user DSQ or on the BPF side, the kernel neither
+ * consumes nor decides on the fields. Synchronizing the writers is the BPF
+ * scheduler's responsibility. An rq-locked scx_bpf_task_set_slice() write and a
+ * concurrent DSQ insertion commit can race each other and whichever lands last
+ * wins.
*
- * Running: the slice must be changed under the task's rq lock, because:
+ * A DSQ insert kfunc doesn't update the fields directly. The verdict carries
+ * the values and apply_slice_vtime() commits them at the insertion.
*
- * - Raising it extends occupancy, allowed only with %SCX_CAP_BASE on the cpu,
- * and that cap check is coherent only under the rq lock. Shortening is always
- * allowed.
+ * scx_bpf_task_set_slice() may be called from any context and writes directly
+ * only if @p's rq lock is already held, otherwise it bounces through
+ * p->scx.slice_oob, applied under @p's rq lock at the next slice consideration.
*
- * - The kernel decrements it there as the task runs. The decrement is a
- * read-modify-write, so a racing write can be clobbered.
- *
- * scx_bpf_task_set_slice() writes directly only when @p is queued or running
- * on the rq lock it holds. That is the only state where the lock keeps us @p's
- * owner: @p can't move to another rq without it. A task that isn't queued here
- * can instead be woken onto a different rq without taking this lock, and that
- * dispatch sets its slice - so a direct write would race. Those cases stash
- * into p->scx.slice_oob to be applied under @p's actual rq lock. A later in-band
- * write supersedes a stash, and a stash whose scheduler id no longer matches
- * @p's owner is dropped.
+ * dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so
+ * scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct
+ * BPF stores from non-cid-form schedulers are outside these rules.
*/
/* clear a pending slice request */
@@ -1244,6 +1239,7 @@ static void clear_task_slice_oob(struct task_struct *p)
/* set @p's slice, leaving any pending out-of-band request in place */
static void set_task_slice_keep_oob(struct task_struct *p, u64 slice)
{
+ lockdep_assert_rq_held(task_rq(p));
p->scx.slice = slice;
}
@@ -1254,7 +1250,7 @@ void scx_set_task_slice(struct task_struct *p, u64 slice)
clear_task_slice_oob(p);
}
-/* request @p's slice to be set to @slice, see the slice write rules above */
+/* request @p's slice to be set to @slice, see the write rules above */
static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64 slice)
{
u64 dur;
@@ -1276,7 +1272,7 @@ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64
* Apply a pending out-of-band slice request under @rq's lock. A request whose
* packed id no longer matches @p's current owner is dropped. An extension needs
* baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials.
- * Shortening is always allowed. See the slice write rules above.
+ * Shortening is always allowed. See the write rules above.
*/
static void apply_task_slice_oob(struct rq *rq, struct task_struct *p)
{
@@ -1305,7 +1301,30 @@ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p)
return;
}
- p->scx.slice = slice;
+ set_task_slice_keep_oob(p, slice);
+}
+
+/*
+ * A dsq insert kfunc doesn't write slice or dsq_vtime. The verdict carries them
+ * and they are committed here, at the insertion. A zero @slice keeps the
+ * current value, floored at 1 so the task isn't treated as expired.
+ */
+static void apply_slice_vtime(struct task_struct *p, u64 slice, u64 vtime, u64 enq_flags)
+{
+ if (slice) {
+ p->scx.slice = slice;
+ /*
+ * An explicit slice supersedes a pending oob request. A carried
+ * default refill is not an explicit request and must keep it.
+ */
+ if (!(enq_flags & SCX_ENQ_SLICE_DFL))
+ clear_task_slice_oob(p);
+ } else if (!p->scx.slice) {
+ p->scx.slice = 1;
+ }
+
+ if (enq_flags & SCX_ENQ_DSQ_PRIQ)
+ p->scx.dsq_vtime = vtime;
}
static void update_curr_scx(struct rq *rq)
@@ -1502,7 +1521,7 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
struct scx_dispatch_q *dsq, struct task_struct *p,
- u64 enq_flags)
+ u64 slice, u64 vtime, u64 enq_flags)
{
bool is_rq_owned = false;
@@ -1541,6 +1560,13 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
enq_flags &= ~SCX_ENQ_DSQ_PRIQ;
}
+ /*
+ * @dsq is locked and @enq_flags is sanitized. Commit the carried slice
+ * and vtime before the PRIQ insertion below reads the new dsq_vtime.
+ */
+ if (enq_flags & SCX_ENQ_APPLY_SLICE)
+ apply_slice_vtime(p, slice, vtime, enq_flags);
+
if (enq_flags & SCX_ENQ_DSQ_PRIQ) {
struct rb_node *rbp;
@@ -1763,7 +1789,7 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,
static void mark_direct_dispatch(struct scx_sched *sch,
struct task_struct *ddsp_task,
struct task_struct *p, u64 dsq_id,
- u64 enq_flags)
+ u64 slice, u64 vtime, u64 enq_flags)
{
/*
* Mark that dispatch already happened from ops.select_cpu() or
@@ -1787,6 +1813,8 @@ static void mark_direct_dispatch(struct scx_sched *sch,
WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID);
WARN_ON_ONCE(p->scx.ddsp_enq_flags);
+ p->scx.ddsp_slice = slice;
+ p->scx.ddsp_vtime = vtime;
p->scx.ddsp_dsq_id = dsq_id;
p->scx.ddsp_enq_flags = enq_flags;
}
@@ -1818,7 +1846,7 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
struct rq *rq = task_rq(p);
struct scx_dispatch_q *dsq =
find_dsq_for_dispatch(sch, rq, p->scx.ddsp_dsq_id, task_cpu(p));
- u64 ddsp_enq_flags;
+ u64 ddsp_enq_flags, slice, vtime;
touch_core_sched_dispatch(rq, p);
@@ -1860,9 +1888,12 @@ static void direct_dispatch(struct scx_sched *sch, struct task_struct *p,
}
ddsp_enq_flags = p->scx.ddsp_enq_flags;
+ slice = p->scx.ddsp_slice;
+ vtime = p->scx.ddsp_vtime;
clear_direct_dispatch(p);
- scx_dispatch_enqueue(sch, rq, dsq, p, ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS);
+ scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
+ ddsp_enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS);
}
bool scx_rq_online(struct rq *rq)
@@ -1983,7 +2014,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
direct_dispatch(sch, p, enq_flags);
return;
local_norefill:
- scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, enq_flags);
+ scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0, enq_flags);
return;
local:
dsq = &rq->scx.local_dsq;
@@ -2004,7 +2035,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
touch_core_sched(rq, p);
refill_task_slice_dfl(sch, p);
clear_direct_dispatch(p);
- scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags);
+ scx_dispatch_enqueue(sch, rq, dsq, p, 0, 0, enq_flags);
}
static bool task_runnable(const struct task_struct *p)
@@ -2542,7 +2573,7 @@ static struct rq *move_task_between_dsqs(struct scx_sched *sch,
dispatch_dequeue_locked(p, src_dsq);
raw_spin_unlock(&src_dsq->lock);
- scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, enq_flags);
+ scx_dispatch_enqueue(sch, dst_rq, dst_dsq, p, 0, 0, enq_flags);
}
return dst_rq;
@@ -2608,6 +2639,8 @@ bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq)
* @rq: current rq which is locked
* @dst_dsq: destination DSQ
* @p: task to dispatch
+ * @slice: slice carried by the insert verdict, 0 keeps the current value
+ * @vtime: vtime carried by the insert verdict, committed on PRIQ inserts
* @enq_flags: %SCX_ENQ_*
*
* We're holding @rq lock and want to dispatch @p to @dst_dsq which is a local
@@ -2618,8 +2651,8 @@ bool scx_consume_global_dsq(struct scx_sched *sch, struct rq *rq)
* %SCX_OPSS_DISPATCHING).
*/
static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
- struct scx_dispatch_q *dst_dsq,
- struct task_struct *p, u64 enq_flags)
+ struct scx_dispatch_q *dst_dsq, struct task_struct *p,
+ u64 slice, u64 vtime, u64 enq_flags)
{
struct rq *src_rq = task_rq(p);
struct rq *dst_rq = container_of(dst_dsq, struct rq, scx.local_dsq);
@@ -2632,8 +2665,8 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
* If dispatching to @rq that @p is already on, no lock dancing needed.
*/
if (rq == src_rq && rq == dst_rq) {
- scx_dispatch_enqueue(sch, rq, dst_dsq, p,
- enq_flags | SCX_ENQ_CLEAR_OPSS);
+ scx_dispatch_enqueue(sch, rq, dst_dsq, p, slice, vtime,
+ enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS);
return;
}
@@ -2671,13 +2704,16 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
if (src_rq == dst_rq) {
p->scx.holding_cpu = -1;
scx_dispatch_enqueue(sch, dst_rq, &dst_rq->scx.local_dsq, p,
- enq_flags);
+ slice, vtime, enq_flags | SCX_ENQ_APPLY_SLICE);
} else if (unlikely(!task_can_run_on_remote_rq(sch, p, dst_rq, true))) {
p->scx.holding_cpu = -1;
fallback = true;
scx_dispatch_enqueue(sch, src_rq, find_global_dsq(sch, task_cpu(p)),
- p, enq_flags | SCX_ENQ_GDSQ_FALLBACK);
+ p, slice, vtime,
+ enq_flags | SCX_ENQ_APPLY_SLICE |
+ SCX_ENQ_GDSQ_FALLBACK);
} else {
+ apply_slice_vtime(p, slice, vtime, enq_flags);
move_remote_task_to_local_dsq(sch, p, enq_flags, src_rq, dst_rq);
/* task has been moved to dst_rq, which is now locked */
locked_rq = dst_rq;
@@ -2713,10 +2749,9 @@ static void dispatch_to_local_dsq(struct scx_sched *sch, struct rq *rq,
* was valid in the first place. Make sure that the task is still owned by the
* BPF scheduler and claim the ownership before dispatching.
*/
-static void finish_dispatch(struct scx_sched *sch, struct rq *rq,
- struct task_struct *p,
- unsigned long qseq_at_dispatch,
- u64 dsq_id, u64 enq_flags)
+static void finish_dispatch(struct scx_sched *sch, struct rq *rq, struct task_struct *p,
+ unsigned long qseq_at_dispatch, u64 dsq_id,
+ u64 slice, u64 vtime, u64 enq_flags)
{
struct scx_dispatch_q *dsq;
unsigned long opss;
@@ -2776,9 +2811,10 @@ static void finish_dispatch(struct scx_sched *sch, struct rq *rq,
dsq = find_dsq_for_dispatch(sch, this_rq(), dsq_id, task_cpu(p));
if (dsq->id == SCX_DSQ_LOCAL)
- dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags);
+ dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
else
- scx_dispatch_enqueue(sch, rq, dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
+ scx_dispatch_enqueue(sch, rq, dsq, p, slice, vtime,
+ enq_flags | SCX_ENQ_APPLY_SLICE | SCX_ENQ_CLEAR_OPSS);
}
void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
@@ -2790,7 +2826,7 @@ void scx_flush_dispatch_buf(struct scx_sched *sch, struct rq *rq)
struct scx_dsp_buf_ent *ent = &dspc->buf[u];
finish_dispatch(sch, rq, ent->task, ent->qseq, ent->dsq_id,
- ent->enq_flags);
+ ent->slice, ent->vtime, ent->enq_flags);
}
dspc->nr_tasks += dspc->cursor;
@@ -3035,7 +3071,8 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
} else {
- scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, SCX_ENQ_HEAD);
+ scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
+ SCX_ENQ_HEAD);
}
goto switch_class;
}
@@ -3319,7 +3356,13 @@ static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flag
cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, NULL, 0);
if (cpu >= 0) {
- refill_task_slice_dfl(sch, p);
+ /*
+ * Carry the slice refill and let the insertion commit
+ * it under rq lock. See the write rules.
+ */
+ __scx_add_event(sch, SCX_EV_REFILL_SLICE_DFL, 1);
+ p->scx.ddsp_slice = READ_ONCE(sch->slice_dfl);
+ p->scx.ddsp_enq_flags = SCX_ENQ_SLICE_DFL;
p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;
} else {
cpu = prev_cpu;
@@ -4051,13 +4094,15 @@ static void process_ddsp_deferred_locals(struct rq *rq)
struct scx_dispatch_q *dsq;
u64 dsq_id = p->scx.ddsp_dsq_id;
u64 enq_flags = p->scx.ddsp_enq_flags;
+ u64 slice = p->scx.ddsp_slice;
+ u64 vtime = p->scx.ddsp_vtime;
list_del_init(&p->scx.dsq_list.node);
clear_direct_dispatch(p);
dsq = find_dsq_for_dispatch(sch, rq, dsq_id, task_cpu(p));
if (!WARN_ON_ONCE(dsq->id != SCX_DSQ_LOCAL))
- dispatch_to_local_dsq(sch, rq, dsq, p, enq_flags);
+ dispatch_to_local_dsq(sch, rq, dsq, p, slice, vtime, enq_flags);
}
}
@@ -5487,7 +5532,7 @@ static u32 bypass_lb_cpu(struct scx_sched *sch, s32 donor,
* between bypass DSQs.
*/
dispatch_dequeue_locked(p, donor_dsq);
- scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, SCX_ENQ_NESTED);
+ scx_dispatch_enqueue(sch, cpu_rq(donee), donee_dsq, p, 0, 0, SCX_ENQ_NESTED);
/*
* $donee might have been idle and need to be woken up. No need
@@ -8434,14 +8479,14 @@ static bool scx_dsq_insert_preamble(struct scx_sched *sch, struct task_struct *p
}
static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p,
- u64 dsq_id, u64 enq_flags)
+ u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags)
{
struct scx_dsp_ctx *dspc = &this_cpu_ptr(sch->pcpu)->dsp_ctx;
struct task_struct *ddsp_task;
ddsp_task = __this_cpu_read(direct_dispatch_task);
if (ddsp_task) {
- mark_direct_dispatch(sch, ddsp_task, p, dsq_id, enq_flags);
+ mark_direct_dispatch(sch, ddsp_task, p, dsq_id, slice, vtime, enq_flags);
return;
}
@@ -8454,6 +8499,8 @@ static void scx_dsq_insert_commit(struct scx_sched *sch, struct task_struct *p,
.task = p,
.qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK,
.dsq_id = dsq_id,
+ .slice = slice,
+ .vtime = vtime,
.enq_flags = enq_flags,
};
}
@@ -8514,12 +8561,7 @@ __bpf_kfunc bool scx_bpf_dsq_insert___v2(struct task_struct *p, u64 dsq_id,
if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
return false;
- if (slice)
- scx_set_task_slice(p, slice);
- else
- set_task_slice_keep_oob(p, p->scx.slice ?: 1);
-
- scx_dsq_insert_commit(sch, p, dsq_id, enq_flags);
+ scx_dsq_insert_commit(sch, p, dsq_id, slice, 0, enq_flags);
return true;
}
@@ -8540,14 +8582,7 @@ static bool scx_dsq_insert_vtime(struct scx_sched *sch, struct task_struct *p,
if (!scx_dsq_insert_preamble(sch, p, dsq_id, &enq_flags))
return false;
- if (slice)
- scx_set_task_slice(p, slice);
- else
- set_task_slice_keep_oob(p, p->scx.slice ?: 1);
-
- p->scx.dsq_vtime = vtime;
-
- scx_dsq_insert_commit(sch, p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
+ scx_dsq_insert_commit(sch, p, dsq_id, slice, vtime, enq_flags | SCX_ENQ_DSQ_PRIQ);
return true;
}
@@ -8723,9 +8758,9 @@ static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,
dst_dsq = find_dsq_for_dispatch(sch, this_rq, dsq_id, task_cpu(p));
/*
- * Apply vtime and slice updates before moving so that the new time is
- * visible before inserting into $dst_dsq. @p is still on $src_dsq but
- * this is safe as we're locking it.
+ * Apply vtime and slice updates before moving. @p is still on $src_dsq
+ * with both $src_dsq and its task_rq locked, satisfying the write
+ * rules, and the PRIQ insertion into $dst_dsq reads the new vtime.
*/
if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)
p->scx.dsq_vtime = kit->vtime;
@@ -9135,7 +9170,16 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
/*
* Directly write only when we hold the lock of the rq @p is queued or
- * running on. See the slice write rules above.
+ * running on. See the write rules above.
+ *
+ * While @p is queued on a user DSQ or in the BPF scheduler,
+ * synchronization is the scheduler's responsibility. This write can
+ * race a concurrent dispatch's commit, see apply_slice_vtime().
+ *
+ * Making this kfunc always go through the oob stash would leave the
+ * commit as the only direct writer and close the race, but that would
+ * require two more oob application points - the dispatch keep-prev test
+ * and the tick-time expiry check.
*/
locked_rq = scx_locked_rq();
if (!locked_rq ||
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 9e500d58908e..f97c2c1cd914 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -1246,6 +1246,8 @@ struct scx_dsp_buf_ent {
struct task_struct *task;
unsigned long qseq;
u64 dsq_id;
+ u64 slice;
+ u64 vtime;
u64 enq_flags;
};
@@ -1680,6 +1682,8 @@ enum scx_enq_flags {
SCX_ENQ_NESTED = 1LLU << 58,
SCX_ENQ_GDSQ_FALLBACK = 1LLU << 59, /* fell back to global DSQ */
SCX_ENQ_IGNORE_CAPS = 1LLU << 60, /* admit to local DSQ ignoring caps */
+ SCX_ENQ_APPLY_SLICE = 1LLU << 61, /* apply carried slice/vtime at insertion */
+ SCX_ENQ_SLICE_DFL = 1LLU << 62, /* carried slice is a default refill */
};
enum scx_deq_flags {
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 07/12] sched_ext: Add SCX_TASK_PROTECTED
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (5 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 06/12] sched_ext: Synchronize slice and dsq_vtime writes Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 08/12] sched_ext: Add bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (4 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
A BPF scheduler can displace any of its tasks at will - cut a running one's
slice with an SCX_ENQ_PREEMPT dispatch, an SCX_KICK_PREEMPT kick or a direct
shortening, and jump a queued one with HEAD insertions. Sometimes the kernel
needs a slice and a DSQ position to stick regardless.
Add SCX_TASK_PROTECTED, guarding both:
- The slice becomes immutable. Every scheduler-reachable write is refused
and counted as SCX_EV_SLICE_DENIED. Higher scheduling classes are
unaffected. PREEMPT|IMMED can't preempt a running protected task and gets
reenqueued.
- A protected task that reached the head of its DSQ keeps it - HEAD
insertions land behind the leading run of protected tasks and reenqueue
sweeps skip them. Only rq-owned DSQs can hold protected tasks, so the walk
runs only for them.
The bit lives in p->scx.flags so that both the refusal and the head walk
read it under the rq lock that protects it.
Protection ends when the slice is consumed, when the task leaves the rq
except for a save/restore on the running task, on a yield, when the
scheduler enters bypass, and when the task leaves scx. The flag is
kernel-internal and not used yet.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
include/linux/sched/ext.h | 1 +
kernel/sched/ext/ext.c | 165 +++++++++++++++++++++++++++++++-----
kernel/sched/ext/internal.h | 2 +-
3 files changed, 144 insertions(+), 24 deletions(-)
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index 3166a0c3d892..b519fbc88e17 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -102,6 +102,7 @@ enum scx_ent_flags {
SCX_TASK_DEQD_FOR_SLEEP = 1 << 3, /* last dequeue was for SLEEP */
SCX_TASK_SUB_INIT = 1 << 4, /* task being initialized for a sub sched */
SCX_TASK_IMMED = 1 << 5, /* task is on local DSQ with %SCX_ENQ_IMMED */
+ SCX_TASK_PROTECTED = 1 << 6, /* slice and DSQ head position protected */
/*
* Bits 8 to 10 are used to carry task state:
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 4503655e7b6b..eca507e888fc 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -386,8 +386,16 @@ static bool rq_is_open(struct rq *rq, u64 enq_flags)
* so allow it to avoid spuriously triggering reenq on a combined
* PREEMPT|IMMED insertion.
*/
- if (enq_flags & SCX_ENQ_PREEMPT)
- return true;
+ if (enq_flags & SCX_ENQ_PREEMPT) {
+ struct task_struct *curr = rq->curr;
+
+ /*
+ * A protected slice refuses the preemption and the cpu stays
+ * occupied. See rq_owned_post_enq().
+ */
+ return curr->sched_class != &ext_sched_class ||
+ likely(!(curr->scx.flags & SCX_TASK_PROTECTED));
+ }
/*
* @rq is either in transition to or running an SCX task and can't go
@@ -1224,6 +1232,9 @@ enum scx_slice_oob_consts {
* only if @p's rq lock is already held, otherwise it bounces through
* p->scx.slice_oob, applied under @p's rq lock at the next slice consideration.
*
+ * While %SCX_TASK_PROTECTED is set, every scheduler-reachable slice update is
+ * refused. See set_task_slice_keep_oob().
+ *
* dsq_vtime orders the next PRIQ insertion and has no running-side consumer, so
* scx_bpf_task_set_dsq_vtime() writes it directly. Fork-time init and direct
* BPF stores from non-cid-form schedulers are outside these rules.
@@ -1236,18 +1247,92 @@ static void clear_task_slice_oob(struct task_struct *p)
atomic64_set(&p->scx.slice_oob, 0);
}
-/* set @p's slice, leaving any pending out-of-band request in place */
-static void set_task_slice_keep_oob(struct task_struct *p, u64 slice)
+/**
+ * dsq_insert_head - FIFO head insertion honoring %SCX_TASK_PROTECTED
+ * @dsq: DSQ to insert into
+ * @p: task being inserted
+ *
+ * A HEAD insert should land behind any leading protected tasks. Return %true
+ * indicates whether @p became the first entry.
+ */
+static bool dsq_insert_head(struct scx_dispatch_q *dsq, struct task_struct *p)
+{
+ struct list_head *pos = &dsq->list;
+ struct scx_dsq_list_node *node;
+
+ /*
+ * Only rq-owned DSQs can hold protected tasks and the associated rq
+ * lock keeps their flags stable.
+ */
+ if (!dsq_is_rq_owned(dsq)) {
+ list_add(&p->scx.dsq_list.node, &dsq->list);
+ return true;
+ }
+
+ list_for_each_entry(node, &dsq->list, node) {
+ struct task_struct *q;
+
+ if (WARN_ON_ONCE(node->flags & SCX_DSQ_LNODE_ITER_CURSOR))
+ continue;
+
+ q = container_of(node, struct task_struct, scx.dsq_list);
+ if (!(q->scx.flags & SCX_TASK_PROTECTED))
+ break;
+
+ pos = &node->node;
+ }
+
+ list_add(&p->scx.dsq_list.node, pos);
+
+ return pos == &dsq->list;
+}
+
+/**
+ * set_task_slice_keep_oob - Set @p's slice, leaving any pending oob request
+ * @p: task of interest
+ * @slice: slice to set
+ *
+ * While %SCX_TASK_PROTECTED is set, BPF schedulers may not modify the slice.
+ * Refuse and return %false.
+ */
+static bool set_task_slice_keep_oob(struct task_struct *p, u64 slice)
{
lockdep_assert_rq_held(task_rq(p));
+
+ if (unlikely(p->scx.flags & SCX_TASK_PROTECTED))
+ return false;
+
p->scx.slice = slice;
+ return true;
}
/* set @p's slice, superseding any pending out-of-band request */
-void scx_set_task_slice(struct task_struct *p, u64 slice)
+bool scx_set_task_slice(struct task_struct *p, u64 slice)
{
- set_task_slice_keep_oob(p, slice);
+ if (!set_task_slice_keep_oob(p, slice))
+ return false;
clear_task_slice_oob(p);
+ return true;
+}
+
+/**
+ * scx_task_slice_ended - @p's slice is consumed or given up
+ * @rq: rq @p is on
+ * @p: task of interest
+ *
+ * End what rides on the slice - the protection.
+ *
+ * A dequeue normally ends the slice too. The exception is a save/restore pair
+ * on the running task. Attribute changes like renice cycle the task through
+ * dequeue and enqueue while it keeps executing, so the slice continues. A
+ * queued task instead loses its DSQ position on any dequeue and the slice ends
+ * with it.
+ */
+static void scx_task_slice_ended(struct rq *rq, struct task_struct *p)
+{
+ lockdep_assert_rq_held(rq);
+
+ p->scx.flags &= ~SCX_TASK_PROTECTED;
}
/* request @p's slice to be set to @slice, see the write rules above */
@@ -1271,8 +1356,9 @@ static void set_task_slice_oob(struct scx_sched *sch, struct task_struct *p, u64
/*
* Apply a pending out-of-band slice request under @rq's lock. A request whose
* packed id no longer matches @p's current owner is dropped. An extension needs
- * baseline cpu access on @p's cid. %SCX_EV_SLICE_DENIED counts the denials.
- * Shortening is always allowed. See the write rules above.
+ * baseline cpu access on @p's cid, shortening is always allowed, and a
+ * protected slice refuses both. %SCX_EV_SLICE_DENIED counts the denials. See
+ * the write rules above.
*/
static void apply_task_slice_oob(struct rq *rq, struct task_struct *p)
{
@@ -1301,7 +1387,8 @@ static void apply_task_slice_oob(struct rq *rq, struct task_struct *p)
return;
}
- set_task_slice_keep_oob(p, slice);
+ if (unlikely(!set_task_slice_keep_oob(p, slice)))
+ __scx_add_event(scx_task_sched(p), SCX_EV_SLICE_DENIED, 1);
}
/*
@@ -1514,8 +1601,10 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
rq->curr->sched_class == &ext_sched_class) {
- scx_set_task_slice(rq->curr, 0);
- resched_curr(rq);
+ if (likely(scx_set_task_slice(rq->curr, 0)))
+ resched_curr(rq);
+ else
+ __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
}
}
@@ -1606,9 +1695,8 @@ static void scx_dispatch_enqueue(struct scx_sched *sch, struct rq *rq,
dsq->id);
if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT)) {
- list_add(&p->scx.dsq_list.node, &dsq->list);
/* new task inserted at head - use fastpath */
- if (!(dsq->id & SCX_DSQ_FLAG_BUILTIN))
+ if (dsq_insert_head(dsq, p) && !(dsq->id & SCX_DSQ_FLAG_BUILTIN))
rcu_assign_pointer(dsq->first_task, p);
} else {
/*
@@ -2254,6 +2342,11 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
sub_nr_running(rq, 1);
scx_dispatch_dequeue(rq, p);
+
+ /* see scx_task_slice_ended() for the save/restore exception */
+ if (!((deq_flags & DEQUEUE_SAVE) && task_current(rq, p)))
+ scx_task_slice_ended(rq, p);
+
clear_direct_dispatch(p);
return true;
}
@@ -2263,6 +2356,9 @@ static void yield_task_scx(struct rq *rq)
struct task_struct *p = rq->donor;
struct scx_sched *sch = scx_task_sched(p);
+ /* a yield gives the slice up */
+ scx_task_slice_ended(rq, p);
+
if (SCX_HAS_OP(sch, yield))
SCX_CALL_OP_2TASKS_RET(sch, yield, rq, p, NULL);
else
@@ -2274,6 +2370,9 @@ static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
struct task_struct *from = rq->donor;
struct scx_sched *sch = scx_task_sched(from);
+ /* like a plain yield, giving the slice up ends the protection */
+ scx_task_slice_ended(rq, from);
+
if (SCX_HAS_OP(sch, yield) && sch == scx_task_sched(to))
return SCX_CALL_OP_2TASKS_RET(sch, yield, rq, from, to);
else
@@ -2317,7 +2416,7 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct
WARN_ON_ONCE(p->scx.holding_cpu >= 0);
if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
- list_add(&p->scx.dsq_list.node, &dst_dsq->list);
+ dsq_insert_head(dst_dsq, p);
else
list_add_tail(&p->scx.dsq_list.node, &dst_dsq->list);
@@ -3051,6 +3150,10 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
update_curr_scx(rq);
+ /* the slice is consumed, protection ends with it */
+ if (!p->scx.slice)
+ scx_task_slice_ended(rq, p);
+
/* see dequeue_task_scx() on why we skip when !QUEUED */
if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED))
SCX_CALL_OP_TASK(sch, stopping, rq, p, true);
@@ -3204,8 +3307,11 @@ do_pick_task_scx(struct rq *rq, struct rq_flags *rf, bool force_scx)
*/
if (keep_prev) {
p = prev;
- if (!p->scx.slice)
+ if (!p->scx.slice) {
+ /* the slice is consumed, protection ends */
+ scx_task_slice_ended(rq, p);
refill_task_slice_dfl(scx_task_sched(p), p);
+ }
} else {
p = first_local_task(rq);
if (!p)
@@ -3725,6 +3831,7 @@ static void scx_disable_task(struct scx_sched *sch, struct task_struct *p)
* control, after ops.disable() has observed their final values.
*/
p->scx.dsq_vtime = 0;
+ scx_task_slice_ended(rq, p);
scx_set_task_slice(p, 0);
p->scx.reenq_cnt = 0;
@@ -4137,6 +4244,9 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p,
first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST);
*reenq_flags |= SCX_REENQ_TSR_NOT_FIRST;
+ if (unlikely(p->scx.flags & SCX_TASK_PROTECTED))
+ return false;
+
*reason = SCX_TASK_REENQ_KFUNC;
if ((p->scx.flags & SCX_TASK_IMMED) &&
@@ -5901,6 +6011,13 @@ void scx_bypass(struct scx_sched *sch, bool bypass)
if (!scx_is_descendant(scx_task_sched(p), sch))
continue;
+ /*
+ * Bypass trumps protection. Cycling clears for queued
+ * tasks but current task needs explicit stripping.
+ */
+ if (bypass && task_current(rq, p))
+ scx_task_slice_ended(rq, p);
+
/* cycling deq/enq is enough, see the function comment */
scoped_guard (sched_change, p, DEQUEUE_SAVE | DEQUEUE_MOVE) {
/* nothing */ ;
@@ -8196,11 +8313,10 @@ static bool kick_one_cpu(s32 cpu, struct scx_sched_pcpu *pcpu, struct rq *this_r
if (cur_class == &ext_sched_class) {
u64 caps = scx_caps_for_preempt(pcpu->sch, rq, 0);
- if (likely(!scx_missing_caps(pcpu->sch, cpu, caps)))
- scx_set_task_slice(rq->curr, 0);
- else
- __scx_add_event(pcpu->sch,
- SCX_EV_SUB_PREEMPT_DENIED, 1);
+ if (unlikely(scx_missing_caps(pcpu->sch, cpu, caps)))
+ __scx_add_event(pcpu->sch, SCX_EV_SUB_PREEMPT_DENIED, 1);
+ else if (unlikely(!scx_set_task_slice(rq->curr, 0)))
+ __scx_add_event(pcpu->sch, SCX_EV_SLICE_DENIED, 1);
}
cpumask_clear_cpu(cpu, pcpu->cpus_to_preempt);
}
@@ -9191,10 +9307,13 @@ __bpf_kfunc bool scx_bpf_task_set_slice(struct task_struct *p, u64 slice,
/* under the rq lock: apply now, extensions gated on baseline access */
if (slice > p->scx.slice &&
- unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE)))
+ unlikely(scx_missing_caps(sch, cpu_of(locked_rq), SCX_CAP_BASE))) {
+ __scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
+ return true;
+ }
+
+ if (unlikely(!scx_set_task_slice(p, slice)))
__scx_add_event(sch, SCX_EV_SLICE_DENIED, 1);
- else
- scx_set_task_slice(p, slice);
return true;
}
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index f97c2c1cd914..18bbe249e6e3 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -1972,7 +1972,7 @@ void scx_task_iter_start(struct scx_task_iter *iter, struct cgroup *cgrp);
void scx_task_iter_unlock(struct scx_task_iter *iter);
void scx_task_iter_stop(struct scx_task_iter *iter);
struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter);
-void scx_set_task_slice(struct task_struct *p, u64 slice);
+bool scx_set_task_slice(struct task_struct *p, u64 slice);
void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq);
void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p);
void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 08/12] sched_ext: Add bandwidth-limited rescue execution for stranded tasks
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (6 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 07/12] sched_ext: Add SCX_TASK_PROTECTED Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 09/12] sched_ext: Eject the top rescue consumer on overload Tejun Heo
` (3 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
A local DSQ insert lacking the needed caps is diverted to the reject DSQ and
bounced back through ops.enqueue() so the scheduler can re-decide. That
recovery assumes the scheduler has somewhere legal to send the task. When it
doesn't, e.g. when the task's affinity is restricted to cids delegated away,
the task starves until the stall watchdog ejects the scheduler. An exiting
task is worse - it skips ops.enqueue() and the rejection becomes a
self-requeuing cycle that burns the CPU until the watchdog fires.
Add SCX_ENQ_RESCUE, a fallback modifier on local DSQ inserts. When the
insert would be rejected for missing caps, the kernel takes over and runs
the task on the target CPU without consulting the owning scheduler. The
kernel sets the flag itself when enqueueing an exiting task.
Rescue is a last-resort forward-progress backstop with a persistent
disadvantage, not a way around cap enforcement. A per-CPU token bucket
accrues rescue_bandwidth_ppt (default 2%) of CPU time and rescues run one at
a time in arrival order. Each is granted a slice of the rescue_quantum_us
(default 5ms) quantum divided across the waiters, waits at the tail of the
local DSQ claiming no priority, and rejoins its scheduler as a fresh arrival
once the slice is served.
The schedulers keep their normal control over an admitted rescuee and may
preempt or reslice it. Service is measured on CPU time actually received, so
neither shortens the rescue. Prolonged denial escalates - the remaining
slice turns into protected execution (SCX_TASK_PROTECTED) and the rescuee
preempts the current task. Escalation is paced by the same bucket, and
delivered service converges on the configured bandwidth no matter how
aggressively the schedulers dispatch.
Both knobs are root-only and SCX_RESCUE_DISABLE turns rescue off, making
SCX_ENQ_RESCUE inserts reject as usual.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
include/linux/sched/ext.h | 1 +
kernel/sched/ext/ext.c | 112 +++++++++--
kernel/sched/ext/internal.h | 56 +++++-
kernel/sched/ext/sub.c | 376 ++++++++++++++++++++++++++++++++++--
kernel/sched/ext/sub.h | 26 +++
kernel/sched/ext/types.h | 9 +
kernel/sched/sched.h | 12 ++
7 files changed, 559 insertions(+), 33 deletions(-)
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index b519fbc88e17..a6aabbefd185 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -59,6 +59,7 @@ enum scx_dsq_id_flags {
SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2,
SCX_DSQ_BYPASS = SCX_DSQ_FLAG_BUILTIN | 3,
SCX_DSQ_REJECT = SCX_DSQ_FLAG_BUILTIN | 4, /* internal - see find_dsq_for_dispatch() */
+ SCX_DSQ_RESCUE = SCX_DSQ_FLAG_BUILTIN | 5, /* internal - see find_dsq_for_dispatch() */
SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON,
SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU,
};
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index eca507e888fc..1bae142c4e6d 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -111,6 +111,7 @@ static bool dsq_is_rq_owned(struct scx_dispatch_q *dsq)
switch (dsq->id) {
case SCX_DSQ_LOCAL:
case SCX_DSQ_REJECT:
+ case SCX_DSQ_RESCUE:
return true;
default:
return false;
@@ -1320,7 +1321,8 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice)
* @rq: rq @p is on
* @p: task of interest
*
- * End what rides on the slice - the protection.
+ * End what rides on the slice - the protection, and the rescue if @p is being
+ * rescued.
*
* A dequeue normally ends the slice too. The exception is a save/restore pair
* on the running task. Attribute changes like renice cycle the task through
@@ -1328,11 +1330,13 @@ bool scx_set_task_slice(struct task_struct *p, u64 slice)
* queued task instead loses its DSQ position on any dequeue and the slice ends
* with it.
*/
-static void scx_task_slice_ended(struct rq *rq, struct task_struct *p)
+void scx_task_slice_ended(struct rq *rq, struct task_struct *p)
{
lockdep_assert_rq_held(rq);
p->scx.flags &= ~SCX_TASK_PROTECTED;
+ if (unlikely(p == scx_rescuee(rq)))
+ scx_rescue_end(rq);
}
/* request @p's slice to be set to @slice, see the write rules above */
@@ -1432,6 +1436,9 @@ static void update_curr_scx(struct rq *rq)
touch_core_sched(rq, curr);
}
+ if (unlikely(curr == scx_rescuee(rq)))
+ scx_rescue_charge(rq, delta_exec);
+
dl_server_update(&rq->ext_server, delta_exec);
}
@@ -1547,9 +1554,13 @@ static void rq_owned_post_enq(struct scx_sched *sch, struct rq *rq,
{
call_task_dequeue(sch, rq, p, 0);
- /* rejected: kick the deferred reenq, skip wakeup/preemption */
- if (unlikely(dsq->id == SCX_DSQ_REJECT)) {
- schedule_deferred_locked(rq);
+ /*
+ * Only local inserts get the wakeup treatment below. Rejects kick the
+ * deferred reenq and rescue parks are paced by the rescue timer.
+ */
+ if (unlikely(dsq->id != SCX_DSQ_LOCAL)) {
+ if (dsq->id == SCX_DSQ_REJECT)
+ schedule_deferred_locked(rq);
return;
}
@@ -1863,8 +1874,8 @@ static struct scx_dispatch_q *find_dsq_for_dispatch(struct scx_sched *sch,
dsq = find_user_dsq(sch, dsq_id);
/*
- * Built-in DSQs are never inserted into dsq_hash, so REJECT hits the
- * error below. It cannot be reached with an ID.
+ * Built-in DSQs are never inserted into dsq_hash, so REJECT and RESCUE
+ * hit the error below. They cannot be reached with an ID.
*/
if (unlikely(!dsq)) {
scx_error(sch, "non-existent DSQ 0x%llx", dsq_id);
@@ -2056,6 +2067,7 @@ void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
if (!(sch->ops.flags & SCX_OPS_ENQ_EXITING) &&
unlikely(p->flags & PF_EXITING)) {
__scx_add_event(sch, SCX_EV_ENQ_SKIP_EXITING, 1);
+ enq_flags |= SCX_ENQ_RESCUE; /* avoid looping on cap rejection */
goto local;
}
@@ -2324,9 +2336,11 @@ static bool dequeue_task_scx(struct rq *rq, struct task_struct *p, int core_deq_
* information meaningful to the BPF scheduler and can be suppressed by
* skipping the callbacks if the task is !QUEUED.
*/
- if (SCX_HAS_OP(sch, stopping) && task_current(rq, p)) {
+ if (task_current(rq, p) &&
+ (SCX_HAS_OP(sch, stopping) || unlikely(p == scx_rescuee(rq)))) {
update_curr_scx(rq);
- SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
+ if (SCX_HAS_OP(sch, stopping))
+ SCX_CALL_OP_TASK(sch, stopping, rq, p, false);
}
if (SCX_HAS_OP(sch, quiescent) && !task_on_rq_migrating(p))
@@ -2409,8 +2423,9 @@ void scx_move_local_task_to_local_dsq(struct scx_sched *sch, struct task_struct
{
struct scx_dispatch_q *dst_dsq = scx_resolve_local_dsq(sch, dst_rq, p, &enq_flags);
- /* @dsq is locked and @p is on @dst_rq */
- lockdep_assert_held(&src_dsq->lock);
+ /* @p is on @dst_rq, an rq-owned @src_dsq is covered by the rq lock */
+ if (!dsq_is_rq_owned(src_dsq))
+ lockdep_assert_held(&src_dsq->lock);
lockdep_assert_rq_held(dst_rq);
WARN_ON_ONCE(p->scx.holding_cpu >= 0);
@@ -3144,15 +3159,23 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
struct task_struct *next)
{
struct scx_sched *sch = scx_task_sched(p);
+ bool rescue_keep = false;
/* see kick_sync_wait_bal_cb() */
smp_store_release(&rq->scx.kick_sync, rq->scx.kick_sync + 1);
update_curr_scx(rq);
- /* the slice is consumed, protection ends with it */
- if (!p->scx.slice)
- scx_task_slice_ended(rq, p);
+ /*
+ * If the slice is consumed, protection ends with it. A rescuee
+ * preempted beforehand keeps going, see scx_rescue_keep().
+ */
+ if (!p->scx.slice) {
+ if (unlikely(p == scx_rescuee(rq)))
+ rescue_keep = scx_rescue_keep(rq, p);
+ if (!rescue_keep)
+ scx_task_slice_ended(rq, p);
+ }
/* see dequeue_task_scx() on why we skip when !QUEUED */
if (SCX_HAS_OP(sch, stopping) && (p->scx.flags & SCX_TASK_QUEUED))
@@ -3167,15 +3190,34 @@ static void put_prev_task_scx(struct rq *rq, struct task_struct *p,
* forcing a different task. Leave it at the head of the local
* DSQ unless it was an IMMED task. IMMED tasks should not
* linger on a busy CPU, reenqueue them to the BPF scheduler.
+ *
+ * An open rescue must keep @p on the local DSQ even if the
+ * scheduler zeroed the slice in ops.stopping() above.
*/
- if (p->scx.slice && !scx_bypassing(sch, cpu_of(rq))) {
+ if ((p->scx.slice || unlikely(p == scx_rescuee(rq))) &&
+ !scx_bypassing(sch, cpu_of(rq))) {
if (p->scx.flags & SCX_TASK_IMMED) {
p->scx.flags |= SCX_TASK_REENQ_PREEMPTED;
scx_do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
p->scx.flags &= ~SCX_TASK_REENQ_REASON_MASK;
} else {
+ u64 enq_flags = 0;
+
+ /*
+ * Keep a preempted rescue going. If preempted
+ * by another SCX task, append to the local DSQ,
+ * see scx_rescue_keep().
+ */
+ if (unlikely(p == scx_rescuee(rq))) {
+ enq_flags |= SCX_ENQ_IGNORE_CAPS;
+ if (!rescue_keep)
+ enq_flags |= SCX_ENQ_HEAD;
+ } else {
+ enq_flags |= SCX_ENQ_HEAD;
+ }
+
scx_dispatch_enqueue(sch, rq, &rq->scx.local_dsq, p, 0, 0,
- SCX_ENQ_HEAD);
+ enq_flags);
}
goto switch_class;
}
@@ -3575,6 +3617,7 @@ static void rq_online_scx(struct rq *rq)
static void rq_offline_scx(struct rq *rq)
{
rq->scx.flags &= ~SCX_RQ_ONLINE;
+ scx_rescue_flush(rq);
}
static bool check_rq_for_timeouts(struct rq *rq)
@@ -4244,7 +4287,7 @@ static bool local_task_should_reenq(struct rq *rq, struct task_struct *p,
first = !(*reenq_flags & SCX_REENQ_TSR_NOT_FIRST);
*reenq_flags |= SCX_REENQ_TSR_NOT_FIRST;
- if (unlikely(p->scx.flags & SCX_TASK_PROTECTED))
+ if (unlikely((p->scx.flags & SCX_TASK_PROTECTED) || p == scx_rescuee(rq)))
return false;
*reason = SCX_TASK_REENQ_KFUNC;
@@ -4527,6 +4570,13 @@ bool scx_can_stop_tick(struct rq *rq)
if (scx_bypassing(sch, cpu_of(rq)))
return false;
+ /*
+ * A running rescuee's charging and expiry are tick-driven, see
+ * scx_rescue_charge(). Keep the tick while rescue is in progress.
+ */
+ if (unlikely(p == scx_rescuee(rq)))
+ return false;
+
/*
* @rq can dispatch from different DSQs, so we can't tell whether it
* needs the tick or not by looking at nr_running. Allow stopping ticks
@@ -6722,6 +6772,7 @@ static void scx_dump_cpu(struct scx_sched *sch, struct seq_buf *s,
scx_dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu ksync=%lu",
cpu, rq->scx.nr_running, rq->scx.flags, rq->scx.cpu_released,
rq->scx.ops_qseq, rq->scx.kick_sync);
+ scx_rescue_dump(&ns, rq);
scx_dump_line(&ns, " curr=%s[%d] class=%ps",
rq->curr->comm, rq->curr->pid, rq->curr->sched_class);
if (!cpumask_empty(pcpu->cpus_to_kick))
@@ -7393,6 +7444,7 @@ static void scx_root_enable_workfn(struct kthread_work *work)
}
scx_discard_stale_ecaps_syncs();
+ scx_rescue_set_knobs(sch);
/*
* Keep CPUs stable during enable so that the BPF scheduler can track
@@ -7898,6 +7950,24 @@ static int bpf_scx_init_member(const struct btf_type *t,
case offsetof(struct sched_ext_ops, cid_shard_size):
ops->cid_shard_size = *(u32 *)(udata + moff);
return 1;
+ case offsetof(struct sched_ext_ops, rescue_bandwidth_ppt): {
+ u32 bw_ppt = *(u32 *)(udata + moff);
+
+ if (bw_ppt > SCX_RESCUE_MAX_BW_PPT && bw_ppt != SCX_RESCUE_DISABLE)
+ return -E2BIG;
+ ops->rescue_bandwidth_ppt = bw_ppt;
+ return 1;
+ }
+ case offsetof(struct sched_ext_ops, rescue_quantum_us): {
+ u32 quantum_us = *(u32 *)(udata + moff);
+
+ if (quantum_us > SCX_RESCUE_MAX_QUANTUM_US)
+ return -E2BIG;
+ if (quantum_us && quantum_us < SCX_RESCUE_MIN_QUANTUM_US)
+ return -EINVAL;
+ ops->rescue_quantum_us = quantum_us;
+ return 1;
+ }
#ifdef CONFIG_EXT_SUB_SCHED
case offsetof(struct sched_ext_ops, sub_cgroup_id):
ops->sub_cgroup_id = *(u64 *)(udata + moff);
@@ -8520,6 +8590,7 @@ void __init init_sched_ext_class(void)
BUG_ON(scx_init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL, NULL));
#ifdef CONFIG_EXT_SUB_SCHED
BUG_ON(scx_init_dsq(&rq->scx.reject_dsq, SCX_DSQ_REJECT, NULL));
+ scx_rescue_init(rq);
#endif
INIT_LIST_HEAD(&rq->scx.runnable_list);
@@ -8569,6 +8640,11 @@ static bool scx_vet_enq_flags(struct scx_sched *sch, u64 dsq_id, u64 *enq_flags)
*enq_flags |= SCX_ENQ_IMMED;
}
+ if (unlikely((*enq_flags & SCX_ENQ_RESCUE) && !is_local)) {
+ scx_error(sch, "SCX_ENQ_RESCUE on a non-local DSQ 0x%llx", dsq_id);
+ return false;
+ }
+
return true;
}
@@ -10750,6 +10826,8 @@ static int __init scx_init(void)
CID_OFFSET_MATCH(exit_dump_len, exit_dump_len);
CID_OFFSET_MATCH(hotplug_seq, hotplug_seq);
CID_OFFSET_MATCH(cid_shard_size, cid_shard_size);
+ CID_OFFSET_MATCH(rescue_bandwidth_ppt, rescue_bandwidth_ppt);
+ CID_OFFSET_MATCH(rescue_quantum_us, rescue_quantum_us);
CID_OFFSET_MATCH(sub_cgroup_id, sub_cgroup_id);
/* shared callbacks: the union view requires byte-for-byte offset match */
CID_OFFSET_MATCH(enqueue, enqueue);
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 18bbe249e6e3..7a2b0357b04f 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -924,6 +924,37 @@ struct sched_ext_ops {
*/
u32 cid_shard_size;
+ /**
+ * @rescue_bandwidth_ppt: Rescue execution bandwidth in parts per thousand
+ *
+ * The fraction of each CPU's time that may be consumed running tasks
+ * from its rescue DSQ. A higher bandwidth admits and escalates rescues
+ * faster, see @rescue_quantum_us.
+ *
+ * Only the root scheduler's value is used. 0 means the default of 20
+ * (2%). May not exceed 250 (25%). %SCX_RESCUE_DISABLE disables rescue -
+ * %SCX_ENQ_RESCUE inserts are then rejected like any other insert
+ * lacking the caps.
+ */
+ u32 rescue_bandwidth_ppt;
+
+ /**
+ * @rescue_quantum_us: Rescue execution quantum in microseconds
+ *
+ * How much CPU time each rescue gets. Rescues run one at a time per CPU
+ * and admissions are paced to keep rescue execution within
+ * @rescue_bandwidth_ppt - with the defaults, one 5ms rescue every
+ * 250ms. A crowded queue round-robins on the quantum divided across the
+ * waiters, floored at 1ms. A stuck rescue eventually escalates to
+ * forced execution. A larger quantum interrupts the CPU less often but
+ * for longer and spaces rescues further apart.
+ *
+ * Only the root scheduler's value is used. 0 means the default (5000).
+ * Non-zero values must be within [1000, 100000]. Values too short for
+ * the kernel to meter are lifted silently.
+ */
+ u32 rescue_quantum_us;
+
/**
* @cgroup_id: When >1, attach the scheduler as a sub-scheduler on the
* specified cgroup.
@@ -1058,6 +1089,8 @@ struct sched_ext_ops_cid {
u32 exit_dump_len;
u64 hotplug_seq;
u32 cid_shard_size;
+ u32 rescue_bandwidth_ppt;
+ u32 rescue_quantum_us;
u64 sub_cgroup_id;
char name[SCX_OPS_NAME_LEN];
@@ -1211,6 +1244,12 @@ struct scx_event_stats {
* sub-sched lacked SCX_CAP_PERF on the target cid.
*/
s64 SCX_EV_SUB_CIDPERF_DENIED;
+
+ /*
+ * The number of times an insert carrying %SCX_ENQ_RESCUE lacked the
+ * caps for its cid and the task entered the rescue path.
+ */
+ s64 SCX_EV_SUB_RESCUE;
};
#define SCX_EVENTS_LIST(SCX_EVENT) \
@@ -1233,7 +1272,8 @@ struct scx_event_stats {
SCX_EVENT(SCX_EV_SUB_PREEMPT_DENIED); \
SCX_EVENT(SCX_EV_SUB_KICK_DENIED); \
SCX_EVENT(SCX_EV_SUB_REENQ_DENIED); \
- SCX_EVENT(SCX_EV_SUB_CIDPERF_DENIED)
+ SCX_EVENT(SCX_EV_SUB_CIDPERF_DENIED); \
+ SCX_EVENT(SCX_EV_SUB_RESCUE)
struct scx_sched;
@@ -1656,6 +1696,17 @@ enum scx_enq_flags {
*/
SCX_ENQ_IMMED = 1LLU << 33,
+ /*
+ * Only allowed on local DSQs. If the insert lacks the caps for the
+ * target cid, divert the task to the CPU's rescue path instead of
+ * rejecting and reenqueueing, e.g. when the task's affinity is
+ * restricted to cids the scheduler doesn't hold. The kernel runs
+ * rescued tasks on the target CPU. Rescue execution is guaranteed to
+ * make forward progress and is bandwidth-limited, see the
+ * rescue_bandwidth_ppt and rescue_quantum_us ops fields.
+ */
+ SCX_ENQ_RESCUE = 1LLU << 34,
+
/*
* The task being enqueued was previously enqueued on a DSQ, but was
* removed and is being re-enqueued. See SCX_TASK_REENQ_* flags to find
@@ -1973,6 +2024,7 @@ void scx_task_iter_unlock(struct scx_task_iter *iter);
void scx_task_iter_stop(struct scx_task_iter *iter);
struct task_struct *scx_task_iter_next_locked(struct scx_task_iter *iter);
bool scx_set_task_slice(struct task_struct *p, u64 slice);
+void scx_task_slice_ended(struct rq *rq, struct task_struct *p);
void scx_task_unlink_from_dsq(struct task_struct *p, struct scx_dispatch_q *dsq);
void scx_dispatch_dequeue(struct rq *rq, struct task_struct *p);
void scx_do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
@@ -2371,6 +2423,7 @@ static inline struct scx_sched *scx_parent(struct scx_sched *sch)
else
return NULL;
}
+
#else /* CONFIG_EXT_SUB_SCHED */
static inline bool scx_has_subs(void) { return false; }
@@ -2402,6 +2455,7 @@ static inline struct scx_sched *scx_prog_sched(const struct bpf_prog_aux *aux)
}
static inline struct scx_sched *scx_parent(struct scx_sched *sch) { return NULL; }
+
#endif /* CONFIG_EXT_SUB_SCHED */
#endif /* _KERNEL_SCHED_EXT_INTERNAL_H */
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index c30f48ee07f9..3c1f11268e7f 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -28,6 +28,11 @@
*/
DEFINE_STATIC_KEY_FALSE(__scx_has_subs);
+/* latched at root enable before any rescue runs */
+static s32 scx_rescue_bw_1024;
+static s64 scx_rescue_quantum_ns;
+static s64 scx_rescue_sat_delta_ns;
+
/**
* scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
* @pos: current position
@@ -229,18 +234,350 @@ void scx_init_root_caps(struct scx_sched *sch)
}
}
+/* unserved remainder of @rq's rescuee's admitted slice, 0 once fully served */
+static s64 scx_rescue_slice_remaining(struct rq *rq)
+{
+ s64 served = rq->scx.rescue.curr->se.sum_exec_runtime - rq->scx.rescue.exec_snap;
+
+ return max(rq->scx.rescue.slice - served, 0);
+}
+
+/**
+ * scx_rescue_charge - Charge the rescuee's runtime
+ * @rq: rq the rescuee is running on
+ * @delta_exec: runtime being charged
+ *
+ * Also ends the rescue once the admitted slice has been served in full. Ending
+ * on served time rather than slice exhaustion bounds both the rescue and the
+ * charging when a scheduler extends the rescuee's slice.
+ */
+void scx_rescue_charge(struct rq *rq, s64 delta_exec)
+{
+ lockdep_assert_rq_held(rq);
+
+ /*
+ * A rescue slice is bounded by one quantum and tick-driven expiry can
+ * overshoot by up to a tick. Clamp to avoid wild over-charges on VMs.
+ */
+ delta_exec = min_t(s64, delta_exec, scx_rescue_quantum_ns + TICK_NSEC);
+
+ rq->scx.rescue.budget -= delta_exec;
+
+ if (!scx_rescue_slice_remaining(rq))
+ scx_task_slice_ended(rq, rq->scx.rescue.curr);
+}
+
/**
- * scx_resolve_local_dsq - Pick the local or reject DSQ for an insert
+ * scx_rescue_end - End the rescue execution on @rq
+ * @rq: rq of interest
+ *
+ * When no rescuee is left pending, the session is over and the balance above
+ * one quantum dies with it - it would otherwise become a banked license to
+ * preempt the cid owner long after the starvation ended. While waiters remain,
+ * the accrued deficit belongs to the queue and carries into the next rescue.
+ */
+void scx_rescue_end(struct rq *rq)
+{
+ lockdep_assert_rq_held(rq);
+
+ rq->scx.rescue.curr = NULL;
+ if (list_empty(&rq->scx.rescue.dsq.list))
+ rq->scx.rescue.budget = min(rq->scx.rescue.budget, scx_rescue_quantum_ns);
+}
+
+/**
+ * scx_rescue_keep - Keep the rescue going for a preempted-out rescuee
+ * @rq: rq @p is running on
+ * @p: task under rescue whose slice is exhausted
+ *
+ * Called from put_prev_task_scx() to decide what an exhausted slice means for
+ * the rescuee. scx_rescue_charge() ends the rescue the moment the admitted
+ * slice is fully served, so arriving here with the rescue still open means @p
+ * was preempted. Restore the unserved remainder and return %true - @p stays the
+ * rescuee and the caller reinserts it at the tail of the local DSQ, behind
+ * whatever preempted the rescuee.
+ *
+ * Return %false to end the rescue instead - the slice is already fully served,
+ * @p is leaving the rq or bypass is dismantling rescues.
+ */
+bool scx_rescue_keep(struct rq *rq, struct task_struct *p)
+{
+ s64 remaining = scx_rescue_slice_remaining(rq);
+
+ lockdep_assert_rq_held(rq);
+
+ if (!remaining || !(p->scx.flags & SCX_TASK_QUEUED) ||
+ scx_bypassing(scx_task_sched(p), cpu_of(rq)))
+ return false;
+
+ scx_set_task_slice(p, remaining);
+ return true;
+}
+
+/**
+ * scx_rescue_accrue - Accrue budget at the configured fraction of elapsed time
+ * @rq: rq of interest
+ *
+ * A session spans from the first arrival until no rescuee is left, pending or
+ * admitted. While one is active the cap is three quanta and the balance drives
+ * escalation, see scx_rescue_timerfn(). Outside a session the cap is one
+ * quantum, so an idle gap funds the next arrival's admission but never an
+ * escalation.
+ */
+static void scx_rescue_accrue(struct rq *rq)
+{
+ bool in_session = rq->scx.rescue.curr || !list_empty(&rq->scx.rescue.dsq.list);
+ s64 cap = in_session ? 3 * scx_rescue_quantum_ns : scx_rescue_quantum_ns;
+ s64 delta;
+ u64 now;
+
+ lockdep_assert_rq_held(rq);
+
+ /* not every path here holds an updated rq clock, use __scx_bpf_now() */
+ now = __scx_bpf_now(rq);
+ delta = now - rq->scx.rescue.clock;
+ rq->scx.rescue.clock = now;
+
+ /*
+ * Avoid multiplication overflows by taking a shortcut when the gap is
+ * large enough to fill the budget.
+ */
+ if (delta >= scx_rescue_sat_delta_ns)
+ rq->scx.rescue.budget = cap;
+ else
+ rq->scx.rescue.budget =
+ min(cap, rq->scx.rescue.budget +
+ ((delta * scx_rescue_bw_1024) >> SCHED_CAPACITY_SHIFT));
+}
+
+/*
+ * The slice for the next admission - the quantum divided across the stranded
+ * tasks so that a crowded queue round-robins on shorter slices.
+ */
+static s64 scx_rescue_next_slice(struct rq *rq)
+{
+ s64 min_slice = max_t(s64, SCX_RESCUE_MIN_SLICE_US * NSEC_PER_USEC, TICK_NSEC);
+ u32 depth = rq->scx.rescue.dsq.nr ?: 1;
+
+ return clamp(div_s64(scx_rescue_quantum_ns, depth), min_slice, scx_rescue_quantum_ns);
+}
+
+static void scx_rescue_timer_arm(struct rq *rq)
+{
+ struct timer_list *timer = &rq->scx.rescue.timer;
+ s64 delay = scx_rescue_quantum_ns / 4; /* should be granular enough */
+
+ if (timer_pending(timer))
+ return;
+
+ /*
+ * While the head waiter can't be admitted because the bucket is short
+ * of a full quantum, stretch to the full funding delay.
+ */
+ if (!rq->scx.rescue.curr && rq->scx.rescue.budget < scx_rescue_quantum_ns) {
+ s64 deficit = scx_rescue_quantum_ns - rq->scx.rescue.budget;
+
+ delay = max(delay,
+ div_s64(deficit << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024));
+ }
+
+ /* +1 rounds up so the beat is due by the time the timer fires */
+ timer->expires = jiffies + nsecs_to_jiffies(delay) + 1;
+ add_timer_on(timer, cpu_of(rq));
+}
+
+/**
+ * scx_rescue_admit - Start rescuing @p on @rq
+ * @rq: rq @p is being admitted on
+ * @p: task being admitted, off any DSQ
+ * @slice: CPU time to grant
+ *
+ * The schedulers keep their normal control over @p and may preempt or reslice
+ * it. @slice is measured on served CPU time against the snapshot taken here, so
+ * neither shortens the rescue, see scx_rescue_charge() and scx_rescue_keep().
+ * Prolonged denial escalates into protected execution, see
+ * scx_rescue_timerfn().
+ */
+static void scx_rescue_admit(struct rq *rq, struct task_struct *p, s64 slice)
+{
+ lockdep_assert_rq_held(rq);
+ WARN_ON_ONCE(rq->scx.rescue.curr);
+
+ rq->scx.rescue.curr = p;
+ rq->scx.rescue.slice = slice;
+ rq->scx.rescue.exec_snap = p->se.sum_exec_runtime;
+ scx_set_task_slice(p, slice);
+ scx_rescue_timer_arm(rq);
+}
+
+/**
+ * scx_rescue_try_admit - Try to admit a freshly stranded task
+ * @rq: rq @p is being inserted on
+ * @p: stranded task being diverted to rescue
+ *
+ * One rescue at a time and earlier arrivals go first. Admission needs a full
+ * quantum of budget, spent as the rescue runs. Return %true if @p was admitted
+ * and should be inserted at the tail of @rq's local DSQ, %false if it has to
+ * park on the rescue DSQ, with the timer armed to admit it later.
+ */
+static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p)
+{
+ scx_rescue_accrue(rq);
+
+ if (!rq->scx.rescue.curr && list_empty(&rq->scx.rescue.dsq.list) &&
+ rq->scx.rescue.budget >= scx_rescue_quantum_ns) {
+ scx_rescue_admit(rq, p, scx_rescue_quantum_ns);
+ return true;
+ }
+
+ scx_rescue_timer_arm(rq);
+ return false;
+}
+
+/**
+ * scx_rescue_timerfn - Drive and pace rescue execution
+ * @timer: rq->scx.rescue.timer
+ *
+ * Runs every quarter quantum while a rescuee exists, pending or admitted, see
+ * scx_rescue_timer_arm(). The head waiter is admitted once the bucket holds a
+ * full quantum and granted its slice, see scx_rescue_next_slice(). A session
+ * whose budget accumulates over two quanta with the admitted rescuee still
+ * waiting escalates - the rescuee's remaining slice turns into protected
+ * execution and it preempts the current task.
+ */
+static void scx_rescue_timerfn(struct timer_list *timer)
+{
+ struct rq *rq = timer_container_of(rq, timer, scx.rescue.timer);
+ struct task_struct *p;
+
+ guard(rq_lock_irqsave)(rq);
+
+ p = rq->scx.rescue.curr;
+ if (!p && list_empty(&rq->scx.rescue.dsq.list))
+ return;
+
+ scx_rescue_accrue(rq);
+
+ if (!p) {
+ s64 slice = scx_rescue_next_slice(rq);
+
+ /* no rescue in progress */
+ if (rq->scx.rescue.budget < scx_rescue_quantum_ns)
+ goto out_arm;
+
+ /* there's enough budget to start rescuing the next one */
+ p = list_first_entry(&rq->scx.rescue.dsq.list, struct task_struct,
+ scx.dsq_list.node);
+ scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
+ scx_rescue_admit(rq, p, slice);
+ scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS,
+ &rq->scx.rescue.dsq, rq);
+ if (sched_class_above(&ext_sched_class, rq->curr->sched_class))
+ resched_curr(rq);
+ } else if (p->scx.dsq && rq->scx.rescue.budget > 2 * scx_rescue_quantum_ns) {
+ /*
+ * The rescuee waited for the CPU for too long. Escalate - grant
+ * the unserved remainder, protect it from the schedulers and
+ * preempt the current task. The slice is set before the
+ * protection. Repeat beats only repeat the head move - the
+ * slice write is refused on a protected task.
+ */
+ scx_set_task_slice(p, scx_rescue_slice_remaining(rq));
+ p->scx.flags |= SCX_TASK_PROTECTED;
+ scx_task_unlink_from_dsq(p, &rq->scx.local_dsq);
+ scx_move_local_task_to_local_dsq(scx_task_sched(p), p,
+ SCX_ENQ_HEAD | SCX_ENQ_PREEMPT | SCX_ENQ_IGNORE_CAPS,
+ &rq->scx.local_dsq, rq);
+ }
+out_arm:
+ scx_rescue_timer_arm(rq);
+}
+
+/* flush out tasks waiting for rescue before a CPU goes down */
+void scx_rescue_flush(struct rq *rq)
+{
+ struct task_struct *p, *n;
+
+ lockdep_assert_rq_held(rq);
+
+ /* sched domain rebuilds call rq_offline with the CPU staying alive */
+ if (cpu_active(cpu_of(rq)))
+ return;
+
+ /* end the current rescue */
+ if (rq->scx.rescue.curr)
+ scx_task_slice_ended(rq, rq->scx.rescue.curr);
+
+ /* and flush out all pending ones */
+ list_for_each_entry_safe(p, n, &rq->scx.rescue.dsq.list, scx.dsq_list.node) {
+ scx_task_unlink_from_dsq(p, &rq->scx.rescue.dsq);
+ scx_move_local_task_to_local_dsq(scx_task_sched(p), p, SCX_ENQ_IGNORE_CAPS,
+ &rq->scx.rescue.dsq, rq);
+ }
+
+ timer_delete(&rq->scx.rescue.timer);
+}
+
+void scx_rescue_dump(struct seq_buf *s, struct rq *rq)
+{
+ struct task_struct *p = rq->scx.rescue.curr;
+
+ scx_dump_line(s, " rescue=%u budget=%lldus rescuing=%s[%d]",
+ rq->scx.rescue.dsq.nr,
+ div_s64(rq->scx.rescue.budget, NSEC_PER_USEC),
+ p ? p->comm : "none", p ? p->pid : -1);
+}
+
+/* latch the rescue parameters on root scheduler enable */
+void scx_rescue_set_knobs(struct scx_sched *sch)
+{
+ s32 bw_ppt = sch->ops.rescue_bandwidth_ppt ?: SCX_RESCUE_DFL_BW_PPT;
+ s64 quantum_us = sch->ops.rescue_quantum_us ?: SCX_RESCUE_DFL_QUANTUM_US;
+
+ if (sch->ops.rescue_bandwidth_ppt == SCX_RESCUE_DISABLE) {
+ scx_rescue_bw_1024 = 0;
+ return;
+ }
+
+ scx_rescue_bw_1024 = bw_ppt * SCHED_CAPACITY_SCALE / 1000;
+ scx_rescue_quantum_ns = max(quantum_us * NSEC_PER_USEC, TICK_NSEC);
+ scx_rescue_sat_delta_ns =
+ div_s64((4 * scx_rescue_quantum_ns + TICK_NSEC) << SCHED_CAPACITY_SHIFT,
+ scx_rescue_bw_1024);
+
+ /*
+ * A rescued task is guaranteed to run after two full periods - one to
+ * be admitted, one more to escalate. Require the two periods to fit in
+ * a quarter of the watchdog timeout, so one full period may take at
+ * most an eighth.
+ */
+ if (div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024) >
+ jiffies_to_nsecs(sch->watchdog_timeout) / 8)
+ pr_warn("sched_ext: rescue may not run a stuck task before the %ums watchdog timeout, decrease rescue_quantum_us or increase rescue_bandwidth_ppt\n",
+ jiffies_to_msecs(sch->watchdog_timeout));
+}
+
+void scx_rescue_init(struct rq *rq)
+{
+ BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL));
+ timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED);
+}
+
+/**
+ * scx_resolve_local_dsq - Pick the local, rescue or reject DSQ for an insert
* @sch: enqueuing sub-sched
* @rq: rq whose local DSQ @p targets
* @p: task being inserted
* @enq_flags: in/out, unhonored flags are cleared
*
- * Return @rq's local DSQ if @sch holds the required caps on @rq's cid,
- * otherwise @rq's reject DSQ after recording the reenq reason on @p.
+ * Return @rq's local DSQ if @sch holds the required caps on @rq's cid.
+ * Otherwise, return @rq's rescue DSQ if the insert carries %SCX_ENQ_RESCUE and
+ * rescue is enabled, or @rq's reject DSQ after recording the reenq reason on
+ * @p.
*
- * %SCX_ENQ_IMMED and %SCX_ENQ_PREEMPT are cleared when diverting to reject.
- * %SCX_ENQ_PREEMPT is also cleared on a fallback migration-disabled admission.
+ * %SCX_ENQ_IMMED, %SCX_ENQ_PREEMPT and %SCX_ENQ_HEAD are cleared when diverting
+ * to rescue or reject. %SCX_ENQ_PREEMPT is also cleared on a fallback
+ * migration-disabled admission.
*
* Bypass doesn't need special-casing as a bypassing sched's tasks are enqueued
* to and run by its nearest non-bypassing ancestor. If root is bypassing, it
@@ -282,18 +619,27 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
return &rq->scx.local_dsq;
}
- p->scx.reenq_reason_caps = missing;
- p->scx.reenq_reason_cid = cid;
-
/*
- * Only local DSQ can honor IMMED and dsq_inc_nr() WARNs on IMMED into
- * others. Strip both the enq flag and the sticky task flag - the
- * latter can carry in from an earlier admitted IMMED insert. Strip
- * PREEMPT too.
+ * Diverting to rescue or reject, neither of which honors IMMED, PREEMPT
+ * or HEAD - a diversion has no priority and IMMED is not allowed on
+ * non-local DSQs. Strip the enq and task flags along with the slice.
*/
- *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT);
+ *enq_flags &= ~(SCX_ENQ_IMMED | SCX_ENQ_PREEMPT | SCX_ENQ_HEAD |
+ SCX_ENQ_APPLY_SLICE | SCX_ENQ_SLICE_DFL);
p->scx.flags &= ~SCX_TASK_IMMED;
+ /* the enqueuer opted for rescue instead of rejection and reenqueue */
+ if ((*enq_flags & SCX_ENQ_RESCUE) && likely(scx_rescue_bw_1024)) {
+ __scx_add_event(sch, SCX_EV_SUB_RESCUE, 1);
+ if (scx_rescue_try_admit(rq, p))
+ return &rq->scx.local_dsq;
+ else
+ return &rq->scx.rescue.dsq;
+ }
+
+ p->scx.reenq_reason_caps = missing;
+ p->scx.reenq_reason_cid = cid;
+
return &rq->scx.reject_dsq;
}
@@ -302,8 +648,8 @@ bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p)
{
u64 missing;
- /* migration-disabled tasks are admitted regardless of caps */
- if (is_migration_disabled(p))
+ /* migration-disabled tasks and the rescuee are admitted capless */
+ if (is_migration_disabled(p) || p == scx_rescuee(rq))
return false;
missing = scx_missing_caps(scx_task_sched(p), cpu_of(rq), scx_caps_for_task(p));
diff --git a/kernel/sched/ext/sub.h b/kernel/sched/ext/sub.h
index fe1d82e6c1d5..f7bcdfda8dd8 100644
--- a/kernel/sched/ext/sub.h
+++ b/kernel/sched/ext/sub.h
@@ -38,6 +38,13 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
struct task_struct *p, u64 *enq_flags);
bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p);
void scx_reenq_reject(struct rq *rq);
+void scx_rescue_charge(struct rq *rq, s64 delta_exec);
+void scx_rescue_end(struct rq *rq);
+bool scx_rescue_keep(struct rq *rq, struct task_struct *p);
+void scx_rescue_flush(struct rq *rq);
+void scx_rescue_dump(struct seq_buf *s, struct rq *rq);
+void scx_rescue_set_knobs(struct scx_sched *sch);
+void scx_rescue_init(struct rq *rq);
/*
* cgrp->scx_sched is written by root/sub enable/disable under all of
@@ -87,6 +94,13 @@ static inline void scx_discard_stale_ecaps_syncs(void) {}
static inline struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *rq, struct task_struct *p, u64 *enq_flags) { return &rq->scx.local_dsq; }
static inline bool scx_task_reenq_on_cap_revoke(struct rq *rq, struct task_struct *p) { return false; }
static inline void scx_reenq_reject(struct rq *rq) {}
+static inline void scx_rescue_charge(struct rq *rq, s64 delta_exec) {}
+static inline void scx_rescue_end(struct rq *rq) {}
+static inline bool scx_rescue_keep(struct rq *rq, struct task_struct *p) { return false; }
+static inline void scx_rescue_flush(struct rq *rq) {}
+static inline void scx_rescue_dump(struct seq_buf *s, struct rq *rq) {}
+static inline void scx_rescue_set_knobs(struct scx_sched *sch) {}
+static inline void scx_rescue_init(struct rq *rq) {}
static inline void scx_dec_has_subs(struct scx_sched *sch) {}
#endif /* CONFIG_EXT_SUB_SCHED */
@@ -195,11 +209,23 @@ static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p
return likely(!scx_missing_caps(scx_task_sched(p), cpu_of(rq), SCX_CAP_BASE));
}
+/* the task admitted for rescue on @rq, NULL if none */
+static inline struct task_struct *scx_rescuee(struct rq *rq)
+{
+ lockdep_assert_rq_held(rq);
+
+ if (!scx_has_subs())
+ return NULL;
+
+ return rq->scx.rescue.curr;
+}
+
#else /* CONFIG_EXT_SUB_SCHED */
static inline u64 scx_missing_caps(struct scx_sched *sch, s32 cpu, u64 needed) { return 0; }
static inline u64 scx_caps_for_preempt(struct scx_sched *sch, struct rq *rq, u64 enq_flags) { return 0; }
static inline bool scx_task_can_stay_on_cpu(struct rq *rq, struct task_struct *p) { return true; }
+static inline struct task_struct *scx_rescuee(struct rq *rq) { return NULL; }
#endif /* CONFIG_EXT_SUB_SCHED */
diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h
index b94ddee21c57..d39588717e9b 100644
--- a/kernel/sched/ext/types.h
+++ b/kernel/sched/ext/types.h
@@ -19,6 +19,15 @@ enum scx_consts {
SCX_DSP_MAX_LOOPS = 32,
SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ,
+ /* rescue knob defaults and limits, see scx_rescue_timerfn() */
+ SCX_RESCUE_DFL_BW_PPT = 20, /* parts per thousand, 2% */
+ SCX_RESCUE_MAX_BW_PPT = 250, /* 25% */
+ SCX_RESCUE_DISABLE = U32_MAX, /* disables rescue */
+ SCX_RESCUE_DFL_QUANTUM_US = 5000,
+ SCX_RESCUE_MIN_QUANTUM_US = 1000,
+ SCX_RESCUE_MAX_QUANTUM_US = 100000,
+ SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */
+
/* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */
SCX_TID_CHUNK = 1024,
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index c0cb879d75f0..289e298df628 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -794,10 +794,22 @@ enum scx_rq_flags {
SCX_RQ_IN_BALANCE = 1 << 17,
};
+/* per-rq rescue execution state, see scx_rescue_timerfn() */
+struct scx_rq_rescue {
+ struct scx_dispatch_q dsq; /* stranded tasks awaiting rescue */
+ s64 budget; /* execution token bucket, ns */
+ u64 clock; /* last budget accrual timestamp */
+ struct task_struct *curr; /* task being rescued, one at a time */
+ s64 slice; /* curr's admitted slice */
+ u64 exec_snap; /* sum_exec_runtime at admission */
+ struct timer_list timer; /* paces admission and escalation */
+};
+
struct scx_rq {
struct scx_dispatch_q local_dsq;
#ifdef CONFIG_EXT_SUB_SCHED
struct scx_dispatch_q reject_dsq; /* staging for cap-rejected tasks */
+ struct scx_rq_rescue rescue;
#endif
struct list_head runnable_list; /* runnable tasks on this rq */
struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 09/12] sched_ext: Eject the top rescue consumer on overload
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (7 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 08/12] sched_ext: Add bandwidth-limited rescue execution for stranded tasks Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 10/12] sched_ext: Sync tools autogen enum headers Tejun Heo
` (2 subsequent siblings)
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
When rescue demand on a cpu persistently exceeds the configured bandwidth,
tasks age on that cpu's rescue DSQ until the stall watchdog fires. The
watchdog blames the waiting task's owner, but the misbehaving party is
whoever floods the queue, not whoever happens to time out.
Track each sched's recent rescue consumption per cpu as a decaying average.
Once the oldest waiter on a cpu's rescue DSQ has been queued past a
threshold derived from the rescue knobs (4s at the defaults), the rescue
timer ejects the sub with the highest recent consumption on that cpu with
SCX_EXIT_ERROR_RESCUE. With no recent consumer there is no victim and
nothing is ejected - the generic stall watchdog eventually blames the
waiter's owner instead. Ejections on a cpu are spaced one threshold apart so
the freed bandwidth can drain the backlog before another sub is judged.
The overload check only wins the race against the stall watchdog when the
watchdog timeout clears the threshold, and a single in-budget wait must not
cross the trigger on its own. Warn on a scheduler whose timeout doesn't fit
and on knobs whose funding period exceeds half the threshold.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
include/linux/sched/ext.h | 3 +
kernel/sched/ext/ext.c | 2 +
kernel/sched/ext/internal.h | 9 +++
kernel/sched/ext/sub.c | 136 +++++++++++++++++++++++++++++++++---
kernel/sched/ext/types.h | 3 +
kernel/sched/sched.h | 1 +
6 files changed, 143 insertions(+), 11 deletions(-)
diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
index a6aabbefd185..a3ec980e2925 100644
--- a/include/linux/sched/ext.h
+++ b/include/linux/sched/ext.h
@@ -215,6 +215,9 @@ struct sched_ext_entity {
#ifdef CONFIG_SCHED_CORE
u64 core_sched_at; /* see scx_prio_less() */
#endif
+#ifdef CONFIG_EXT_SUB_SCHED
+ unsigned long rescue_at; /* queued on a rescue DSQ at, jiffies */
+#endif
/*
* Unique non-zero task ID assigned at fork. Persists across exec and
diff --git a/kernel/sched/ext/ext.c b/kernel/sched/ext/ext.c
index 1bae142c4e6d..89b490b287b9 100644
--- a/kernel/sched/ext/ext.c
+++ b/kernel/sched/ext/ext.c
@@ -6140,6 +6140,8 @@ static const char *scx_exit_reason(enum scx_exit_kind kind)
return "runnable task stall";
case SCX_EXIT_ERROR_REENQ:
return "reenqueue limit";
+ case SCX_EXIT_ERROR_RESCUE:
+ return "rescue bandwidth overload";
default:
return "<UNKNOWN>";
}
diff --git a/kernel/sched/ext/internal.h b/kernel/sched/ext/internal.h
index 7a2b0357b04f..75ec512d664f 100644
--- a/kernel/sched/ext/internal.h
+++ b/kernel/sched/ext/internal.h
@@ -57,6 +57,7 @@ enum scx_exit_kind {
SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */
SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */
SCX_EXIT_ERROR_REENQ, /* task hit reenqueue limit without running */
+ SCX_EXIT_ERROR_RESCUE, /* ejected for overloading rescue execution */
};
/*
@@ -1340,6 +1341,14 @@ struct scx_sched_pcpu {
bool idle_renotify;
/* effective caps as of the last sub_ecaps_updated() delivery */
u64 reported_ecaps;
+
+ /*
+ * Decaying rescue runtime consumed on this cpu, see
+ * scx_rescue_decay_avg(). Overload on this cpu ejects the sub with the
+ * largest value. Accessed only under this cpu's rq lock.
+ */
+ u64 rescue_avg;
+ unsigned long rescue_avg_at; /* last decay, jiffies */
#endif
/*
diff --git a/kernel/sched/ext/sub.c b/kernel/sched/ext/sub.c
index 3c1f11268e7f..fdbe1c1bfaa8 100644
--- a/kernel/sched/ext/sub.c
+++ b/kernel/sched/ext/sub.c
@@ -32,6 +32,8 @@ DEFINE_STATIC_KEY_FALSE(__scx_has_subs);
static s32 scx_rescue_bw_1024;
static s64 scx_rescue_quantum_ns;
static s64 scx_rescue_sat_delta_ns;
+static unsigned long scx_rescue_decay_halflife;
+static unsigned long scx_rescue_overload_after;
/**
* scx_skip_subtree_pre - Skip @pos's subtree in a pre-order walk
@@ -242,6 +244,23 @@ static s64 scx_rescue_slice_remaining(struct rq *rq)
return max(rq->scx.rescue.slice - served, 0);
}
+/*
+ * Decay @pcpu's rescue usage average in place, halving per the knob-derived
+ * halflife, see scx_rescue_set_knobs(). The timestamp advances only by whole
+ * halflives.
+ */
+static u64 scx_rescue_decay_avg(struct scx_sched_pcpu *pcpu)
+{
+ unsigned long halflife = scx_rescue_decay_halflife;
+ unsigned long n = (jiffies - pcpu->rescue_avg_at) / halflife;
+
+ if (n) {
+ pcpu->rescue_avg = n < 64 ? pcpu->rescue_avg >> n : 0;
+ pcpu->rescue_avg_at += n * halflife;
+ }
+ return pcpu->rescue_avg;
+}
+
/**
* scx_rescue_charge - Charge the rescuee's runtime
* @rq: rq the rescuee is running on
@@ -253,6 +272,8 @@ static s64 scx_rescue_slice_remaining(struct rq *rq)
*/
void scx_rescue_charge(struct rq *rq, s64 delta_exec)
{
+ struct scx_sched_pcpu *pcpu;
+
lockdep_assert_rq_held(rq);
/*
@@ -263,6 +284,10 @@ void scx_rescue_charge(struct rq *rq, s64 delta_exec)
rq->scx.rescue.budget -= delta_exec;
+ /* per-cpu usage average feeds the overload victim pick */
+ pcpu = per_cpu_ptr(scx_task_sched(rq->curr)->pcpu, cpu_of(rq));
+ pcpu->rescue_avg = scx_rescue_decay_avg(pcpu) + delta_exec;
+
if (!scx_rescue_slice_remaining(rq))
scx_task_slice_ended(rq, rq->scx.rescue.curr);
}
@@ -434,6 +459,63 @@ static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p)
return false;
}
+/**
+ * scx_rescue_check_overload - Eject the top rescue consumer on a stuck rescue
+ * @rq: rq whose rescue timer fired
+ *
+ * If the oldest waiter on @rq's rescue DSQ has been queued for too long, rescue
+ * demand on this cpu persistently exceeds the configured bandwidth. Eject the
+ * sub with the highest recent rescue consumption instead of letting the
+ * scheduler stall path blame the waiter's owner, who may just be crowded out.
+ */
+static void scx_rescue_check_overload(struct rq *rq)
+{
+ struct scx_sched *victim = NULL, *pos;
+ struct task_struct *p;
+ int cpu = cpu_of(rq);
+ u64 max_avg = 0;
+ u32 dur_ms;
+
+ lockdep_assert_rq_held(rq);
+
+ p = list_first_entry_or_null(&rq->scx.rescue.dsq.list, struct task_struct,
+ scx.dsq_list.node);
+ if (!p)
+ return;
+
+ /* has the head waiter been queued for longer than the threshold? */
+ if (time_before(jiffies, p->scx.rescue_at + scx_rescue_overload_after))
+ return;
+
+ /*
+ * Grace period after the last ejection on this cpu - the freed
+ * bandwidth gets one threshold's worth of time to drain the backlog
+ * before another sub is judged.
+ */
+ if (time_before(jiffies, rq->scx.rescue.kill_at + scx_rescue_overload_after))
+ return;
+
+ list_for_each_entry_rcu(pos, &scx_sched_all, all) {
+ u64 avg = scx_rescue_decay_avg(per_cpu_ptr(pos->pcpu, cpu));
+
+ /* skip an already-exiting sub, else the ejection is wasted */
+ if (pos->level && avg > max_avg &&
+ atomic_read(&pos->exit_kind) == SCX_EXIT_NONE) {
+ max_avg = avg;
+ victim = pos;
+ }
+ }
+ if (!victim)
+ return;
+
+ rq->scx.rescue.kill_at = jiffies;
+ dur_ms = jiffies_to_msecs(jiffies - p->scx.rescue_at);
+ __scx_exit(victim, SCX_EXIT_ERROR_RESCUE, 0, cpu,
+ "used too much rescue CPU time (%llums) while %s[%d] waited %u.%03us to be rescued",
+ div_u64(max_avg, NSEC_PER_MSEC), p->comm, p->pid, dur_ms / 1000,
+ dur_ms % 1000);
+}
+
/**
* scx_rescue_timerfn - Drive and pace rescue execution
* @timer: rq->scx.rescue.timer
@@ -443,7 +525,8 @@ static bool scx_rescue_try_admit(struct rq *rq, struct task_struct *p)
* full quantum and granted its slice, see scx_rescue_next_slice(). A session
* whose budget accumulates over two quanta with the admitted rescuee still
* waiting escalates - the rescuee's remaining slice turns into protected
- * execution and it preempts the current task.
+ * execution and it preempts the current task. An overloaded rescue queue ejects
+ * the top consumer, see scx_rescue_check_overload().
*/
static void scx_rescue_timerfn(struct timer_list *timer)
{
@@ -457,6 +540,7 @@ static void scx_rescue_timerfn(struct timer_list *timer)
return;
scx_rescue_accrue(rq);
+ scx_rescue_check_overload(rq);
if (!p) {
s64 slice = scx_rescue_next_slice(rq);
@@ -528,11 +612,28 @@ void scx_rescue_dump(struct seq_buf *s, struct rq *rq)
p ? p->comm : "none", p ? p->pid : -1);
}
+/*
+ * A scheduler whose stall watchdog is shorter than the overload threshold gets
+ * stall-killed over its parked waiters before the overload check can eject the
+ * actual top consumer. The root's knobs set the threshold, warn on any
+ * scheduler that doesn't fit it.
+ */
+static void scx_rescue_check_timeout(struct scx_sched *sch)
+{
+ if (!scx_rescue_bw_1024 || sch->watchdog_timeout > scx_rescue_overload_after)
+ return;
+
+ pr_warn("sched_ext: %s: watchdog timeout %ums <= rescue overload threshold %ums\n",
+ sch->ops.name, jiffies_to_msecs(sch->watchdog_timeout),
+ jiffies_to_msecs(scx_rescue_overload_after));
+}
+
/* latch the rescue parameters on root scheduler enable */
void scx_rescue_set_knobs(struct scx_sched *sch)
{
s32 bw_ppt = sch->ops.rescue_bandwidth_ppt ?: SCX_RESCUE_DFL_BW_PPT;
s64 quantum_us = sch->ops.rescue_quantum_us ?: SCX_RESCUE_DFL_QUANTUM_US;
+ s64 period_ns;
if (sch->ops.rescue_bandwidth_ppt == SCX_RESCUE_DISABLE) {
scx_rescue_bw_1024 = 0;
@@ -546,21 +647,30 @@ void scx_rescue_set_knobs(struct scx_sched *sch)
scx_rescue_bw_1024);
/*
- * A rescued task is guaranteed to run after two full periods - one to
- * be admitted, one more to escalate. Require the two periods to fit in
- * a quarter of the watchdog timeout, so one full period may take at
- * most an eighth.
+ * The overload threshold and the decay halflife scale with the funding
+ * period - the time the bucket takes to fund one full quantum.
*/
- if (div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024) >
- jiffies_to_nsecs(sch->watchdog_timeout) / 8)
- pr_warn("sched_ext: rescue may not run a stuck task before the %ums watchdog timeout, decrease rescue_quantum_us or increase rescue_bandwidth_ppt\n",
- jiffies_to_msecs(sch->watchdog_timeout));
+ period_ns = div_s64(scx_rescue_quantum_ns << SCHED_CAPACITY_SHIFT, scx_rescue_bw_1024);
+ scx_rescue_overload_after =
+ clamp(nsecs_to_jiffies(SCX_RESCUE_OVERLOAD_MULT * period_ns),
+ msecs_to_jiffies(SCX_RESCUE_MIN_OVERLOAD_MS),
+ msecs_to_jiffies(SCX_RESCUE_MAX_OVERLOAD_MS));
+ scx_rescue_decay_halflife = scx_rescue_overload_after / 4;
+
+ /* a single in-budget wait must not cross the overload trigger */
+ if (nsecs_to_jiffies(period_ns) > scx_rescue_overload_after / 2)
+ pr_warn("sched_ext: %s: rescue funding period %lldms > overload threshold %ums / 2\n",
+ sch->ops.name, div_s64(period_ns, NSEC_PER_MSEC),
+ jiffies_to_msecs(scx_rescue_overload_after));
+
+ scx_rescue_check_timeout(sch);
}
void scx_rescue_init(struct rq *rq)
{
BUG_ON(scx_init_dsq(&rq->scx.rescue.dsq, SCX_DSQ_RESCUE, NULL));
timer_setup(&rq->scx.rescue.timer, scx_rescue_timerfn, TIMER_PINNED);
+ rq->scx.rescue.kill_at = jiffies;
}
/**
@@ -633,8 +743,10 @@ struct scx_dispatch_q *scx_resolve_local_dsq(struct scx_sched *sch, struct rq *r
__scx_add_event(sch, SCX_EV_SUB_RESCUE, 1);
if (scx_rescue_try_admit(rq, p))
return &rq->scx.local_dsq;
- else
- return &rq->scx.rescue.dsq;
+
+ /* queueing, the overload trigger measures the wait from here */
+ p->scx.rescue_at = jiffies;
+ return &rq->scx.rescue.dsq;
}
p->scx.reenq_reason_caps = missing;
@@ -1654,6 +1766,8 @@ void scx_sub_enable_workfn(struct kthread_work *work)
if (ret)
goto err_disable;
+ scx_rescue_check_timeout(sch);
+
/*
* Allocate pshard[] before scx_link_sched() publishes @sch into the
* parent's RCU children list. A concurrent revoke walking the tree
diff --git a/kernel/sched/ext/types.h b/kernel/sched/ext/types.h
index d39588717e9b..1eb3ac8508f6 100644
--- a/kernel/sched/ext/types.h
+++ b/kernel/sched/ext/types.h
@@ -27,6 +27,9 @@ enum scx_consts {
SCX_RESCUE_MIN_QUANTUM_US = 1000,
SCX_RESCUE_MAX_QUANTUM_US = 100000,
SCX_RESCUE_MIN_SLICE_US = 1000, /* floor of the divided slice */
+ SCX_RESCUE_OVERLOAD_MULT = 16, /* overload threshold in funding periods */
+ SCX_RESCUE_MIN_OVERLOAD_MS = 1000,
+ SCX_RESCUE_MAX_OVERLOAD_MS = 15000,
/* per-CPU chunk size for p->scx.tid allocation, see scx_alloc_tid() */
SCX_TID_CHUNK = 1024,
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 289e298df628..6d796f9ba17a 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -803,6 +803,7 @@ struct scx_rq_rescue {
s64 slice; /* curr's admitted slice */
u64 exec_snap; /* sum_exec_runtime at admission */
struct timer_list timer; /* paces admission and escalation */
+ unsigned long kill_at; /* last ejection, init before any */
};
struct scx_rq {
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 10/12] sched_ext: Sync tools autogen enum headers
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (8 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 09/12] sched_ext: Eject the top rescue consumer on overload Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 11/12] sched_ext: scx_qmap - Idle-check pinned tasks before direct dispatch Tejun Heo
2026-08-01 8:51 ` [PATCH 12/12] sched_ext: scx_qmap - Add rescue support Tejun Heo
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
Mechanical regeneration for the enumerators added by the slice write rework
and the rescue machinery. BPF schedulers reference SCX_ENQ_RESCUE, the rest
are enum_defs presence markers.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
tools/sched_ext/include/scx/enum_defs.autogen.h | 16 ++++++++++++++++
tools/sched_ext/include/scx/enums.autogen.bpf.h | 3 +++
tools/sched_ext/include/scx/enums.autogen.h | 1 +
3 files changed, 20 insertions(+)
diff --git a/tools/sched_ext/include/scx/enum_defs.autogen.h b/tools/sched_ext/include/scx/enum_defs.autogen.h
index 0379eff117c9..c05d4b572955 100644
--- a/tools/sched_ext/include/scx/enum_defs.autogen.h
+++ b/tools/sched_ext/include/scx/enum_defs.autogen.h
@@ -26,6 +26,16 @@
#define HAVE_SCX_DSP_DFL_MAX_BATCH
#define HAVE_SCX_DSP_MAX_LOOPS
#define HAVE_SCX_WATCHDOG_MAX_TIMEOUT
+#define HAVE_SCX_RESCUE_DFL_BW_PPT
+#define HAVE_SCX_RESCUE_MAX_BW_PPT
+#define HAVE_SCX_RESCUE_DISABLE
+#define HAVE_SCX_RESCUE_DFL_QUANTUM_US
+#define HAVE_SCX_RESCUE_MIN_QUANTUM_US
+#define HAVE_SCX_RESCUE_MAX_QUANTUM_US
+#define HAVE_SCX_RESCUE_MIN_SLICE_US
+#define HAVE_SCX_RESCUE_OVERLOAD_MULT
+#define HAVE_SCX_RESCUE_MIN_OVERLOAD_MS
+#define HAVE_SCX_RESCUE_MAX_OVERLOAD_MS
#define HAVE_SCX_TID_CHUNK
#define HAVE_SCX_EXIT_BT_LEN
#define HAVE_SCX_EXIT_MSG_LEN
@@ -53,6 +63,7 @@
#define HAVE_SCX_DSQ_LOCAL
#define HAVE_SCX_DSQ_BYPASS
#define HAVE_SCX_DSQ_REJECT
+#define HAVE_SCX_DSQ_RESCUE
#define HAVE_SCX_DSQ_LOCAL_ON
#define HAVE_SCX_DSQ_LOCAL_CPU_MASK
#define HAVE_SCX_DSQ_ITER_REV
@@ -71,6 +82,7 @@
#define HAVE_SCX_ENQ_CPU_SELECTED
#define HAVE_SCX_ENQ_PREEMPT
#define HAVE_SCX_ENQ_IMMED
+#define HAVE_SCX_ENQ_RESCUE
#define HAVE_SCX_ENQ_REENQ
#define HAVE_SCX_ENQ_LAST
#define HAVE___SCX_ENQ_INTERNAL_MASK
@@ -79,6 +91,8 @@
#define HAVE_SCX_ENQ_NESTED
#define HAVE_SCX_ENQ_GDSQ_FALLBACK
#define HAVE_SCX_ENQ_IGNORE_CAPS
+#define HAVE_SCX_ENQ_APPLY_SLICE
+#define HAVE_SCX_ENQ_SLICE_DFL
#define HAVE_SCX_TASK_DSQ_ON_PRIQ
#define HAVE_SCX_TASK_QUEUED
#define HAVE_SCX_TASK_IN_CUSTODY
@@ -86,6 +100,7 @@
#define HAVE_SCX_TASK_DEQD_FOR_SLEEP
#define HAVE_SCX_TASK_SUB_INIT
#define HAVE_SCX_TASK_IMMED
+#define HAVE_SCX_TASK_PROTECTED
#define HAVE_SCX_TASK_STATE_SHIFT
#define HAVE_SCX_TASK_STATE_BITS
#define HAVE_SCX_TASK_STATE_MASK
@@ -120,6 +135,7 @@
#define HAVE_SCX_EXIT_ERROR_BPF
#define HAVE_SCX_EXIT_ERROR_STALL
#define HAVE_SCX_EXIT_ERROR_REENQ
+#define HAVE_SCX_EXIT_ERROR_RESCUE
#define HAVE_SCX_KF_ALLOW_UNLOCKED
#define HAVE_SCX_KF_ALLOW_INIT_CIDS
#define HAVE_SCX_KF_ALLOW_CPU_RELEASE
diff --git a/tools/sched_ext/include/scx/enums.autogen.bpf.h b/tools/sched_ext/include/scx/enums.autogen.bpf.h
index dafccbb6b69d..11bd9b70811a 100644
--- a/tools/sched_ext/include/scx/enums.autogen.bpf.h
+++ b/tools/sched_ext/include/scx/enums.autogen.bpf.h
@@ -124,6 +124,9 @@ const volatile u64 __SCX_ENQ_PREEMPT __weak;
const volatile u64 __SCX_ENQ_IMMED __weak;
#define SCX_ENQ_IMMED __SCX_ENQ_IMMED
+const volatile u64 __SCX_ENQ_RESCUE __weak;
+#define SCX_ENQ_RESCUE __SCX_ENQ_RESCUE
+
const volatile u64 __SCX_ENQ_REENQ __weak;
#define SCX_ENQ_REENQ __SCX_ENQ_REENQ
diff --git a/tools/sched_ext/include/scx/enums.autogen.h b/tools/sched_ext/include/scx/enums.autogen.h
index bbd4901f4fce..0cc4f21c6b6a 100644
--- a/tools/sched_ext/include/scx/enums.autogen.h
+++ b/tools/sched_ext/include/scx/enums.autogen.h
@@ -45,6 +45,7 @@
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_HEAD); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_PREEMPT); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_IMMED); \
+ SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_RESCUE); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_REENQ); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_LAST); \
SCX_ENUM_SET(skel, scx_enq_flags, SCX_ENQ_CLEAR_OPSS); \
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 11/12] sched_ext: scx_qmap - Idle-check pinned tasks before direct dispatch
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (9 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 10/12] sched_ext: Sync tools autogen enum headers Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
2026-08-01 8:51 ` [PATCH 12/12] sched_ext: scx_qmap - Add rescue support Tejun Heo
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
pick_direct_dispatch_cid() sent a pinned task straight to its only cpu
without an idle check. An insert onto a time-shared cid is forced IMMED,
which may not wait on a busy cpu - the kernel bounces the task back to
ops.enqueue(), the shortcut re-inserts it, and the loop runs into the
reenqueue repeat limit.
Run pinned tasks through the same idle test as everyone else and queue them
when the cpu is busy. always_enq_immed forces IMMED on every local insert
and already skipped the shortcut for the same reason.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
tools/sched_ext/scx_qmap.bpf.c | 3 ---
1 file changed, 3 deletions(-)
diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
index 8822ed11c0d8..9a0321e84e88 100644
--- a/tools/sched_ext/scx_qmap.bpf.c
+++ b/tools/sched_ext/scx_qmap.bpf.c
@@ -230,9 +230,6 @@ static s32 pick_direct_dispatch_cid(struct task_struct *p, s32 prev_cid,
s32 cid;
u32 i;
- if (!always_enq_immed && p->nr_cpus_allowed == 1)
- return prev_cid;
-
if (cmask_test(prev_cid, &qa.self_cids.mask) &&
cmask_test_and_clear(prev_cid, &qa.idle_cids.mask))
return prev_cid;
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread* [PATCH 12/12] sched_ext: scx_qmap - Add rescue support
2026-08-01 8:51 [PATCHSET sched_ext/for-7.3] sched_ext: Bandwidth-limited rescue execution for stranded tasks Tejun Heo
` (10 preceding siblings ...)
2026-08-01 8:51 ` [PATCH 11/12] sched_ext: scx_qmap - Idle-check pinned tasks before direct dispatch Tejun Heo
@ 2026-08-01 8:51 ` Tejun Heo
11 siblings, 0 replies; 13+ messages in thread
From: Tejun Heo @ 2026-08-01 8:51 UTC (permalink / raw)
To: David Vernet, Andrea Righi, Changwoo Min
Cc: sched-ext, Emil Tsalapatis, linux-kernel, Tejun Heo
A sched holds only the cids its parent granted and nothing guarantees that
they cover its tasks' affinities. A task that can run on none of them has
nowhere to go and qmap stalls out: it force-inserts the task onto its first
allowed cid, but the kernel bounces the insert back and the task parks in
SHARED_DSQ, which is drained only on self cids it can't run on.
Set SCX_ENQ_RESCUE on these inserts so the kernel diverts such tasks to its
rescue path instead of bouncing them. The force-insert covers scheds with
and without children and fires on re-enqueues, and the SHARED_DSQ scan on
every dispatch rescues tasks stranded there - the enqueue-time check misses
a task whose cids were lost while it was already queued. The wrong-cid fault
injection carries the flag too and doubles as a deterministic rescue-traffic
generator.
-B and -q set the root-only rescue bandwidth and quantum ops knobs. -B 0
maps to SCX_RESCUE_DISABLE and turns rescue off kernel-side. Rescue inserts
are counted and reported in the hier stats line.
Signed-off-by: Tejun Heo <tj@kernel.org>
---
tools/sched_ext/scx_qmap.bpf.c | 85 +++++++++++++++++++++++-----------
tools/sched_ext/scx_qmap.c | 23 +++++++--
tools/sched_ext/scx_qmap.h | 1 +
3 files changed, 79 insertions(+), 30 deletions(-)
diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
index 9a0321e84e88..b6e7b004611c 100644
--- a/tools/sched_ext/scx_qmap.bpf.c
+++ b/tools/sched_ext/scx_qmap.bpf.c
@@ -449,31 +449,33 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
taskc->core_sched_seq = qa.core_sched_tail_seqs[idx]++;
/*
- * A node with children delegates most cids. A task of ours that can run
- * on none of our self cids (e.g. a per-NUMA kthread pinned to delegated
- * cids) would starve in SHARED/FIFO since we never pull those on a
- * delegated cid. Force it onto its first allowed cid's local DSQ with
- * needs_immed(): if we hold access there it runs, else the kernel
- * rejects and bounces it back via REENQ_CAP. Best-effort
- * anti-starvation nudge.
+ * A task of ours that can run on none of our self cids - the parent
+ * didn't grant them or we delegated them to children - would starve in
+ * SHARED/FIFO since we only pull from those on self cids.
+ *
+ * Force it onto its first allowed cid's local DSQ. If we hold that cid
+ * it runs. Otherwise the insert carries SCX_ENQ_RESCUE and the kernel
+ * diverts the task to its rescue path.
*/
- if (qa.nr_sub_scheds && !(enq_flags & SCX_ENQ_REENQ) &&
- !cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) {
+ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) {
s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
if (c >= 0 && c < scx_bpf_nr_cids()) {
taskc->force_local = false;
+ __sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | c, slice_ns,
- enq_flags | needs_immed(c));
+ enq_flags | needs_immed(c) | SCX_ENQ_RESCUE);
return;
}
}
/*
* Fault injection: deliberately dispatch one of our own tasks to a cid
- * we don't hold. The kernel cap check must reject it and re-enqueue
- * with SCX_TASK_REENQ_CAP, so nr_inject_attempts tracks nr_reenq_cap
- * and proves delivery-time enforcement. Throttled.
+ * we don't hold. The inserts carry SCX_ENQ_RESCUE and divert to the
+ * kernel rescue path, a deterministic rescue-traffic generator. Under
+ * -B 0 the kernel cap check rejects and re-enqueues them instead, so
+ * nr_inject_attempts tracks nr_reenq_cap 1:1 and proves delivery-time
+ * enforcement. Throttled.
*/
if (qa.inject_mode == QMAP_INJ_WRONG_CID && p->nr_cpus_allowed > 1 &&
!(enq_flags & SCX_ENQ_REENQ)) {
@@ -484,8 +486,9 @@ void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
if (bad >= 0 && cmask_test(bad, &taskc->cpus_allowed)) {
__sync_fetch_and_add(&qa.nr_inject_attempts, 1);
- scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | bad,
- slice_ns, enq_flags);
+ __sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
+ scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL_ON | bad, slice_ns,
+ enq_flags | SCX_ENQ_RESCUE);
return;
}
}
@@ -588,29 +591,47 @@ static void update_core_sched_head_seq(struct task_struct *p)
}
/*
+ * One pass over SHARED_DSQ: rescue stranded tasks and boost highpri ones. A
+ * task whose cids were lost while it was queued in the fifos would strand on
+ * SHARED_DSQ, which is consumed only on self cids it can't run on - move it to
+ * the kernel rescue path. One whose cids were lost after the highpri cull is
+ * likewise rescued out of HIGHPRI_DSQ below.
+ *
* To demonstrate the use of scx_bpf_dsq_move(), implement silly selective
- * priority boosting mechanism by scanning SHARED_DSQ looking for highpri tasks,
- * moving them to HIGHPRI_DSQ and then consuming them first. This makes minor
- * difference only when dsp_batch is larger than 1.
+ * priority boosting mechanism by moving highpri tasks to HIGHPRI_DSQ and then
+ * consuming them first. This makes minor difference only when dsp_batch is
+ * larger than 1.
*
* scx_bpf_dsq_move[_vtime]() are allowed both from ops.dispatch() and
* non-rq-lock holding BPF programs. As demonstration, this function is called
* from qmap_dispatch() and monitor_timerfn().
*/
-static bool dispatch_highpri(bool from_timer)
+static bool scan_shared_dsq(bool from_timer)
{
struct task_struct *p;
s32 this_cid = scx_bpf_this_cid();
u32 nr_cids = scx_bpf_nr_cids();
- /* scan SHARED_DSQ and move highpri tasks to HIGHPRI_DSQ */
+ /* rescue strands and move highpri tasks to HIGHPRI_DSQ */
bpf_for_each(scx_dsq, p, SHARED_DSQ, 0) {
static u64 highpri_seq;
task_ctx_t *taskc;
+ s32 c;
if (!(taskc = lookup_task_ctx(p)))
return false;
+ /* stranded? rescue - it can't be dispatched here either way */
+ if (!cmask_intersects(&taskc->cpus_allowed, &qa.self_cids.mask)) {
+ c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
+ if (c >= 0 && c < scx_bpf_nr_cids()) {
+ __sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
+ scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c,
+ needs_immed(c) | SCX_ENQ_RESCUE);
+ }
+ continue;
+ }
+
if (taskc->highpri) {
/* exercise the set_*() and vtime interface too */
scx_bpf_dsq_move_set_slice(BPF_FOR_EACH_ITER, slice_ns * 2);
@@ -640,8 +661,17 @@ static bool dispatch_highpri(bool from_timer)
cid = cmask_next_and_set_wrap(&taskc->cpus_allowed,
&qa.self_cids.mask,
this_cid + 1);
- if (cid >= nr_cids)
+ if (cid >= nr_cids) {
+ /* stranded after the cull - rescue it from here */
+ s32 c = cmask_next_set_wrap(&taskc->cpus_allowed, 0);
+
+ if (c >= 0 && c < nr_cids) {
+ __sync_fetch_and_add(&qa.nr_rescue_dsp, 1);
+ scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | c,
+ needs_immed(c) | SCX_ENQ_RESCUE);
+ }
continue;
+ }
if (scx_bpf_dsq_move(BPF_FOR_EACH_ITER, p, SCX_DSQ_LOCAL_ON | cid,
SCX_ENQ_PREEMPT | needs_immed(cid))) {
@@ -670,10 +700,9 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
struct cpu_ctx __arena *cpuc;
task_ctx_t *taskc;
u32 batch = dsp_batch ?: 1;
- s32 owner;
- s32 i;
+ s32 owner, i;
- if (dispatch_highpri(false))
+ if (scan_shared_dsq(false))
return;
/*
@@ -785,11 +814,10 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
*/
if (!cmask_test(cid, &taskc->cpus_allowed))
scx_bpf_kick_cid(scx_bpf_task_cid(p), 0);
-
batch--;
cpuc->dsp_cnt--;
if (!batch || !scx_bpf_dispatch_nr_slots()) {
- if (dispatch_highpri(false))
+ if (scan_shared_dsq(false))
return;
scx_bpf_dsq_move_to_local(SHARED_DSQ, needs_immed(cid));
return;
@@ -801,6 +829,9 @@ void BPF_STRUCT_OPS(qmap_dispatch, s32 cid, struct task_struct *prev)
cpuc->dsp_cnt = 0;
}
+ if (scan_shared_dsq(false))
+ return;
+
/*
* No other tasks. @prev will keep running. Update its core_sched_seq as
* if the task were enqueued and dispatched immediately.
@@ -1185,7 +1216,7 @@ static void dump_shared_dsq(void)
static int monitor_timerfn(void *map, int *key, struct bpf_timer *timer)
{
bpf_rcu_read_lock();
- dispatch_highpri(true);
+ scan_shared_dsq(true);
bpf_rcu_read_unlock();
monitor_cpuperf();
diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c
index 3f54796e48be..189c8fed2f68 100644
--- a/tools/sched_ext/scx_qmap.c
+++ b/tools/sched_ext/scx_qmap.c
@@ -72,6 +72,8 @@ const char help_fmt[] =
" -J MODE Fault injection (wrong-cid: dispatch to a cid not held,\n"
" init-fail/cgrp-init-fail: fail init_task/cpuctl_init for\n"
" \"qmfail*\" comms/cgroups)\n"
+" -B PPT Rescue bandwidth in parts per thousand, 0 disables (root only, default 20)\n"
+" -q US Rescue batch quantum in microseconds (root only, default 5000)\n"
" -v Print libbpf debug messages\n"
" -h Display this help and exit\n";
@@ -106,6 +108,7 @@ struct hier_prev {
u64 nr_reenq_cap;
u64 nr_reenq_immed;
u64 nr_inject_attempts;
+ u64 nr_rescue_dsp;
};
/* current wall-clock time as "HH:MM:SS" for the startup and interval headers */
@@ -187,14 +190,16 @@ static void print_hier(struct qmap_arena *qa, struct hier_prev *prev, u64 own_cg
}
format_cid_ranges(qa, CID_SHARED, ranges, sizeof(ranges));
- printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu\n",
+ printf("hier : nsub=%llu excl=%u shared=%s rr=%s reenq cap/immed +%llu/+%llu inj=+%llu rescue=+%llu\n",
(unsigned long long)qa->nr_sub_scheds, qa->part.nr_excl, ranges, rr,
(unsigned long long)(qa->nr_reenq_cap - prev->nr_reenq_cap),
(unsigned long long)(qa->nr_reenq_immed - prev->nr_reenq_immed),
- (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts));
+ (unsigned long long)(qa->nr_inject_attempts - prev->nr_inject_attempts),
+ (unsigned long long)(qa->nr_rescue_dsp - prev->nr_rescue_dsp));
prev->nr_reenq_cap = qa->nr_reenq_cap;
prev->nr_reenq_immed = qa->nr_reenq_immed;
prev->nr_inject_attempts = qa->nr_inject_attempts;
+ prev->nr_rescue_dsp = qa->nr_rescue_dsp;
printf("hier : %-4s %10s %4s %6s %8s %s\n",
"", "cgroup", "w", "alloc", "disp/s", "cids");
@@ -256,7 +261,8 @@ int main(int argc, char **argv)
skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
skel->rodata->max_tasks = 16384;
- while ((opt = getopt(argc, argv, "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:vh")) != -1) {
+ while ((opt = getopt(argc, argv,
+ "s:e:t:T:l:b:N:PMHc:d:D:SpIF:C:i:R:J:B:q:vh")) != -1) {
switch (opt) {
case 's':
skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
@@ -400,6 +406,17 @@ int main(int argc, char **argv)
else
inject_mode = strtoul(optarg, NULL, 0);
break;
+ case 'B': {
+ u32 ppt = strtoul(optarg, NULL, 0);
+
+ if (!ppt)
+ ppt = __COMPAT_ENUM_OR_ZERO("scx_consts", "SCX_RESCUE_DISABLE");
+ skel->struct_ops.qmap_ops->rescue_bandwidth_ppt = ppt;
+ break;
+ }
+ case 'q':
+ skel->struct_ops.qmap_ops->rescue_quantum_us = strtoul(optarg, NULL, 0);
+ break;
case 'v':
verbose = true;
break;
diff --git a/tools/sched_ext/scx_qmap.h b/tools/sched_ext/scx_qmap.h
index c42f7ef74b89..c8f602d58ca3 100644
--- a/tools/sched_ext/scx_qmap.h
+++ b/tools/sched_ext/scx_qmap.h
@@ -175,6 +175,7 @@ struct qmap_arena {
u64 nr_reenq_cap; /* SCX_TASK_REENQ_CAP bounces */
u64 nr_reenq_immed; /* SCX_TASK_REENQ_IMMED bounces */
u64 nr_inject_attempts; /* fault-injection: dispatches to an unheld cid */
+ u64 nr_rescue_dsp; /* SCX_ENQ_RESCUE dispatch attempts */
u32 inject_mode; /* fault-injection mode (QMAP_INJ_*) */
};
--
2.55.0
^ permalink raw reply related [flat|nested] 13+ messages in thread