* [PATCH 1/4] tools/sched_ext: add scx_nest scheduler
2026-01-12 21:41 [PATCH 0/4] tools/sched_ext: Add example C schedulers Emil Tsalapatis
@ 2026-01-12 21:41 ` Emil Tsalapatis
2026-01-12 21:41 ` [PATCH 2/4] tools/sched_ext: add scx_userland scheduler Emil Tsalapatis
` (3 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: Emil Tsalapatis @ 2026-01-12 21:41 UTC (permalink / raw)
To: sched-ext; +Cc: changwoo, arighi, tj, void, Emil Tsalapatis, David Vernet
Add the Nest C scheduler that tries to keep tasks compacted
into a few warm (frequency-wise) CPUs.
Cc: Tejun Heo <tj@kernel.org>
Cc: David Vernet <dvernet@meta.com>
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
tools/sched_ext/Makefile | 2 +-
tools/sched_ext/scx_nest.bpf.c | 652 +++++++++++++++++++++++++
tools/sched_ext/scx_nest.c | 240 +++++++++
tools/sched_ext/scx_nest.h | 18 +
tools/sched_ext/scx_nest_stats_table.h | 20 +
5 files changed, 931 insertions(+), 1 deletion(-)
create mode 100644 tools/sched_ext/scx_nest.bpf.c
create mode 100644 tools/sched_ext/scx_nest.c
create mode 100644 tools/sched_ext/scx_nest.h
create mode 100644 tools/sched_ext/scx_nest_stats_table.h
diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile
index e4bda2474060..2439a311723c 100644
--- a/tools/sched_ext/Makefile
+++ b/tools/sched_ext/Makefile
@@ -189,7 +189,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
-c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg
+c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_nest
$(addprefix $(BINDIR)/,$(c-sched-targets)): \
$(BINDIR)/%: \
diff --git a/tools/sched_ext/scx_nest.bpf.c b/tools/sched_ext/scx_nest.bpf.c
new file mode 100644
index 000000000000..2992f90bf3d7
--- /dev/null
+++ b/tools/sched_ext/scx_nest.bpf.c
@@ -0,0 +1,652 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * As described in [0], a Nest scheduler which encourages task placement on
+ * cores that are likely to be running at higher frequency, based upon recent usage.
+ *
+ * [0]: https://hal.inria.fr/hal-03612592/file/paper.pdf
+ *
+ * It operates as a global weighted vtime scheduler (similarly to CFS), while
+ * using the Nest algorithm to choose idle cores at wakeup time.
+ *
+ * It also demonstrates the following niceties.
+ *
+ * - More robust task placement policies.
+ * - Termination notification for userspace.
+ *
+ * While rather simple, this scheduler should work reasonably well on CPUs with
+ * a uniform L3 cache topology. While preemption is not implemented, the fact
+ * that the scheduling queue is shared across all CPUs means that whatever is
+ * at the front of the queue is likely to be executed fairly quickly given
+ * enough number of CPUs.
+ *
+ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
+ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
+ */
+#include <scx/common.bpf.h>
+
+#include "scx_nest.h"
+
+#define TASK_DEAD 0x00000080
+
+char _license[] SEC("license") = "GPL";
+
+enum {
+ FALLBACK_DSQ_ID = 0,
+ MSEC_PER_SEC = 1000LLU,
+ USEC_PER_MSEC = 1000LLU,
+ NSEC_PER_USEC = 1000LLU,
+ NSEC_PER_MSEC = USEC_PER_MSEC * NSEC_PER_USEC,
+ USEC_PER_SEC = USEC_PER_MSEC * MSEC_PER_SEC,
+ NSEC_PER_SEC = NSEC_PER_USEC * USEC_PER_SEC,
+};
+
+#define CLOCK_BOOTTIME 7
+
+const volatile u64 p_remove_ns = 2 * NSEC_PER_MSEC;
+const volatile u64 r_max = 5;
+const volatile u64 r_impatient = 2;
+const volatile u64 slice_ns;
+const volatile bool find_fully_idle = false;
+const volatile u64 sampling_cadence_ns = 1 * NSEC_PER_SEC;
+const volatile u64 r_depth = 5;
+
+// Used for stats tracking. May be stale at any given time.
+u64 stats_primary_mask, stats_reserved_mask, stats_other_mask, stats_idle_mask;
+
+// Used for internal tracking.
+static s32 nr_reserved;
+
+static u64 vtime_now;
+UEI_DEFINE(uei);
+
+extern unsigned long CONFIG_HZ __kconfig;
+
+/* Per-task scheduling context */
+struct task_ctx {
+ /*
+ * A temporary cpumask for calculating a task's primary and reserve
+ * mask.
+ */
+ struct bpf_cpumask __kptr *tmp_mask;
+
+ /*
+ * The number of times that a task observes that its previous core is
+ * not idle. If this occurs r_impatient times in a row, a core is
+ * attempted to be retrieved from either the reserve nest, or the
+ * fallback nest.
+ */
+ u32 prev_misses;
+
+ /*
+ * A core that the task is "attached" to, meaning the last core that it
+ * executed on at least twice in a row, and the core that it first
+ * tries to migrate to on wakeup. The task only migrates to the
+ * attached core if it is idle and in the primary nest.
+ */
+ s32 attached_core;
+
+ /*
+ * The last core that the task executed on. This is used to determine
+ * if the task should attach to the core that it will execute on next.
+ */
+ s32 prev_cpu;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
+ __uint(map_flags, BPF_F_NO_PREALLOC);
+ __type(key, int);
+ __type(value, struct task_ctx);
+} task_ctx_stor SEC(".maps");
+
+struct pcpu_ctx {
+ /* The timer used to compact the core from the primary nest. */
+ struct bpf_timer timer;
+
+ /* Whether the current core has been scheduled for compaction. */
+ bool scheduled_compaction;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1024);
+ __type(key, s32);
+ __type(value, struct pcpu_ctx);
+} pcpu_ctxs SEC(".maps");
+
+struct stats_timer {
+ struct bpf_timer timer;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, u32);
+ __type(value, struct stats_timer);
+} stats_timer SEC(".maps");
+
+const volatile u32 nr_cpus = 1; /* !0 for veristat, set during init. */
+
+private(NESTS) struct bpf_cpumask __kptr *primary_cpumask;
+private(NESTS) struct bpf_cpumask __kptr *reserve_cpumask;
+
+struct {
+ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+ __uint(key_size, sizeof(u32));
+ __uint(value_size, sizeof(u64));
+ __uint(max_entries, NEST_STAT(NR));
+} stats SEC(".maps");
+
+
+static __always_inline void stat_inc(u32 idx)
+{
+ u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx);
+ if (cnt_p)
+ (*cnt_p)++;
+}
+
+static inline bool vtime_before(u64 a, u64 b)
+{
+ return (s64)(a - b) < 0;
+}
+
+static __always_inline void
+try_make_core_reserved(s32 cpu, struct bpf_cpumask * reserved, bool promotion)
+{
+ s32 tmp_nr_reserved;
+
+ /*
+ * This check is racy, but that's OK. If we incorrectly fail to promote
+ * a core to reserve, it's because another context added or removed a
+ * core from reserved in this small window. It will balance out over
+ * subsequent wakeups.
+ */
+ tmp_nr_reserved = nr_reserved;
+ if (tmp_nr_reserved < r_max) {
+ /*
+ * It's possible that we could exceed r_max for a time here,
+ * but that should balance out as more cores are either demoted
+ * or fail to be promoted into the reserve nest.
+ */
+ __sync_fetch_and_add(&nr_reserved, 1);
+ bpf_cpumask_set_cpu(cpu, reserved);
+ if (promotion)
+ stat_inc(NEST_STAT(PROMOTED_TO_RESERVED));
+ else
+ stat_inc(NEST_STAT(DEMOTED_TO_RESERVED));
+ } else {
+ bpf_cpumask_clear_cpu(cpu, reserved);
+ stat_inc(NEST_STAT(RESERVED_AT_CAPACITY));
+ }
+}
+
+static void update_attached(struct task_ctx *tctx, s32 prev_cpu, s32 new_cpu)
+{
+ if (tctx->prev_cpu == new_cpu)
+ tctx->attached_core = new_cpu;
+ tctx->prev_cpu = prev_cpu;
+}
+
+static int compact_primary_core(void *map, int *key, struct bpf_timer *timer)
+{
+ struct bpf_cpumask *primary, *reserve;
+ s32 cpu = bpf_get_smp_processor_id();
+ struct pcpu_ctx *pcpu_ctx;
+
+ stat_inc(NEST_STAT(CALLBACK_COMPACTED));
+ /*
+ * If we made it to this callback, it means that the timer callback was
+ * never cancelled, and so the core needs to be demoted from the
+ * primary nest.
+ */
+ pcpu_ctx = bpf_map_lookup_elem(&pcpu_ctxs, &cpu);
+ if (!pcpu_ctx) {
+ scx_bpf_error("Couldn't lookup pcpu ctx");
+ return 0;
+ }
+ bpf_rcu_read_lock();
+ primary = primary_cpumask;
+ reserve = reserve_cpumask;
+ if (!primary || !reserve) {
+ scx_bpf_error("Couldn't find primary or reserve");
+ bpf_rcu_read_unlock();
+ return 0;
+ }
+
+ bpf_cpumask_clear_cpu(cpu, primary);
+ try_make_core_reserved(cpu, reserve, false);
+ bpf_rcu_read_unlock();
+ pcpu_ctx->scheduled_compaction = false;
+ return 0;
+}
+
+s32 BPF_STRUCT_OPS(nest_select_cpu, struct task_struct *p, s32 prev_cpu,
+ u64 wake_flags)
+{
+ struct bpf_cpumask *p_mask, *primary, *reserve;
+ s32 cpu;
+ struct task_ctx *tctx;
+ struct pcpu_ctx *pcpu_ctx;
+ bool direct_to_primary = false, reset_impatient = true;
+
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
+ if (!tctx)
+ return -ENOENT;
+
+ bpf_rcu_read_lock();
+ p_mask = tctx->tmp_mask;
+ primary = primary_cpumask;
+ reserve = reserve_cpumask;
+ if (!p_mask || !primary || !reserve) {
+ bpf_rcu_read_unlock();
+ return -ENOENT;
+ }
+
+ tctx->prev_cpu = prev_cpu;
+
+ bpf_cpumask_and(p_mask, p->cpus_ptr, cast_mask(primary));
+
+ /* First try to wake the task on its attached core. */
+ if (bpf_cpumask_test_cpu(tctx->attached_core, cast_mask(p_mask)) &&
+ scx_bpf_test_and_clear_cpu_idle(tctx->attached_core)) {
+ cpu = tctx->attached_core;
+ stat_inc(NEST_STAT(WAKEUP_ATTACHED));
+ goto migrate_primary;
+ }
+
+ /*
+ * Try to stay on the previous core if it's in the primary set, and
+ * there's no hypertwin. If the previous core is the core the task is
+ * attached to, don't bother as we already just tried that above.
+ */
+ if (prev_cpu != tctx->attached_core &&
+ bpf_cpumask_test_cpu(prev_cpu, cast_mask(p_mask)) &&
+ scx_bpf_test_and_clear_cpu_idle(prev_cpu)) {
+ cpu = prev_cpu;
+ stat_inc(NEST_STAT(WAKEUP_PREV_PRIMARY));
+ goto migrate_primary;
+ }
+
+ if (find_fully_idle) {
+ /* Then try any fully idle core in primary. */
+ cpu = scx_bpf_pick_idle_cpu(cast_mask(p_mask),
+ SCX_PICK_IDLE_CORE);
+ if (cpu >= 0) {
+ stat_inc(NEST_STAT(WAKEUP_FULLY_IDLE_PRIMARY));
+ goto migrate_primary;
+ }
+ }
+
+ /* Then try _any_ idle core in primary, even if its hypertwin is active. */
+ cpu = scx_bpf_pick_idle_cpu(cast_mask(p_mask), 0);
+ if (cpu >= 0) {
+ stat_inc(NEST_STAT(WAKEUP_ANY_IDLE_PRIMARY));
+ goto migrate_primary;
+ }
+
+ if (r_impatient > 0 && ++tctx->prev_misses >= r_impatient) {
+ direct_to_primary = true;
+ tctx->prev_misses = 0;
+ stat_inc(NEST_STAT(TASK_IMPATIENT));
+ }
+
+ reset_impatient = false;
+
+ /* Then try any fully idle core in reserve. */
+ bpf_cpumask_and(p_mask, p->cpus_ptr, cast_mask(reserve));
+ if (find_fully_idle) {
+ cpu = scx_bpf_pick_idle_cpu(cast_mask(p_mask),
+ SCX_PICK_IDLE_CORE);
+ if (cpu >= 0) {
+ stat_inc(NEST_STAT(WAKEUP_FULLY_IDLE_RESERVE));
+ goto promote_to_primary;
+ }
+ }
+
+ /* Then try _any_ idle core in reserve, even if its hypertwin is active. */
+ cpu = scx_bpf_pick_idle_cpu(cast_mask(p_mask), 0);
+ if (cpu >= 0) {
+ stat_inc(NEST_STAT(WAKEUP_ANY_IDLE_RESERVE));
+ goto promote_to_primary;
+ }
+
+ /* Then try _any_ idle core in the task's cpumask. */
+ cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
+ if (cpu >= 0) {
+ /*
+ * We found a core that (we didn't _think_) is in any nest.
+ * This means that we need to either promote the core to the
+ * reserve nest, or if we're going direct to primary due to
+ * r_impatient being exceeded, promote directly to primary.
+ *
+ * We have to do one final check here to see if the core is in
+ * the primary or reserved cpumask because we could potentially
+ * race with the core changing states between AND'ing the
+ * primary and reserve masks with p->cpus_ptr above, and
+ * atomically reserving it from the idle mask with
+ * scx_bpf_pick_idle_cpu(). This is also technically true of
+ * the checks above, but in all of those cases we just put the
+ * core directly into the primary mask so it's not really that
+ * big of a problem. Here, we want to make sure that we don't
+ * accidentally put a core into the reserve nest that was e.g.
+ * already in the primary nest. This is unlikely, but we check
+ * for it on what should be a relatively cold path regardless.
+ */
+ stat_inc(NEST_STAT(WAKEUP_IDLE_OTHER));
+ if (bpf_cpumask_test_cpu(cpu, cast_mask(primary)))
+ goto migrate_primary;
+ else if (bpf_cpumask_test_cpu(cpu, cast_mask(reserve)))
+ goto promote_to_primary;
+ else if (direct_to_primary)
+ goto promote_to_primary;
+ else
+ try_make_core_reserved(cpu, reserve, true);
+ bpf_rcu_read_unlock();
+ return cpu;
+ }
+
+ bpf_rcu_read_unlock();
+ return prev_cpu;
+
+promote_to_primary:
+ stat_inc(NEST_STAT(PROMOTED_TO_PRIMARY));
+migrate_primary:
+ if (reset_impatient)
+ tctx->prev_misses = 0;
+ pcpu_ctx = bpf_map_lookup_elem(&pcpu_ctxs, &cpu);
+ if (pcpu_ctx) {
+ if (pcpu_ctx->scheduled_compaction) {
+ if (bpf_timer_cancel(&pcpu_ctx->timer) < 0)
+ scx_bpf_error("Failed to cancel pcpu timer");
+ if (bpf_timer_set_callback(&pcpu_ctx->timer, compact_primary_core))
+ scx_bpf_error("Failed to re-arm pcpu timer");
+ pcpu_ctx->scheduled_compaction = false;
+ stat_inc(NEST_STAT(CANCELLED_COMPACTION));
+ }
+ } else {
+ scx_bpf_error("Failed to lookup pcpu ctx");
+ }
+ bpf_cpumask_set_cpu(cpu, primary);
+ /*
+ * Check to see whether the CPU is in the reserved nest. This can
+ * happen if the core is compacted concurrently with us trying to place
+ * the currently-waking task onto it. Similarly, this is the expected
+ * state of the core if we found the core in the reserve nest and are
+ * promoting it.
+ *
+ * We don't have to worry about racing with any other waking task here
+ * because we've atomically reserved the core with (some variant of)
+ * scx_bpf_pick_idle_cpu().
+ */
+ if (bpf_cpumask_test_cpu(cpu, cast_mask(reserve))) {
+ __sync_sub_and_fetch(&nr_reserved, 1);
+ bpf_cpumask_clear_cpu(cpu, reserve);
+ }
+ bpf_rcu_read_unlock();
+ update_attached(tctx, prev_cpu, cpu);
+ scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, slice_ns, 0);
+ return cpu;
+}
+
+void BPF_STRUCT_OPS(nest_enqueue, struct task_struct *p, u64 enq_flags)
+{
+ struct task_ctx *tctx;
+ u64 vtime = p->scx.dsq_vtime;
+
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
+ if (!tctx) {
+ scx_bpf_error("Unable to find task ctx");
+ return;
+ }
+
+ /*
+ * Limit the amount of budget that an idling task can accumulate
+ * to one slice.
+ */
+ if (vtime_before(vtime, vtime_now - slice_ns))
+ vtime = vtime_now - slice_ns;
+
+ scx_bpf_dsq_insert_vtime(p, FALLBACK_DSQ_ID, slice_ns, vtime,
+ enq_flags);
+}
+
+void BPF_STRUCT_OPS(nest_dispatch, s32 cpu, struct task_struct *prev)
+{
+ struct pcpu_ctx *pcpu_ctx;
+ struct bpf_cpumask *primary, *reserve;
+ s32 key = cpu;
+ bool in_primary;
+
+ primary = primary_cpumask;
+ reserve = reserve_cpumask;
+ if (!primary || !reserve) {
+ scx_bpf_error("No primary or reserve cpumask");
+ return;
+ }
+
+ pcpu_ctx = bpf_map_lookup_elem(&pcpu_ctxs, &key);
+ if (!pcpu_ctx) {
+ scx_bpf_error("Failed to lookup pcpu ctx");
+ return;
+ }
+
+ if (!scx_bpf_dsq_move_to_local(FALLBACK_DSQ_ID)) {
+ in_primary = bpf_cpumask_test_cpu(cpu, cast_mask(primary));
+
+ if (prev && (prev->scx.flags & SCX_TASK_QUEUED) && in_primary) {
+ scx_bpf_dsq_insert(prev, SCX_DSQ_LOCAL, slice_ns, 0);
+ return;
+ }
+
+ stat_inc(NEST_STAT(NOT_CONSUMED));
+ if (in_primary) {
+ /*
+ * Immediately demote a primary core if the previous
+ * task on it is dying
+ *
+ * Note that we elect to not compact the "first" CPU in
+ * the mask so as to encourage at least one core to
+ * remain in the nest. It would be better to check for
+ * whether there is only one core remaining in the
+ * nest, but BPF doesn't yet have a kfunc for querying
+ * cpumask weight.
+ */
+ if ((prev && prev->__state == TASK_DEAD) &&
+ (cpu != bpf_cpumask_first(cast_mask(primary)))) {
+ stat_inc(NEST_STAT(EAGERLY_COMPACTED));
+ bpf_cpumask_clear_cpu(cpu, primary);
+ try_make_core_reserved(cpu, reserve, false);
+ } else {
+ pcpu_ctx->scheduled_compaction = true;
+ /*
+ * The core isn't being used anymore. Set a
+ * timer to remove the core from the nest in
+ * p_remove if it's still unused by that point.
+ */
+ bpf_timer_start(&pcpu_ctx->timer, p_remove_ns,
+ BPF_F_TIMER_CPU_PIN);
+ stat_inc(NEST_STAT(SCHEDULED_COMPACTION));
+ }
+ }
+ return;
+ }
+ stat_inc(NEST_STAT(CONSUMED));
+}
+
+void BPF_STRUCT_OPS(nest_running, struct task_struct *p)
+{
+ /*
+ * Global vtime always progresses forward as tasks start executing. The
+ * test and update can be performed concurrently from multiple CPUs and
+ * thus racy. Any error should be contained and temporary. Let's just
+ * live with it.
+ */
+ if (vtime_before(vtime_now, p->scx.dsq_vtime))
+ vtime_now = p->scx.dsq_vtime;
+}
+
+void BPF_STRUCT_OPS(nest_stopping, struct task_struct *p, bool runnable)
+{
+ /* scale the execution time by the inverse of the weight and charge */
+ p->scx.dsq_vtime += (slice_ns - p->scx.slice) * 100 / p->scx.weight;
+}
+
+s32 BPF_STRUCT_OPS(nest_init_task, struct task_struct *p,
+ struct scx_init_task_args *args)
+{
+ struct task_ctx *tctx;
+ struct bpf_cpumask *cpumask;
+
+ /*
+ * @p is new. Let's ensure that its task_ctx is available. We can sleep
+ * in this function and the following will automatically use GFP_KERNEL.
+ */
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0,
+ BPF_LOCAL_STORAGE_GET_F_CREATE);
+ if (!tctx)
+ return -ENOMEM;
+
+ cpumask = bpf_cpumask_create();
+ if (!cpumask)
+ return -ENOMEM;
+
+ cpumask = bpf_kptr_xchg(&tctx->tmp_mask, cpumask);
+ if (cpumask)
+ bpf_cpumask_release(cpumask);
+
+ tctx->attached_core = -1;
+ tctx->prev_cpu = -1;
+
+ return 0;
+}
+
+void BPF_STRUCT_OPS(nest_enable, struct task_struct *p)
+{
+ p->scx.dsq_vtime = vtime_now;
+}
+
+static int stats_timerfn(void *map, int *key, struct bpf_timer *timer)
+{
+ s32 cpu;
+ struct bpf_cpumask *primary, *reserve;
+ const struct cpumask *idle;
+ stats_primary_mask = 0;
+ stats_reserved_mask = 0;
+ stats_other_mask = 0;
+ stats_idle_mask = 0;
+ long err;
+
+ bpf_rcu_read_lock();
+ primary = primary_cpumask;
+ reserve = reserve_cpumask;
+ if (!primary || !reserve) {
+ bpf_rcu_read_unlock();
+ scx_bpf_error("Failed to lookup primary or reserve");
+ return 0;
+ }
+
+ idle = scx_bpf_get_idle_cpumask();
+ bpf_for(cpu, 0, nr_cpus) {
+ if (bpf_cpumask_test_cpu(cpu, cast_mask(primary)))
+ stats_primary_mask |= (1ULL << cpu);
+ else if (bpf_cpumask_test_cpu(cpu, cast_mask(reserve)))
+ stats_reserved_mask |= (1ULL << cpu);
+ else
+ stats_other_mask |= (1ULL << cpu);
+
+ if (bpf_cpumask_test_cpu(cpu, idle))
+ stats_idle_mask |= (1ULL << cpu);
+ }
+ bpf_rcu_read_unlock();
+ scx_bpf_put_idle_cpumask(idle);
+
+ err = bpf_timer_start(timer, sampling_cadence_ns - 5000, 0);
+ if (err)
+ scx_bpf_error("Failed to arm stats timer");
+
+ return 0;
+}
+
+s32 BPF_STRUCT_OPS_SLEEPABLE(nest_init)
+{
+ struct bpf_cpumask *cpumask;
+ s32 cpu;
+ int err;
+ struct bpf_timer *timer;
+ u32 key = 0;
+
+ err = scx_bpf_create_dsq(FALLBACK_DSQ_ID, NUMA_NO_NODE);
+ if (err) {
+ scx_bpf_error("Failed to create fallback DSQ");
+ return err;
+ }
+
+ cpumask = bpf_cpumask_create();
+ if (!cpumask)
+ return -ENOMEM;
+ bpf_cpumask_clear(cpumask);
+ cpumask = bpf_kptr_xchg(&primary_cpumask, cpumask);
+ if (cpumask)
+ bpf_cpumask_release(cpumask);
+
+ cpumask = bpf_cpumask_create();
+ if (!cpumask)
+ return -ENOMEM;
+
+ bpf_cpumask_clear(cpumask);
+ cpumask = bpf_kptr_xchg(&reserve_cpumask, cpumask);
+ if (cpumask)
+ bpf_cpumask_release(cpumask);
+
+ bpf_for(cpu, 0, nr_cpus) {
+ s32 key = cpu;
+ struct pcpu_ctx *ctx = bpf_map_lookup_elem(&pcpu_ctxs, &key);
+
+ if (!ctx) {
+ scx_bpf_error("Failed to lookup pcpu_ctx");
+ return -ENOENT;
+ }
+ ctx->scheduled_compaction = false;
+ if (bpf_timer_init(&ctx->timer, &pcpu_ctxs, CLOCK_BOOTTIME)) {
+ scx_bpf_error("Failed to initialize pcpu timer");
+ return -EINVAL;
+ }
+ err = bpf_timer_set_callback(&ctx->timer, compact_primary_core);
+ if (err) {
+ scx_bpf_error("Failed to set pcpu timer callback");
+ return -EINVAL;
+ }
+ }
+
+ timer = bpf_map_lookup_elem(&stats_timer, &key);
+ if (!timer) {
+ scx_bpf_error("Failed to lookup central timer");
+ return -ESRCH;
+ }
+ bpf_timer_init(timer, &stats_timer, CLOCK_BOOTTIME);
+ bpf_timer_set_callback(timer, stats_timerfn);
+ err = bpf_timer_start(timer, sampling_cadence_ns - 5000, 0);
+ if (err)
+ scx_bpf_error("Failed to arm stats timer");
+
+ return err;
+}
+
+void BPF_STRUCT_OPS(nest_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SCX_OPS_DEFINE(nest_ops,
+ .select_cpu = (void *)nest_select_cpu,
+ .enqueue = (void *)nest_enqueue,
+ .dispatch = (void *)nest_dispatch,
+ .running = (void *)nest_running,
+ .stopping = (void *)nest_stopping,
+ .init_task = (void *)nest_init_task,
+ .enable = (void *)nest_enable,
+ .init = (void *)nest_init,
+ .exit = (void *)nest_exit,
+ .flags = 0,
+ .name = "nest");
diff --git a/tools/sched_ext/scx_nest.c b/tools/sched_ext/scx_nest.c
new file mode 100644
index 000000000000..d91142b4af71
--- /dev/null
+++ b/tools/sched_ext/scx_nest.c
@@ -0,0 +1,240 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
+ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <assert.h>
+#include <libgen.h>
+#include <bpf/bpf.h>
+#include <scx/common.h>
+
+#include "scx_nest.bpf.skel.h"
+#include "scx_nest.h"
+
+#define SAMPLING_CADENCE_S 2
+
+const char help_fmt[] =
+"A Nest sched_ext scheduler.\n"
+"\n"
+"See the top-level comment in .bpf.c for more details.\n"
+"\n"
+"Usage: %s [-p] [-d DELAY] [-m <max>] [-i ITERS]\n"
+"\n"
+" -d DELAY_US Delay (us), before removing an idle core from the primary nest (default 2000us / 2ms)\n"
+" -m R_MAX Maximum number of cores in the reserve nest (default 5)\n"
+" -i ITERS Number of successive placement failures tolerated before trying to aggressively expand primary nest (default 2), or 0 to disable\n"
+" -s SLICE_US Override slice duration in us (default 20000us / 20ms)\n"
+" -I First try to find a fully idle core, and then any idle core, when searching nests. Default behavior is to ignore hypertwins and check for any idle core.\n"
+" -v Print libbpf debug messages\n"
+" -h Display this help and exit\n";
+
+static bool verbose;
+static volatile int exit_req;
+
+static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
+{
+ if (level == LIBBPF_DEBUG && !verbose)
+ return 0;
+ return vfprintf(stderr, format, args);
+}
+
+static void sigint_handler(int nest)
+{
+ exit_req = 1;
+}
+
+struct nest_stat {
+ const char *label;
+ enum nest_stat_group group;
+ enum nest_stat_idx idx;
+};
+
+#define NEST_ST(__stat, __grp, __desc) { \
+ .label = #__stat, \
+ .group = __grp, \
+ .idx = NEST_STAT(__stat) \
+},
+static struct nest_stat nest_stats[NEST_STAT(NR)] = {
+#include "scx_nest_stats_table.h"
+};
+#undef NEST_ST
+
+static void read_stats(struct scx_nest *skel, u64 *stats)
+{
+ int nr_cpus = libbpf_num_possible_cpus();
+ assert(nr_cpus > 0);
+ u64 cnts[NEST_STAT(NR)][nr_cpus];
+ u32 idx;
+
+ memset(stats, 0, sizeof(stats[0]) * NEST_STAT(NR));
+
+ for (idx = 0; idx < NEST_STAT(NR); idx++) {
+ int ret, cpu;
+
+ ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats),
+ &idx, cnts[idx]);
+ if (ret < 0)
+ continue;
+ for (cpu = 0; cpu < nr_cpus; cpu++)
+ stats[idx] += cnts[idx][cpu];
+ }
+}
+
+static void print_underline(const char *str)
+{
+ char buf[64];
+ size_t len;
+
+ len = strlen(str);
+ memset(buf, '-', len);
+ buf[len] = '\0';
+ printf("\n\n%s\n%s\n", str, buf);
+}
+
+static void print_stat_grp(enum nest_stat_group grp)
+{
+ const char *group;
+
+ switch (grp) {
+ case STAT_GRP_WAKEUP:
+ group = "Wakeup stats";
+ break;
+ case STAT_GRP_NEST:
+ group = "Nest stats";
+ break;
+ case STAT_GRP_CONSUME:
+ group = "Consume stats";
+ break;
+ default:
+ group = "Unknown stats";
+ break;
+ }
+
+ print_underline(group);
+}
+
+static void print_active_nests(const struct scx_nest *skel)
+{
+ u64 primary = skel->bss->stats_primary_mask;
+ u64 reserved = skel->bss->stats_reserved_mask;
+ u64 other = skel->bss->stats_other_mask;
+ u64 idle = skel->bss->stats_idle_mask;
+ u32 nr_cpus = skel->rodata->nr_cpus, cpu;
+ int idx;
+ char cpus[nr_cpus + 1];
+
+ memset(cpus, 0, nr_cpus + 1);
+ print_underline("Masks");
+ for (idx = 0; idx < 4; idx++) {
+ const char *mask_str;
+ u64 mask, total = 0;
+
+ memset(cpus, '-', nr_cpus);
+ if (idx == 0) {
+ mask_str = "PRIMARY";
+ mask = primary;
+ } else if (idx == 1) {
+ mask_str = "RESERVED";
+ mask = reserved;
+ } else if (idx == 2) {
+ mask_str = "OTHER";
+ mask = other;
+ } else {
+ mask_str = "IDLE";
+ mask = idle;
+ }
+ for (cpu = 0; cpu < nr_cpus; cpu++) {
+ if (mask & (1ULL << cpu)) {
+ cpus[cpu] = '*';
+ total++;
+ }
+ }
+ printf("%-9s(%2" PRIu64 "): | %s |\n", mask_str, total, cpus);
+ }
+}
+
+int main(int argc, char **argv)
+{
+ struct scx_nest *skel;
+ struct bpf_link *link;
+ __u32 opt;
+ __u64 ecode;
+
+ libbpf_set_print(libbpf_print_fn);
+ signal(SIGINT, sigint_handler);
+ signal(SIGTERM, sigint_handler);
+restart:
+ skel = SCX_OPS_OPEN(nest_ops, scx_nest);
+
+ skel->rodata->nr_cpus = libbpf_num_possible_cpus();
+ assert(skel->rodata->nr_cpus > 0);
+ skel->rodata->sampling_cadence_ns = SAMPLING_CADENCE_S * 1000 * 1000 * 1000;
+ skel->rodata->slice_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
+
+ while ((opt = getopt(argc, argv, "d:m:i:Is:vh")) != -1) {
+ switch (opt) {
+ case 'd':
+ skel->rodata->p_remove_ns = strtoull(optarg, NULL, 0) * 1000;
+ break;
+ case 'm':
+ skel->rodata->r_max = strtoull(optarg, NULL, 0);
+ break;
+ case 'i':
+ skel->rodata->r_impatient = strtoull(optarg, NULL, 0);
+ break;
+ case 'I':
+ skel->rodata->find_fully_idle = true;
+ break;
+ case 's':
+ skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
+ break;
+ case 'v':
+ verbose = true;
+ break;
+ default:
+ fprintf(stderr, help_fmt, basename(argv[0]));
+ return opt != 'h';
+ }
+ }
+
+ SCX_OPS_LOAD(skel, nest_ops, scx_nest, uei);
+ link = SCX_OPS_ATTACH(skel, nest_ops, scx_nest);
+
+ while (!exit_req && !UEI_EXITED(skel, uei)) {
+ u64 stats[NEST_STAT(NR)];
+ enum nest_stat_idx i;
+ enum nest_stat_group last_grp = -1;
+
+ read_stats(skel, stats);
+ for (i = 0; i < NEST_STAT(NR); i++) {
+ struct nest_stat *nest_stat;
+
+ nest_stat = &nest_stats[i];
+ if (nest_stat->group != last_grp) {
+ print_stat_grp(nest_stat->group);
+ last_grp = nest_stat->group;
+ }
+ printf("%s=%" PRIu64 "\n", nest_stat->label, stats[nest_stat->idx]);
+ }
+ printf("\n");
+ print_active_nests(skel);
+ printf("\n");
+ printf("\n");
+ printf("\n");
+ fflush(stdout);
+ sleep(SAMPLING_CADENCE_S);
+ }
+
+ bpf_link__destroy(link);
+ ecode = UEI_REPORT(skel, uei);
+ scx_nest__destroy(skel);
+
+ if (UEI_ECODE_RESTART(ecode))
+ goto restart;
+ return 0;
+}
diff --git a/tools/sched_ext/scx_nest.h b/tools/sched_ext/scx_nest.h
new file mode 100644
index 000000000000..060444f81b88
--- /dev/null
+++ b/tools/sched_ext/scx_nest.h
@@ -0,0 +1,18 @@
+#ifndef __SCX_NEST_H
+#define __SCX_NEST_H
+
+enum nest_stat_group {
+ STAT_GRP_WAKEUP,
+ STAT_GRP_NEST,
+ STAT_GRP_CONSUME,
+};
+
+#define NEST_STAT(__stat) BPFSTAT_##__stat
+#define NEST_ST(__stat, __grp, __desc) NEST_STAT(__stat),
+enum nest_stat_idx {
+#include "scx_nest_stats_table.h"
+ NEST_ST(NR, 0, 0)
+};
+#undef NEST_ST
+
+#endif /* __SCX_NEST_H */
diff --git a/tools/sched_ext/scx_nest_stats_table.h b/tools/sched_ext/scx_nest_stats_table.h
new file mode 100644
index 000000000000..6625f705448a
--- /dev/null
+++ b/tools/sched_ext/scx_nest_stats_table.h
@@ -0,0 +1,20 @@
+NEST_ST(WAKEUP_ATTACHED, STAT_GRP_WAKEUP, "Attached CPU was idle, and in primary nest")
+NEST_ST(WAKEUP_PREV_PRIMARY, STAT_GRP_WAKEUP, "Previous CPU was idle, and in primary nest")
+NEST_ST(WAKEUP_FULLY_IDLE_PRIMARY, STAT_GRP_WAKEUP, "Woken up to fully idle primary nest core")
+NEST_ST(WAKEUP_ANY_IDLE_PRIMARY, STAT_GRP_WAKEUP, "Woken up to idle logical primary nest core")
+NEST_ST(WAKEUP_FULLY_IDLE_RESERVE, STAT_GRP_WAKEUP, "Woken up to fully idle reserve nest core")
+NEST_ST(WAKEUP_ANY_IDLE_RESERVE, STAT_GRP_WAKEUP, "Woken up to idle logical reserve nest core")
+NEST_ST(WAKEUP_IDLE_OTHER, STAT_GRP_WAKEUP, "Woken to any idle logical core in p->cpus_ptr")
+
+NEST_ST(TASK_IMPATIENT, STAT_GRP_NEST, "A task was found to be impatient")
+NEST_ST(PROMOTED_TO_PRIMARY, STAT_GRP_NEST, "A core was promoted into the primary nest")
+NEST_ST(PROMOTED_TO_RESERVED, STAT_GRP_NEST, "A core was promoted into the reserve nest")
+NEST_ST(DEMOTED_TO_RESERVED, STAT_GRP_NEST, "A core was demoted into the reserve nest")
+NEST_ST(RESERVED_AT_CAPACITY, STAT_GRP_NEST, "Reserved nest was at capacity")
+NEST_ST(SCHEDULED_COMPACTION, STAT_GRP_NEST, "Scheduled a primary core to be compacted")
+NEST_ST(CANCELLED_COMPACTION, STAT_GRP_NEST, "Cancelled a primary core from being compacted at task wakeup time")
+NEST_ST(EAGERLY_COMPACTED, STAT_GRP_NEST, "A core was compacted in ops.dispatch()")
+NEST_ST(CALLBACK_COMPACTED, STAT_GRP_NEST, "A core was compacted in the scheduled timer callback")
+
+NEST_ST(CONSUMED, STAT_GRP_CONSUME, "A task was consumed from the global DSQ")
+NEST_ST(NOT_CONSUMED, STAT_GRP_CONSUME, "There was no task in the global DSQ")
--
2.49.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 2/4] tools/sched_ext: add scx_userland scheduler
2026-01-12 21:41 [PATCH 0/4] tools/sched_ext: Add example C schedulers Emil Tsalapatis
2026-01-12 21:41 ` [PATCH 1/4] tools/sched_ext: add scx_nest scheduler Emil Tsalapatis
@ 2026-01-12 21:41 ` Emil Tsalapatis
2026-01-12 21:41 ` [PATCH 3/4] tools/sched_ext: add scx_pair scheduler Emil Tsalapatis
` (2 subsequent siblings)
4 siblings, 0 replies; 8+ messages in thread
From: Emil Tsalapatis @ 2026-01-12 21:41 UTC (permalink / raw)
To: sched-ext; +Cc: changwoo, arighi, tj, void, Emil Tsalapatis, David Vernet
Add in the scx_userland scheduler that does vruntime-based
scheduling in userspace code and communicates scheduling
decisions to BPF by accessing and modifying globals through
the skeleton.
Cc: Tejun Heo <tj@kernel.org>
Cc: David Vernet <dvernet@meta.com>
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
tools/sched_ext/Makefile | 2 +-
tools/sched_ext/scx_userland.bpf.c | 344 +++++++++++++++++++++++
tools/sched_ext/scx_userland.c | 437 +++++++++++++++++++++++++++++
tools/sched_ext/scx_userland.h | 17 ++
4 files changed, 799 insertions(+), 1 deletion(-)
create mode 100644 tools/sched_ext/scx_userland.bpf.c
create mode 100644 tools/sched_ext/scx_userland.c
create mode 100644 tools/sched_ext/scx_userland.h
diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile
index 2439a311723c..2f06e0587fbb 100644
--- a/tools/sched_ext/Makefile
+++ b/tools/sched_ext/Makefile
@@ -189,7 +189,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
-c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_nest
+c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_nest scx_userland
$(addprefix $(BINDIR)/,$(c-sched-targets)): \
$(BINDIR)/%: \
diff --git a/tools/sched_ext/scx_userland.bpf.c b/tools/sched_ext/scx_userland.bpf.c
new file mode 100644
index 000000000000..f29862b89386
--- /dev/null
+++ b/tools/sched_ext/scx_userland.bpf.c
@@ -0,0 +1,344 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * A minimal userland scheduler.
+ *
+ * In terms of scheduling, this provides two different types of behaviors:
+ * 1. A global FIFO scheduling order for _any_ tasks that have CPU affinity.
+ * All such tasks are direct-dispatched from the kernel, and are never
+ * enqueued in user space.
+ * 2. A primitive vruntime scheduler that is implemented in user space, for all
+ * other tasks.
+ *
+ * Some parts of this example user space scheduler could be implemented more
+ * efficiently using more complex and sophisticated data structures. For
+ * example, rather than using BPF_MAP_TYPE_QUEUE's,
+ * BPF_MAP_TYPE_{USER_}RINGBUF's could be used for exchanging messages between
+ * user space and kernel space. Similarly, we use a simple vruntime-sorted list
+ * in user space, but an rbtree could be used instead.
+ *
+ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
+ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
+ */
+#include <scx/common.bpf.h>
+#include "scx_userland.h"
+
+/*
+ * Maximum amount of tasks enqueued/dispatched between kernel and user-space.
+ */
+#define MAX_ENQUEUED_TASKS 4096
+
+char _license[] SEC("license") = "GPL";
+
+const volatile s32 usersched_pid;
+
+/* !0 for veristat, set during init */
+const volatile u32 num_possible_cpus = 64;
+
+/* Stats that are printed by user space. */
+u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues;
+
+/*
+ * Number of tasks that are queued for scheduling.
+ *
+ * This number is incremented by the BPF component when a task is queued to the
+ * user-space scheduler and it must be decremented by the user-space scheduler
+ * when a task is consumed.
+ */
+volatile u64 nr_queued;
+
+/*
+ * Number of tasks that are waiting for scheduling.
+ *
+ * This number must be updated by the user-space scheduler to keep track if
+ * there is still some scheduling work to do.
+ */
+volatile u64 nr_scheduled;
+
+UEI_DEFINE(uei);
+
+/*
+ * The map containing tasks that are enqueued in user space from the kernel.
+ *
+ * This map is drained by the user space scheduler.
+ */
+struct {
+ __uint(type, BPF_MAP_TYPE_QUEUE);
+ __uint(max_entries, MAX_ENQUEUED_TASKS);
+ __type(value, struct scx_userland_enqueued_task);
+} enqueued SEC(".maps");
+
+/*
+ * The map containing tasks that are dispatched to the kernel from user space.
+ *
+ * Drained by the kernel in userland_dispatch().
+ */
+struct {
+ __uint(type, BPF_MAP_TYPE_QUEUE);
+ __uint(max_entries, MAX_ENQUEUED_TASKS);
+ __type(value, s32);
+} dispatched SEC(".maps");
+
+/* Per-task scheduling context */
+struct task_ctx {
+ bool force_local; /* Dispatch directly to local DSQ */
+};
+
+/* Map that contains task-local storage. */
+struct {
+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
+ __uint(map_flags, BPF_F_NO_PREALLOC);
+ __type(key, int);
+ __type(value, struct task_ctx);
+} task_ctx_stor SEC(".maps");
+
+/*
+ * Flag used to wake-up the user-space scheduler.
+ */
+static volatile u32 usersched_needed;
+
+/*
+ * Set user-space scheduler wake-up flag (equivalent to an atomic release
+ * operation).
+ */
+static void set_usersched_needed(void)
+{
+ __sync_fetch_and_or(&usersched_needed, 1);
+}
+
+/*
+ * Check and clear user-space scheduler wake-up flag (equivalent to an atomic
+ * acquire operation).
+ */
+static bool test_and_clear_usersched_needed(void)
+{
+ return __sync_fetch_and_and(&usersched_needed, 0) == 1;
+}
+
+static bool is_usersched_task(const struct task_struct *p)
+{
+ return p->pid == usersched_pid;
+}
+
+static bool keep_in_kernel(const struct task_struct *p)
+{
+ return p->nr_cpus_allowed < num_possible_cpus;
+}
+
+static struct task_struct *usersched_task(void)
+{
+ struct task_struct *p;
+
+ p = bpf_task_from_pid(usersched_pid);
+ /*
+ * Should never happen -- the usersched task should always be managed
+ * by sched_ext.
+ */
+ if (!p)
+ scx_bpf_error("Failed to find usersched task %d", usersched_pid);
+
+ return p;
+}
+
+s32 BPF_STRUCT_OPS(userland_select_cpu, struct task_struct *p,
+ s32 prev_cpu, u64 wake_flags)
+{
+ if (keep_in_kernel(p)) {
+ s32 cpu;
+ struct task_ctx *tctx;
+
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
+ if (!tctx) {
+ scx_bpf_error("Failed to look up task-local storage for %s", p->comm);
+ return -ESRCH;
+ }
+
+ if (p->nr_cpus_allowed == 1 ||
+ scx_bpf_test_and_clear_cpu_idle(prev_cpu)) {
+ tctx->force_local = true;
+ return prev_cpu;
+ }
+
+ cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
+ if (cpu >= 0) {
+ tctx->force_local = true;
+ return cpu;
+ }
+ }
+
+ return prev_cpu;
+}
+
+static void dispatch_user_scheduler(void)
+{
+ struct task_struct *p;
+
+ p = usersched_task();
+ if (p) {
+ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
+ bpf_task_release(p);
+ }
+}
+
+static void enqueue_task_in_user_space(struct task_struct *p, u64 enq_flags)
+{
+ struct scx_userland_enqueued_task task = {};
+
+ task.pid = p->pid;
+ task.sum_exec_runtime = p->se.sum_exec_runtime;
+ task.weight = p->scx.weight;
+
+ if (bpf_map_push_elem(&enqueued, &task, 0)) {
+ /*
+ * If we fail to enqueue the task in user space, put it
+ * directly on the global DSQ.
+ */
+ __sync_fetch_and_add(&nr_failed_enqueues, 1);
+ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
+ } else {
+ __sync_fetch_and_add(&nr_user_enqueues, 1);
+ set_usersched_needed();
+ }
+}
+
+void BPF_STRUCT_OPS(userland_enqueue, struct task_struct *p, u64 enq_flags)
+{
+ if (keep_in_kernel(p)) {
+ u64 dsq_id = SCX_DSQ_GLOBAL;
+ struct task_ctx *tctx;
+
+ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
+ if (!tctx) {
+ scx_bpf_error("Failed to lookup task ctx for %s", p->comm);
+ return;
+ }
+
+ if (tctx->force_local)
+ dsq_id = SCX_DSQ_LOCAL;
+ tctx->force_local = false;
+ scx_bpf_dsq_insert(p, dsq_id, SCX_SLICE_DFL, enq_flags);
+ __sync_fetch_and_add(&nr_kernel_enqueues, 1);
+ return;
+ } else if (!is_usersched_task(p)) {
+ enqueue_task_in_user_space(p, enq_flags);
+ }
+}
+
+void BPF_STRUCT_OPS(userland_dispatch, s32 cpu, struct task_struct *prev)
+{
+ if (test_and_clear_usersched_needed())
+ dispatch_user_scheduler();
+
+ bpf_repeat(MAX_ENQUEUED_TASKS) {
+ s32 pid;
+ struct task_struct *p;
+
+ if (bpf_map_pop_elem(&dispatched, &pid))
+ break;
+
+ /*
+ * The task could have exited by the time we get around to
+ * dispatching it. Treat this as a normal occurrence, and simply
+ * move onto the next iteration.
+ */
+ p = bpf_task_from_pid(pid);
+ if (!p)
+ continue;
+
+ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
+ bpf_task_release(p);
+ }
+}
+
+/*
+ * A CPU is about to change its idle state. If the CPU is going idle, ensure
+ * that the user-space scheduler has a chance to run if there is any remaining
+ * work to do.
+ */
+void BPF_STRUCT_OPS(userland_update_idle, s32 cpu, bool idle)
+{
+ /*
+ * Don't do anything if we exit from and idle state, a CPU owner will
+ * be assigned in .running().
+ */
+ if (!idle)
+ return;
+ /*
+ * A CPU is now available, notify the user-space scheduler that tasks
+ * can be dispatched, if there is at least one task waiting to be
+ * scheduled, either queued (accounted in nr_queued) or scheduled
+ * (accounted in nr_scheduled).
+ *
+ * NOTE: nr_queued is incremented by the BPF component, more exactly in
+ * enqueue(), when a task is sent to the user-space scheduler, then
+ * the scheduler drains the queued tasks (updating nr_queued) and adds
+ * them to its internal data structures / state; at this point tasks
+ * become "scheduled" and the user-space scheduler will take care of
+ * updating nr_scheduled accordingly; lastly tasks will be dispatched
+ * and the user-space scheduler will update nr_scheduled again.
+ *
+ * Checking both counters allows to determine if there is still some
+ * pending work to do for the scheduler: new tasks have been queued
+ * since last check, or there are still tasks "queued" or "scheduled"
+ * since the previous user-space scheduler run. If the counters are
+ * both zero it is pointless to wake-up the scheduler (even if a CPU
+ * becomes idle), because there is nothing to do.
+ *
+ * Keep in mind that update_idle() doesn't run concurrently with the
+ * user-space scheduler (that is single-threaded): this function is
+ * naturally serialized with the user-space scheduler code, therefore
+ * this check here is also safe from a concurrency perspective.
+ */
+ if (nr_queued || nr_scheduled) {
+ /*
+ * Kick the CPU to make it immediately ready to accept
+ * dispatched tasks.
+ */
+ set_usersched_needed();
+ scx_bpf_kick_cpu(cpu, 0);
+ }
+}
+
+s32 BPF_STRUCT_OPS(userland_init_task, struct task_struct *p,
+ struct scx_init_task_args *args)
+{
+ if (bpf_task_storage_get(&task_ctx_stor, p, 0,
+ BPF_LOCAL_STORAGE_GET_F_CREATE))
+ return 0;
+ else
+ return -ENOMEM;
+}
+
+s32 BPF_STRUCT_OPS(userland_init)
+{
+ if (num_possible_cpus == 0) {
+ scx_bpf_error("User scheduler # CPUs uninitialized (%d)",
+ num_possible_cpus);
+ return -EINVAL;
+ }
+
+ if (usersched_pid <= 0) {
+ scx_bpf_error("User scheduler pid uninitialized (%d)",
+ usersched_pid);
+ return -EINVAL;
+ }
+
+ return 0;
+}
+
+void BPF_STRUCT_OPS(userland_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SCX_OPS_DEFINE(userland_ops,
+ .select_cpu = (void *)userland_select_cpu,
+ .enqueue = (void *)userland_enqueue,
+ .dispatch = (void *)userland_dispatch,
+ .update_idle = (void *)userland_update_idle,
+ .init_task = (void *)userland_init_task,
+ .init = (void *)userland_init,
+ .exit = (void *)userland_exit,
+ .flags = SCX_OPS_ENQ_LAST |
+ SCX_OPS_KEEP_BUILTIN_IDLE,
+ .name = "userland");
diff --git a/tools/sched_ext/scx_userland.c b/tools/sched_ext/scx_userland.c
new file mode 100644
index 000000000000..10b31020f44f
--- /dev/null
+++ b/tools/sched_ext/scx_userland.c
@@ -0,0 +1,437 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * A demo sched_ext user space scheduler which provides vruntime semantics
+ * using a simple ordered-list implementation.
+ *
+ * Each CPU in the system resides in a single, global domain. This precludes
+ * the need to do any load balancing between domains. The scheduler could
+ * easily be extended to support multiple domains, with load balancing
+ * happening in user space.
+ *
+ * Any task which has any CPU affinity is scheduled entirely in BPF. This
+ * program only schedules tasks which may run on any CPU.
+ *
+ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
+ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <sched.h>
+#include <signal.h>
+#include <assert.h>
+#include <libgen.h>
+#include <pthread.h>
+#include <bpf/bpf.h>
+#include <sys/mman.h>
+#include <sys/queue.h>
+#include <sys/syscall.h>
+
+#include <scx/common.h>
+#include "scx_userland.h"
+#include "scx_userland.bpf.skel.h"
+
+const char help_fmt[] =
+"A minimal userland sched_ext scheduler.\n"
+"\n"
+"See the top-level comment in .bpf.c for more details.\n"
+"\n"
+"Try to reduce `sysctl kernel.pid_max` if this program triggers OOMs.\n"
+"\n"
+"Usage: %s [-b BATCH]\n"
+"\n"
+" -b BATCH The number of tasks to batch when dispatching (default: 8)\n"
+" -v Print libbpf debug messages\n"
+" -h Display this help and exit\n";
+
+/* Defined in UAPI */
+#define SCHED_EXT 7
+
+/* Number of tasks to batch when dispatching to user space. */
+static __u32 batch_size = 8;
+
+static bool verbose;
+static volatile int exit_req;
+static int enqueued_fd, dispatched_fd;
+
+static struct scx_userland *skel;
+static struct bpf_link *ops_link;
+
+/* Stats collected in user space. */
+static __u64 nr_vruntime_enqueues, nr_vruntime_dispatches, nr_vruntime_failed;
+
+/* Number of tasks currently enqueued. */
+static __u64 nr_curr_enqueued;
+
+/* The data structure containing tasks that are enqueued in user space. */
+struct enqueued_task {
+ LIST_ENTRY(enqueued_task) entries;
+ __u64 sum_exec_runtime;
+ double vruntime;
+};
+
+/*
+ * Use a vruntime-sorted list to store tasks. This could easily be extended to
+ * a more optimal data structure, such as an rbtree as is done in CFS. We
+ * currently elect to use a sorted list to simplify the example for
+ * illustrative purposes.
+ */
+LIST_HEAD(listhead, enqueued_task);
+
+/*
+ * A vruntime-sorted list of tasks. The head of the list contains the task with
+ * the lowest vruntime. That is, the task that has the "highest" claim to be
+ * scheduled.
+ */
+static struct listhead vruntime_head = LIST_HEAD_INITIALIZER(vruntime_head);
+
+/*
+ * The main array of tasks. The array is allocated all at once during
+ * initialization, based on /proc/sys/kernel/pid_max, to avoid having to
+ * dynamically allocate memory on the enqueue path, which could cause a
+ * deadlock. A more substantive user space scheduler could e.g. provide a hook
+ * for newly enabled tasks that are passed to the scheduler from the
+ * .prep_enable() callback to allows the scheduler to allocate on safe paths.
+ */
+struct enqueued_task *tasks;
+static int pid_max;
+
+static double min_vruntime;
+
+static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
+{
+ if (level == LIBBPF_DEBUG && !verbose)
+ return 0;
+ return vfprintf(stderr, format, args);
+}
+
+static void sigint_handler(int userland)
+{
+ exit_req = 1;
+}
+
+static int get_pid_max(void)
+{
+ FILE *fp;
+ int pid_max;
+
+ fp = fopen("/proc/sys/kernel/pid_max", "r");
+ if (fp == NULL) {
+ fprintf(stderr, "Error opening /proc/sys/kernel/pid_max\n");
+ return -1;
+ }
+ if (fscanf(fp, "%d", &pid_max) != 1) {
+ fprintf(stderr, "Error reading from /proc/sys/kernel/pid_max\n");
+ fclose(fp);
+ return -1;
+ }
+ fclose(fp);
+
+ return pid_max;
+}
+
+static int init_tasks(void)
+{
+ pid_max = get_pid_max();
+ if (pid_max < 0)
+ return pid_max;
+
+ tasks = calloc(pid_max, sizeof(*tasks));
+ if (!tasks) {
+ fprintf(stderr, "Error allocating tasks array\n");
+ return -ENOMEM;
+ }
+
+ return 0;
+}
+
+static __u32 task_pid(const struct enqueued_task *task)
+{
+ return ((uintptr_t)task - (uintptr_t)tasks) / sizeof(*task);
+}
+
+static int dispatch_task(__s32 pid)
+{
+ int err;
+
+ err = bpf_map_update_elem(dispatched_fd, NULL, &pid, 0);
+ if (err) {
+ nr_vruntime_failed++;
+ } else {
+ nr_vruntime_dispatches++;
+ }
+
+ return err;
+}
+
+static struct enqueued_task *get_enqueued_task(__s32 pid)
+{
+ if (pid >= pid_max)
+ return NULL;
+
+ return &tasks[pid];
+}
+
+static double calc_vruntime_delta(__u64 weight, __u64 delta)
+{
+ double weight_f = (double)weight / 100.0;
+ double delta_f = (double)delta;
+
+ return delta_f / weight_f;
+}
+
+static void update_enqueued(struct enqueued_task *enqueued, const struct scx_userland_enqueued_task *bpf_task)
+{
+ __u64 delta;
+
+ delta = bpf_task->sum_exec_runtime - enqueued->sum_exec_runtime;
+
+ enqueued->vruntime += calc_vruntime_delta(bpf_task->weight, delta);
+ if (min_vruntime > enqueued->vruntime)
+ enqueued->vruntime = min_vruntime;
+ enqueued->sum_exec_runtime = bpf_task->sum_exec_runtime;
+}
+
+static int vruntime_enqueue(const struct scx_userland_enqueued_task *bpf_task)
+{
+ struct enqueued_task *curr, *enqueued, *prev;
+
+ curr = get_enqueued_task(bpf_task->pid);
+ if (!curr)
+ return ENOENT;
+
+ update_enqueued(curr, bpf_task);
+ nr_vruntime_enqueues++;
+ nr_curr_enqueued++;
+
+ /*
+ * Enqueue the task in a vruntime-sorted list. A more optimal data
+ * structure such as an rbtree could easily be used as well. We elect
+ * to use a list here simply because it's less code, and thus the
+ * example is less convoluted and better serves to illustrate what a
+ * user space scheduler could look like.
+ */
+
+ if (LIST_EMPTY(&vruntime_head)) {
+ LIST_INSERT_HEAD(&vruntime_head, curr, entries);
+ return 0;
+ }
+
+ LIST_FOREACH(enqueued, &vruntime_head, entries) {
+ if (curr->vruntime <= enqueued->vruntime) {
+ LIST_INSERT_BEFORE(enqueued, curr, entries);
+ return 0;
+ }
+ prev = enqueued;
+ }
+
+ LIST_INSERT_AFTER(prev, curr, entries);
+
+ return 0;
+}
+
+static void drain_enqueued_map(void)
+{
+ while (1) {
+ struct scx_userland_enqueued_task task;
+ int err;
+
+ if (bpf_map_lookup_and_delete_elem(enqueued_fd, NULL, &task)) {
+ skel->bss->nr_queued = 0;
+ skel->bss->nr_scheduled = nr_curr_enqueued;
+ return;
+ }
+
+ err = vruntime_enqueue(&task);
+ if (err) {
+ fprintf(stderr, "Failed to enqueue task %d: %s\n",
+ task.pid, strerror(err));
+ exit_req = 1;
+ return;
+ }
+ }
+}
+
+static void dispatch_batch(void)
+{
+ __u32 i;
+
+ for (i = 0; i < batch_size; i++) {
+ struct enqueued_task *task;
+ int err;
+ __s32 pid;
+
+ task = LIST_FIRST(&vruntime_head);
+ if (!task)
+ break;
+
+ min_vruntime = task->vruntime;
+ pid = task_pid(task);
+ LIST_REMOVE(task, entries);
+ err = dispatch_task(pid);
+ if (err) {
+ /*
+ * If we fail to dispatch, put the task back to the
+ * vruntime_head list and stop dispatching additional
+ * tasks in this batch.
+ */
+ LIST_INSERT_HEAD(&vruntime_head, task, entries);
+ break;
+ }
+ nr_curr_enqueued--;
+ }
+ skel->bss->nr_scheduled = nr_curr_enqueued;
+}
+
+static void *run_stats_printer(void *arg)
+{
+ while (!exit_req) {
+ __u64 nr_failed_enqueues, nr_kernel_enqueues, nr_user_enqueues, total;
+
+ nr_failed_enqueues = skel->bss->nr_failed_enqueues;
+ nr_kernel_enqueues = skel->bss->nr_kernel_enqueues;
+ nr_user_enqueues = skel->bss->nr_user_enqueues;
+ total = nr_failed_enqueues + nr_kernel_enqueues + nr_user_enqueues;
+
+ printf("o-----------------------o\n");
+ printf("| BPF ENQUEUES |\n");
+ printf("|-----------------------|\n");
+ printf("| kern: %10llu |\n", nr_kernel_enqueues);
+ printf("| user: %10llu |\n", nr_user_enqueues);
+ printf("| failed: %10llu |\n", nr_failed_enqueues);
+ printf("| -------------------- |\n");
+ printf("| total: %10llu |\n", total);
+ printf("| |\n");
+ printf("|-----------------------|\n");
+ printf("| VRUNTIME / USER |\n");
+ printf("|-----------------------|\n");
+ printf("| enq: %10llu |\n", nr_vruntime_enqueues);
+ printf("| disp: %10llu |\n", nr_vruntime_dispatches);
+ printf("| failed: %10llu |\n", nr_vruntime_failed);
+ printf("o-----------------------o\n");
+ printf("\n\n");
+ fflush(stdout);
+ sleep(1);
+ }
+
+ return NULL;
+}
+
+static int spawn_stats_thread(void)
+{
+ pthread_t stats_printer;
+
+ return pthread_create(&stats_printer, NULL, run_stats_printer, NULL);
+}
+
+static void pre_bootstrap(int argc, char **argv)
+{
+ int err;
+ __u32 opt;
+ struct sched_param sched_param = {
+ .sched_priority = sched_get_priority_max(SCHED_EXT),
+ };
+
+ err = init_tasks();
+ if (err)
+ exit(err);
+
+ libbpf_set_print(libbpf_print_fn);
+ signal(SIGINT, sigint_handler);
+ signal(SIGTERM, sigint_handler);
+
+ /*
+ * Enforce that the user scheduler task is managed by sched_ext. The
+ * task eagerly drains the list of enqueued tasks in its main work
+ * loop, and then yields the CPU. The BPF scheduler only schedules the
+ * user space scheduler task when at least one other task in the system
+ * needs to be scheduled.
+ */
+ err = syscall(__NR_sched_setscheduler, getpid(), SCHED_EXT, &sched_param);
+ SCX_BUG_ON(err, "Failed to set scheduler to SCHED_EXT");
+
+ while ((opt = getopt(argc, argv, "b:vh")) != -1) {
+ switch (opt) {
+ case 'b':
+ batch_size = strtoul(optarg, NULL, 0);
+ break;
+ case 'v':
+ verbose = true;
+ break;
+ default:
+ fprintf(stderr, help_fmt, basename(argv[0]));
+ exit(opt != 'h');
+ }
+ }
+
+ /*
+ * It's not always safe to allocate in a user space scheduler, as an
+ * enqueued task could hold a lock that we require in order to be able
+ * to allocate.
+ */
+ err = mlockall(MCL_CURRENT | MCL_FUTURE);
+ SCX_BUG_ON(err, "Failed to prefault and lock address space");
+}
+
+static void bootstrap(char *comm)
+{
+ skel = SCX_OPS_OPEN(userland_ops, scx_userland);
+
+ skel->rodata->num_possible_cpus = libbpf_num_possible_cpus();
+ assert(skel->rodata->num_possible_cpus > 0);
+ skel->rodata->usersched_pid = getpid();
+ assert(skel->rodata->usersched_pid > 0);
+
+ SCX_OPS_LOAD(skel, userland_ops, scx_userland, uei);
+
+ enqueued_fd = bpf_map__fd(skel->maps.enqueued);
+ dispatched_fd = bpf_map__fd(skel->maps.dispatched);
+ assert(enqueued_fd > 0);
+ assert(dispatched_fd > 0);
+
+ SCX_BUG_ON(spawn_stats_thread(), "Failed to spawn stats thread");
+
+ ops_link = SCX_OPS_ATTACH(skel, userland_ops, scx_userland);
+}
+
+static void sched_main_loop(void)
+{
+ while (!exit_req) {
+ /*
+ * Perform the following work in the main user space scheduler
+ * loop:
+ *
+ * 1. Drain all tasks from the enqueued map, and enqueue them
+ * to the vruntime sorted list.
+ *
+ * 2. Dispatch a batch of tasks from the vruntime sorted list
+ * down to the kernel.
+ *
+ * 3. Yield the CPU back to the system. The BPF scheduler will
+ * reschedule the user space scheduler once another task has
+ * been enqueued to user space.
+ */
+ drain_enqueued_map();
+ dispatch_batch();
+ sched_yield();
+ }
+}
+
+int main(int argc, char **argv)
+{
+ __u64 ecode;
+
+ pre_bootstrap(argc, argv);
+restart:
+ bootstrap(argv[0]);
+ sched_main_loop();
+
+ exit_req = 1;
+ bpf_link__destroy(ops_link);
+ ecode = UEI_REPORT(skel, uei);
+ scx_userland__destroy(skel);
+
+ if (UEI_ECODE_RESTART(ecode))
+ goto restart;
+ return 0;
+}
diff --git a/tools/sched_ext/scx_userland.h b/tools/sched_ext/scx_userland.h
new file mode 100644
index 000000000000..684fb2dd5de9
--- /dev/null
+++ b/tools/sched_ext/scx_userland.h
@@ -0,0 +1,17 @@
+// SPDX-License-Identifier: GPL-2.0
+/* Copyright (c) 2022 Meta, Inc */
+
+#ifndef __SCX_USERLAND_COMMON_H
+#define __SCX_USERLAND_COMMON_H
+
+/*
+ * An instance of a task that has been enqueued by the kernel for consumption
+ * by a user space global scheduler thread.
+ */
+struct scx_userland_enqueued_task {
+ __s32 pid;
+ u64 sum_exec_runtime;
+ u64 weight;
+};
+
+#endif // __SCX_USERLAND_COMMON_H
--
2.49.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 3/4] tools/sched_ext: add scx_pair scheduler
2026-01-12 21:41 [PATCH 0/4] tools/sched_ext: Add example C schedulers Emil Tsalapatis
2026-01-12 21:41 ` [PATCH 1/4] tools/sched_ext: add scx_nest scheduler Emil Tsalapatis
2026-01-12 21:41 ` [PATCH 2/4] tools/sched_ext: add scx_userland scheduler Emil Tsalapatis
@ 2026-01-12 21:41 ` Emil Tsalapatis
2026-01-12 21:41 ` [PATCH 4/4] tools/sched_ext: add arena based scheduler Emil Tsalapatis
2026-01-13 5:57 ` [PATCH 0/4] tools/sched_ext: Add example C schedulers Changwoo Min
4 siblings, 0 replies; 8+ messages in thread
From: Emil Tsalapatis @ 2026-01-12 21:41 UTC (permalink / raw)
To: sched-ext; +Cc: changwoo, arighi, tj, void, Emil Tsalapatis, David Vernet
Add the scx_pair cgroup-based core scheduler.
Cc: Tejun Heo <tj@kernel.org>
Cc: David Vernet <dvernet@meta.com>
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
tools/sched_ext/Makefile | 2 +-
tools/sched_ext/scx_pair.bpf.c | 610 +++++++++++++++++++++++++++++++++
tools/sched_ext/scx_pair.c | 180 ++++++++++
tools/sched_ext/scx_pair.h | 9 +
4 files changed, 800 insertions(+), 1 deletion(-)
create mode 100644 tools/sched_ext/scx_pair.bpf.c
create mode 100644 tools/sched_ext/scx_pair.c
create mode 100644 tools/sched_ext/scx_pair.h
diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile
index 2f06e0587fbb..32867f468db2 100644
--- a/tools/sched_ext/Makefile
+++ b/tools/sched_ext/Makefile
@@ -189,7 +189,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
-c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_nest scx_userland
+c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_nest scx_userland scx_pair
$(addprefix $(BINDIR)/,$(c-sched-targets)): \
$(BINDIR)/%: \
diff --git a/tools/sched_ext/scx_pair.bpf.c b/tools/sched_ext/scx_pair.bpf.c
new file mode 100644
index 000000000000..267011b57cba
--- /dev/null
+++ b/tools/sched_ext/scx_pair.bpf.c
@@ -0,0 +1,610 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * A demo sched_ext core-scheduler which always makes every sibling CPU pair
+ * execute from the same CPU cgroup.
+ *
+ * This scheduler is a minimal implementation and would need some form of
+ * priority handling both inside each cgroup and across the cgroups to be
+ * practically useful.
+ *
+ * Each CPU in the system is paired with exactly one other CPU, according to a
+ * "stride" value that can be specified when the BPF scheduler program is first
+ * loaded. Throughout the runtime of the scheduler, these CPU pairs guarantee
+ * that they will only ever schedule tasks that belong to the same CPU cgroup.
+ *
+ * Scheduler Initialization
+ * ------------------------
+ *
+ * The scheduler BPF program is first initialized from user space, before it is
+ * enabled. During this initialization process, each CPU on the system is
+ * assigned several values that are constant throughout its runtime:
+ *
+ * 1. *Pair CPU*: The CPU that it synchronizes with when making scheduling
+ * decisions. Paired CPUs always schedule tasks from the same
+ * CPU cgroup, and synchronize with each other to guarantee
+ * that this constraint is not violated.
+ * 2. *Pair ID*: Each CPU pair is assigned a Pair ID, which is used to access
+ * a struct pair_ctx object that is shared between the pair.
+ * 3. *In-pair-index*: An index, 0 or 1, that is assigned to each core in the
+ * pair. Each struct pair_ctx has an active_mask field,
+ * which is a bitmap used to indicate whether each core
+ * in the pair currently has an actively running task.
+ * This index specifies which entry in the bitmap corresponds
+ * to each CPU in the pair.
+ *
+ * During this initialization, the CPUs are paired according to a "stride" that
+ * may be specified when invoking the user space program that initializes and
+ * loads the scheduler. By default, the stride is 1/2 the total number of CPUs.
+ *
+ * Tasks and cgroups
+ * -----------------
+ *
+ * Every cgroup in the system is registered with the scheduler using the
+ * pair_cgroup_init() callback, and every task in the system is associated with
+ * exactly one cgroup. At a high level, the idea with the pair scheduler is to
+ * always schedule tasks from the same cgroup within a given CPU pair. When a
+ * task is enqueued (i.e. passed to the pair_enqueue() callback function), its
+ * cgroup ID is read from its task struct, and then a corresponding queue map
+ * is used to FIFO-enqueue the task for that cgroup.
+ *
+ * If you look through the implementation of the scheduler, you'll notice that
+ * there is quite a bit of complexity involved with looking up the per-cgroup
+ * FIFO queue that we enqueue tasks in. For example, there is a cgrp_q_idx_hash
+ * BPF hash map that is used to map a cgroup ID to a globally unique ID that's
+ * allocated in the BPF program. This is done because we use separate maps to
+ * store the FIFO queue of tasks, and the length of that map, per cgroup. This
+ * complexity is only present because of current deficiencies in BPF that will
+ * soon be addressed. The main point to keep in mind is that newly enqueued
+ * tasks are added to their cgroup's FIFO queue.
+ *
+ * Dispatching tasks
+ * -----------------
+ *
+ * This section will describe how enqueued tasks are dispatched and scheduled.
+ * Tasks are dispatched in pair_dispatch(), and at a high level the workflow is
+ * as follows:
+ *
+ * 1. Fetch the struct pair_ctx for the current CPU. As mentioned above, this is
+ * the structure that's used to synchronize amongst the two pair CPUs in their
+ * scheduling decisions. After any of the following events have occurred:
+ *
+ * - The cgroup's slice run has expired, or
+ * - The cgroup becomes empty, or
+ * - Either CPU in the pair is preempted by a higher priority scheduling class
+ *
+ * The cgroup transitions to the draining state and stops executing new tasks
+ * from the cgroup.
+ *
+ * 2. If the pair is still executing a task, mark the pair_ctx as draining, and
+ * wait for the pair CPU to be preempted.
+ *
+ * 3. Otherwise, if the pair CPU is not running a task, we can move onto
+ * scheduling new tasks. Pop the next cgroup id from the top_q queue.
+ *
+ * 4. Pop a task from that cgroup's FIFO task queue, and begin executing it.
+ *
+ * Note again that this scheduling behavior is simple, but the implementation
+ * is complex mostly because this it hits several BPF shortcomings and has to
+ * work around in often awkward ways. Most of the shortcomings are expected to
+ * be resolved in the near future which should allow greatly simplifying this
+ * scheduler.
+ *
+ * Dealing with preemption
+ * -----------------------
+ *
+ * SCX is the lowest priority sched_class, and could be preempted by them at
+ * any time. To address this, the scheduler implements pair_cpu_release() and
+ * pair_cpu_acquire() callbacks which are invoked by the core scheduler when
+ * the scheduler loses and gains control of the CPU respectively.
+ *
+ * In pair_cpu_release(), we mark the pair_ctx as having been preempted, and
+ * then invoke:
+ *
+ * scx_bpf_kick_cpu(pair_cpu, SCX_KICK_PREEMPT | SCX_KICK_WAIT);
+ *
+ * This preempts the pair CPU, and waits until it has re-entered the scheduler
+ * before returning. This is necessary to ensure that the higher priority
+ * sched_class that preempted our scheduler does not schedule a task
+ * concurrently with our pair CPU.
+ *
+ * When the CPU is re-acquired in pair_cpu_acquire(), we unmark the preemption
+ * in the pair_ctx, and send another resched IPI to the pair CPU to re-enable
+ * pair scheduling.
+ *
+ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
+ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
+ */
+#include <scx/common.bpf.h>
+#include "scx_pair.h"
+
+char _license[] SEC("license") = "GPL";
+
+/* !0 for veristat, set during init */
+const volatile u32 nr_cpu_ids = 1;
+
+/* a pair of CPUs stay on a cgroup for this duration */
+const volatile u32 pair_batch_dur_ns;
+
+/* cpu ID -> pair cpu ID */
+const volatile s32 RESIZABLE_ARRAY(rodata, pair_cpu);
+
+/* cpu ID -> pair_id */
+const volatile u32 RESIZABLE_ARRAY(rodata, pair_id);
+
+/* CPU ID -> CPU # in the pair (0 or 1) */
+const volatile u32 RESIZABLE_ARRAY(rodata, in_pair_idx);
+
+struct pair_ctx {
+ struct bpf_spin_lock lock;
+
+ /* the cgroup the pair is currently executing */
+ u64 cgid;
+
+ /* the pair started executing the current cgroup at */
+ u64 started_at;
+
+ /* whether the current cgroup is draining */
+ bool draining;
+
+ /* the CPUs that are currently active on the cgroup */
+ u32 active_mask;
+
+ /*
+ * the CPUs that are currently preempted and running tasks in a
+ * different scheduler.
+ */
+ u32 preempted_mask;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __type(key, u32);
+ __type(value, struct pair_ctx);
+} pair_ctx SEC(".maps");
+
+/* queue of cgrp_q's possibly with tasks on them */
+struct {
+ __uint(type, BPF_MAP_TYPE_QUEUE);
+ /*
+ * Because it's difficult to build strong synchronization encompassing
+ * multiple non-trivial operations in BPF, this queue is managed in an
+ * opportunistic way so that we guarantee that a cgroup w/ active tasks
+ * is always on it but possibly multiple times. Once we have more robust
+ * synchronization constructs and e.g. linked list, we should be able to
+ * do this in a prettier way but for now just size it big enough.
+ */
+ __uint(max_entries, 4 * MAX_CGRPS);
+ __type(value, u64);
+} top_q SEC(".maps");
+
+/* per-cgroup q which FIFOs the tasks from the cgroup */
+struct cgrp_q {
+ __uint(type, BPF_MAP_TYPE_QUEUE);
+ __uint(max_entries, MAX_QUEUED);
+ __type(value, u32);
+};
+
+/*
+ * Ideally, we want to allocate cgrp_q and cgrq_q_len in the cgroup local
+ * storage; however, a cgroup local storage can only be accessed from the BPF
+ * progs attached to the cgroup. For now, work around by allocating array of
+ * cgrp_q's and then allocating per-cgroup indices.
+ *
+ * Another caveat: It's difficult to populate a large array of maps statically
+ * or from BPF. Initialize it from userland.
+ */
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
+ __uint(max_entries, MAX_CGRPS);
+ __type(key, s32);
+ __array(values, struct cgrp_q);
+} cgrp_q_arr SEC(".maps");
+
+static u64 cgrp_q_len[MAX_CGRPS];
+
+/*
+ * This and cgrp_q_idx_hash combine into a poor man's IDR. This likely would be
+ * useful to have as a map type.
+ */
+static u32 cgrp_q_idx_cursor;
+static u64 cgrp_q_idx_busy[MAX_CGRPS];
+
+/*
+ * All added up, the following is what we do:
+ *
+ * 1. When a cgroup is enabled, RR cgroup_q_idx_busy array doing cmpxchg looking
+ * for a free ID. If not found, fail cgroup creation with -EBUSY.
+ *
+ * 2. Hash the cgroup ID to the allocated cgrp_q_idx in the following
+ * cgrp_q_idx_hash.
+ *
+ * 3. Whenever a cgrp_q needs to be accessed, first look up the cgrp_q_idx from
+ * cgrp_q_idx_hash and then access the corresponding entry in cgrp_q_arr.
+ *
+ * This is sadly complicated for something pretty simple. Hopefully, we should
+ * be able to simplify in the future.
+ */
+struct {
+ __uint(type, BPF_MAP_TYPE_HASH);
+ __uint(max_entries, MAX_CGRPS);
+ __uint(key_size, sizeof(u64)); /* cgrp ID */
+ __uint(value_size, sizeof(s32)); /* cgrp_q idx */
+} cgrp_q_idx_hash SEC(".maps");
+
+/* statistics */
+u64 nr_total, nr_dispatched, nr_missing, nr_kicks, nr_preemptions;
+u64 nr_exps, nr_exp_waits, nr_exp_empty;
+u64 nr_cgrp_next, nr_cgrp_coll, nr_cgrp_empty;
+
+UEI_DEFINE(uei);
+
+void BPF_STRUCT_OPS(pair_enqueue, struct task_struct *p, u64 enq_flags)
+{
+ struct cgroup *cgrp;
+ struct cgrp_q *cgq;
+ s32 pid = p->pid;
+ u64 cgid;
+ u32 *q_idx;
+ u64 *cgq_len;
+
+ __sync_fetch_and_add(&nr_total, 1);
+
+ cgrp = scx_bpf_task_cgroup(p);
+ cgid = cgrp->kn->id;
+ bpf_cgroup_release(cgrp);
+
+ /* find the cgroup's q and push @p into it */
+ q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid);
+ if (!q_idx) {
+ scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid);
+ return;
+ }
+
+ cgq = bpf_map_lookup_elem(&cgrp_q_arr, q_idx);
+ if (!cgq) {
+ scx_bpf_error("failed to lookup q_arr for cgroup[%llu] q_idx[%u]",
+ cgid, *q_idx);
+ return;
+ }
+
+ if (bpf_map_push_elem(cgq, &pid, 0)) {
+ scx_bpf_error("cgroup[%llu] queue overflow", cgid);
+ return;
+ }
+
+ /* bump q len, if going 0 -> 1, queue cgroup into the top_q */
+ cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]);
+ if (!cgq_len) {
+ scx_bpf_error("MEMBER_VTPR malfunction");
+ return;
+ }
+
+ if (!__sync_fetch_and_add(cgq_len, 1) &&
+ bpf_map_push_elem(&top_q, &cgid, 0)) {
+ scx_bpf_error("top_q overflow");
+ return;
+ }
+}
+
+static int lookup_pairc_and_mask(s32 cpu, struct pair_ctx **pairc, u32 *mask)
+{
+ u32 *vptr;
+
+ vptr = (u32 *)ARRAY_ELEM_PTR(pair_id, cpu, nr_cpu_ids);
+ if (!vptr)
+ return -EINVAL;
+
+ *pairc = bpf_map_lookup_elem(&pair_ctx, vptr);
+ if (!(*pairc))
+ return -EINVAL;
+
+ vptr = (u32 *)ARRAY_ELEM_PTR(in_pair_idx, cpu, nr_cpu_ids);
+ if (!vptr)
+ return -EINVAL;
+
+ *mask = 1U << *vptr;
+
+ return 0;
+}
+
+__attribute__((noinline))
+static int try_dispatch(s32 cpu)
+{
+ struct pair_ctx *pairc;
+ struct bpf_map *cgq_map;
+ struct task_struct *p;
+ u64 now = scx_bpf_now();
+ bool kick_pair = false;
+ bool expired, pair_preempted;
+ u32 *vptr, in_pair_mask;
+ s32 pid, q_idx;
+ u64 cgid;
+ int ret;
+
+ ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask);
+ if (ret) {
+ scx_bpf_error("failed to lookup pairc and in_pair_mask for cpu[%d]",
+ cpu);
+ return -ENOENT;
+ }
+
+ bpf_spin_lock(&pairc->lock);
+ pairc->active_mask &= ~in_pair_mask;
+
+ expired = time_before(pairc->started_at + pair_batch_dur_ns, now);
+ if (expired || pairc->draining) {
+ u64 new_cgid = 0;
+
+ __sync_fetch_and_add(&nr_exps, 1);
+
+ /*
+ * We're done with the current cgid. An obvious optimization
+ * would be not draining if the next cgroup is the current one.
+ * For now, be dumb and always expire.
+ */
+ pairc->draining = true;
+
+ pair_preempted = pairc->preempted_mask;
+ if (pairc->active_mask || pair_preempted) {
+ /*
+ * The other CPU is still active, or is no longer under
+ * our control due to e.g. being preempted by a higher
+ * priority sched_class. We want to wait until this
+ * cgroup expires, or until control of our pair CPU has
+ * been returned to us.
+ *
+ * If the pair controls its CPU, and the time already
+ * expired, kick. When the other CPU arrives at
+ * dispatch and clears its active mask, it'll push the
+ * pair to the next cgroup and kick this CPU.
+ */
+ __sync_fetch_and_add(&nr_exp_waits, 1);
+ bpf_spin_unlock(&pairc->lock);
+ if (expired && !pair_preempted)
+ kick_pair = true;
+ goto out_maybe_kick;
+ }
+
+ bpf_spin_unlock(&pairc->lock);
+
+ /*
+ * Pick the next cgroup. It'd be easier / cleaner to not drop
+ * pairc->lock and use stronger synchronization here especially
+ * given that we'll be switching cgroups significantly less
+ * frequently than tasks. Unfortunately, bpf_spin_lock can't
+ * really protect anything non-trivial. Let's do opportunistic
+ * operations instead.
+ */
+ bpf_repeat(BPF_MAX_LOOPS) {
+ u32 *q_idx;
+ u64 *cgq_len;
+
+ if (bpf_map_pop_elem(&top_q, &new_cgid)) {
+ /* no active cgroup, go idle */
+ __sync_fetch_and_add(&nr_exp_empty, 1);
+ return 0;
+ }
+
+ q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &new_cgid);
+ if (!q_idx)
+ continue;
+
+ /*
+ * This is the only place where empty cgroups are taken
+ * off the top_q.
+ */
+ cgq_len = MEMBER_VPTR(cgrp_q_len, [*q_idx]);
+ if (!cgq_len || !*cgq_len)
+ continue;
+
+ /*
+ * If it has any tasks, requeue as we may race and not
+ * execute it.
+ */
+ bpf_map_push_elem(&top_q, &new_cgid, 0);
+ break;
+ }
+
+ bpf_spin_lock(&pairc->lock);
+
+ /*
+ * The other CPU may already have started on a new cgroup while
+ * we dropped the lock. Make sure that we're still draining and
+ * start on the new cgroup.
+ */
+ if (pairc->draining && !pairc->active_mask) {
+ __sync_fetch_and_add(&nr_cgrp_next, 1);
+ pairc->cgid = new_cgid;
+ pairc->started_at = now;
+ pairc->draining = false;
+ kick_pair = true;
+ } else {
+ __sync_fetch_and_add(&nr_cgrp_coll, 1);
+ }
+ }
+
+ cgid = pairc->cgid;
+ pairc->active_mask |= in_pair_mask;
+ bpf_spin_unlock(&pairc->lock);
+
+ /* again, it'd be better to do all these with the lock held, oh well */
+ vptr = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid);
+ if (!vptr) {
+ scx_bpf_error("failed to lookup q_idx for cgroup[%llu]", cgid);
+ return -ENOENT;
+ }
+ q_idx = *vptr;
+
+ /* claim one task from cgrp_q w/ q_idx */
+ bpf_repeat(BPF_MAX_LOOPS) {
+ u64 *cgq_len, len;
+
+ cgq_len = MEMBER_VPTR(cgrp_q_len, [q_idx]);
+ if (!cgq_len || !(len = *(volatile u64 *)cgq_len)) {
+ /* the cgroup must be empty, expire and repeat */
+ __sync_fetch_and_add(&nr_cgrp_empty, 1);
+ bpf_spin_lock(&pairc->lock);
+ pairc->draining = true;
+ pairc->active_mask &= ~in_pair_mask;
+ bpf_spin_unlock(&pairc->lock);
+ return -EAGAIN;
+ }
+
+ if (__sync_val_compare_and_swap(cgq_len, len, len - 1) != len)
+ continue;
+
+ break;
+ }
+
+ cgq_map = bpf_map_lookup_elem(&cgrp_q_arr, &q_idx);
+ if (!cgq_map) {
+ scx_bpf_error("failed to lookup cgq_map for cgroup[%llu] q_idx[%d]",
+ cgid, q_idx);
+ return -ENOENT;
+ }
+
+ if (bpf_map_pop_elem(cgq_map, &pid)) {
+ scx_bpf_error("cgq_map is empty for cgroup[%llu] q_idx[%d]",
+ cgid, q_idx);
+ return -ENOENT;
+ }
+
+ p = bpf_task_from_pid(pid);
+ if (p) {
+ __sync_fetch_and_add(&nr_dispatched, 1);
+ scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
+ bpf_task_release(p);
+ } else {
+ /* we don't handle dequeues, retry on lost tasks */
+ __sync_fetch_and_add(&nr_missing, 1);
+ return -EAGAIN;
+ }
+
+out_maybe_kick:
+ if (kick_pair) {
+ s32 *pair = (s32 *)ARRAY_ELEM_PTR(pair_cpu, cpu, nr_cpu_ids);
+ if (pair) {
+ __sync_fetch_and_add(&nr_kicks, 1);
+ scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT);
+ }
+ }
+ return 0;
+}
+
+void BPF_STRUCT_OPS(pair_dispatch, s32 cpu, struct task_struct *prev)
+{
+ bpf_repeat(BPF_MAX_LOOPS) {
+ if (try_dispatch(cpu) != -EAGAIN)
+ break;
+ }
+}
+
+void BPF_STRUCT_OPS(pair_cpu_acquire, s32 cpu, struct scx_cpu_acquire_args *args)
+{
+ int ret;
+ u32 in_pair_mask;
+ struct pair_ctx *pairc;
+ bool kick_pair;
+
+ ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask);
+ if (ret)
+ return;
+
+ bpf_spin_lock(&pairc->lock);
+ pairc->preempted_mask &= ~in_pair_mask;
+ /* Kick the pair CPU, unless it was also preempted. */
+ kick_pair = !pairc->preempted_mask;
+ bpf_spin_unlock(&pairc->lock);
+
+ if (kick_pair) {
+ s32 *pair = (s32 *)ARRAY_ELEM_PTR(pair_cpu, cpu, nr_cpu_ids);
+
+ if (pair) {
+ __sync_fetch_and_add(&nr_kicks, 1);
+ scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT);
+ }
+ }
+}
+
+void BPF_STRUCT_OPS(pair_cpu_release, s32 cpu, struct scx_cpu_release_args *args)
+{
+ int ret;
+ u32 in_pair_mask;
+ struct pair_ctx *pairc;
+ bool kick_pair;
+
+ ret = lookup_pairc_and_mask(cpu, &pairc, &in_pair_mask);
+ if (ret)
+ return;
+
+ bpf_spin_lock(&pairc->lock);
+ pairc->preempted_mask |= in_pair_mask;
+ pairc->active_mask &= ~in_pair_mask;
+ /* Kick the pair CPU if it's still running. */
+ kick_pair = pairc->active_mask;
+ pairc->draining = true;
+ bpf_spin_unlock(&pairc->lock);
+
+ if (kick_pair) {
+ s32 *pair = (s32 *)ARRAY_ELEM_PTR(pair_cpu, cpu, nr_cpu_ids);
+
+ if (pair) {
+ __sync_fetch_and_add(&nr_kicks, 1);
+ scx_bpf_kick_cpu(*pair, SCX_KICK_PREEMPT | SCX_KICK_WAIT);
+ }
+ }
+ __sync_fetch_and_add(&nr_preemptions, 1);
+}
+
+s32 BPF_STRUCT_OPS(pair_cgroup_init, struct cgroup *cgrp)
+{
+ u64 cgid = cgrp->kn->id;
+ s32 i, q_idx;
+
+ bpf_for(i, 0, MAX_CGRPS) {
+ q_idx = __sync_fetch_and_add(&cgrp_q_idx_cursor, 1) % MAX_CGRPS;
+ if (!__sync_val_compare_and_swap(&cgrp_q_idx_busy[q_idx], 0, 1))
+ break;
+ }
+ if (i == MAX_CGRPS)
+ return -EBUSY;
+
+ if (bpf_map_update_elem(&cgrp_q_idx_hash, &cgid, &q_idx, BPF_ANY)) {
+ u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [q_idx]);
+ if (busy)
+ *busy = 0;
+ return -EBUSY;
+ }
+
+ return 0;
+}
+
+void BPF_STRUCT_OPS(pair_cgroup_exit, struct cgroup *cgrp)
+{
+ u64 cgid = cgrp->kn->id;
+ s32 *q_idx;
+
+ q_idx = bpf_map_lookup_elem(&cgrp_q_idx_hash, &cgid);
+ if (q_idx) {
+ u64 *busy = MEMBER_VPTR(cgrp_q_idx_busy, [*q_idx]);
+ if (busy)
+ *busy = 0;
+ bpf_map_delete_elem(&cgrp_q_idx_hash, &cgid);
+ }
+}
+
+void BPF_STRUCT_OPS(pair_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SCX_OPS_DEFINE(pair_ops,
+ .enqueue = (void *)pair_enqueue,
+ .dispatch = (void *)pair_dispatch,
+ .cpu_acquire = (void *)pair_cpu_acquire,
+ .cpu_release = (void *)pair_cpu_release,
+ .cgroup_init = (void *)pair_cgroup_init,
+ .cgroup_exit = (void *)pair_cgroup_exit,
+ .exit = (void *)pair_exit,
+ .name = "pair");
diff --git a/tools/sched_ext/scx_pair.c b/tools/sched_ext/scx_pair.c
new file mode 100644
index 000000000000..d3e97faa6334
--- /dev/null
+++ b/tools/sched_ext/scx_pair.c
@@ -0,0 +1,180 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
+ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <assert.h>
+#include <libgen.h>
+#include <bpf/bpf.h>
+#include <scx/common.h>
+#include "scx_pair.h"
+#include "scx_pair.bpf.skel.h"
+
+const char help_fmt[] =
+"A demo sched_ext core-scheduler which always makes every sibling CPU pair\n"
+"execute from the same CPU cgroup.\n"
+"\n"
+"See the top-level comment in .bpf.c for more details.\n"
+"\n"
+"Usage: %s [-S STRIDE]\n"
+"\n"
+" -S STRIDE Override CPU pair stride (default: nr_cpus_ids / 2)\n"
+" -v Print libbpf debug messages\n"
+" -h Display this help and exit\n";
+
+static bool verbose;
+static volatile int exit_req;
+
+static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
+{
+ if (level == LIBBPF_DEBUG && !verbose)
+ return 0;
+ return vfprintf(stderr, format, args);
+}
+
+static void sigint_handler(int dummy)
+{
+ exit_req = 1;
+}
+
+int main(int argc, char **argv)
+{
+ struct scx_pair *skel;
+ struct bpf_link *link;
+ __u64 seq = 0, ecode;
+ __s32 stride, i, opt, outer_fd;
+
+ libbpf_set_print(libbpf_print_fn);
+ signal(SIGINT, sigint_handler);
+ signal(SIGTERM, sigint_handler);
+restart:
+ skel = SCX_OPS_OPEN(pair_ops, scx_pair);
+
+ skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus();
+ assert(skel->rodata->nr_cpu_ids > 0);
+ skel->rodata->pair_batch_dur_ns = __COMPAT_ENUM_OR_ZERO("scx_public_consts", "SCX_SLICE_DFL");
+
+ /* pair up the earlier half to the latter by default, override with -s */
+ stride = skel->rodata->nr_cpu_ids / 2;
+
+ while ((opt = getopt(argc, argv, "S:vh")) != -1) {
+ switch (opt) {
+ case 'S':
+ stride = strtoul(optarg, NULL, 0);
+ break;
+ case 'v':
+ verbose = true;
+ break;
+ default:
+ fprintf(stderr, help_fmt, basename(argv[0]));
+ return opt != 'h';
+ }
+ }
+
+ bpf_map__set_max_entries(skel->maps.pair_ctx, skel->rodata->nr_cpu_ids / 2);
+
+ /* Resize arrays so their element count is equal to cpu count. */
+ RESIZE_ARRAY(skel, rodata, pair_cpu, skel->rodata->nr_cpu_ids);
+ RESIZE_ARRAY(skel, rodata, pair_id, skel->rodata->nr_cpu_ids);
+ RESIZE_ARRAY(skel, rodata, in_pair_idx, skel->rodata->nr_cpu_ids);
+
+ for (i = 0; i < skel->rodata->nr_cpu_ids; i++)
+ skel->rodata_pair_cpu->pair_cpu[i] = -1;
+
+ printf("Pairs: ");
+ for (i = 0; i < skel->rodata->nr_cpu_ids; i++) {
+ int j = (i + stride) % skel->rodata->nr_cpu_ids;
+
+ if (skel->rodata_pair_cpu->pair_cpu[i] >= 0)
+ continue;
+
+ SCX_BUG_ON(i == j,
+ "Invalid stride %d - CPU%d wants to be its own pair",
+ stride, i);
+
+ SCX_BUG_ON(skel->rodata_pair_cpu->pair_cpu[j] >= 0,
+ "Invalid stride %d - three CPUs (%d, %d, %d) want to be a pair",
+ stride, i, j, skel->rodata_pair_cpu->pair_cpu[j]);
+
+ skel->rodata_pair_cpu->pair_cpu[i] = j;
+ skel->rodata_pair_cpu->pair_cpu[j] = i;
+ skel->rodata_pair_id->pair_id[i] = i;
+ skel->rodata_pair_id->pair_id[j] = i;
+ skel->rodata_in_pair_idx->in_pair_idx[i] = 0;
+ skel->rodata_in_pair_idx->in_pair_idx[j] = 1;
+
+ printf("[%d, %d] ", i, j);
+ }
+ printf("\n");
+
+ SCX_OPS_LOAD(skel, pair_ops, scx_pair, uei);
+
+ /*
+ * Populate the cgrp_q_arr map which is an array containing per-cgroup
+ * queues. It'd probably be better to do this from BPF but there are too
+ * many to initialize statically and there's no way to dynamically
+ * populate from BPF.
+ */
+ outer_fd = bpf_map__fd(skel->maps.cgrp_q_arr);
+ SCX_BUG_ON(outer_fd < 0, "Failed to get outer_fd: %d", outer_fd);
+
+ printf("Initializing");
+ for (i = 0; i < MAX_CGRPS; i++) {
+ __s32 inner_fd;
+
+ if (exit_req)
+ break;
+
+ inner_fd = bpf_map_create(BPF_MAP_TYPE_QUEUE, NULL, 0,
+ sizeof(__u32), MAX_QUEUED, NULL);
+ SCX_BUG_ON(inner_fd < 0, "Failed to get inner_fd: %d",
+ inner_fd);
+ SCX_BUG_ON(bpf_map_update_elem(outer_fd, &i, &inner_fd, BPF_ANY),
+ "Failed to set inner map");
+ close(inner_fd);
+
+ if (!(i % 10))
+ printf(".");
+ fflush(stdout);
+ }
+ printf("\n");
+
+ /*
+ * Fully initialized, attach and run.
+ */
+ link = SCX_OPS_ATTACH(skel, pair_ops, scx_pair);
+
+ while (!exit_req && !UEI_EXITED(skel, uei)) {
+ printf("[SEQ %llu]\n", seq++);
+ printf(" total:%10" PRIu64 " dispatch:%10" PRIu64 " missing:%10" PRIu64 "\n",
+ skel->bss->nr_total,
+ skel->bss->nr_dispatched,
+ skel->bss->nr_missing);
+ printf(" kicks:%10" PRIu64 " preemptions:%7" PRIu64 "\n",
+ skel->bss->nr_kicks,
+ skel->bss->nr_preemptions);
+ printf(" exp:%10" PRIu64 " exp_wait:%10" PRIu64 " exp_empty:%10" PRIu64 "\n",
+ skel->bss->nr_exps,
+ skel->bss->nr_exp_waits,
+ skel->bss->nr_exp_empty);
+ printf("cgnext:%10" PRIu64 " cgcoll:%10" PRIu64 " cgempty:%10" PRIu64 "\n",
+ skel->bss->nr_cgrp_next,
+ skel->bss->nr_cgrp_coll,
+ skel->bss->nr_cgrp_empty);
+ fflush(stdout);
+ sleep(1);
+ }
+
+ bpf_link__destroy(link);
+ ecode = UEI_REPORT(skel, uei);
+ scx_pair__destroy(skel);
+
+ if (UEI_ECODE_RESTART(ecode))
+ goto restart;
+ return 0;
+}
diff --git a/tools/sched_ext/scx_pair.h b/tools/sched_ext/scx_pair.h
new file mode 100644
index 000000000000..d9666a447d3f
--- /dev/null
+++ b/tools/sched_ext/scx_pair.h
@@ -0,0 +1,9 @@
+#ifndef __SCX_EXAMPLE_PAIR_H
+#define __SCX_EXAMPLE_PAIR_H
+
+enum {
+ MAX_QUEUED = 4096,
+ MAX_CGRPS = 4096,
+};
+
+#endif /* __SCX_EXAMPLE_PAIR_H */
--
2.49.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* [PATCH 4/4] tools/sched_ext: add arena based scheduler
2026-01-12 21:41 [PATCH 0/4] tools/sched_ext: Add example C schedulers Emil Tsalapatis
` (2 preceding siblings ...)
2026-01-12 21:41 ` [PATCH 3/4] tools/sched_ext: add scx_pair scheduler Emil Tsalapatis
@ 2026-01-12 21:41 ` Emil Tsalapatis
2026-01-13 5:57 ` [PATCH 0/4] tools/sched_ext: Add example C schedulers Changwoo Min
4 siblings, 0 replies; 8+ messages in thread
From: Emil Tsalapatis @ 2026-01-12 21:41 UTC (permalink / raw)
To: sched-ext; +Cc: changwoo, arighi, tj, void, Emil Tsalapatis
Add a scheduler that uses BPF arenas to manage task context data.
Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
---
tools/sched_ext/Makefile | 2 +-
tools/sched_ext/scx_sdt.bpf.c | 710 ++++++++++++++++++++++++++++++++++
tools/sched_ext/scx_sdt.c | 101 +++++
tools/sched_ext/scx_sdt.h | 113 ++++++
4 files changed, 925 insertions(+), 1 deletion(-)
create mode 100644 tools/sched_ext/scx_sdt.bpf.c
create mode 100644 tools/sched_ext/scx_sdt.c
create mode 100644 tools/sched_ext/scx_sdt.h
diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile
index 32867f468db2..c8cfe326c556 100644
--- a/tools/sched_ext/Makefile
+++ b/tools/sched_ext/Makefile
@@ -189,7 +189,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
-c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_nest scx_userland scx_pair
+c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_nest scx_userland scx_pair scx_sdt
$(addprefix $(BINDIR)/,$(c-sched-targets)): \
$(BINDIR)/%: \
diff --git a/tools/sched_ext/scx_sdt.bpf.c b/tools/sched_ext/scx_sdt.bpf.c
new file mode 100644
index 000000000000..78431d97165e
--- /dev/null
+++ b/tools/sched_ext/scx_sdt.bpf.c
@@ -0,0 +1,710 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Arena-based task data scheduler. This is a variation of scx_simple
+ * that uses a combined allocator and indexing structure to organize
+ * task data. Task context allocation is done when a task enters the
+ * scheduler, while freeing is done when it exits. Task contexts are
+ * retrieved from task-local storage, pointing to the allocated memory.
+ *
+ * The main purpose of this scheduler is to demostrate arena memory
+ * management.
+ *
+ * Copyright (c) 2024-2025 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2024-2025 Emil Tsalapatis <etsal@meta.com>
+ * Copyright (c) 2024-2025 Tejun Heo <tj@kernel.org>
+ *
+ */
+#include <scx/common.bpf.h>
+#include <scx/bpf_arena_common.bpf.h>
+
+#include "scx_sdt.h"
+
+char _license[] SEC("license") = "GPL";
+
+UEI_DEFINE(uei);
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARENA);
+ __uint(map_flags, BPF_F_MMAPABLE);
+#if defined(__TARGET_ARCH_arm64) || defined(__aarch64__)
+ __uint(max_entries, 1 << 16); /* number of pages */
+ __ulong(map_extra, (1ull << 32)); /* start of mmap() region */
+#else
+ __uint(max_entries, 1 << 20); /* number of pages */
+ __ulong(map_extra, (1ull << 44)); /* start of mmap() region */
+#endif
+} arena __weak SEC(".maps");
+
+#define SHARED_DSQ 0
+
+#define DEFINE_SDT_STAT(metric) \
+static inline void \
+stat_inc_##metric(struct scx_stats __arena *stats) \
+{ \
+ cast_kern(stats); \
+ stats->metric += 1; \
+} \
+__u64 stat_##metric; \
+
+DEFINE_SDT_STAT(enqueue);
+DEFINE_SDT_STAT(init);
+DEFINE_SDT_STAT(exit);
+DEFINE_SDT_STAT(select_idle_cpu);
+DEFINE_SDT_STAT(select_busy_cpu);
+
+/*
+ * Necessary for cond_break/can_loop's semantics. According to kernel commit
+ * 011832b, the loop counter variable must be seen as imprecise and bounded
+ * by the verifier. Initializing it from a constant (e.g., i = 0;), then,
+ * makes it precise and prevents may_goto from helping with converging the
+ * loop. For these loops we must initialize the loop counter from a variable
+ * whose value the verifier cannot reason about when checking the program, so
+ * that the loop counter's value is imprecise.
+ */
+static __u64 zero = 0;
+
+/*
+ * XXX Hack to get the verifier to find the arena for sdt_exit_task.
+ * As of 6.12-rc5, The verifier associates arenas with programs by
+ * checking LD.IMM instruction operands for an arena and populating
+ * the program state with the first instance it finds. This requires
+ * accessing our global arena variable, but scx methods do not necessarily
+ * do so while still using pointers from that arena. Insert a bpf_printk
+ * statement that triggers at most once to generate an LD.IMM instruction
+ * to access the arena and help the verifier.
+ */
+static volatile bool scx_arena_verify_once;
+
+__hidden void scx_arena_subprog_init(void)
+{
+ if (scx_arena_verify_once)
+ return;
+
+ bpf_printk("%s: arena pointer %p", __func__, &arena);
+ scx_arena_verify_once = true;
+}
+
+
+private(LOCK) struct bpf_spin_lock alloc_lock;
+private(POOL_LOCK) struct bpf_spin_lock alloc_pool_lock;
+
+/* allocation pools */
+struct sdt_pool desc_pool;
+struct sdt_pool chunk_pool;
+
+/* Protected by alloc_lock. */
+struct scx_alloc_stats alloc_stats;
+
+
+/* Allocate element from the pool. Must be called with a then pool lock held. */
+static
+void __arena *scx_alloc_from_pool(struct sdt_pool *pool)
+{
+ __u64 elem_size, max_elems;
+ void __arena *slab;
+ void __arena *ptr;
+
+ elem_size = pool->elem_size;
+ max_elems = pool->max_elems;
+
+ /* If the chunk is spent, get a new one. */
+ if (pool->idx >= max_elems) {
+ slab = bpf_arena_alloc_pages(&arena, NULL,
+ div_round_up(max_elems * elem_size, PAGE_SIZE), NUMA_NO_NODE, 0);
+ if (!slab)
+ return NULL;
+
+ pool->slab = slab;
+ pool->idx = 0;
+ }
+
+ ptr = (void __arena *)((__u64) pool->slab + elem_size * pool->idx);
+ pool->idx += 1;
+
+ return ptr;
+}
+
+/* Alloc desc and associated chunk. Called with the allocator spinlock held. */
+static sdt_desc_t *scx_alloc_chunk(void)
+{
+ struct sdt_chunk __arena *chunk;
+ sdt_desc_t *desc;
+ sdt_desc_t *out;
+
+ chunk = scx_alloc_from_pool(&chunk_pool);
+ if (!chunk)
+ return NULL;
+
+ desc = scx_alloc_from_pool(&desc_pool);
+ if (!desc) {
+ /*
+ * Effectively frees the previous chunk allocation.
+ * Index cannot be 0, so decrementing is always
+ * valid.
+ */
+ chunk_pool.idx -= 1;
+ return NULL;
+ }
+
+ out = desc;
+
+ desc->nr_free = SDT_TASK_ENTS_PER_CHUNK;
+ desc->chunk = chunk;
+
+ alloc_stats.chunk_allocs += 1;
+
+ return out;
+}
+
+static int pool_set_size(struct sdt_pool *pool, __u64 data_size, __u64 nr_pages)
+{
+ if (unlikely(data_size % 8))
+ return -EINVAL;
+
+ if (unlikely(nr_pages == 0))
+ return -EINVAL;
+
+ pool->elem_size = data_size;
+ pool->max_elems = (PAGE_SIZE * nr_pages) / pool->elem_size;
+ /* Populate the pool slab on the first allocation. */
+ pool->idx = pool->max_elems;
+
+ return 0;
+}
+
+/* Initialize both the base pool allocators and the root chunk of the index. */
+__hidden int
+scx_alloc_init(struct scx_allocator *alloc, __u64 data_size)
+{
+ size_t min_chunk_size;
+ int ret;
+
+ _Static_assert(sizeof(struct sdt_chunk) <= PAGE_SIZE,
+ "chunk size must fit into a page");
+
+ ret = pool_set_size(&chunk_pool, sizeof(struct sdt_chunk), 1);
+ if (ret != 0)
+ return ret;
+
+ ret = pool_set_size(&desc_pool, sizeof(struct sdt_desc), 1);
+ if (ret != 0)
+ return ret;
+
+ /* Wrap data into a descriptor and word align. */
+ data_size += sizeof(struct sdt_data);
+ data_size = round_up(data_size, 8);
+
+ /*
+ * Ensure we allocate large enough chunks from the arena to avoid excessive
+ * internal fragmentation when turning chunks it into structs.
+ */
+ min_chunk_size = div_round_up(SDT_TASK_MIN_ELEM_PER_ALLOC * data_size, PAGE_SIZE);
+ ret = pool_set_size(&alloc->pool, data_size, min_chunk_size);
+ if (ret != 0)
+ return ret;
+
+ bpf_spin_lock(&alloc_lock);
+ alloc->root = scx_alloc_chunk();
+ bpf_spin_unlock(&alloc_lock);
+ if (!alloc->root)
+ return -ENOMEM;
+
+ return 0;
+}
+
+static
+int set_idx_state(sdt_desc_t *desc, __u64 pos, bool state)
+{
+ __u64 __arena *allocated = desc->allocated;
+ __u64 bit;
+
+ if (unlikely(pos >= SDT_TASK_ENTS_PER_CHUNK))
+ return -EINVAL;
+
+ bit = (__u64)1 << (pos % 64);
+
+ if (state)
+ allocated[pos / 64] |= bit;
+ else
+ allocated[pos / 64] &= ~bit;
+
+ return 0;
+}
+
+static __noinline
+int mark_nodes_avail(sdt_desc_t *lv_desc[SDT_TASK_LEVELS], __u64 lv_pos[SDT_TASK_LEVELS])
+{
+ sdt_desc_t *desc;
+ __u64 u, level;
+ int ret;
+
+ for (u = zero; u < SDT_TASK_LEVELS && can_loop; u++) {
+ level = SDT_TASK_LEVELS - 1 - u;
+
+ /* Only propagate upwards if we are the parent's only free chunk. */
+ desc = lv_desc[level];
+
+ ret = set_idx_state(desc, lv_pos[level], false);
+ if (unlikely(ret != 0))
+ return ret;
+
+ desc->nr_free += 1;
+ if (desc->nr_free > 1)
+ return 0;
+ }
+
+ return 0;
+}
+
+/*
+ * Free the allocated struct with the given index. Called with the
+ * allocator lock taken.
+ */
+__hidden
+int scx_alloc_free_idx(struct scx_allocator *alloc, __u64 idx)
+{
+ const __u64 mask = (1 << SDT_TASK_ENTS_PER_PAGE_SHIFT) - 1;
+ sdt_desc_t *lv_desc[SDT_TASK_LEVELS];
+ sdt_desc_t * __arena *desc_children;
+ struct sdt_chunk __arena *chunk;
+ sdt_desc_t *desc;
+ struct sdt_data __arena *data;
+ __u64 level, shift, pos;
+ __u64 lv_pos[SDT_TASK_LEVELS];
+ int ret;
+ int i;
+
+ if (!alloc)
+ return 0;
+
+ desc = alloc->root;
+ if (unlikely(!desc))
+ return -EINVAL;
+
+ /* To appease the verifier. */
+ for (level = zero; level < SDT_TASK_LEVELS && can_loop; level++) {
+ lv_desc[level] = NULL;
+ lv_pos[level] = 0;
+ }
+
+ /* Find the leaf node containing the index. */
+ for (level = zero; level < SDT_TASK_LEVELS && can_loop; level++) {
+ shift = (SDT_TASK_LEVELS - 1 - level) * SDT_TASK_ENTS_PER_PAGE_SHIFT;
+ pos = (idx >> shift) & mask;
+
+ lv_desc[level] = desc;
+ lv_pos[level] = pos;
+
+ if (level == SDT_TASK_LEVELS - 1)
+ break;
+
+ chunk = desc->chunk;
+
+ desc_children = (sdt_desc_t * __arena *)chunk->descs;
+ desc = desc_children[pos];
+
+ if (unlikely(!desc))
+ return -EINVAL;
+ }
+
+ chunk = desc->chunk;
+
+ pos = idx & mask;
+ data = chunk->data[pos];
+ if (likely(!data)) {
+ data[pos] = (struct sdt_data) {
+ .tid.genn = data->tid.genn + 1,
+ };
+
+ /* Zero out one word at a time. */
+ for (i = zero; i < alloc->pool.elem_size / 8 && can_loop; i++) {
+ data->payload[i] = 0;
+ }
+ }
+
+ ret = mark_nodes_avail(lv_desc, lv_pos);
+ if (unlikely(ret != 0))
+ return ret;
+
+ alloc_stats.active_allocs -= 1;
+ alloc_stats.free_ops += 1;
+
+ return 0;
+}
+
+static inline
+int ffs(__u64 word)
+{
+ unsigned int num = 0;
+
+ if ((word & 0xffffffff) == 0) {
+ num += 32;
+ word >>= 32;
+ }
+
+ if ((word & 0xffff) == 0) {
+ num += 16;
+ word >>= 16;
+ }
+
+ if ((word & 0xff) == 0) {
+ num += 8;
+ word >>= 8;
+ }
+
+ if ((word & 0xf) == 0) {
+ num += 4;
+ word >>= 4;
+ }
+
+ if ((word & 0x3) == 0) {
+ num += 2;
+ word >>= 2;
+ }
+
+ if ((word & 0x1) == 0) {
+ num += 1;
+ word >>= 1;
+ }
+
+ return num;
+}
+
+
+/* find the first empty slot */
+__hidden
+__u64 chunk_find_empty(sdt_desc_t __arg_arena *desc)
+{
+ __u64 freeslots;
+ __u64 i;
+
+ for (i = 0; i < SDT_TASK_CHUNK_BITMAP_U64S; i++) {
+ freeslots = ~desc->allocated[i];
+ if (freeslots == (__u64)0)
+ continue;
+
+ return (i * 64) + ffs(freeslots);
+ }
+
+ return SDT_TASK_ENTS_PER_CHUNK;
+}
+
+/*
+ * Find and return an available idx on the allocator.
+ * Called with the task spinlock held.
+ */
+static sdt_desc_t * desc_find_empty(sdt_desc_t *desc, __u64 *idxp)
+{
+ sdt_desc_t *lv_desc[SDT_TASK_LEVELS];
+ sdt_desc_t * __arena *desc_children;
+ struct sdt_chunk __arena *chunk;
+ sdt_desc_t *tmp;
+ __u64 lv_pos[SDT_TASK_LEVELS];
+ __u64 u, pos, level;
+ __u64 idx = 0;
+ int ret;
+
+ for (level = zero; level < SDT_TASK_LEVELS && can_loop; level++) {
+ pos = chunk_find_empty(desc);
+
+ /* If we error out, something has gone very wrong. */
+ if (unlikely(pos > SDT_TASK_ENTS_PER_CHUNK))
+ return NULL;
+
+ if (pos == SDT_TASK_ENTS_PER_CHUNK)
+ return NULL;
+
+ idx <<= SDT_TASK_ENTS_PER_PAGE_SHIFT;
+ idx |= pos;
+
+ /* Log the levels to complete allocation. */
+ lv_desc[level] = desc;
+ lv_pos[level] = pos;
+
+ /* The rest of the loop is for internal node traversal. */
+ if (level == SDT_TASK_LEVELS - 1)
+ break;
+
+ /* Allocate an internal node if necessary. */
+ chunk = desc->chunk;
+ desc_children = (sdt_desc_t * __arena *)chunk->descs;
+
+ desc = desc_children[pos];
+ if (!desc) {
+ desc = scx_alloc_chunk();
+ if (!desc)
+ return NULL;
+
+ desc_children[pos] = desc;
+ }
+ }
+
+ /*
+ * Finding the descriptor along with any internal node
+ * allocations was successful. Update all levels with
+ * the new allocation.
+ */
+ bpf_for(u, 0, SDT_TASK_LEVELS) {
+ level = SDT_TASK_LEVELS - 1 - u;
+ tmp = lv_desc[level];
+
+ ret = set_idx_state(tmp, lv_pos[level], true);
+ if (ret != 0)
+ break;
+
+ tmp->nr_free -= 1;
+ if (tmp->nr_free > 0)
+ break;
+ }
+
+ *idxp = idx;
+
+ return desc;
+}
+
+__hidden
+void __arena *scx_alloc(struct scx_allocator *alloc)
+{
+ struct sdt_data __arena *data = NULL;
+ struct sdt_chunk __arena *chunk;
+ sdt_desc_t *desc;
+ __u64 idx, pos;
+
+ if (!alloc)
+ return NULL;
+
+ bpf_spin_lock(&alloc_lock);
+
+ /* We unlock if we encounter an error in the function. */
+ desc = desc_find_empty(alloc->root, &idx);
+ if (unlikely(desc == NULL)) {
+ bpf_spin_unlock(&alloc_lock);
+ return NULL;
+ }
+
+ chunk = desc->chunk;
+
+ /* Populate the leaf node if necessary. */
+ pos = idx & (SDT_TASK_ENTS_PER_CHUNK - 1);
+ data = chunk->data[pos];
+ if (!data) {
+ data = scx_alloc_from_pool(&alloc->pool);
+ if (!data) {
+ scx_alloc_free_idx(alloc, idx);
+ bpf_spin_unlock(&alloc_lock);
+ return NULL;
+ }
+ }
+
+ chunk->data[pos] = data;
+
+ /* The data counts as a chunk */
+ alloc_stats.data_allocs += 1;
+ alloc_stats.alloc_ops += 1;
+ alloc_stats.active_allocs += 1;
+
+ data->tid.idx = idx;
+
+ bpf_spin_unlock(&alloc_lock);
+
+ return data;
+}
+
+/*
+ * Task BPF map entry recording the task's assigned ID and pointing to the data
+ * area allocated in arena.
+ */
+struct scx_task_map_val {
+ union sdt_id tid;
+ __u64 tptr;
+ struct sdt_data __arena *data;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
+ __uint(map_flags, BPF_F_NO_PREALLOC);
+ __type(key, int);
+ __type(value, struct scx_task_map_val);
+} scx_task_map SEC(".maps");
+
+static struct scx_allocator scx_task_allocator;
+
+__hidden
+void __arena *scx_task_alloc(struct task_struct *p)
+{
+ struct sdt_data __arena *data = NULL;
+ struct scx_task_map_val *mval;
+
+ mval = bpf_task_storage_get(&scx_task_map, p, 0,
+ BPF_LOCAL_STORAGE_GET_F_CREATE);
+ if (!mval)
+ return NULL;
+
+ data = scx_alloc(&scx_task_allocator);
+ if (unlikely(!data))
+ return NULL;
+
+ mval->tid = data->tid;
+ mval->tptr = (__u64) p;
+ mval->data = data;
+
+ return (void __arena *)data->payload;
+}
+
+__hidden
+int scx_task_init(__u64 data_size)
+{
+ return scx_alloc_init(&scx_task_allocator, data_size);
+}
+
+__hidden
+void __arena *scx_task_data(struct task_struct *p)
+{
+ struct sdt_data __arena *data;
+ struct scx_task_map_val *mval;
+
+ scx_arena_subprog_init();
+
+ mval = bpf_task_storage_get(&scx_task_map, p, 0, 0);
+ if (!mval)
+ return NULL;
+
+ data = mval->data;
+
+ return (void __arena *)data->payload;
+}
+
+__hidden
+void scx_task_free(struct task_struct *p)
+{
+ struct scx_task_map_val *mval;
+
+ scx_arena_subprog_init();
+
+ mval = bpf_task_storage_get(&scx_task_map, p, 0, 0);
+ if (!mval)
+ return;
+
+ bpf_spin_lock(&alloc_lock);
+ scx_alloc_free_idx(&scx_task_allocator, mval->tid.idx);
+ bpf_spin_unlock(&alloc_lock);
+
+ bpf_task_storage_delete(&scx_task_map, p);
+}
+
+static inline void
+scx_stat_global_update(struct scx_stats __arena *stats)
+{
+ cast_kern(stats);
+ __sync_fetch_and_add(&stat_enqueue, stats->enqueue);
+ __sync_fetch_and_add(&stat_init, stats->init);
+ __sync_fetch_and_add(&stat_exit, stats->exit);
+ __sync_fetch_and_add(&stat_select_idle_cpu, stats->select_idle_cpu);
+ __sync_fetch_and_add(&stat_select_busy_cpu, stats->select_busy_cpu);
+}
+
+s32 BPF_STRUCT_OPS(sdt_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
+{
+ struct scx_stats __arena *stats;
+ bool is_idle = false;
+ s32 cpu;
+
+ stats = scx_task_data(p);
+ if (!stats) {
+ scx_bpf_error("%s: no stats for pid %d", __func__, p->pid);
+ return 0;
+ }
+
+ cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
+ if (is_idle) {
+ stat_inc_select_idle_cpu(stats);
+ scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
+ } else {
+ stat_inc_select_busy_cpu(stats);
+ }
+
+ return cpu;
+}
+
+void BPF_STRUCT_OPS(sdt_enqueue, struct task_struct *p, u64 enq_flags)
+{
+ struct scx_stats __arena *stats;
+
+ stats = scx_task_data(p);
+ if (!stats) {
+ scx_bpf_error("%s: no stats for pid %d", __func__, p->pid);
+ return;
+ }
+
+ stat_inc_enqueue(stats);
+
+ scx_bpf_dsq_insert(p, SHARED_DSQ, SCX_SLICE_DFL, enq_flags);
+}
+
+void BPF_STRUCT_OPS(sdt_dispatch, s32 cpu, struct task_struct *prev)
+{
+ scx_bpf_dsq_move_to_local(SHARED_DSQ);
+}
+
+s32 BPF_STRUCT_OPS_SLEEPABLE(sdt_init_task, struct task_struct *p,
+ struct scx_init_task_args *args)
+{
+ struct scx_stats __arena *stats;
+
+ stats = scx_task_alloc(p);
+ if (!stats) {
+ scx_bpf_error("arena allocator out of memory");
+ return -ENOMEM;
+ }
+
+ stats->pid = p->pid;
+
+ stat_inc_init(stats);
+
+ return 0;
+}
+
+void BPF_STRUCT_OPS(sdt_exit_task, struct task_struct *p,
+ struct scx_exit_task_args *args)
+{
+ struct scx_stats __arena *stats;
+
+ stats = scx_task_data(p);
+ if (!stats) {
+ scx_bpf_error("%s: no stats for pid %d", __func__, p->pid);
+ return;
+ }
+
+ stat_inc_exit(stats);
+ scx_stat_global_update(stats);
+
+ scx_task_free(p);
+}
+
+s32 BPF_STRUCT_OPS_SLEEPABLE(sdt_init)
+{
+ int ret;
+
+ ret = scx_task_init(sizeof(struct scx_stats));
+ if (ret < 0) {
+ scx_bpf_error("%s: failed with %d", __func__, ret);
+ return ret;
+ }
+
+ return scx_bpf_create_dsq(SHARED_DSQ, -1);
+}
+
+void BPF_STRUCT_OPS(sdt_exit, struct scx_exit_info *ei)
+{
+ UEI_RECORD(uei, ei);
+}
+
+SCX_OPS_DEFINE(sdt_ops,
+ .select_cpu = (void *)sdt_select_cpu,
+ .enqueue = (void *)sdt_enqueue,
+ .dispatch = (void *)sdt_dispatch,
+ .init_task = (void *)sdt_init_task,
+ .exit_task = (void *)sdt_exit_task,
+ .init = (void *)sdt_init,
+ .exit = (void *)sdt_exit,
+ .name = "sdt");
diff --git a/tools/sched_ext/scx_sdt.c b/tools/sched_ext/scx_sdt.c
new file mode 100644
index 000000000000..b0363363476d
--- /dev/null
+++ b/tools/sched_ext/scx_sdt.c
@@ -0,0 +1,101 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2024 Emil Tsalapatis <etsal@meta.com>
+ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
+ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <signal.h>
+#include <libgen.h>
+#include <bpf/bpf.h>
+#include <scx/common.h>
+
+#include "scx_sdt.h"
+#include "scx_sdt.bpf.skel.h"
+
+const char help_fmt[] =
+"A simple arena-based sched_ext scheduler.\n"
+"\n"
+"Modified version of scx_simple that demonstrates arena-based data structures.\n"
+"\n"
+"Usage: %s [-f] [-v]\n"
+"\n"
+" -v Print libbpf debug messages\n"
+" -h Display this help and exit\n";
+
+static bool verbose;
+static volatile int exit_req;
+
+static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
+{
+ if (level == LIBBPF_DEBUG && !verbose)
+ return 0;
+ return vfprintf(stderr, format, args);
+}
+
+static void sigint_handler(int sig)
+{
+ exit_req = 1;
+}
+
+int main(int argc, char **argv)
+{
+ struct scx_sdt *skel;
+ struct bpf_link *link;
+ __u32 opt;
+ __u64 ecode;
+
+ libbpf_set_print(libbpf_print_fn);
+ signal(SIGINT, sigint_handler);
+ signal(SIGTERM, sigint_handler);
+restart:
+ skel = SCX_OPS_OPEN(sdt_ops, scx_sdt);
+
+ while ((opt = getopt(argc, argv, "fvh")) != -1) {
+ switch (opt) {
+ case 'v':
+ verbose = true;
+ break;
+ default:
+ fprintf(stderr, help_fmt, basename(argv[0]));
+ return opt != 'h';
+ }
+ }
+
+ SCX_OPS_LOAD(skel, sdt_ops, scx_sdt, uei);
+ link = SCX_OPS_ATTACH(skel, sdt_ops, scx_sdt);
+
+ while (!exit_req && !UEI_EXITED(skel, uei)) {
+ printf("====SCHEDULING STATS====\n");
+ printf("enqueues=%llu\t", skel->bss->stat_enqueue);
+ printf("inits=%llu\t", skel->bss->stat_init);
+ printf("exits=%llu\t", skel->bss->stat_exit);
+ printf("\n");
+
+ printf("select_idle_cpu=%llu\t", skel->bss->stat_select_idle_cpu);
+ printf("select_busy_cpu=%llu\t", skel->bss->stat_select_busy_cpu);
+ printf("\n");
+
+ printf("====ALLOCATION STATS====\n");
+ printf("chunk allocs=%llu\t", skel->bss->alloc_stats.chunk_allocs);
+ printf("data_allocs=%llu\n", skel->bss->alloc_stats.data_allocs);
+ printf("alloc_ops=%llu\t", skel->bss->alloc_stats.alloc_ops);
+ printf("free_ops=%llu\t", skel->bss->alloc_stats.free_ops);
+ printf("active_allocs=%llu\t", skel->bss->alloc_stats.active_allocs);
+ printf("arena_pages_used=%llu\t", skel->bss->alloc_stats.arena_pages_used);
+ printf("\n\n");
+
+ fflush(stdout);
+ sleep(1);
+ }
+
+ bpf_link__destroy(link);
+ ecode = UEI_REPORT(skel, uei);
+ scx_sdt__destroy(skel);
+
+ if (UEI_ECODE_RESTART(ecode))
+ goto restart;
+ return 0;
+}
diff --git a/tools/sched_ext/scx_sdt.h b/tools/sched_ext/scx_sdt.h
new file mode 100644
index 000000000000..67982ce9bc9b
--- /dev/null
+++ b/tools/sched_ext/scx_sdt.h
@@ -0,0 +1,113 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0
+ * Copyright (c) 2025 Meta Platforms, Inc. and affiliates.
+ * Copyright (c) 2025 Tejun Heo <tj@kernel.org>
+ * Copyright (c) 2025 Emil Tsalapatis <etsal@meta.com>
+ */
+#pragma once
+
+#ifndef __BPF__
+#define __arena
+#endif /* __BPF__ */
+
+struct scx_alloc_stats {
+ __u64 chunk_allocs;
+ __u64 data_allocs;
+ __u64 alloc_ops;
+ __u64 free_ops;
+ __u64 active_allocs;
+ __u64 arena_pages_used;
+};
+
+struct sdt_pool {
+ void __arena *slab;
+ __u64 elem_size;
+ __u64 max_elems;
+ __u64 idx;
+};
+
+#ifndef div_round_up
+#define div_round_up(a, b) (((a) + (b) - 1) / (b))
+#endif
+
+#ifndef round_up
+#define round_up(a, b) (div_round_up((a), (b)) * (b))
+#endif
+
+typedef struct sdt_desc __arena sdt_desc_t;
+
+enum sdt_consts {
+ SDT_TASK_ENTS_PER_PAGE_SHIFT = 9,
+ SDT_TASK_LEVELS = 3,
+ SDT_TASK_ENTS_PER_CHUNK = 1 << SDT_TASK_ENTS_PER_PAGE_SHIFT,
+ SDT_TASK_CHUNK_BITMAP_U64S = div_round_up(SDT_TASK_ENTS_PER_CHUNK, 64),
+ SDT_TASK_MIN_ELEM_PER_ALLOC = 8,
+};
+
+union sdt_id {
+ __s64 val;
+ struct {
+ __s32 idx; /* index in the radix tree */
+ __s32 genn; /* ++'d on recycle so that it forms unique'ish 64bit ID */
+ };
+};
+
+struct sdt_chunk;
+
+/*
+ * Each index page is described by the following descriptor which carries the
+ * bitmap. This way the actual index can host power-of-two numbers of entries
+ * which makes indexing cheaper.
+ */
+struct sdt_desc {
+ __u64 allocated[SDT_TASK_CHUNK_BITMAP_U64S];
+ __u64 nr_free;
+ struct sdt_chunk __arena *chunk;
+};
+
+/*
+ * Leaf node containing per-task data.
+ */
+struct sdt_data {
+ union sdt_id tid;
+ __u64 payload[];
+};
+
+/*
+ * Intermediate node pointing to another intermediate node or leaf node.
+ */
+struct sdt_chunk {
+ union {
+ sdt_desc_t * descs[SDT_TASK_ENTS_PER_CHUNK];
+ struct sdt_data __arena *data[SDT_TASK_ENTS_PER_CHUNK];
+ };
+};
+
+struct scx_allocator {
+ struct sdt_pool pool;
+ sdt_desc_t *root;
+};
+
+struct scx_stats {
+ int seq;
+ pid_t pid;
+ __u64 enqueue;
+ __u64 exit;
+ __u64 init;
+ __u64 select_busy_cpu;
+ __u64 select_idle_cpu;
+};
+
+#ifdef __BPF__
+
+void __arena *scx_task_data(struct task_struct *p);
+int scx_task_init(__u64 data_size);
+void __arena *scx_task_alloc(struct task_struct *p);
+void scx_task_free(struct task_struct *p);
+void scx_arena_subprog_init(void);
+
+int scx_alloc_init(struct scx_allocator *alloc, __u64 data_size);
+u64 scx_alloc_internal(struct scx_allocator *alloc);
+int scx_alloc_free_idx(struct scx_allocator *alloc, __u64 idx);
+
+#endif /* __BPF__ */
--
2.49.0
^ permalink raw reply related [flat|nested] 8+ messages in thread* Re: [PATCH 0/4] tools/sched_ext: Add example C schedulers
2026-01-12 21:41 [PATCH 0/4] tools/sched_ext: Add example C schedulers Emil Tsalapatis
` (3 preceding siblings ...)
2026-01-12 21:41 ` [PATCH 4/4] tools/sched_ext: add arena based scheduler Emil Tsalapatis
@ 2026-01-13 5:57 ` Changwoo Min
2026-01-13 6:33 ` Andrea Righi
4 siblings, 1 reply; 8+ messages in thread
From: Changwoo Min @ 2026-01-13 5:57 UTC (permalink / raw)
To: Emil Tsalapatis, sched-ext; +Cc: arighi, tj, void
Hi Emil,
On 1/13/26 6:41 AM, Emil Tsalapatis wrote:
> Add to the tree several C sched-ext schedulers developed and maintained
> out-of-tree by the sched-ext project over the years. These schedulers
> demonstrate sched-ext's feature set and provide a starting point for
> scheduler development. Place the schedulers together with the existing
> examples in tools/sched_ext.
In general, I am okay to include more examples of C schedulers in the
kernel tree. However, I am doubtful whether it is worth including the
nest scheduler, as it has been barely maintained in recent times.
Regards,
Changwoo Min
>
> Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
>
> Emil Tsalapatis (4):
> tools/sched_ext: add scx_nest scheduler
> tools/sched_ext: add scx_userland scheduler
> tools/sched_ext: add scx_pair scheduler
> tools/sched_ext: add arena based scheduler
>
> tools/sched_ext/Makefile | 2 +-
> tools/sched_ext/scx_nest.bpf.c | 652 +++++++++++++++++++++++
> tools/sched_ext/scx_nest.c | 240 +++++++++
> tools/sched_ext/scx_nest.h | 18 +
> tools/sched_ext/scx_nest_stats_table.h | 20 +
> tools/sched_ext/scx_pair.bpf.c | 610 +++++++++++++++++++++
> tools/sched_ext/scx_pair.c | 180 +++++++
> tools/sched_ext/scx_pair.h | 9 +
> tools/sched_ext/scx_sdt.bpf.c | 710 +++++++++++++++++++++++++
> tools/sched_ext/scx_sdt.c | 101 ++++
> tools/sched_ext/scx_sdt.h | 113 ++++
> tools/sched_ext/scx_userland.bpf.c | 344 ++++++++++++
> tools/sched_ext/scx_userland.c | 437 +++++++++++++++
> tools/sched_ext/scx_userland.h | 17 +
> 14 files changed, 3452 insertions(+), 1 deletion(-)
> create mode 100644 tools/sched_ext/scx_nest.bpf.c
> create mode 100644 tools/sched_ext/scx_nest.c
> create mode 100644 tools/sched_ext/scx_nest.h
> create mode 100644 tools/sched_ext/scx_nest_stats_table.h
> create mode 100644 tools/sched_ext/scx_pair.bpf.c
> create mode 100644 tools/sched_ext/scx_pair.c
> create mode 100644 tools/sched_ext/scx_pair.h
> create mode 100644 tools/sched_ext/scx_sdt.bpf.c
> create mode 100644 tools/sched_ext/scx_sdt.c
> create mode 100644 tools/sched_ext/scx_sdt.h
> create mode 100644 tools/sched_ext/scx_userland.bpf.c
> create mode 100644 tools/sched_ext/scx_userland.c
> create mode 100644 tools/sched_ext/scx_userland.h
>
^ permalink raw reply [flat|nested] 8+ messages in thread* Re: [PATCH 0/4] tools/sched_ext: Add example C schedulers
2026-01-13 5:57 ` [PATCH 0/4] tools/sched_ext: Add example C schedulers Changwoo Min
@ 2026-01-13 6:33 ` Andrea Righi
2026-01-13 16:05 ` Emil Tsalapatis
0 siblings, 1 reply; 8+ messages in thread
From: Andrea Righi @ 2026-01-13 6:33 UTC (permalink / raw)
To: Changwoo Min; +Cc: Emil Tsalapatis, sched-ext, tj, void
On Tue, Jan 13, 2026 at 02:57:13PM +0900, Changwoo Min wrote:
> Hi Emil,
>
> On 1/13/26 6:41 AM, Emil Tsalapatis wrote:
> > Add to the tree several C sched-ext schedulers developed and maintained
> > out-of-tree by the sched-ext project over the years. These schedulers
> > demonstrate sched-ext's feature set and provide a starting point for
> > scheduler development. Place the schedulers together with the existing
> > examples in tools/sched_ext.
>
> In general, I am okay to include more examples of C schedulers in the
> kernel tree. However, I am doubtful whether it is worth including the
> nest scheduler, as it has been barely maintained in recent times.
+1 on that, even if it's a good example, it's still unmaintained code.
Maybe we should just move it in a separate github repo.
For the rest of the schedulers I'm ok to include them in-tree (still under
the assumption that they should be considered examples).
Thanks,
-Andrea
>
> Regards,
> Changwoo Min
>
> >
> > Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
> >
> > Emil Tsalapatis (4):
> > tools/sched_ext: add scx_nest scheduler
> > tools/sched_ext: add scx_userland scheduler
> > tools/sched_ext: add scx_pair scheduler
> > tools/sched_ext: add arena based scheduler
> >
> > tools/sched_ext/Makefile | 2 +-
> > tools/sched_ext/scx_nest.bpf.c | 652 +++++++++++++++++++++++
> > tools/sched_ext/scx_nest.c | 240 +++++++++
> > tools/sched_ext/scx_nest.h | 18 +
> > tools/sched_ext/scx_nest_stats_table.h | 20 +
> > tools/sched_ext/scx_pair.bpf.c | 610 +++++++++++++++++++++
> > tools/sched_ext/scx_pair.c | 180 +++++++
> > tools/sched_ext/scx_pair.h | 9 +
> > tools/sched_ext/scx_sdt.bpf.c | 710 +++++++++++++++++++++++++
> > tools/sched_ext/scx_sdt.c | 101 ++++
> > tools/sched_ext/scx_sdt.h | 113 ++++
> > tools/sched_ext/scx_userland.bpf.c | 344 ++++++++++++
> > tools/sched_ext/scx_userland.c | 437 +++++++++++++++
> > tools/sched_ext/scx_userland.h | 17 +
> > 14 files changed, 3452 insertions(+), 1 deletion(-)
> > create mode 100644 tools/sched_ext/scx_nest.bpf.c
> > create mode 100644 tools/sched_ext/scx_nest.c
> > create mode 100644 tools/sched_ext/scx_nest.h
> > create mode 100644 tools/sched_ext/scx_nest_stats_table.h
> > create mode 100644 tools/sched_ext/scx_pair.bpf.c
> > create mode 100644 tools/sched_ext/scx_pair.c
> > create mode 100644 tools/sched_ext/scx_pair.h
> > create mode 100644 tools/sched_ext/scx_sdt.bpf.c
> > create mode 100644 tools/sched_ext/scx_sdt.c
> > create mode 100644 tools/sched_ext/scx_sdt.h
> > create mode 100644 tools/sched_ext/scx_userland.bpf.c
> > create mode 100644 tools/sched_ext/scx_userland.c
> > create mode 100644 tools/sched_ext/scx_userland.h
> >
>
^ permalink raw reply [flat|nested] 8+ messages in thread
* Re: [PATCH 0/4] tools/sched_ext: Add example C schedulers
2026-01-13 6:33 ` Andrea Righi
@ 2026-01-13 16:05 ` Emil Tsalapatis
0 siblings, 0 replies; 8+ messages in thread
From: Emil Tsalapatis @ 2026-01-13 16:05 UTC (permalink / raw)
To: Andrea Righi, Changwoo Min; +Cc: sched-ext, tj, void
On Tue Jan 13, 2026 at 1:33 AM EST, Andrea Righi wrote:
> On Tue, Jan 13, 2026 at 02:57:13PM +0900, Changwoo Min wrote:
>> Hi Emil,
>>
>> On 1/13/26 6:41 AM, Emil Tsalapatis wrote:
>> > Add to the tree several C sched-ext schedulers developed and maintained
>> > out-of-tree by the sched-ext project over the years. These schedulers
>> > demonstrate sched-ext's feature set and provide a starting point for
>> > scheduler development. Place the schedulers together with the existing
>> > examples in tools/sched_ext.
>>
>> In general, I am okay to include more examples of C schedulers in the
>> kernel tree. However, I am doubtful whether it is worth including the
>> nest scheduler, as it has been barely maintained in recent times.
>
> +1 on that, even if it's a good example, it's still unmaintained code.
> Maybe we should just move it in a separate github repo.
>
> For the rest of the schedulers I'm ok to include them in-tree (still under
> the assumption that they should be considered examples).
>
> Thanks,
> -Andrea
>
Makes sense, I will remove the nest schedulers and resubmit. Also wrt
testing, I'd like to note scx_sdt depends on changes in the current
bpf-next to verify, so it should be fine by the end of the merge window.
>>
>> Regards,
>> Changwoo Min
>>
>> >
>> > Signed-off-by: Emil Tsalapatis <emil@etsalapatis.com>
>> >
>> > Emil Tsalapatis (4):
>> > tools/sched_ext: add scx_nest scheduler
>> > tools/sched_ext: add scx_userland scheduler
>> > tools/sched_ext: add scx_pair scheduler
>> > tools/sched_ext: add arena based scheduler
>> >
>> > tools/sched_ext/Makefile | 2 +-
>> > tools/sched_ext/scx_nest.bpf.c | 652 +++++++++++++++++++++++
>> > tools/sched_ext/scx_nest.c | 240 +++++++++
>> > tools/sched_ext/scx_nest.h | 18 +
>> > tools/sched_ext/scx_nest_stats_table.h | 20 +
>> > tools/sched_ext/scx_pair.bpf.c | 610 +++++++++++++++++++++
>> > tools/sched_ext/scx_pair.c | 180 +++++++
>> > tools/sched_ext/scx_pair.h | 9 +
>> > tools/sched_ext/scx_sdt.bpf.c | 710 +++++++++++++++++++++++++
>> > tools/sched_ext/scx_sdt.c | 101 ++++
>> > tools/sched_ext/scx_sdt.h | 113 ++++
>> > tools/sched_ext/scx_userland.bpf.c | 344 ++++++++++++
>> > tools/sched_ext/scx_userland.c | 437 +++++++++++++++
>> > tools/sched_ext/scx_userland.h | 17 +
>> > 14 files changed, 3452 insertions(+), 1 deletion(-)
>> > create mode 100644 tools/sched_ext/scx_nest.bpf.c
>> > create mode 100644 tools/sched_ext/scx_nest.c
>> > create mode 100644 tools/sched_ext/scx_nest.h
>> > create mode 100644 tools/sched_ext/scx_nest_stats_table.h
>> > create mode 100644 tools/sched_ext/scx_pair.bpf.c
>> > create mode 100644 tools/sched_ext/scx_pair.c
>> > create mode 100644 tools/sched_ext/scx_pair.h
>> > create mode 100644 tools/sched_ext/scx_sdt.bpf.c
>> > create mode 100644 tools/sched_ext/scx_sdt.c
>> > create mode 100644 tools/sched_ext/scx_sdt.h
>> > create mode 100644 tools/sched_ext/scx_userland.bpf.c
>> > create mode 100644 tools/sched_ext/scx_userland.c
>> > create mode 100644 tools/sched_ext/scx_userland.h
>> >
>>
^ permalink raw reply [flat|nested] 8+ messages in thread