Re: [PATCH] tools/sched_ext: Add scx_priority dual-queue priority CPU scheduler

[Andrea Righi <arighi@nvidia.com>]

On Tue, Sep 22, 2026 at 02:45:57PM +0600, rahadbhuiya wrote:
> Add scx_priority, a dual-queue priority CPU scheduler using sched_ext.
> It differentiates between latency-sensitive/interactive tasks (nice < 0)
> and normal/batch tasks, dispatching high-priority tasks with boosted
> slices and draining the high-priority queue first upon core availability.
> 
> - Add scx_priority.bpf.c with two DSQs (high priority and normal).
> - Add scx_priority.c userspace monitor tracking live throughput.
> - Update Makefile and README.md.
> 
> Signed-off-by: rahadbhuiya <rahadbhuiya2021@gmail.com>

Personally, I don't see a strong motivation to include this in the kernel,
schedulers under tools/sched_ext are primarily intended as examples to
demonstrate how to use the sched_ext API, and the functionality covered here is
already well represented by the existing examples.

If your goal is to offer a scheduler for people to actually use it, I'd
recommend submitting a PR to the community scheduler repository:
https://github.com/sched-ext/scx instead.

-Andrea

> ---
>  tools/sched_ext/Makefile           |   2 +-
>  tools/sched_ext/README.md          |   7 ++
>  tools/sched_ext/scx_priority.bpf.c | 117 ++++++++++++++++++++++++++
>  tools/sched_ext/scx_priority.c     | 128 +++++++++++++++++++++++++++++
>  4 files changed, 253 insertions(+), 1 deletion(-)
>  create mode 100644 tools/sched_ext/scx_priority.bpf.c
>  create mode 100644 tools/sched_ext/scx_priority.c
> 
> diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile
> index 21554f089692..41b80e164cda 100644
> --- a/tools/sched_ext/Makefile
> +++ b/tools/sched_ext/Makefile
> @@ -191,7 +191,7 @@ $(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BP
>  
>  SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
>  
> -c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_userland scx_pair scx_sdt
> +c-sched-targets = scx_simple scx_cpu0 scx_qmap scx_central scx_flatcg scx_userland scx_pair scx_sdt scx_priority
>  
>  $(addprefix $(BINDIR)/,$(c-sched-targets)): \
>  	$(BINDIR)/%: \
> diff --git a/tools/sched_ext/README.md b/tools/sched_ext/README.md
> index 0ee5a3d997e5..18b80eacc28a 100644
> --- a/tools/sched_ext/README.md
> +++ b/tools/sched_ext/README.md
> @@ -164,6 +164,13 @@ scx_simple can be run in either global weighted vtime mode, or FIFO mode.
>  Though very simple, in limited scenarios, this scheduler can perform reasonably
>  well on single-socket systems with a unified L3 cache.
>  
> +## scx_priority
> +
> +A dual-queue priority scheduler that separates latency-sensitive and interactive
> +tasks from normal/batch tasks. Tasks with higher priority (nice < 0) are queued
> +to a dedicated high-priority DSQ with boosted time slices and drained first upon
> +dispatch.
> +
>  ## scx_qmap
>  
>  Another simple, yet slightly more complex scheduler that provides an example of
> diff --git a/tools/sched_ext/scx_priority.bpf.c b/tools/sched_ext/scx_priority.bpf.c
> new file mode 100644
> index 000000000000..816b197c4019
> --- /dev/null
> +++ b/tools/sched_ext/scx_priority.bpf.c
> @@ -0,0 +1,117 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * A dual-queue priority scheduler based on sched_ext.
> + *
> + * Dispatches latency-sensitive / interactive tasks (nice < 0) to a high-priority
> + * DSQ, and batch / normal tasks to a standard DSQ. When a CPU core becomes
> + * available, the high-priority queue is drained first before serving normal tasks.
> + *
> + * Copyright (c) 2026 Rahad Bhuiya <rahadbhuiya2021@gmail.com>
> + */
> +#include <scx/common.bpf.h>
> +
> +char _license[] SEC("license") = "GPL";
> +
> +#define PRIO_DSQ_HIGH	0
> +#define PRIO_DSQ_LOW	1
> +
> +/*
> + * Stats tracking:
> + * [0] - High priority / interactive tasks queued
> + * [1] - Standard / batch tasks queued
> + */
> +struct {
> +	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
> +	__uint(key_size, sizeof(u32));
> +	__uint(value_size, sizeof(u64));
> +	__uint(max_entries, 2);
> +} stats SEC(".maps");
> +
> +static void stat_inc(u32 idx)
> +{
> +	u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx);
> +
> +	if (cnt_p)
> +		(*cnt_p)++;
> +}
> +
> +static bool is_high_prio(const struct task_struct *p)
> +{
> +	/*
> +	 * In the Linux kernel, static_prio maps nice -20..19 to 100..139.
> +	 * Default nice 0 corresponds to static_prio 120. Tasks with nice < 0
> +	 * (static_prio < 120) or real-time policies are prioritized.
> +	 */
> +	return p->static_prio < 120;
> +}
> +
> +s32 BPF_STRUCT_OPS(prio_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
> +{
> +	bool is_idle = false;
> +	s32 cpu;
> +
> +	cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
> +	if (is_idle) {
> +		u64 slice = is_high_prio(p) ? (2 * SCX_SLICE_DFL) : SCX_SLICE_DFL;
> +
> +		stat_inc(is_high_prio(p) ? 0 : 1);
> +		scx_bpf_dsq_insert(p, SCX_DSQ_LOCAL, slice, 0);
> +	}
> +
> +	return cpu;
> +}
> +
> +void BPF_STRUCT_OPS(prio_enqueue, struct task_struct *p, u64 enq_flags)
> +{
> +	if (is_high_prio(p)) {
> +		stat_inc(0);
> +		scx_bpf_dsq_insert(p, PRIO_DSQ_HIGH, 2 * SCX_SLICE_DFL, enq_flags);
> +	} else {
> +		stat_inc(1);
> +		scx_bpf_dsq_insert(p, PRIO_DSQ_LOW, SCX_SLICE_DFL, enq_flags);
> +	}
> +}
> +
> +void BPF_STRUCT_OPS(prio_dispatch, s32 cpu, struct task_struct *prev)
> +{
> +	/* First drain high-priority tasks if any are waiting */
> +	if (scx_bpf_dsq_move_to_local(PRIO_DSQ_HIGH, 0))
> +		return;
> +
> +	/* Otherwise drain standard priority tasks */
> +	scx_bpf_dsq_move_to_local(PRIO_DSQ_LOW, 0);
> +}
> +
> +s32 BPF_STRUCT_OPS_SLEEPABLE(prio_init)
> +{
> +	int ret;
> +
> +	ret = scx_bpf_create_dsq(PRIO_DSQ_HIGH, -1);
> +	if (ret) {
> +		scx_bpf_error("failed to create high priority DSQ (%d)", ret);
> +		return ret;
> +	}
> +
> +	ret = scx_bpf_create_dsq(PRIO_DSQ_LOW, -1);
> +	if (ret) {
> +		scx_bpf_error("failed to create low priority DSQ (%d)", ret);
> +		return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +UEI_DEFINE(uei);
> +
> +void BPF_STRUCT_OPS(prio_exit, struct scx_exit_info *ei)
> +{
> +	UEI_RECORD(uei, ei);
> +}
> +
> +SCX_OPS_DEFINE(priority_ops,
> +	       .select_cpu	= (void *)prio_select_cpu,
> +	       .enqueue		= (void *)prio_enqueue,
> +	       .dispatch	= (void *)prio_dispatch,
> +	       .init		= (void *)prio_init,
> +	       .exit		= (void *)prio_exit,
> +	       .name		= "priority");
> diff --git a/tools/sched_ext/scx_priority.c b/tools/sched_ext/scx_priority.c
> new file mode 100644
> index 000000000000..3cf975be3a1e
> --- /dev/null
> +++ b/tools/sched_ext/scx_priority.c
> @@ -0,0 +1,128 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Userspace controller and monitor for scx_priority scheduler.
> + *
> + * Copyright (c) 2026 Rahad Bhuiya <rahadbhuiya2021@gmail.com>
> + */
> +#include <stdio.h>
> +#include <stdlib.h>
> +#include <unistd.h>
> +#include <signal.h>
> +#include <assert.h>
> +#include <libgen.h>
> +#include <bpf/bpf.h>
> +#include <scx/common.h>
> +#include "scx_priority.bpf.skel.h"
> +
> +const char help_fmt[] =
> +"A dual-queue priority sched_ext scheduler.\n"
> +"\n"
> +"Usage: %s [-i INTERVAL] [-v] [-h]\n"
> +"\n"
> +"  -i INTERVAL   Stats monitoring interval in seconds (default: 1)\n"
> +"  -v            Print libbpf debug messages\n"
> +"  -h            Display this help and exit\n";
> +
> +static bool verbose;
> +static sig_atomic_t exit_req;
> +
> +static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
> +{
> +	if (level == LIBBPF_DEBUG && !verbose)
> +		return 0;
> +	return vfprintf(stderr, format, args);
> +}
> +
> +static void sigint_handler(int sig)
> +{
> +	exit_req = 1;
> +}
> +
> +static void read_stats(struct scx_priority *skel, __u64 *stats)
> +{
> +	int nr_cpus = libbpf_num_possible_cpus();
> +	__u64 *cnts[2];
> +	__u32 idx;
> +
> +	assert(nr_cpus > 0);
> +	cnts[0] = calloc(nr_cpus, sizeof(__u64));
> +	cnts[1] = calloc(nr_cpus, sizeof(__u64));
> +	if (!cnts[0] || !cnts[1]) {
> +		free(cnts[0]);
> +		free(cnts[1]);
> +		return;
> +	}
> +
> +	memset(stats, 0, sizeof(stats[0]) * 2);
> +
> +	for (idx = 0; idx < 2; idx++) {
> +		int ret, cpu;
> +
> +		ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats),
> +					  &idx, cnts[idx]);
> +		if (ret < 0)
> +			continue;
> +		for (cpu = 0; cpu < nr_cpus; cpu++)
> +			stats[idx] += cnts[idx][cpu];
> +	}
> +
> +	free(cnts[0]);
> +	free(cnts[1]);
> +}
> +
> +int main(int argc, char **argv)
> +{
> +	struct scx_priority *skel;
> +	struct bpf_link *link;
> +	__s32 opt;
> +	__u64 ecode;
> +	int interval = 1;
> +
> +	libbpf_set_print(libbpf_print_fn);
> +	signal(SIGINT, sigint_handler);
> +	signal(SIGTERM, sigint_handler);
> +
> +restart:
> +	optind = 1;
> +	skel = SCX_OPS_OPEN(priority_ops, scx_priority);
> +
> +	while ((opt = getopt(argc, argv, "i:vh")) != -1) {
> +		switch (opt) {
> +		case 'i':
> +			interval = atoi(optarg);
> +			if (interval <= 0)
> +				interval = 1;
> +			break;
> +		case 'v':
> +			verbose = true;
> +			break;
> +		default:
> +			fprintf(stderr, help_fmt, basename(argv[0]));
> +			return opt != 'h';
> +		}
> +	}
> +
> +	SCX_OPS_LOAD(skel, priority_ops, scx_priority, uei);
> +	link = SCX_OPS_ATTACH(skel, priority_ops, scx_priority);
> +
> +	printf("scx_priority started (interval: %ds). Press Ctrl-C to stop.\n", interval);
> +	printf("%-15s %-15s %-15s\n", "HIGH_PRIO(UI)", "LOW_PRIO(BATCH)", "TOTAL_DISPATCH");
> +
> +	while (!exit_req && !UEI_EXITED(skel, uei)) {
> +		__u64 stats[2];
> +
> +		read_stats(skel, stats);
> +		printf("%-15llu %-15llu %-15llu\n",
> +		       stats[0], stats[1], stats[0] + stats[1]);
> +		fflush(stdout);
> +		sleep(interval);
> +	}
> +
> +	bpf_link__destroy(link);
> +	ecode = UEI_REPORT(skel, uei);
> +	scx_priority__destroy(skel);
> +
> +	if (!exit_req && UEI_ECODE_RESTART(ecode))
> +		goto restart;
> +	return 0;
> +}
> -- 
> 2.54.0.windows.1
>