Re: [PATCH sched_ext/for-7.3-fixes] sched_ext: Fix use-after-free of a local reenq node left on an inactive cpu

[Andrea Righi <arighi@nvidia.com>]

Hi Tejun,

On Thu, Oct 15, 2026 at 07:18:05AM -1000, Tejun Heo wrote:
> A local DSQ reenq is queued on the rq through a per-sched node and run from
> deferred work. When the reenq is requested from dispatch, the work is a
> balance callback queued once dispatch is done. A cpu going down installs
> balance push early in its teardown, and queue_balance_callback() refuses new
> callbacks from then on. ops.dispatch() keeps running on that cpu until the
> rq goes offline after an RCU grace period, so a reenq it requests in that
> window may never run. The IMMED handling in dispatch_one() requests one the
> same way.
> 
> A reenq requested from elsewhere for a cpu that is already offline goes
> through irq_work_queue_on(), which warns, and its work doesn't run until the
> cpu comes back.
> 
> Either way the node stays on the rq's list. When the scheduler is freed,
> scx_sched_free_rcu_work() warns and frees the pcpu with the node still
> linked, and the list is walked again once the cpu comes back.
> 
> An inactive cpu's local DSQ holds only tasks that are bound to it or are
> being moved off, so the reenq is pointless. Ignore reenqs of an inactive
> cpu's local DSQ.
> 
> The test is lockless. The hotplug thread of a cpu going down runs on that
> cpu, and it marks the cpu inactive before it installs balance push. A
> dispatch on the cpu runs to completion before that thread can run there, so
> a dispatch that saw the cpu active has queued its callback by then. Requests
> from outside dispatch run from the wakeup path or through irq_work, which
> balance push doesn't affect. The grace period after the cpu goes inactive
> waits for any requester that saw it active, so that irq_work is queued while
> the cpu is online.
> 
> Fixes: 238eba8c210d ("sched_ext: Use schedule_deferred_locked() in schedule_dsq_reenq()")
> Fixes: 76edc2761ab8 ("sched_ext: Use irq_work_queue_on() in schedule_deferred()")
> Cc: stable@vger.kernel.org # v7.1+
> Signed-off-by: Tejun Heo <tj@kernel.org>

Looks good and validated in virtme-ng with the following AI-generated
reproducer.

Reviewed-by: Andrea Righi <arighi@nvidia.com>
Tested-by: Andrea Righi <arighi@nvidia.com>

Thanks,
-Andrea

diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile
index af919a56a4f4f..f0af0c3b32076 100644
--- a/tools/testing/selftests/sched_ext/Makefile
+++ b/tools/testing/selftests/sched_ext/Makefile
@@ -184,6 +184,7 @@ auto-test-targets :=			\
 	allowed_cpus			\
 	peek_dsq			\
 	prog_run			\
+	reenq_offline			\
 	reload_loop			\
 	select_cpu_dfl			\
 	select_cpu_dfl_nodispatch	\
diff --git a/tools/testing/selftests/sched_ext/reenq_offline.bpf.c b/tools/testing/selftests/sched_ext/reenq_offline.bpf.c
new file mode 100644
index 0000000000000..a3fff2c0e0de6
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/reenq_offline.bpf.c
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <scx/common.bpf.h>
+
+char _license[] SEC("license") = "GPL";
+
+volatile bool trigger_reenq;
+volatile u64 reenq_attempts;
+
+void BPF_STRUCT_OPS(reenq_offline_enqueue, struct task_struct *p, u64 enq_flags)
+{
+	if (trigger_reenq) {
+		scx_bpf_dsq_reenq(SCX_DSQ_LOCAL_ON | 1, 0);
+		__sync_fetch_and_add(&reenq_attempts, 1);
+	}
+
+	scx_bpf_dsq_insert(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
+}
+
+void BPF_STRUCT_OPS_SLEEPABLE(reenq_offline_cpu_offline, s32 cpu)
+{}
+
+SEC(".struct_ops.link")
+struct sched_ext_ops reenq_offline_ops = {
+	.enqueue = (void *)reenq_offline_enqueue,
+	.cpu_offline = (void *)reenq_offline_cpu_offline,
+	.name = "reenq_offline",
+};
diff --git a/tools/testing/selftests/sched_ext/reenq_offline.c b/tools/testing/selftests/sched_ext/reenq_offline.c
new file mode 100644
index 0000000000000..739d5b4e22cf7
--- /dev/null
+++ b/tools/testing/selftests/sched_ext/reenq_offline.c
@@ -0,0 +1,76 @@
+// SPDX-License-Identifier: GPL-2.0
+#include <bpf/bpf.h>
+#include <scx/common.h>
+#include <sched.h>
+#include <unistd.h>
+
+#include "reenq_offline.bpf.skel.h"
+#include "scx_test.h"
+#include "util.h"
+
+#define CPU1_ONLINE "/sys/devices/system/cpu/cpu1/online"
+
+static enum scx_test_status run(void *ctx)
+{
+	struct reenq_offline *skel = NULL;
+	struct bpf_link *link = NULL;
+	cpu_set_t mask, original_mask;
+	enum scx_test_status status = SCX_TEST_FAIL;
+	int i;
+
+	if (access(CPU1_ONLINE, W_OK) || file_read_long(CPU1_ONLINE) != 1)
+		return SCX_TEST_SKIP;
+
+	if (sched_getaffinity(0, sizeof(original_mask), &original_mask))
+		return SCX_TEST_SKIP;
+
+	CPU_ZERO(&mask);
+	CPU_SET(0, &mask);
+	if (sched_setaffinity(0, sizeof(mask), &mask))
+		return SCX_TEST_SKIP;
+
+	skel = reenq_offline__open();
+	if (!skel)
+		goto out;
+	SCX_ENUM_INIT(skel);
+	if (reenq_offline__load(skel))
+		goto out;
+
+	link = bpf_map__attach_struct_ops(skel->maps.reenq_offline_ops);
+	if (!link)
+		goto out;
+
+	if (file_write_long(CPU1_ONLINE, 0))
+		goto out;
+
+	skel->bss->trigger_reenq = true;
+	for (i = 0; i < 1000 && !skel->bss->reenq_attempts; i++)
+		usleep(1000);
+	skel->bss->trigger_reenq = false;
+
+	if (!skel->bss->reenq_attempts) {
+		SCX_ERR("No enqueue reached the offline CPU reenqueue request");
+		goto out;
+	}
+
+	status = SCX_TEST_PASS;
+out:
+	/* Detach while CPU 1 is offline to expose a stranded reenq node. */
+	bpf_link__destroy(link);
+	if (link)
+		sleep(1);
+	if (file_read_long(CPU1_ONLINE) == 0 &&
+	    file_write_long(CPU1_ONLINE, 1))
+		status = SCX_TEST_FAIL;
+	if (sched_setaffinity(0, sizeof(original_mask), &original_mask))
+		status = SCX_TEST_FAIL;
+	reenq_offline__destroy(skel);
+	return status;
+}
+
+struct scx_test reenq_offline = {
+	.name = "reenq_offline",
+	.description = "Reenqueue an offline CPU local DSQ and detach",
+	.run = run,
+};
+REGISTER_SCX_TEST(&reenq_offline)