Re: [PATCH 10/11] sched_ext: scx_qmap: Add proxy execution support

[Andrea Righi <arighi@nvidia.com>]

Hi John,

On Fri, Jul 17, 2026 at 07:28:49PM -0700, John Stultz wrote:
> On Thu, Jul 16, 2026 at 6:23 AM Andrea Righi <arighi@nvidia.com> wrote:
> >
> > Add a -B option to opt scx_qmap into queueing mutex-blocked tasks for
> > proxy execution. Without the option, SCX_OPS_ENQ_BLOCKED remains clear
> > and mutex waiters block normally. With -B, blocked donors are passed to
> > qmap_enqueue() with SCX_ENQ_BLOCKED.
> >
> > When scx_qmap receives a blocked donor, dispatch it directly to the
> > local DSQ of its current cid with a fresh slice and SCX_ENQ_PREEMPT.
> > This places the donor at the head of the DSQ and requests an immediate
> > reschedule, allowing the core proxy-exec path to run the mutex owner
> > using the donor's scheduling context as soon as the donor is selected.
> 
> Hey Andrea!
> 
> Thanks again for your great work on all this, I'm working to test your
> changes with my full proxy-exec tree, but before that I caught one
> issue here.
> 
> I was excited to try this out, but when building and running this from
> your kernel tree in a VM, (who's kernel is running from the same
> kernel tree, built on the host), I get the following error (with or
> without -B). The other scx_ schedulers in tools/sched_ext/ seem to
> load and run ok.  Maybe is there something I'm forgetting to do here?
> 
> thanks
> -john
> 
> # ./build/bin/scx_qmap -B
> libbpf: prog 'qmap_dequeue': BPF program load failed: -EACCES
> libbpf: prog 'qmap_dequeue': -- BEGIN PROG LOAD LOG --
> 0: R1=ctx() R10=fp0
> ; void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64
> deq_flags) @ scx_qmap.bpf.c:584
> 0: (79) r2 = *(u64 *)(r1 +0)
> func 'dequeue' arg0 has btf_id 142670 type STRUCT 'task_struct'
> 1: R1=ctx() R2=trusted_ptr_task_struct()
> 1: (79) r4 = *(u64 *)(r1 +8)          ; R1=ctx() R4=scalar()
> ; __sync_fetch_and_add(&qa.nr_dequeued, 1); @ scx_qmap.bpf.c:588
> 2: (18) r1 = 0x10000fff7000           ; R1=scalar()
> 4: (18) r3 = 0x10000fff7000           ; R3=scalar()
> 6: (07) r3 += 32                      ; R3=scalar()
> 7: (bf) r5 = addr_space_cast(r3, 0, 1)        ; R3=scalar() R5=arena
> 8: (b7) r3 = 1                        ; R3=1
> 9: (b7) r0 = 1                        ; R0=1
> 10: (db) lock *(u64 *)(r5 +0) += r0   ; R0=1 R5=arena
> 11: (18) r5 = 0x100000000             ; R5=0x100000000
> ; if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC) @ scx_qmap.bpf.c:589
> 13: (5f) r4 &= r5                     ;
> R4=scalar(smin=smin32=0,smax=umax=0x100000000,smax32=umax32=0,var_off=(0x0;
> 0x100000000))
> 14: (15) if r4 == 0x0 goto pc+3       ; R4=0x100000000
> ; __sync_fetch_and_add(&qa.nr_core_sched_execed, 1); @ scx_qmap.bpf.c:590
> 15: (07) r1 += 48                     ; R1=scalar()
> 16: (bf) r1 = addr_space_cast(r1, 0, 1)       ; R1=arena
> 17: (db) lock *(u64 *)(r1 +0) += r3   ;
> ; QMAP_TOUCH_ARENA(); @ scx_qmap.bpf.c:268
> 18: (18) r1 = 0xffff888104842c00      ; R1=map_ptr(map=arena,ks=0,vs=0)
> ; v = bpf_task_storage_get(&task_ctx_stor, p, 0, 0); @ scx_qmap.bpf.c:270
> 20: (18) r1 = 0xffff88810c333600      ; R1=map_ptr(map=task_ctx_stor,ks=4,vs=8)
> 22: (b7) r3 = 0                       ; R3=0
> 23: (b7) r4 = 0                       ; R4=0
> 24: (85) call bpf_task_storage_get#156        ;
> R0=map_value_or_null(id=1,map=task_ctx_stor,ks=4,vs=8)
> ; if (!v || !v->taskc) @ scx_qmap.bpf.c:271
> 25: (15) if r0 == 0x0 goto pc+76      ;
> R0=map_value(id=1,map=task_ctx_stor,ks=4,vs=8)
> 26: (79) r7 = *(u64 *)(r0 +0)         ;
> R0=map_value(id=1,map=task_ctx_stor,ks=4,vs=8) R7=scalar()
> 27: (15) if r7 == 0x0 goto pc+74      ; R7=scalar(umin=1)
> 28: (bf) r9 = addr_space_cast(r7, 0, 1)       ; R7=scalar(umin=1) R9=arena
> ; if (taskc && taskc->fifo) { @ scx_qmap.bpf.c:593
> 29: (79) r8 = *(u64 *)(r9 +24)        ; R8=scalar() R9=arena
> 30: (15) if r8 == 0x0 goto pc+71      ; R8=scalar(umin=1)
> ; if (taskc->highpri) @ scx_qmap.bpf.c:594
> 31: (71) r1 = *(u8 *)(r9 +45)         ;
> R1=scalar(smin=smin32=0,smax=umax=smax32=umax32=255,var_off=(0x0;
> 0xff)) R9=arena
> 32: (56) if w1 != 0x1 goto pc+9       ; R1=1
> 33: (b7) r1 = 1                       ; R1=1
> ; __sync_fetch_and_sub(&qa.nr_highpri_queued, 1); @ scx_qmap.bpf.c:595
> 34: (87) r1 = -r1                     ; R1=-1
> 35: (18) r2 = 0x10000fff7000          ; R2=scalar()
> 37: (07) r2 += 88                     ; R2=scalar()
> 38: (bf) r2 = addr_space_cast(r2, 0, 1)       ; R2=arena
> 39: (db) lock *(u64 *)(r2 +0) += r1   ; R1=-1 R2=arena
> ; struct qmap_fifo __arena *fifo = taskc->fifo; @ scx_qmap.bpf.c:320
> 40: (79) r8 = *(u64 *)(r9 +24)        ; R8=scalar() R9=arena
> ; if (!fifo) @ scx_qmap.bpf.c:323
> 41: (15) if r8 == 0x0 goto pc+60      ; R8=scalar(umin=1)
> ; lock = qa_q_lock(fifo->idx); @ scx_qmap.bpf.c:326
> 42: (bf) r2 = addr_space_cast(r8, 0, 1)       ; R2=arena R8=scalar(umin=1)
> 43: (61) r1 = *(u32 *)(r2 +16)        ;
> R1=scalar(smin=0,smax=umax=0xffffffff,var_off=(0x0; 0xffffffff))
> R2=arena
> ; switch (qid) { @ scx_qmap.bpf.c:129
> 44: (66) if w1 s> 0x1 goto pc+8       ;
> R1=scalar(smin=0,smax=umax=0xffffffff,smax32=1,var_off=(0x0;
> 0xffffffff))
> 45: (18) r6 = 0xffff88810c3339f8      ;
> R6=map_value(map=.data.qa_q_lock,ks=4,vs=4)
> 47: (16) if w1 == 0x0 goto pc+17      ;
> R1=scalar(smin=0,smax=umax=0xffffffff,smax32=1,var_off=(0x0;
> 0xffffffff))
> 48: (16) if w1 == 0x1 goto pc+1 50: R2=arena R7=scalar(umin=1)
> R8=scalar(umin=1) R9=arena R10=fp0
> ; switch (qid) { @ scx_qmap.bpf.c:129
> 50: (18) r6 = 0xffff88810c333bf8      ;
> R6=map_value(map=.data.qa_q_lock,ks=4,vs=4)
> 52: (05) goto pc+12
> 65: (7b) *(u64 *)(r10 -16) = r2       ; R2=arena R10=fp0 fp-16=arena
> ; if (bpf_res_spin_lock(lock)) { @ scx_qmap.bpf.c:212
> 66: (bf) r1 = r6                      ;
> R1=map_value(map=.data.qa_q_lock,ks=4,vs=4)
> R6=map_value(map=.data.qa_q_lock,ks=4,vs=4)
> 67: (85) call bpf_res_spin_lock#85478         ; R0=0
> 68: (16) if w0 == 0x0 goto pc+9       ; R0=0
> ; if (taskc->fifo != fifo) { @ scx_qmap.bpf.c:331
> 78: (79) r1 = *(u64 *)(r9 +24)        ; R1=scalar() R9=arena
> 79: (1d) if r1 == r8 goto pc+1 81:
> R6=map_value(map=.data.qa_q_lock,ks=4,vs=4) R7=scalar(umin=1)
> R8=scalar(umin=1) R9=arena R10=fp0 fp-16=arena
> ; bpf_res_spin_unlock(lock); @ scx_qmap.bpf.c:332
> 81: (79) r2 = *(u64 *)(r9 +16)        ; R2=scalar() R9=arena
> ; if (taskc->q_next) @ scx_qmap.bpf.c:336
> 82: (79) r1 = *(u64 *)(r9 +8)         ; R1=scalar() R9=arena
> 83: (15) if r1 == 0x0 goto pc+4       ; R1=scalar(umin=1)
> 84: (bf) r3 = addr_space_cast(r1, 0, 1)       ; R1=scalar(umin=1) R3=arena
> ; taskc->q_next->q_prev = taskc->q_prev; @ scx_qmap.bpf.c:337
> 85: (7b) *(u64 *)(r3 +16) = r2        ; R2=scalar() R3=arena
> ; if (taskc->q_prev) @ scx_qmap.bpf.c:340
> 86: (79) r2 = *(u64 *)(r9 +16)        ; R2=scalar() R9=arena
> ; taskc->q_next->q_prev = taskc->q_prev; @ scx_qmap.bpf.c:337
> 87: (05) goto pc+2
> ; fifo->tail = taskc->q_prev; @ scx_qmap.bpf.c:339
> 90: (07) r7 += 8                      ; R7=scalar()
> ; if (taskc->q_prev) @ scx_qmap.bpf.c:340
> 91: (15) if r2 == 0x0 goto pc+2       ; R2=scalar(umin=1)
> 92: (07) r2 += 8                      ; R2=scalar()
> 93: (bf) r8 = r2                      ; R8=scalar(id=2)
> 94: (bf) r2 = addr_space_cast(r8, 0, 1)       ; R2=arena R8=scalar(id=2)
> 95: (7b) *(u64 *)(r2 +0) = r1         ; R1=scalar(umin=1) R2=arena
> 96: (b7) r1 = 0                       ; R1=0
> ; taskc->q_prev = NULL; @ scx_qmap.bpf.c:345
> 97: (7b) *(u64 *)(r7 +16) = r1
> R7 invalid mem access 'scalar'

This looks like something related to BPF arena pointers and incorrectly
generated BPF code (the other schedulers don't use BPF arenas, so that'd explain
why you only see this with scx_qmap).

Which version of clang are you using?

Thanks,
-Andrea