Re: [RFC PATCH] sched: Add scx_cpuperf_target in sched_cpu_util()

Peter Zijlstra
2026-03-19T09:03:00Z

On Wed, Mar 18, 2026 at 03:08:43PM -1000, Tejun Heo wrote:
> On Wed, Mar 18, 2026 at 01:47:18PM +0100, Peter Zijlstra wrote:
> > On Wed, Mar 18, 2026 at 08:17:55PM +0800, Xuewen Yan wrote:
> > > diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> > > index bf948db905ed..20adb6fede2a 100644
> > > --- a/kernel/sched/fair.c
> > > +++ b/kernel/sched/fair.c
> > > @@ -8198,7 +8198,12 @@ unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
> > >
> > > unsigned long sched_cpu_util(int cpu)
> > > {
> > > - return effective_cpu_util(cpu, cpu_util_cfs(cpu), NULL, NULL);
> > > + unsigned long util = scx_cpuperf_target(cpu);
> > > +
> > > + if (!scx_switched_all())
> > > + util += cpu_util_cfs(cpu);
> > > +
> > > + return effective_cpu_util(cpu, util, NULL, NULL);
> > > }
> >
> > This puts the common case of no ext muck into the slow path of that
> > static_branch.
> >
> > This wants to be something like:
> >
> > unsigned long sched_cpu_util(int cpu)
> > {
> > unsigned long util = cpu_util_cfs(cpu);
> >
> > if (scx_enabled()) {
> > unsigned long scx_util = scx_cpuperf_target(cpu);
> >
> > if (!scx_switched_all())
> > scx_util += util;
> >
> > util = scx_util;
> > }
> >
> > return effective_cpu_util(cpu, util, NULL, NULL);
> > }
>
> scx_switched_all() is an unlikely static branch just like scx_enabled() and
> scx_cpuperf_target() has scx_enabled() in it too, so the difference for the
> fair path between the two versions is two noop run-throughs vs. one. Either
> way is fine but it is more code for likely no discernible gain.

(added noinline to effective_cpu_util() for clarity)

So the original patch generates this:

sched_cpu_util:
1c5240: sched_cpu_util+0x0 endbr64
1c5244: sched_cpu_util+0x4 call 0x1c5249 <__fentry__>
1c5249: sched_cpu_util+0x9 push %rbp
1c524a: sched_cpu_util+0xa push %rbx
1c524b: sched_cpu_util+0xb mov %edi,%ebx
1c524d: sched_cpu_util+0xd <jump_table.1c524d>
= nop2 (if DEFAULT)
= jmp 1c5271 <sched_cpu_util+0x31> (if JUMP)
1c524f: sched_cpu_util+0xf xor %ebp,%ebp
1c5251: sched_cpu_util+0x11 <jump_table.1c5251>
= nop2 (if DEFAULT)
= jmp 1c5261 <sched_cpu_util+0x21> (if JUMP)
1c5253: sched_cpu_util+0x13 xor %edx,%edx
1c5255: sched_cpu_util+0x15 xor %esi,%esi
1c5257: sched_cpu_util+0x17 mov %ebx,%edi
1c5259: sched_cpu_util+0x19 call 0x1bc5b0 <cpu_util.constprop.0>
1c525e: sched_cpu_util+0x1e add %rax,%rbp
1c5261: sched_cpu_util+0x21 mov %rbp,%rsi
1c5264: sched_cpu_util+0x24 mov %ebx,%edi
1c5266: sched_cpu_util+0x26 xor %ecx,%ecx
1c5268: sched_cpu_util+0x28 pop %rbx
1c5269: sched_cpu_util+0x29 xor %edx,%edx
1c526b: sched_cpu_util+0x2b pop %rbp
1c526c: sched_cpu_util+0x2c jmp 0x1c5160 <effective_cpu_util>

(slowpath)

1c5271: sched_cpu_util+0x31 movslq %edi,%rdx
1c5274: sched_cpu_util+0x34 mov $0x0,%rax
1c527b: sched_cpu_util+0x3b mov 0x0(,%rdx,8),%rdx
1c5283: sched_cpu_util+0x43 mov 0xa34(%rdx,%rax,1),%ebp
1c528a: sched_cpu_util+0x4a jmp 0x1c5251 <sched_cpu_util+0x11>

While my proposal generates this:

sched_cpu_util:
1c5240: sched_cpu_util+0x0 endbr64
1c5244: sched_cpu_util+0x4 call 0x1c5249 <__fentry__>
1c5249: sched_cpu_util+0x9 push %rbx
1c524a: sched_cpu_util+0xa xor %esi,%esi
1c524c: sched_cpu_util+0xc xor %edx,%edx
1c524e: sched_cpu_util+0xe mov %edi,%ebx
1c5250: sched_cpu_util+0x10 call 0x1bc5b0 <cpu_util.constprop.0>
1c5255: sched_cpu_util+0x15 mov %rax,%rsi
1c5258: sched_cpu_util+0x18 <jump_table.1c5258>
= nop2 (if DEFAULT)
= jmp 1c5266 <sched_cpu_util+0x26> (if JUMP)
1c525a: sched_cpu_util+0x1a mov %ebx,%edi
1c525c: sched_cpu_util+0x1c xor %ecx,%ecx
1c525e: sched_cpu_util+0x1e xor %edx,%edx
1c5260: sched_cpu_util+0x20 pop %rbx
1c5261: sched_cpu_util+0x21 jmp 0x1c5160 <effective_cpu_util>

(slowpath)

1c5266: sched_cpu_util+0x26 <jump_table.1c5266>
= nop2 (if DEFAULT)
= jmp 1c527b <sched_cpu_util+0x3b> (if JUMP)
1c5268: sched_cpu_util+0x28 xor %eax,%eax
1c526a: sched_cpu_util+0x2a <jump_table.1c526a>
= nop2 (if DEFAULT)
= jmp 1c5296 <sched_cpu_util+0x56> (if JUMP)
1c526c: sched_cpu_util+0x2c mov %ebx,%edi
1c526e: sched_cpu_util+0x2e add %rax,%rsi
1c5271: sched_cpu_util+0x31 xor %ecx,%ecx
1c5273: sched_cpu_util+0x33 xor %edx,%edx
1c5275: sched_cpu_util+0x35 pop %rbx
1c5276: sched_cpu_util+0x36 jmp 0x1c5160 <effective_cpu_util>
1c527b: sched_cpu_util+0x3b movslq %ebx,%rdx
1c527e: sched_cpu_util+0x3e mov $0x0,%rax
1c5285: sched_cpu_util+0x45 mov 0x0(,%rdx,8),%rdx
1c528d: sched_cpu_util+0x4d mov 0xa34(%rdx,%rax,1),%eax
1c5294: sched_cpu_util+0x54 jmp 0x1c526a <sched_cpu_util+0x2a>
1c5296: sched_cpu_util+0x56 mov %ebx,%edi
1c5298: sched_cpu_util+0x58 mov %rax,%rsi
1c529b: sched_cpu_util+0x5b xor %ecx,%ecx
1c529d: sched_cpu_util+0x5d xor %edx,%edx
1c529f: sched_cpu_util+0x5f pop %rbx
1c52a0: sched_cpu_util+0x60 jmp 0x1c5160 <effective_cpu_util>

That fastpath is definitely better; the slowpath is worse, but that is
in part because the compilers are stupid and cannot eliminate
static_branch().

/me goes try again .. Yeah, the below patch does nothing :-( It will
happily emit scx_enabled() twice.

---
diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h
index 05b16299588d..47cd1a1f9784 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -32,7 +32,7 @@
JUMP_TABLE_ENTRY(key, label)
#endif /* CONFIG_HAVE_JUMP_LABEL_HACK */

-static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch)
+static __always_inline __const bool arch_static_branch(struct static_key * const key, const bool branch)
{
asm goto(ARCH_STATIC_BRANCH_ASM("%c0 + %c1", "%l[l_yes]")
: : "i" (key), "i" (branch) : : l_yes);
@@ -42,7 +42,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key, co
return true;
}

-static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch)
+static __always_inline __const bool arch_static_branch_jump(struct static_key * const key, const bool branch)
{
asm goto("1:"
"jmp %l[l_yes]\n\t"
diff --git a/include/linux/compiler_attributes.h b/include/linux/compiler_attributes.h
index c16d4199bf92..553fc9f3f7eb 100644
--- a/include/linux/compiler_attributes.h
+++ b/include/linux/compiler_attributes.h
@@ -312,6 +312,7 @@
* gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-pure-function-attribute
*/
#define __pure __attribute__((__pure__))
+#define __const __attribute__((__const__))

/*
* gcc: https://gcc.gnu.org/onlinedocs/gcc/Common-Function-Attributes.html#index-section-function-attribute