Re: [Patch v4 02/22] sched/cache: Limit the scan number of CPUs when calculating task occupancy

Luo Gengkun
2026-04-10T10:12:45Z

On 2026/4/9 21:41, Peter Zijlstra wrote:
> On Thu, Apr 09, 2026 at 09:17:10PM +0800, Luo Gengkun wrote:
>>
>>
>> On 2026/4/2 5:52, Tim Chen wrote:
>>> From: Chen Yu <yu.c.chen@intel.com>
>>>
>>> When NUMA balancing is enabled, the kernel currently iterates over all
>>> online CPUs to aggregate process-wide occupancy data. On large systems,
>>> this global scan introduces significant overhead.
>>>
>>> To reduce scan latency, limit the search to a subset of relevant CPUs:
>>> 1. The task's preferred NUMA node.
>>> 2. The node where the task is currently running.
>>> 3. The node that contains the task's current preferred LLC..
>>>
>>> While focusing solely on the preferred NUMA node is ideal, a
>>> process-wide scan must remain flexible because the "preferred node"
>>> is a per-task attribute. Different threads within the same process may
>>> have different preferred nodes, causing the process-wide preference to
>>> migrate. Maintaining a mask that covers both the preferred and active
>>> running nodes ensures accuracy while significantly reducing the number of
>>> CPUs inspected.
>>
>> To address the issue of scanning overhead, there is a more targeted
>> approach: only scanning the CPUs actually accessed by the process, and
>> evicting these CPUs when they remain unaccessed for a specific period of
>> time.
>>
>> This significantly reduces unnecessary scanning in most scenario. I have
>> attached the patch below for review. Please feel free to integrate or modify
>> these changes.
>>
>> Thansk!
>> Luo Gengkun
>
> Please fix your MUA, whatever you tried to send is horribly white space
> mangled.
>
Apologies for the formatting mess. The following should display properly.

From de33b29f31a882ac8c0038cac4ec659c7fdfe60d Mon Sep 17 00:00:00 2001
From: Luo Gengkun <luogengkun2@huawei.com>
Date: Thu, 9 Apr 2026 07:22:16 +0000
Subject: [PATCH] sched/cache: Reduce the overhead of task_cache_work by only
scan the visisted cpus.

The overhead of task_cache_work is high, espeically in multi-NUMA system.
Currently, task_cache_work try to find the pref_llc by scan all cpus in the
system. However, most of these scans are meaningless, such as those for
cpus that have never been visited or were accessed a long time ago.

To address this problem, this patch introduces visited_cpus to track the
visited cpus and uses llc_epoch_visited_timeout to evict cpus that have
timed out.

Signed-off-by: Luo Gengkun <luogengkun2@huawei.com>
---
include/linux/sched.h | 1 +
kernel/sched/debug.c | 2 ++
kernel/sched/fair.c | 19 ++++++++++++++++---
kernel/sched/sched.h | 1 +
4 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h
index dfa4bfd099c6..f2327a13fda8 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2390,6 +2390,7 @@ struct sched_cache_time {

struct sched_cache_stat {
struct sched_cache_time __percpu *pcpu_sched;
+ struct cpumask visited_cpus;
raw_spinlock_t lock;
unsigned long epoch;
u64 nr_running_avg;
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 4469e1c152c8..da66acabff4f 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -675,6 +675,8 @@ static __init int sched_init_debug(void)
&llc_epoch_period);
debugfs_create_u32("epoch_affinity_timeout", 0644, llc,
&llc_epoch_affinity_timeout);
+ debugfs_create_u32("epoch_visited_timeout", 0644, llc,
+ &llc_epoch_visited_timeout);
debugfs_create_u32("overaggr_pct", 0644, llc,
&llc_overaggr_pct);
debugfs_create_u32("imb_pct", 0644, llc,
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e4e22696a0b1..4ccabc412aa1 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1285,6 +1285,7 @@ static void set_next_buddy(struct sched_entity *se);
__read_mostly unsigned int llc_aggr_tolerance = 1;
__read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD;
__read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEOUT;
+__read_mostly unsigned int llc_epoch_visited_timeout = EPOCH_LLC_AFFINITY_TIMEOUT;
__read_mostly unsigned int llc_imb_pct = 20;
__read_mostly unsigned int llc_overaggr_pct = 50;

@@ -1466,6 +1467,7 @@ void mm_init_sched(struct mm_struct *mm,
raw_spin_lock_init(&mm->sc_stat.lock);
mm->sc_stat.epoch = epoch;
mm->sc_stat.cpu = -1;
+ cpumask_clear(&mm->sc_stat.visited_cpus);

/*
* The update to mm->sc_stat should not be reordered
@@ -1582,6 +1584,7 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
pcpu_sched->runtime += delta_exec;
rq->cpu_runtime += delta_exec;
epoch = rq->cpu_epoch;
+ cpumask_set_cpu(cpu_of(rq), &mm->sc_stat.visited_cpus);
}

/*
@@ -1724,7 +1727,10 @@ static void task_cache_work(struct callback_head *work)
return;

scoped_guard (cpus_read_lock) {
- get_scan_cpumasks(cpus, p);
+ if (unlikely(llc_epoch_visited_timeout == 0))
+ get_scan_cpumasks(cpus, p);
+ else
+ cpumask_and(cpus, cpu_online_mask, &mm->sc_stat.visited_cpus);

for_each_cpu(cpu, cpus) {
/* XXX sched_cluster_active */
@@ -1736,8 +1742,15 @@ static void task_cache_work(struct callback_head *work)
continue;

for_each_cpu(i, sched_domain_span(sd)) {
- occ = fraction_mm_sched(cpu_rq(i),
- per_cpu_ptr(mm->sc_stat.pcpu_sched, i));
+ struct rq *rq = cpu_rq(i);
+ struct sched_cache_time *pcpu_sched = per_cpu_ptr(mm->sc_stat.pcpu_sched, i);
+ /* Skip the rq that has not been hit for a long time */
+ if (llc_epoch_visited_timeout && (rq->cpu_epoch - pcpu_sched->epoch) >
+ llc_epoch_visited_timeout) {
+ cpumask_clear_cpu(cpu_of(rq), &mm->sc_stat.visited_cpus);
+ continue;
+ }
+ occ = fraction_mm_sched(rq, pcpu_sched);
a_occ += occ;
if (occ > m_occ) {
m_occ = occ;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b757812725f7..93908bb17cc5 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4041,6 +4041,7 @@ extern int sysctl_sched_cache_user;
extern unsigned int llc_aggr_tolerance;
extern unsigned int llc_epoch_period;
extern unsigned int llc_epoch_affinity_timeout;
+extern unsigned int llc_epoch_visited_timeout;
extern unsigned int llc_imb_pct;
extern unsigned int llc_overaggr_pct;

--
2.34.1