Re: [Patch v4 02/22] sched/cache: Limit the scan number of CPUs when calculating task occupancy

Luo Gengkun
2026-04-09T13:17:32Z

On 2026/4/2 5:52, Tim Chen wrote:
> From: Chen Yu <yu.c.chen@intel.com>
>
> When NUMA balancing is enabled, the kernel currently iterates over all
> online CPUs to aggregate process-wide occupancy data. On large systems,
> this global scan introduces significant overhead.
>
> To reduce scan latency, limit the search to a subset of relevant CPUs:
> 1. The task's preferred NUMA node.
> 2. The node where the task is currently running.
> 3. The node that contains the task's current preferred LLC..
>
> While focusing solely on the preferred NUMA node is ideal, a
> process-wide scan must remain flexible because the "preferred node"
> is a per-task attribute. Different threads within the same process may
> have different preferred nodes, causing the process-wide preference to
> migrate. Maintaining a mask that covers both the preferred and active
> running nodes ensures accuracy while significantly reducing the number of
> CPUs inspected.

To address the issue of scanning overhead, there is a more targeted
approach: only scanning the CPUs actually accessed by the process, and
evicting these CPUs when they remain unaccessed for a specific period of
time.

This significantly reduces unnecessary scanning in most scenario. I have
attached the patch below for review. Please feel free to integrate or
modify these changes.

Thansk!
Luo Gengkun

From de33b29f31a882ac8c0038cac4ec659c7fdfe60d Mon Sep 17 00:00:00 2001

From: Luo Gengkun <luogengkun2@huawei.com>

Date: Thu, 9 Apr 2026 07:22:16 +0000

Subject: [PATCH] sched/cache: Reduce the overhead of task_cache_work by
only
scan the visisted cpus.

The overhead of task_cache_work is high, espeically in multi-NUMA
system.
Currently, task_cache_work try to find the pref_llc by scan all cpus in
the
system. However, most of these scans are meaningless, such as those for

cpus that have never been visited or were accessed a long time ago.

To address this problem, this patch introduces visited_cpus to track the

visited cpus and uses llc_epoch_visited_timeout to evict cpus that have

timed out.

Signed-off-by: Luo Gengkun <luogengkun2@huawei.com>

---

include/linux/sched.h | 1 +

kernel/sched/debug.c | 2 ++

kernel/sched/fair.c | 19 ++++++++++++++++---

kernel/sched/sched.h | 1 +

4 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/include/linux/sched.h b/include/linux/sched.h

index dfa4bfd099c6..f2327a13fda8 100644

--- a/include/linux/sched.h

+++ b/include/linux/sched.h

@@ -2390,6 +2390,7 @@ struct sched_cache_time {

struct sched_cache_stat {

struct sched_cache_time __percpu *pcpu_sched;

+ struct cpumask visited_cpus;

raw_spinlock_t lock;

unsigned long epoch;

u64 nr_running_avg;

diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c

index 4469e1c152c8..da66acabff4f 100644

--- a/kernel/sched/debug.c

+++ b/kernel/sched/debug.c

@@ -675,6 +675,8 @@ static __init int sched_init_debug(void)

&llc_epoch_period);

debugfs_create_u32("epoch_affinity_timeout", 0644, llc,

&llc_epoch_affinity_timeout);

+ debugfs_create_u32("epoch_visited_timeout", 0644, llc,

+ &llc_epoch_visited_timeout);

debugfs_create_u32("overaggr_pct", 0644, llc,

&llc_overaggr_pct);

debugfs_create_u32("imb_pct", 0644, llc,

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c

index e4e22696a0b1..4ccabc412aa1 100644

--- a/kernel/sched/fair.c

+++ b/kernel/sched/fair.c

@@ -1285,6 +1285,7 @@ static void set_next_buddy(struct sched_entity
*se);
__read_mostly unsigned int llc_aggr_tolerance = 1;

__read_mostly unsigned int llc_epoch_period = EPOCH_PERIOD;

__read_mostly unsigned int llc_epoch_affinity_timeout =
EPOCH_LLC_AFFINITY_TIMEOUT;
+__read_mostly unsigned int llc_epoch_visited_timeout =
EPOCH_LLC_AFFINITY_TIMEOUT;
__read_mostly unsigned int llc_imb_pct = 20;

__read_mostly unsigned int llc_overaggr_pct = 50;

@@ -1466,6 +1467,7 @@ void mm_init_sched(struct mm_struct *mm,

raw_spin_lock_init(&mm->sc_stat.lock);

mm->sc_stat.epoch = epoch;

mm->sc_stat.cpu = -1;

+ cpumask_clear(&mm->sc_stat.visited_cpus);

/*

* The update to mm->sc_stat should not be reordered

@@ -1582,6 +1584,7 @@ void account_mm_sched(struct rq *rq, struct
task_struct *p, s64 delta_exec)
pcpu_sched->runtime += delta_exec;

rq->cpu_runtime += delta_exec;

epoch = rq->cpu_epoch;

+ cpumask_set_cpu(cpu_of(rq), &mm->sc_stat.visited_cpus);

}

/*

@@ -1724,7 +1727,10 @@ static void task_cache_work(struct callback_head
*work)
return;

scoped_guard (cpus_read_lock) {

- get_scan_cpumasks(cpus, p);

+ if (unlikely(llc_epoch_visited_timeout == 0))

+ get_scan_cpumasks(cpus, p);

+ else

+ cpumask_and(cpus, cpu_online_mask,
&mm->sc_stat.visited_cpus);

for_each_cpu(cpu, cpus) {

/* XXX sched_cluster_active */

@@ -1736,8 +1742,15 @@ static void task_cache_work(struct callback_head
*work)
continue;

for_each_cpu(i, sched_domain_span(sd)) {

- occ = fraction_mm_sched(cpu_rq(i),

-
per_cpu_ptr(mm->sc_stat.pcpu_sched, i));
+ struct rq *rq = cpu_rq(i);

+ struct sched_cache_time *pcpu_sched =
per_cpu_ptr(mm->sc_stat.pcpu_sched, i);
+ /* Skip the rq that has not been hit for
a long time */
+ if (llc_epoch_visited_timeout &&
(rq->cpu_epoch - pcpu_sched->epoch) >
+ llc_epoch_visited_timeout) {

+ cpumask_clear_cpu(cpu_of(rq),
&mm->sc_stat.visited_cpus);
+ continue;

+ }
+ occ = fraction_mm_sched(rq, pcpu_sched);
a_occ += occ;
if (occ > m_occ) {
m_occ = occ;
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b757812725f7..93908bb17cc5 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -4041,6 +4041,7 @@ extern int sysctl_sched_cache_user;
extern unsigned int llc_aggr_tolerance;
extern unsigned int llc_epoch_period;
extern unsigned int llc_epoch_affinity_timeout;
+extern unsigned int llc_epoch_visited_timeout;
extern unsigned int llc_imb_pct;
extern unsigned int llc_overaggr_pct;

--
2.34.1

>
> Future work may integrate numa_group to further refine task aggregation.
>
> Suggested-by: Madadi Vineeth Reddy <vineethr@linux.ibm.com>
> Signed-off-by: Chen Yu <yu.c.chen@intel.com>
> Co-developed-by: Tim Chen <tim.c.chen@linux.intel.com>
> Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
> ---
>
> Notes:
> v3->v4:
> New patch.
>
> kernel/sched/fair.c | 46 ++++++++++++++++++++++++++++++++++++++++++++-
> 1 file changed, 45 insertions(+), 1 deletion(-)
>
> diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
> index eb3cfb852a93..20a33900f4ea 100644
> --- a/kernel/sched/fair.c
> +++ b/kernel/sched/fair.c
> @@ -1431,6 +1431,50 @@ static void task_tick_cache(struct rq *rq, struct task_struct *p)
> }
> }
>
> +static void get_scan_cpumasks(cpumask_var_t cpus, struct task_struct *p)
> +{
> +#ifdef CONFIG_NUMA_BALANCING
> + int cpu, curr_cpu, pref_nid;
> +
> + if (!static_branch_likely(&sched_numa_balancing))
> + goto out;
> +
> + cpu = p->mm->sc_stat.cpu;
> + curr_cpu = task_cpu(p);
> +
> + /*
> + * Scanning in the preferred NUMA node is ideal. However, the NUMA
> + * preferred node is per-task rather than per-process. It is possible
> + * for different threads of the process to have distinct preferred
> + * nodes; consequently, the process-wide preferred LLC may bounce
> + * between different nodes. As a workaround, maintain the scan
> + * CPU mask to also cover the process's current preferred LLC and the
> + * current running node to mitigate the bouncing risk.
> + * TBD: numa_group should be considered during task aggregation.
> + */
> + pref_nid = p->numa_preferred_nid;
> + /* honor the task's preferred node */
> + if (pref_nid == NUMA_NO_NODE)
> + goto out;
> +
> + cpumask_or(cpus, cpus, cpumask_of_node(pref_nid));
> +
> + /* honor the task's preferred LLC CPU */
> + if (cpu != -1 && !cpumask_test_cpu(cpu, cpus))
> + cpumask_or(cpus, cpus,
> + cpumask_of_node(cpu_to_node(cpu)));
> +
> + /* make sure the task's current running node is included */
> + if (!cpumask_test_cpu(curr_cpu, cpus))
> + cpumask_or(cpus, cpus, cpumask_of_node(cpu_to_node(curr_cpu)));
> +
> + return;
> +
> +out:
> +#endif
> + cpumask_copy(cpus, cpu_online_mask);
> +}
> +
> static void task_cache_work(struct callback_head *work)
> {
> struct task_struct *p = current;
> @@ -1451,7 +1495,7 @@ static void task_cache_work(struct callback_head *work)
> return;
>
> scoped_guard (cpus_read_lock) {
> - cpumask_copy(cpus, cpu_online_mask);
> + get_scan_cpumasks(cpus, p);
>
> for_each_cpu(cpu, cpus) {
> /* XXX sched_cluster_active */