[Patch v4 21/22] -- DO NOT APPLY!!! -- sched/cache/debug: Display the per LLC occupancy for each process via proc fs

Tim Chen
2026-04-01T21:47:19Z

From: Chen Yu <yu.c.chen@intel.com>

Debug patch only.

Show the per-LLC occupancy in /proc/{PID}/schedstat, with each column
corresponding to one LLC. This can be used to verify if the cache-aware
load balancer works as expected by aggregating threads onto dedicated LLCs.

Suppose there are 2 LLCs and the sampling duration is 10 seconds:

Enable the cache aware load balance, LLC0 residency delta is 0,
LLC1 is 12 seconds:
0 12281

disable the cache aware load balance, tasks are spread to 2
LLCs:
9299 5435

Note: The race condition is not properly dealt with in this patch;
out-of-bounds access may be triggered during CPU hotplug due to
max_llcs changes. Do not query per-LLC occupancy during CPU hotplug.

Co-developed-by: Aaron Lu <ziqianlu@bytedance.com>
Signed-off-by: Aaron Lu <ziqianlu@bytedance.com>
Signed-off-by: Chen Yu <yu.c.chen@intel.com>
Signed-off-by: Tim Chen <tim.c.chen@linux.intel.com>
---

Notes:
v3->v4:
Minor fix.

fs/proc/base.c | 31 +++++++++++++++++++++++++
include/linux/mm_types.h | 17 +++++++++++---
include/linux/sched.h | 6 +++++
kernel/sched/fair.c | 50 ++++++++++++++++++++++++++++++++++++----
4 files changed, 97 insertions(+), 7 deletions(-)

diff --git a/fs/proc/base.c b/fs/proc/base.c
index 4c863d17dfb4..42629f33e0fa 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -518,6 +518,37 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
(unsigned long long)task->se.sum_exec_runtime,
(unsigned long long)task->sched_info.run_delay,
task->sched_info.pcount);
+#ifdef CONFIG_SCHED_CACHE
+ if (sched_cache_inuse()) {
+ struct mm_struct *mm = task->mm;
+ u64 *llc_runtime;
+ int mm_sched_llc;
+
+ if (!mm)
+ return 0;
+
+ llc_runtime = kcalloc(max_lid + 1, sizeof(u64), GFP_KERNEL);
+ if (!llc_runtime)
+ return 0;
+
+ if (get_mm_per_llc_runtime(task, llc_runtime))
+ goto out;
+
+ if (mm->sc_stat.cpu == -1)
+ mm_sched_llc = -1;
+ else
+ mm_sched_llc = llc_id(mm->sc_stat.cpu);
+
+ for (int i = 0; i <= max_lid; i++)
+ seq_printf(m, "%s%s%llu ",
+ i == task->preferred_llc ? "*" : "",
+ i == mm_sched_llc ? "?" : "",
+ llc_runtime[i]);
+ seq_puts(m, "\n");
+out:
+ kfree(llc_runtime);
+ }
+#endif

return 0;
}
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 67b2dfcc71ea..0eda55f29dd2 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1579,17 +1579,26 @@ static inline unsigned int mm_cid_size(void)

#ifdef CONFIG_SCHED_CACHE
void mm_init_sched(struct mm_struct *mm,
- struct sched_cache_time __percpu *pcpu_sched);
+ struct sched_cache_time __percpu *pcpu_sched,
+ struct sched_cache_time __percpu *pcpu_time);

static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
{
struct sched_cache_time __percpu *pcpu_sched =
- alloc_percpu_noprof(struct sched_cache_time);
+ alloc_percpu_noprof(struct sched_cache_time),
+ *pcpu_time;

if (!pcpu_sched)
return -ENOMEM;

- mm_init_sched(mm, pcpu_sched);
+ pcpu_time = alloc_percpu_noprof(struct sched_cache_time);
+ if (!pcpu_time) {
+ free_percpu(pcpu_sched);
+ return -ENOMEM;
+ }
+
+ mm_init_sched(mm, pcpu_sched, pcpu_time);
+
return 0;
}

@@ -1598,7 +1607,9 @@ static inline int mm_alloc_sched_noprof(struct mm_struct *mm)
static inline void mm_destroy_sched(struct mm_struct *mm)
{
free_percpu(mm->sc_stat.pcpu_sched);
+ free_percpu(mm->sc_stat.pcpu_time);
mm->sc_stat.pcpu_sched = NULL;
+ mm->sc_stat.pcpu_time = NULL;
}
#else /* !CONFIG_SCHED_CACHE */

diff --git a/include/linux/sched.h b/include/linux/sched.h
index dfa4bfd099c6..e24b2b86aba4 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2390,12 +2390,18 @@ struct sched_cache_time {

struct sched_cache_stat {
struct sched_cache_time __percpu *pcpu_sched;
+ struct sched_cache_time __percpu *pcpu_time;
raw_spinlock_t lock;
unsigned long epoch;
u64 nr_running_avg;
int cpu;
} ____cacheline_aligned_in_smp;

+int get_mm_per_llc_runtime(struct task_struct *p, u64 *buf);
+bool sched_cache_inuse(void);
+extern int max_lid;
+int llc_id(int cpu);
+
#else

struct sched_cache_stat { };
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index e4e22696a0b1..2b12918b00fd 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -1288,7 +1288,12 @@ __read_mostly unsigned int llc_epoch_affinity_timeout = EPOCH_LLC_AFFINITY_TIMEO
__read_mostly unsigned int llc_imb_pct = 20;
__read_mostly unsigned int llc_overaggr_pct = 50;

-static int llc_id(int cpu)
+bool sched_cache_inuse(void)
+{
+ return sched_cache_enabled();
+}
+
+int llc_id(int cpu)
{
if (cpu < 0)
return -1;
@@ -1448,18 +1453,21 @@ static void account_llc_dequeue(struct rq *rq, struct task_struct *p)
}

void mm_init_sched(struct mm_struct *mm,
- struct sched_cache_time __percpu *_pcpu_sched)
+ struct sched_cache_time __percpu *_pcpu_sched,
+ struct sched_cache_time __percpu *_pcpu_time)
{
unsigned long epoch = 0;
int i;

for_each_possible_cpu(i) {
struct sched_cache_time *pcpu_sched = per_cpu_ptr(_pcpu_sched, i);
+ struct sched_cache_time *pcpu_time = per_cpu_ptr(_pcpu_time, i);
struct rq *rq = cpu_rq(i);

pcpu_sched->runtime = 0;
/* a slightly stale cpu epoch is acceptible */
pcpu_sched->epoch = rq->cpu_epoch;
+ pcpu_time->runtime = 0;
epoch = rq->cpu_epoch;
}

@@ -1473,6 +1481,8 @@ void mm_init_sched(struct mm_struct *mm,
* the readers may get invalid mm_sched_epoch, etc.
*/
smp_store_release(&mm->sc_stat.pcpu_sched, _pcpu_sched);
+ /* barrier */
+ smp_store_release(&mm->sc_stat.pcpu_time, _pcpu_time);
}

/* because why would C be fully specified */
@@ -1558,7 +1568,8 @@ static unsigned int task_running_on_cpu(int cpu, struct task_struct *p);
static inline
void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
{
- struct sched_cache_time *pcpu_sched;
+ struct sched_cache_time *pcpu_sched,
+ *pcpu_time;
struct mm_struct *mm = p->mm;
int mm_sched_llc = -1;
unsigned long epoch;
@@ -1572,14 +1583,18 @@ void account_mm_sched(struct rq *rq, struct task_struct *p, s64 delta_exec)
* init_task, kthreads and user thread created
* by user_mode_thread() don't have mm.
*/
- if (!mm || !mm->sc_stat.pcpu_sched)
+ if (!mm || !mm->sc_stat.pcpu_sched ||
+ !mm->sc_stat.pcpu_time)
return;

pcpu_sched = per_cpu_ptr(p->mm->sc_stat.pcpu_sched, cpu_of(rq));
+ pcpu_time = per_cpu_ptr(p->mm->sc_stat.pcpu_time, cpu_of(rq));

scoped_guard (raw_spinlock, &rq->cpu_epoch_lock) {
__update_mm_sched(rq, pcpu_sched);
pcpu_sched->runtime += delta_exec;
+ /* pure runtime without decay */
+ pcpu_time->runtime += delta_exec;
rq->cpu_runtime += delta_exec;
epoch = rq->cpu_epoch;
}
@@ -10328,6 +10343,33 @@ static bool migrate_degrades_llc(struct task_struct *p, struct lb_env *env)
return true;
}

+/* p->pi_lock is hold */
+int get_mm_per_llc_runtime(struct task_struct *p, u64 *buf)
+{
+ struct sched_cache_time *pcpu_time;
+ struct mm_struct *mm = p->mm;
+ int cpu;
+
+ if (!mm)
+ return -EINVAL;
+
+ rcu_read_lock();
+ for_each_online_cpu(cpu) {
+ int llc = llc_id(cpu);
+ u64 runtime_ms;
+
+ if (llc < 0 || llc > max_lid)
+ continue;
+
+ pcpu_time = per_cpu_ptr(mm->sc_stat.pcpu_time, cpu);
+ runtime_ms = div_u64(pcpu_time->runtime, NSEC_PER_MSEC);
+ buf[llc] += runtime_ms;
+ }
+ rcu_read_unlock();
+
+ return 0;
+}
+
#else
static inline bool get_llc_stats(int cpu, unsigned long *util,
unsigned long *cap)
--
2.32.0