* [patch V5 01/20] sched/mmcid: Revert the complex CID management
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
@ 2025-11-19 17:26 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-19 17:26 ` [patch V5 02/20] sched/mmcid: Use proper data structures Thomas Gleixner
` (19 subsequent siblings)
20 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:26 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
The CID management is a complex beast, which affects both scheduling and
task migration. The compaction mechanism forces random tasks of a process
into task work on exit to user space causing latency spikes.
Revert back to the initial simple bitmap allocating mechanics, which are
known to have scalability issues as that allows to gradually build up a
replacement functionality in a reviewable way.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V4: Remove barrier comment - Mathieu
---
include/linux/mm_types.h | 53 ----
kernel/fork.c | 5
kernel/sched/core.c | 517 +----------------------------------------------
kernel/sched/sched.h | 289 +++-----------------------
4 files changed, 64 insertions(+), 800 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -922,13 +922,9 @@ struct vm_area_struct {
#define vma_policy(vma) NULL
#endif
-#ifdef CONFIG_SCHED_MM_CID
struct mm_cid {
- u64 time;
- int cid;
- int recent_cid;
+ unsigned int cid;
};
-#endif
/*
* Opaque type representing current mm_struct flag state. Must be accessed via
@@ -1000,12 +996,6 @@ struct mm_struct {
* runqueue locks.
*/
struct mm_cid __percpu *pcpu_cid;
- /*
- * @mm_cid_next_scan: Next mm_cid scan (in jiffies).
- *
- * When the next mm_cid scan is due (in jiffies).
- */
- unsigned long mm_cid_next_scan;
/**
* @nr_cpus_allowed: Number of CPUs allowed for mm.
*
@@ -1014,14 +1004,6 @@ struct mm_struct {
*/
unsigned int nr_cpus_allowed;
/**
- * @max_nr_cid: Maximum number of allowed concurrency
- * IDs allocated.
- *
- * Track the highest number of allowed concurrency IDs
- * allocated for the mm.
- */
- atomic_t max_nr_cid;
- /**
* @cpus_allowed_lock: Lock protecting mm cpus_allowed.
*
* Provide mutual exclusion for mm cpus_allowed and
@@ -1371,35 +1353,7 @@ static inline void vma_iter_init(struct
#ifdef CONFIG_SCHED_MM_CID
-enum mm_cid_state {
- MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */
- MM_CID_LAZY_PUT = (1U << 31),
-};
-
-static inline bool mm_cid_is_unset(int cid)
-{
- return cid == MM_CID_UNSET;
-}
-
-static inline bool mm_cid_is_lazy_put(int cid)
-{
- return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
-}
-
-static inline bool mm_cid_is_valid(int cid)
-{
- return !(cid & MM_CID_LAZY_PUT);
-}
-
-static inline int mm_cid_set_lazy_put(int cid)
-{
- return cid | MM_CID_LAZY_PUT;
-}
-
-static inline int mm_cid_clear_lazy_put(int cid)
-{
- return cid & ~MM_CID_LAZY_PUT;
-}
+#define MM_CID_UNSET (~0U)
/*
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
@@ -1432,11 +1386,8 @@ static inline void mm_init_cid(struct mm
struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
pcpu_cid->cid = MM_CID_UNSET;
- pcpu_cid->recent_cid = MM_CID_UNSET;
- pcpu_cid->time = 0;
}
mm->nr_cpus_allowed = p->nr_cpus_allowed;
- atomic_set(&mm->max_nr_cid, 0);
raw_spin_lock_init(&mm->cpus_allowed_lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
cpumask_clear(mm_cidmask(mm));
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -955,10 +955,9 @@ static struct task_struct *dup_task_stru
#endif
#ifdef CONFIG_SCHED_MM_CID
- tsk->mm_cid = -1;
- tsk->last_mm_cid = -1;
+ tsk->mm_cid = MM_CID_UNSET;
+ tsk->last_mm_cid = MM_CID_UNSET;
tsk->mm_cid_active = 0;
- tsk->migrate_from_cpu = -1;
#endif
return tsk;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2128,8 +2128,6 @@ void activate_task(struct rq *rq, struct
{
if (task_on_rq_migrating(p))
flags |= ENQUEUE_MIGRATED;
- if (flags & ENQUEUE_MIGRATED)
- sched_mm_cid_migrate_to(rq, p);
enqueue_task(rq, p, flags);
@@ -3329,7 +3327,6 @@ void set_task_cpu(struct task_struct *p,
if (p->sched_class->migrate_task_rq)
p->sched_class->migrate_task_rq(p, new_cpu);
p->se.nr_migrations++;
- sched_mm_cid_migrate_from(p);
perf_event_task_migrate(p);
}
@@ -5280,9 +5277,6 @@ context_switch(struct rq *rq, struct tas
*
* kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch
- *
- * switch_mm_cid() needs to be updated if the barriers provided
- * by context_switch() are modified.
*/
if (!next->mm) { // to kernel
enter_lazy_tlb(prev->active_mm, next);
@@ -5312,8 +5306,7 @@ context_switch(struct rq *rq, struct tas
}
}
- /* switch_mm_cid() requires the memory barriers above. */
- switch_mm_cid(rq, prev, next);
+ switch_mm_cid(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -5604,7 +5597,6 @@ void sched_tick(void)
resched_latency = cpu_resched_latency(rq);
calc_global_load_tick(rq);
sched_core_tick(rq);
- task_tick_mm_cid(rq, donor);
scx_tick(rq);
rq_unlock(rq, &rf);
@@ -10376,522 +10368,47 @@ void call_trace_sched_update_nr_running(
}
#ifdef CONFIG_SCHED_MM_CID
-
-/*
- * @cid_lock: Guarantee forward-progress of cid allocation.
- *
- * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock
- * is only used when contention is detected by the lock-free allocation so
- * forward progress can be guaranteed.
- */
-DEFINE_RAW_SPINLOCK(cid_lock);
-
-/*
- * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock.
- *
- * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is
- * detected, it is set to 1 to ensure that all newly coming allocations are
- * serialized by @cid_lock until the allocation which detected contention
- * completes and sets @use_cid_lock back to 0. This guarantees forward progress
- * of a cid allocation.
- */
-int use_cid_lock;
-
-/*
- * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid
- * concurrently with respect to the execution of the source runqueue context
- * switch.
- *
- * There is one basic properties we want to guarantee here:
- *
- * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively
- * used by a task. That would lead to concurrent allocation of the cid and
- * userspace corruption.
- *
- * Provide this guarantee by introducing a Dekker memory ordering to guarantee
- * that a pair of loads observe at least one of a pair of stores, which can be
- * shown as:
- *
- * X = Y = 0
- *
- * w[X]=1 w[Y]=1
- * MB MB
- * r[Y]=y r[X]=x
- *
- * Which guarantees that x==0 && y==0 is impossible. But rather than using
- * values 0 and 1, this algorithm cares about specific state transitions of the
- * runqueue current task (as updated by the scheduler context switch), and the
- * per-mm/cpu cid value.
- *
- * Let's introduce task (Y) which has task->mm == mm and task (N) which has
- * task->mm != mm for the rest of the discussion. There are two scheduler state
- * transitions on context switch we care about:
- *
- * (TSA) Store to rq->curr with transition from (N) to (Y)
- *
- * (TSB) Store to rq->curr with transition from (Y) to (N)
- *
- * On the remote-clear side, there is one transition we care about:
- *
- * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag
- *
- * There is also a transition to UNSET state which can be performed from all
- * sides (scheduler, remote-clear). It is always performed with a cmpxchg which
- * guarantees that only a single thread will succeed:
- *
- * (TMB) cmpxchg to *pcpu_cid to mark UNSET
- *
- * Just to be clear, what we do _not_ want to happen is a transition to UNSET
- * when a thread is actively using the cid (property (1)).
- *
- * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions.
- *
- * Scenario A) (TSA)+(TMA) (from next task perspective)
- *
- * CPU0 CPU1
- *
- * Context switch CS-1 Remote-clear
- * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA)
- * (implied barrier after cmpxchg)
- * - switch_mm_cid()
- * - memory barrier (see switch_mm_cid()
- * comment explaining how this barrier
- * is combined with other scheduler
- * barriers)
- * - mm_cid_get (next)
- * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr)
- *
- * This Dekker ensures that either task (Y) is observed by the
- * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are
- * observed.
- *
- * If task (Y) store is observed by rcu_dereference(), it means that there is
- * still an active task on the cpu. Remote-clear will therefore not transition
- * to UNSET, which fulfills property (1).
- *
- * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(),
- * it will move its state to UNSET, which clears the percpu cid perhaps
- * uselessly (which is not an issue for correctness). Because task (Y) is not
- * observed, CPU1 can move ahead to set the state to UNSET. Because moving
- * state to UNSET is done with a cmpxchg expecting that the old state has the
- * LAZY flag set, only one thread will successfully UNSET.
- *
- * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0
- * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and
- * CPU1 will observe task (Y) and do nothing more, which is fine.
- *
- * What we are effectively preventing with this Dekker is a scenario where
- * neither LAZY flag nor store (Y) are observed, which would fail property (1)
- * because this would UNSET a cid which is actively used.
- */
-
-void sched_mm_cid_migrate_from(struct task_struct *t)
-{
- t->migrate_from_cpu = task_cpu(t);
-}
-
-static
-int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq,
- struct task_struct *t,
- struct mm_cid *src_pcpu_cid)
-{
- struct mm_struct *mm = t->mm;
- struct task_struct *src_task;
- int src_cid, last_mm_cid;
-
- if (!mm)
- return -1;
-
- last_mm_cid = t->last_mm_cid;
- /*
- * If the migrated task has no last cid, or if the current
- * task on src rq uses the cid, it means the source cid does not need
- * to be moved to the destination cpu.
- */
- if (last_mm_cid == -1)
- return -1;
- src_cid = READ_ONCE(src_pcpu_cid->cid);
- if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid)
- return -1;
-
- /*
- * If we observe an active task using the mm on this rq, it means we
- * are not the last task to be migrated from this cpu for this mm, so
- * there is no need to move src_cid to the destination cpu.
- */
- guard(rcu)();
- src_task = rcu_dereference(src_rq->curr);
- if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
- t->last_mm_cid = -1;
- return -1;
- }
-
- return src_cid;
-}
-
-static
-int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq,
- struct task_struct *t,
- struct mm_cid *src_pcpu_cid,
- int src_cid)
-{
- struct task_struct *src_task;
- struct mm_struct *mm = t->mm;
- int lazy_cid;
-
- if (src_cid == -1)
- return -1;
-
- /*
- * Attempt to clear the source cpu cid to move it to the destination
- * cpu.
- */
- lazy_cid = mm_cid_set_lazy_put(src_cid);
- if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid))
- return -1;
-
- /*
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm matches the scheduler barrier in context_switch()
- * between store to rq->curr and load of prev and next task's
- * per-mm/cpu cid.
- *
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm_cid_active matches the barrier in
- * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
- * sched_mm_cid_after_execve() between store to t->mm_cid_active and
- * load of per-mm/cpu cid.
- */
-
- /*
- * If we observe an active task using the mm on this rq after setting
- * the lazy-put flag, this task will be responsible for transitioning
- * from lazy-put flag set to MM_CID_UNSET.
- */
- scoped_guard (rcu) {
- src_task = rcu_dereference(src_rq->curr);
- if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
- /*
- * We observed an active task for this mm, there is therefore
- * no point in moving this cid to the destination cpu.
- */
- t->last_mm_cid = -1;
- return -1;
- }
- }
-
- /*
- * The src_cid is unused, so it can be unset.
- */
- if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
- return -1;
- WRITE_ONCE(src_pcpu_cid->recent_cid, MM_CID_UNSET);
- return src_cid;
-}
-
/*
- * Migration to dst cpu. Called with dst_rq lock held.
- * Interrupts are disabled, which keeps the window of cid ownership without the
- * source rq lock held small.
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
*/
-void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t)
-{
- struct mm_cid *src_pcpu_cid, *dst_pcpu_cid;
- struct mm_struct *mm = t->mm;
- int src_cid, src_cpu;
- bool dst_cid_is_set;
- struct rq *src_rq;
-
- lockdep_assert_rq_held(dst_rq);
-
- if (!mm)
- return;
- src_cpu = t->migrate_from_cpu;
- if (src_cpu == -1) {
- t->last_mm_cid = -1;
- return;
- }
- /*
- * Move the src cid if the dst cid is unset. This keeps id
- * allocation closest to 0 in cases where few threads migrate around
- * many CPUs.
- *
- * If destination cid or recent cid is already set, we may have
- * to just clear the src cid to ensure compactness in frequent
- * migrations scenarios.
- *
- * It is not useful to clear the src cid when the number of threads is
- * greater or equal to the number of allowed CPUs, because user-space
- * can expect that the number of allowed cids can reach the number of
- * allowed CPUs.
- */
- dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq));
- dst_cid_is_set = !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->cid)) ||
- !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->recent_cid));
- if (dst_cid_is_set && atomic_read(&mm->mm_users) >= READ_ONCE(mm->nr_cpus_allowed))
- return;
- src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu);
- src_rq = cpu_rq(src_cpu);
- src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid);
- if (src_cid == -1)
- return;
- src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid,
- src_cid);
- if (src_cid == -1)
- return;
- if (dst_cid_is_set) {
- __mm_cid_put(mm, src_cid);
- return;
- }
- /* Move src_cid to dst cpu. */
- mm_cid_snapshot_time(dst_rq, mm);
- WRITE_ONCE(dst_pcpu_cid->cid, src_cid);
- WRITE_ONCE(dst_pcpu_cid->recent_cid, src_cid);
-}
-
-static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid,
- int cpu)
-{
- struct rq *rq = cpu_rq(cpu);
- struct task_struct *t;
- int cid, lazy_cid;
-
- cid = READ_ONCE(pcpu_cid->cid);
- if (!mm_cid_is_valid(cid))
- return;
-
- /*
- * Clear the cpu cid if it is set to keep cid allocation compact. If
- * there happens to be other tasks left on the source cpu using this
- * mm, the next task using this mm will reallocate its cid on context
- * switch.
- */
- lazy_cid = mm_cid_set_lazy_put(cid);
- if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid))
- return;
-
- /*
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm matches the scheduler barrier in context_switch()
- * between store to rq->curr and load of prev and next task's
- * per-mm/cpu cid.
- *
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm_cid_active matches the barrier in
- * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
- * sched_mm_cid_after_execve() between store to t->mm_cid_active and
- * load of per-mm/cpu cid.
- */
-
- /*
- * If we observe an active task using the mm on this rq after setting
- * the lazy-put flag, that task will be responsible for transitioning
- * from lazy-put flag set to MM_CID_UNSET.
- */
- scoped_guard (rcu) {
- t = rcu_dereference(rq->curr);
- if (READ_ONCE(t->mm_cid_active) && t->mm == mm)
- return;
- }
-
- /*
- * The cid is unused, so it can be unset.
- * Disable interrupts to keep the window of cid ownership without rq
- * lock small.
- */
- scoped_guard (irqsave) {
- if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
- __mm_cid_put(mm, cid);
- }
-}
-
-static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu)
-{
- struct rq *rq = cpu_rq(cpu);
- struct mm_cid *pcpu_cid;
- struct task_struct *curr;
- u64 rq_clock;
-
- /*
- * rq->clock load is racy on 32-bit but one spurious clear once in a
- * while is irrelevant.
- */
- rq_clock = READ_ONCE(rq->clock);
- pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
-
- /*
- * In order to take care of infrequently scheduled tasks, bump the time
- * snapshot associated with this cid if an active task using the mm is
- * observed on this rq.
- */
- scoped_guard (rcu) {
- curr = rcu_dereference(rq->curr);
- if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) {
- WRITE_ONCE(pcpu_cid->time, rq_clock);
- return;
- }
- }
-
- if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS)
- return;
- sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
-}
-
-static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu,
- int weight)
-{
- struct mm_cid *pcpu_cid;
- int cid;
-
- pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
- cid = READ_ONCE(pcpu_cid->cid);
- if (!mm_cid_is_valid(cid) || cid < weight)
- return;
- sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
-}
-
-static void task_mm_cid_work(struct callback_head *work)
-{
- unsigned long now = jiffies, old_scan, next_scan;
- struct task_struct *t = current;
- struct cpumask *cidmask;
- struct mm_struct *mm;
- int weight, cpu;
-
- WARN_ON_ONCE(t != container_of(work, struct task_struct, cid_work));
-
- work->next = work; /* Prevent double-add */
- if (t->flags & PF_EXITING)
- return;
- mm = t->mm;
- if (!mm)
- return;
- old_scan = READ_ONCE(mm->mm_cid_next_scan);
- next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY);
- if (!old_scan) {
- unsigned long res;
-
- res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan);
- if (res != old_scan)
- old_scan = res;
- else
- old_scan = next_scan;
- }
- if (time_before(now, old_scan))
- return;
- if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan))
- return;
- cidmask = mm_cidmask(mm);
- /* Clear cids that were not recently used. */
- for_each_possible_cpu(cpu)
- sched_mm_cid_remote_clear_old(mm, cpu);
- weight = cpumask_weight(cidmask);
- /*
- * Clear cids that are greater or equal to the cidmask weight to
- * recompact it.
- */
- for_each_possible_cpu(cpu)
- sched_mm_cid_remote_clear_weight(mm, cpu, weight);
-}
-
-void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- int mm_users = 0;
-
- if (mm) {
- mm_users = atomic_read(&mm->mm_users);
- if (mm_users == 1)
- mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY);
- }
- t->cid_work.next = &t->cid_work; /* Protect against double add */
- init_task_work(&t->cid_work, task_mm_cid_work);
-}
-
-void task_tick_mm_cid(struct rq *rq, struct task_struct *curr)
-{
- struct callback_head *work = &curr->cid_work;
- unsigned long now = jiffies;
-
- if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) ||
- work->next != work)
- return;
- if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan)))
- return;
-
- /* No page allocation under rq lock */
- task_work_add(curr, work, TWA_RESUME);
-}
-
void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct rq *rq;
- if (!mm)
+ if (!mm || !t->mm_cid_active)
return;
- preempt_disable();
- rq = this_rq();
- guard(rq_lock_irqsave)(rq);
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 0);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- mm_cid_put(mm);
- t->last_mm_cid = t->mm_cid = -1;
+ guard(preempt)();
+ t->mm_cid_active = 0;
+ if (t->mm_cid != MM_CID_UNSET) {
+ cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
+ t->mm_cid = MM_CID_UNSET;
+ }
}
+/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
- struct rq *rq;
-
- if (!mm)
- return;
-
- preempt_disable();
- rq = this_rq();
- guard(rq_lock_irqsave)(rq);
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 0);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- mm_cid_put(mm);
- t->last_mm_cid = t->mm_cid = -1;
+ sched_mm_cid_exit_signals(t);
}
+/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct rq *rq;
if (!mm)
return;
- preempt_disable();
- rq = this_rq();
- scoped_guard (rq_lock_irqsave, rq) {
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 1);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- t->last_mm_cid = t->mm_cid = mm_cid_get(rq, t, mm);
- }
+ guard(preempt)();
+ t->mm_cid_active = 1;
+ mm_cid_select(t);
}
void sched_mm_cid_fork(struct task_struct *t)
{
- WARN_ON_ONCE(!t->mm || t->mm_cid != -1);
+ WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
t->mm_cid_active = 1;
}
#endif /* CONFIG_SCHED_MM_CID */
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,286 +3540,83 @@ extern void sched_dynamic_update(int mod
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
-
-#define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */
-#define MM_CID_SCAN_DELAY 100 /* 100ms */
-
-extern raw_spinlock_t cid_lock;
-extern int use_cid_lock;
-
-extern void sched_mm_cid_migrate_from(struct task_struct *t);
-extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t);
-extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr);
-extern void init_sched_mm_cid(struct task_struct *t);
-
-static inline void __mm_cid_put(struct mm_struct *mm, int cid)
-{
- if (cid < 0)
- return;
- cpumask_clear_cpu(cid, mm_cidmask(mm));
-}
-
-/*
- * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to
- * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to
- * be held to transition to other states.
- *
- * State transitions synchronized with cmpxchg or try_cmpxchg need to be
- * consistent across CPUs, which prevents use of this_cpu_cmpxchg.
- */
-static inline void mm_cid_put_lazy(struct task_struct *t)
+static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid;
+ unsigned int max_cid;
- lockdep_assert_irqs_disabled();
- cid = __this_cpu_read(pcpu_cid->cid);
- if (!mm_cid_is_lazy_put(cid) ||
- !try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
+ if (!mm)
return;
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
-}
-
-static inline int mm_cid_pcpu_unset(struct mm_struct *mm)
-{
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, res;
- lockdep_assert_irqs_disabled();
- cid = __this_cpu_read(pcpu_cid->cid);
- for (;;) {
- if (mm_cid_is_unset(cid))
- return MM_CID_UNSET;
- /*
- * Attempt transition from valid or lazy-put to unset.
- */
- res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
- if (res == cid)
- break;
- cid = res;
- }
- return cid;
+ /* Preset last_mm_cid */
+ max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
+ t->last_mm_cid = max_cid - 1;
}
-static inline void mm_cid_put(struct mm_struct *mm)
+static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
{
- int cid;
+ struct mm_struct *mm = t->mm;
- lockdep_assert_irqs_disabled();
- cid = mm_cid_pcpu_unset(mm);
- if (cid == MM_CID_UNSET)
- return;
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
+ if (cid >= max_cids)
+ return false;
+ if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ return false;
+ t->mm_cid = t->last_mm_cid = cid;
+ __this_cpu_write(mm->pcpu_cid->cid, cid);
+ return true;
}
-static inline int __mm_cid_try_get(struct task_struct *t, struct mm_struct *mm)
+static inline bool mm_cid_get(struct task_struct *t)
{
- struct cpumask *cidmask = mm_cidmask(mm);
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, max_nr_cid, allowed_max_nr_cid;
+ struct mm_struct *mm = t->mm;
+ unsigned int max_cids;
- /*
- * After shrinking the number of threads or reducing the number
- * of allowed cpus, reduce the value of max_nr_cid so expansion
- * of cid allocation will preserve cache locality if the number
- * of threads or allowed cpus increase again.
- */
- max_nr_cid = atomic_read(&mm->max_nr_cid);
- while ((allowed_max_nr_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed),
- atomic_read(&mm->mm_users))),
- max_nr_cid > allowed_max_nr_cid) {
- /* atomic_try_cmpxchg loads previous mm->max_nr_cid into max_nr_cid. */
- if (atomic_try_cmpxchg(&mm->max_nr_cid, &max_nr_cid, allowed_max_nr_cid)) {
- max_nr_cid = allowed_max_nr_cid;
- break;
- }
- }
- /* Try to re-use recent cid. This improves cache locality. */
- cid = __this_cpu_read(pcpu_cid->recent_cid);
- if (!mm_cid_is_unset(cid) && cid < max_nr_cid &&
- !cpumask_test_and_set_cpu(cid, cidmask))
- return cid;
- /*
- * Expand cid allocation if the maximum number of concurrency
- * IDs allocated (max_nr_cid) is below the number cpus allowed
- * and number of threads. Expanding cid allocation as much as
- * possible improves cache locality.
- */
- cid = max_nr_cid;
- while (cid < READ_ONCE(mm->nr_cpus_allowed) && cid < atomic_read(&mm->mm_users)) {
- /* atomic_try_cmpxchg loads previous mm->max_nr_cid into cid. */
- if (!atomic_try_cmpxchg(&mm->max_nr_cid, &cid, cid + 1))
- continue;
- if (!cpumask_test_and_set_cpu(cid, cidmask))
- return cid;
- }
- /*
- * Find the first available concurrency id.
- * Retry finding first zero bit if the mask is temporarily
- * filled. This only happens during concurrent remote-clear
- * which owns a cid without holding a rq lock.
- */
- for (;;) {
- cid = cpumask_first_zero(cidmask);
- if (cid < READ_ONCE(mm->nr_cpus_allowed))
- break;
- cpu_relax();
- }
- if (cpumask_test_and_set_cpu(cid, cidmask))
- return -1;
+ max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
- return cid;
-}
+ /* Try to reuse the last CID of this task */
+ if (__mm_cid_get(t, t->last_mm_cid, max_cids))
+ return true;
-/*
- * Save a snapshot of the current runqueue time of this cpu
- * with the per-cpu cid value, allowing to estimate how recently it was used.
- */
-static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm)
-{
- struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq));
+ /* Try to reuse the last CID of this mm on this CPU */
+ if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
+ return true;
- lockdep_assert_rq_held(rq);
- WRITE_ONCE(pcpu_cid->time, rq->clock);
+ /* Try the first zero bit in the cidmask. */
+ return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
}
-static inline int __mm_cid_get(struct rq *rq, struct task_struct *t,
- struct mm_struct *mm)
+static inline void mm_cid_select(struct task_struct *t)
{
- int cid;
-
- /*
- * All allocations (even those using the cid_lock) are lock-free. If
- * use_cid_lock is set, hold the cid_lock to perform cid allocation to
- * guarantee forward progress.
- */
- if (!READ_ONCE(use_cid_lock)) {
- cid = __mm_cid_try_get(t, mm);
- if (cid >= 0)
- goto end;
- raw_spin_lock(&cid_lock);
- } else {
- raw_spin_lock(&cid_lock);
- cid = __mm_cid_try_get(t, mm);
- if (cid >= 0)
- goto unlock;
- }
-
- /*
- * cid concurrently allocated. Retry while forcing following
- * allocations to use the cid_lock to ensure forward progress.
- */
- WRITE_ONCE(use_cid_lock, 1);
- /*
- * Set use_cid_lock before allocation. Only care about program order
- * because this is only required for forward progress.
- */
- barrier();
- /*
- * Retry until it succeeds. It is guaranteed to eventually succeed once
- * all newcoming allocations observe the use_cid_lock flag set.
- */
- do {
- cid = __mm_cid_try_get(t, mm);
- cpu_relax();
- } while (cid < 0);
/*
- * Allocate before clearing use_cid_lock. Only care about
- * program order because this is for forward progress.
+ * mm_cid_get() can fail when the maximum CID, which is determined
+ * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
+ * That's a transient failure as there cannot be more tasks
+ * concurrently on a CPU (or about to be scheduled in) than that.
*/
- barrier();
- WRITE_ONCE(use_cid_lock, 0);
-unlock:
- raw_spin_unlock(&cid_lock);
-end:
- mm_cid_snapshot_time(rq, mm);
-
- return cid;
-}
-
-static inline int mm_cid_get(struct rq *rq, struct task_struct *t,
- struct mm_struct *mm)
-{
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid;
-
- lockdep_assert_rq_held(rq);
- cid = __this_cpu_read(pcpu_cid->cid);
- if (mm_cid_is_valid(cid)) {
- mm_cid_snapshot_time(rq, mm);
- return cid;
- }
- if (mm_cid_is_lazy_put(cid)) {
- if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
+ for (;;) {
+ if (mm_cid_get(t))
+ break;
}
- cid = __mm_cid_get(rq, t, mm);
- __this_cpu_write(pcpu_cid->cid, cid);
- __this_cpu_write(pcpu_cid->recent_cid, cid);
-
- return cid;
}
-static inline void switch_mm_cid(struct rq *rq,
- struct task_struct *prev,
- struct task_struct *next)
+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
{
- /*
- * Provide a memory barrier between rq->curr store and load of
- * {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition.
- *
- * Should be adapted if context_switch() is modified.
- */
- if (!next->mm) { // to kernel
- /*
- * user -> kernel transition does not guarantee a barrier, but
- * we can use the fact that it performs an atomic operation in
- * mmgrab().
- */
- if (prev->mm) // from user
- smp_mb__after_mmgrab();
- /*
- * kernel -> kernel transition does not change rq->curr->mm
- * state. It stays NULL.
- */
- } else { // to user
- /*
- * kernel -> user transition does not provide a barrier
- * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu].
- * Provide it here.
- */
- if (!prev->mm) { // from kernel
- smp_mb();
- } else { // from user
- /*
- * user->user transition relies on an implicit
- * memory barrier in switch_mm() when
- * current->mm changes. If the architecture
- * switch_mm() does not have an implicit memory
- * barrier, it is emitted here. If current->mm
- * is unchanged, no barrier is needed.
- */
- smp_mb__after_switch_mm();
- }
- }
if (prev->mm_cid_active) {
- mm_cid_snapshot_time(rq, prev->mm);
- mm_cid_put_lazy(prev);
- prev->mm_cid = -1;
+ if (prev->mm_cid != MM_CID_UNSET)
+ cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
+ prev->mm_cid = MM_CID_UNSET;
}
+
if (next->mm_cid_active) {
- next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next, next->mm);
+ mm_cid_select(next);
rseq_sched_set_task_mm_cid(next, next->mm_cid);
}
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { }
-static inline void sched_mm_cid_migrate_from(struct task_struct *t) { }
-static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { }
-static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { }
static inline void init_sched_mm_cid(struct task_struct *t) { }
+static inline void mm_cid_select(struct task_struct *t) { }
+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Revert the complex CID management
2025-11-19 17:26 ` [patch V5 01/20] sched/mmcid: Revert the complex " Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 77d7dc8bef482e987036bc204136bbda552d95cd
Gitweb: https://git.kernel.org/tip/77d7dc8bef482e987036bc204136bbda552d95cd
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:45 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:52 +01:00
sched/mmcid: Revert the complex CID management
The CID management is a complex beast, which affects both scheduling and
task migration. The compaction mechanism forces random tasks of a process
into task work on exit to user space causing latency spikes.
Revert back to the initial simple bitmap allocating mechanics, which are
known to have scalability issues as that allows to gradually build up a
replacement functionality in a reviewable way.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.068197830@linutronix.de
---
include/linux/mm_types.h | 53 +----
kernel/fork.c | 5 +-
kernel/sched/core.c | 517 +-------------------------------------
kernel/sched/sched.h | 289 +++------------------
4 files changed, 64 insertions(+), 800 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 90e5790..63b8c12 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -922,13 +922,9 @@ struct vm_area_struct {
#define vma_policy(vma) NULL
#endif
-#ifdef CONFIG_SCHED_MM_CID
struct mm_cid {
- u64 time;
- int cid;
- int recent_cid;
+ unsigned int cid;
};
-#endif
/*
* Opaque type representing current mm_struct flag state. Must be accessed via
@@ -1000,12 +996,6 @@ struct mm_struct {
* runqueue locks.
*/
struct mm_cid __percpu *pcpu_cid;
- /*
- * @mm_cid_next_scan: Next mm_cid scan (in jiffies).
- *
- * When the next mm_cid scan is due (in jiffies).
- */
- unsigned long mm_cid_next_scan;
/**
* @nr_cpus_allowed: Number of CPUs allowed for mm.
*
@@ -1014,14 +1004,6 @@ struct mm_struct {
*/
unsigned int nr_cpus_allowed;
/**
- * @max_nr_cid: Maximum number of allowed concurrency
- * IDs allocated.
- *
- * Track the highest number of allowed concurrency IDs
- * allocated for the mm.
- */
- atomic_t max_nr_cid;
- /**
* @cpus_allowed_lock: Lock protecting mm cpus_allowed.
*
* Provide mutual exclusion for mm cpus_allowed and
@@ -1371,35 +1353,7 @@ static inline void vma_iter_init(struct vma_iterator *vmi,
#ifdef CONFIG_SCHED_MM_CID
-enum mm_cid_state {
- MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */
- MM_CID_LAZY_PUT = (1U << 31),
-};
-
-static inline bool mm_cid_is_unset(int cid)
-{
- return cid == MM_CID_UNSET;
-}
-
-static inline bool mm_cid_is_lazy_put(int cid)
-{
- return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
-}
-
-static inline bool mm_cid_is_valid(int cid)
-{
- return !(cid & MM_CID_LAZY_PUT);
-}
-
-static inline int mm_cid_set_lazy_put(int cid)
-{
- return cid | MM_CID_LAZY_PUT;
-}
-
-static inline int mm_cid_clear_lazy_put(int cid)
-{
- return cid & ~MM_CID_LAZY_PUT;
-}
+#define MM_CID_UNSET (~0U)
/*
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
@@ -1432,11 +1386,8 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
pcpu_cid->cid = MM_CID_UNSET;
- pcpu_cid->recent_cid = MM_CID_UNSET;
- pcpu_cid->time = 0;
}
mm->nr_cpus_allowed = p->nr_cpus_allowed;
- atomic_set(&mm->max_nr_cid, 0);
raw_spin_lock_init(&mm->cpus_allowed_lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
cpumask_clear(mm_cidmask(mm));
diff --git a/kernel/fork.c b/kernel/fork.c
index 3da0f08..9d9afe4 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -955,10 +955,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#endif
#ifdef CONFIG_SCHED_MM_CID
- tsk->mm_cid = -1;
- tsk->last_mm_cid = -1;
+ tsk->mm_cid = MM_CID_UNSET;
+ tsk->last_mm_cid = MM_CID_UNSET;
tsk->mm_cid_active = 0;
- tsk->migrate_from_cpu = -1;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 579a8e9..11a1735 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2128,8 +2128,6 @@ void activate_task(struct rq *rq, struct task_struct *p, int flags)
{
if (task_on_rq_migrating(p))
flags |= ENQUEUE_MIGRATED;
- if (flags & ENQUEUE_MIGRATED)
- sched_mm_cid_migrate_to(rq, p);
enqueue_task(rq, p, flags);
@@ -3329,7 +3327,6 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu)
if (p->sched_class->migrate_task_rq)
p->sched_class->migrate_task_rq(p, new_cpu);
p->se.nr_migrations++;
- sched_mm_cid_migrate_from(p);
perf_event_task_migrate(p);
}
@@ -5280,9 +5277,6 @@ context_switch(struct rq *rq, struct task_struct *prev,
*
* kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch
- *
- * switch_mm_cid() needs to be updated if the barriers provided
- * by context_switch() are modified.
*/
if (!next->mm) { // to kernel
enter_lazy_tlb(prev->active_mm, next);
@@ -5312,8 +5306,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
}
}
- /* switch_mm_cid() requires the memory barriers above. */
- switch_mm_cid(rq, prev, next);
+ switch_mm_cid(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -5604,7 +5597,6 @@ void sched_tick(void)
resched_latency = cpu_resched_latency(rq);
calc_global_load_tick(rq);
sched_core_tick(rq);
- task_tick_mm_cid(rq, donor);
scx_tick(rq);
rq_unlock(rq, &rf);
@@ -10376,522 +10368,47 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
}
#ifdef CONFIG_SCHED_MM_CID
-
/*
- * @cid_lock: Guarantee forward-progress of cid allocation.
- *
- * Concurrency ID allocation within a bitmap is mostly lock-free. The cid_lock
- * is only used when contention is detected by the lock-free allocation so
- * forward progress can be guaranteed.
- */
-DEFINE_RAW_SPINLOCK(cid_lock);
-
-/*
- * @use_cid_lock: Select cid allocation behavior: lock-free vs spinlock.
- *
- * When @use_cid_lock is 0, the cid allocation is lock-free. When contention is
- * detected, it is set to 1 to ensure that all newly coming allocations are
- * serialized by @cid_lock until the allocation which detected contention
- * completes and sets @use_cid_lock back to 0. This guarantees forward progress
- * of a cid allocation.
- */
-int use_cid_lock;
-
-/*
- * mm_cid remote-clear implements a lock-free algorithm to clear per-mm/cpu cid
- * concurrently with respect to the execution of the source runqueue context
- * switch.
- *
- * There is one basic properties we want to guarantee here:
- *
- * (1) Remote-clear should _never_ mark a per-cpu cid UNSET when it is actively
- * used by a task. That would lead to concurrent allocation of the cid and
- * userspace corruption.
- *
- * Provide this guarantee by introducing a Dekker memory ordering to guarantee
- * that a pair of loads observe at least one of a pair of stores, which can be
- * shown as:
- *
- * X = Y = 0
- *
- * w[X]=1 w[Y]=1
- * MB MB
- * r[Y]=y r[X]=x
- *
- * Which guarantees that x==0 && y==0 is impossible. But rather than using
- * values 0 and 1, this algorithm cares about specific state transitions of the
- * runqueue current task (as updated by the scheduler context switch), and the
- * per-mm/cpu cid value.
- *
- * Let's introduce task (Y) which has task->mm == mm and task (N) which has
- * task->mm != mm for the rest of the discussion. There are two scheduler state
- * transitions on context switch we care about:
- *
- * (TSA) Store to rq->curr with transition from (N) to (Y)
- *
- * (TSB) Store to rq->curr with transition from (Y) to (N)
- *
- * On the remote-clear side, there is one transition we care about:
- *
- * (TMA) cmpxchg to *pcpu_cid to set the LAZY flag
- *
- * There is also a transition to UNSET state which can be performed from all
- * sides (scheduler, remote-clear). It is always performed with a cmpxchg which
- * guarantees that only a single thread will succeed:
- *
- * (TMB) cmpxchg to *pcpu_cid to mark UNSET
- *
- * Just to be clear, what we do _not_ want to happen is a transition to UNSET
- * when a thread is actively using the cid (property (1)).
- *
- * Let's looks at the relevant combinations of TSA/TSB, and TMA transitions.
- *
- * Scenario A) (TSA)+(TMA) (from next task perspective)
- *
- * CPU0 CPU1
- *
- * Context switch CS-1 Remote-clear
- * - store to rq->curr: (N)->(Y) (TSA) - cmpxchg to *pcpu_id to LAZY (TMA)
- * (implied barrier after cmpxchg)
- * - switch_mm_cid()
- * - memory barrier (see switch_mm_cid()
- * comment explaining how this barrier
- * is combined with other scheduler
- * barriers)
- * - mm_cid_get (next)
- * - READ_ONCE(*pcpu_cid) - rcu_dereference(src_rq->curr)
- *
- * This Dekker ensures that either task (Y) is observed by the
- * rcu_dereference() or the LAZY flag is observed by READ_ONCE(), or both are
- * observed.
- *
- * If task (Y) store is observed by rcu_dereference(), it means that there is
- * still an active task on the cpu. Remote-clear will therefore not transition
- * to UNSET, which fulfills property (1).
- *
- * If task (Y) is not observed, but the lazy flag is observed by READ_ONCE(),
- * it will move its state to UNSET, which clears the percpu cid perhaps
- * uselessly (which is not an issue for correctness). Because task (Y) is not
- * observed, CPU1 can move ahead to set the state to UNSET. Because moving
- * state to UNSET is done with a cmpxchg expecting that the old state has the
- * LAZY flag set, only one thread will successfully UNSET.
- *
- * If both states (LAZY flag and task (Y)) are observed, the thread on CPU0
- * will observe the LAZY flag and transition to UNSET (perhaps uselessly), and
- * CPU1 will observe task (Y) and do nothing more, which is fine.
- *
- * What we are effectively preventing with this Dekker is a scenario where
- * neither LAZY flag nor store (Y) are observed, which would fail property (1)
- * because this would UNSET a cid which is actively used.
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
*/
-
-void sched_mm_cid_migrate_from(struct task_struct *t)
-{
- t->migrate_from_cpu = task_cpu(t);
-}
-
-static
-int __sched_mm_cid_migrate_from_fetch_cid(struct rq *src_rq,
- struct task_struct *t,
- struct mm_cid *src_pcpu_cid)
-{
- struct mm_struct *mm = t->mm;
- struct task_struct *src_task;
- int src_cid, last_mm_cid;
-
- if (!mm)
- return -1;
-
- last_mm_cid = t->last_mm_cid;
- /*
- * If the migrated task has no last cid, or if the current
- * task on src rq uses the cid, it means the source cid does not need
- * to be moved to the destination cpu.
- */
- if (last_mm_cid == -1)
- return -1;
- src_cid = READ_ONCE(src_pcpu_cid->cid);
- if (!mm_cid_is_valid(src_cid) || last_mm_cid != src_cid)
- return -1;
-
- /*
- * If we observe an active task using the mm on this rq, it means we
- * are not the last task to be migrated from this cpu for this mm, so
- * there is no need to move src_cid to the destination cpu.
- */
- guard(rcu)();
- src_task = rcu_dereference(src_rq->curr);
- if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
- t->last_mm_cid = -1;
- return -1;
- }
-
- return src_cid;
-}
-
-static
-int __sched_mm_cid_migrate_from_try_steal_cid(struct rq *src_rq,
- struct task_struct *t,
- struct mm_cid *src_pcpu_cid,
- int src_cid)
-{
- struct task_struct *src_task;
- struct mm_struct *mm = t->mm;
- int lazy_cid;
-
- if (src_cid == -1)
- return -1;
-
- /*
- * Attempt to clear the source cpu cid to move it to the destination
- * cpu.
- */
- lazy_cid = mm_cid_set_lazy_put(src_cid);
- if (!try_cmpxchg(&src_pcpu_cid->cid, &src_cid, lazy_cid))
- return -1;
-
- /*
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm matches the scheduler barrier in context_switch()
- * between store to rq->curr and load of prev and next task's
- * per-mm/cpu cid.
- *
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm_cid_active matches the barrier in
- * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
- * sched_mm_cid_after_execve() between store to t->mm_cid_active and
- * load of per-mm/cpu cid.
- */
-
- /*
- * If we observe an active task using the mm on this rq after setting
- * the lazy-put flag, this task will be responsible for transitioning
- * from lazy-put flag set to MM_CID_UNSET.
- */
- scoped_guard (rcu) {
- src_task = rcu_dereference(src_rq->curr);
- if (READ_ONCE(src_task->mm_cid_active) && src_task->mm == mm) {
- /*
- * We observed an active task for this mm, there is therefore
- * no point in moving this cid to the destination cpu.
- */
- t->last_mm_cid = -1;
- return -1;
- }
- }
-
- /*
- * The src_cid is unused, so it can be unset.
- */
- if (!try_cmpxchg(&src_pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
- return -1;
- WRITE_ONCE(src_pcpu_cid->recent_cid, MM_CID_UNSET);
- return src_cid;
-}
-
-/*
- * Migration to dst cpu. Called with dst_rq lock held.
- * Interrupts are disabled, which keeps the window of cid ownership without the
- * source rq lock held small.
- */
-void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t)
-{
- struct mm_cid *src_pcpu_cid, *dst_pcpu_cid;
- struct mm_struct *mm = t->mm;
- int src_cid, src_cpu;
- bool dst_cid_is_set;
- struct rq *src_rq;
-
- lockdep_assert_rq_held(dst_rq);
-
- if (!mm)
- return;
- src_cpu = t->migrate_from_cpu;
- if (src_cpu == -1) {
- t->last_mm_cid = -1;
- return;
- }
- /*
- * Move the src cid if the dst cid is unset. This keeps id
- * allocation closest to 0 in cases where few threads migrate around
- * many CPUs.
- *
- * If destination cid or recent cid is already set, we may have
- * to just clear the src cid to ensure compactness in frequent
- * migrations scenarios.
- *
- * It is not useful to clear the src cid when the number of threads is
- * greater or equal to the number of allowed CPUs, because user-space
- * can expect that the number of allowed cids can reach the number of
- * allowed CPUs.
- */
- dst_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(dst_rq));
- dst_cid_is_set = !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->cid)) ||
- !mm_cid_is_unset(READ_ONCE(dst_pcpu_cid->recent_cid));
- if (dst_cid_is_set && atomic_read(&mm->mm_users) >= READ_ONCE(mm->nr_cpus_allowed))
- return;
- src_pcpu_cid = per_cpu_ptr(mm->pcpu_cid, src_cpu);
- src_rq = cpu_rq(src_cpu);
- src_cid = __sched_mm_cid_migrate_from_fetch_cid(src_rq, t, src_pcpu_cid);
- if (src_cid == -1)
- return;
- src_cid = __sched_mm_cid_migrate_from_try_steal_cid(src_rq, t, src_pcpu_cid,
- src_cid);
- if (src_cid == -1)
- return;
- if (dst_cid_is_set) {
- __mm_cid_put(mm, src_cid);
- return;
- }
- /* Move src_cid to dst cpu. */
- mm_cid_snapshot_time(dst_rq, mm);
- WRITE_ONCE(dst_pcpu_cid->cid, src_cid);
- WRITE_ONCE(dst_pcpu_cid->recent_cid, src_cid);
-}
-
-static void sched_mm_cid_remote_clear(struct mm_struct *mm, struct mm_cid *pcpu_cid,
- int cpu)
-{
- struct rq *rq = cpu_rq(cpu);
- struct task_struct *t;
- int cid, lazy_cid;
-
- cid = READ_ONCE(pcpu_cid->cid);
- if (!mm_cid_is_valid(cid))
- return;
-
- /*
- * Clear the cpu cid if it is set to keep cid allocation compact. If
- * there happens to be other tasks left on the source cpu using this
- * mm, the next task using this mm will reallocate its cid on context
- * switch.
- */
- lazy_cid = mm_cid_set_lazy_put(cid);
- if (!try_cmpxchg(&pcpu_cid->cid, &cid, lazy_cid))
- return;
-
- /*
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm matches the scheduler barrier in context_switch()
- * between store to rq->curr and load of prev and next task's
- * per-mm/cpu cid.
- *
- * The implicit barrier after cmpxchg per-mm/cpu cid before loading
- * rq->curr->mm_cid_active matches the barrier in
- * sched_mm_cid_exit_signals(), sched_mm_cid_before_execve(), and
- * sched_mm_cid_after_execve() between store to t->mm_cid_active and
- * load of per-mm/cpu cid.
- */
-
- /*
- * If we observe an active task using the mm on this rq after setting
- * the lazy-put flag, that task will be responsible for transitioning
- * from lazy-put flag set to MM_CID_UNSET.
- */
- scoped_guard (rcu) {
- t = rcu_dereference(rq->curr);
- if (READ_ONCE(t->mm_cid_active) && t->mm == mm)
- return;
- }
-
- /*
- * The cid is unused, so it can be unset.
- * Disable interrupts to keep the window of cid ownership without rq
- * lock small.
- */
- scoped_guard (irqsave) {
- if (try_cmpxchg(&pcpu_cid->cid, &lazy_cid, MM_CID_UNSET))
- __mm_cid_put(mm, cid);
- }
-}
-
-static void sched_mm_cid_remote_clear_old(struct mm_struct *mm, int cpu)
-{
- struct rq *rq = cpu_rq(cpu);
- struct mm_cid *pcpu_cid;
- struct task_struct *curr;
- u64 rq_clock;
-
- /*
- * rq->clock load is racy on 32-bit but one spurious clear once in a
- * while is irrelevant.
- */
- rq_clock = READ_ONCE(rq->clock);
- pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
-
- /*
- * In order to take care of infrequently scheduled tasks, bump the time
- * snapshot associated with this cid if an active task using the mm is
- * observed on this rq.
- */
- scoped_guard (rcu) {
- curr = rcu_dereference(rq->curr);
- if (READ_ONCE(curr->mm_cid_active) && curr->mm == mm) {
- WRITE_ONCE(pcpu_cid->time, rq_clock);
- return;
- }
- }
-
- if (rq_clock < pcpu_cid->time + SCHED_MM_CID_PERIOD_NS)
- return;
- sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
-}
-
-static void sched_mm_cid_remote_clear_weight(struct mm_struct *mm, int cpu,
- int weight)
-{
- struct mm_cid *pcpu_cid;
- int cid;
-
- pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu);
- cid = READ_ONCE(pcpu_cid->cid);
- if (!mm_cid_is_valid(cid) || cid < weight)
- return;
- sched_mm_cid_remote_clear(mm, pcpu_cid, cpu);
-}
-
-static void task_mm_cid_work(struct callback_head *work)
-{
- unsigned long now = jiffies, old_scan, next_scan;
- struct task_struct *t = current;
- struct cpumask *cidmask;
- struct mm_struct *mm;
- int weight, cpu;
-
- WARN_ON_ONCE(t != container_of(work, struct task_struct, cid_work));
-
- work->next = work; /* Prevent double-add */
- if (t->flags & PF_EXITING)
- return;
- mm = t->mm;
- if (!mm)
- return;
- old_scan = READ_ONCE(mm->mm_cid_next_scan);
- next_scan = now + msecs_to_jiffies(MM_CID_SCAN_DELAY);
- if (!old_scan) {
- unsigned long res;
-
- res = cmpxchg(&mm->mm_cid_next_scan, old_scan, next_scan);
- if (res != old_scan)
- old_scan = res;
- else
- old_scan = next_scan;
- }
- if (time_before(now, old_scan))
- return;
- if (!try_cmpxchg(&mm->mm_cid_next_scan, &old_scan, next_scan))
- return;
- cidmask = mm_cidmask(mm);
- /* Clear cids that were not recently used. */
- for_each_possible_cpu(cpu)
- sched_mm_cid_remote_clear_old(mm, cpu);
- weight = cpumask_weight(cidmask);
- /*
- * Clear cids that are greater or equal to the cidmask weight to
- * recompact it.
- */
- for_each_possible_cpu(cpu)
- sched_mm_cid_remote_clear_weight(mm, cpu, weight);
-}
-
-void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- int mm_users = 0;
-
- if (mm) {
- mm_users = atomic_read(&mm->mm_users);
- if (mm_users == 1)
- mm->mm_cid_next_scan = jiffies + msecs_to_jiffies(MM_CID_SCAN_DELAY);
- }
- t->cid_work.next = &t->cid_work; /* Protect against double add */
- init_task_work(&t->cid_work, task_mm_cid_work);
-}
-
-void task_tick_mm_cid(struct rq *rq, struct task_struct *curr)
-{
- struct callback_head *work = &curr->cid_work;
- unsigned long now = jiffies;
-
- if (!curr->mm || (curr->flags & (PF_EXITING | PF_KTHREAD)) ||
- work->next != work)
- return;
- if (time_before(now, READ_ONCE(curr->mm->mm_cid_next_scan)))
- return;
-
- /* No page allocation under rq lock */
- task_work_add(curr, work, TWA_RESUME);
-}
-
void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct rq *rq;
- if (!mm)
+ if (!mm || !t->mm_cid_active)
return;
- preempt_disable();
- rq = this_rq();
- guard(rq_lock_irqsave)(rq);
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 0);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- mm_cid_put(mm);
- t->last_mm_cid = t->mm_cid = -1;
+ guard(preempt)();
+ t->mm_cid_active = 0;
+ if (t->mm_cid != MM_CID_UNSET) {
+ cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
+ t->mm_cid = MM_CID_UNSET;
+ }
}
+/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
- struct rq *rq;
-
- if (!mm)
- return;
-
- preempt_disable();
- rq = this_rq();
- guard(rq_lock_irqsave)(rq);
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 0);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- mm_cid_put(mm);
- t->last_mm_cid = t->mm_cid = -1;
+ sched_mm_cid_exit_signals(t);
}
+/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct rq *rq;
if (!mm)
return;
- preempt_disable();
- rq = this_rq();
- scoped_guard (rq_lock_irqsave, rq) {
- preempt_enable_no_resched(); /* holding spinlock */
- WRITE_ONCE(t->mm_cid_active, 1);
- /*
- * Store t->mm_cid_active before loading per-mm/cpu cid.
- * Matches barrier in sched_mm_cid_remote_clear_old().
- */
- smp_mb();
- t->last_mm_cid = t->mm_cid = mm_cid_get(rq, t, mm);
- }
+ guard(preempt)();
+ t->mm_cid_active = 1;
+ mm_cid_select(t);
}
void sched_mm_cid_fork(struct task_struct *t)
{
- WARN_ON_ONCE(!t->mm || t->mm_cid != -1);
+ WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
t->mm_cid_active = 1;
}
#endif /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4838dda..bf227c2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,286 +3540,83 @@ extern void sched_dynamic_update(int mode);
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
-
-#define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */
-#define MM_CID_SCAN_DELAY 100 /* 100ms */
-
-extern raw_spinlock_t cid_lock;
-extern int use_cid_lock;
-
-extern void sched_mm_cid_migrate_from(struct task_struct *t);
-extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t);
-extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr);
-extern void init_sched_mm_cid(struct task_struct *t);
-
-static inline void __mm_cid_put(struct mm_struct *mm, int cid)
-{
- if (cid < 0)
- return;
- cpumask_clear_cpu(cid, mm_cidmask(mm));
-}
-
-/*
- * The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to
- * the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to
- * be held to transition to other states.
- *
- * State transitions synchronized with cmpxchg or try_cmpxchg need to be
- * consistent across CPUs, which prevents use of this_cpu_cmpxchg.
- */
-static inline void mm_cid_put_lazy(struct task_struct *t)
+static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid;
+ unsigned int max_cid;
- lockdep_assert_irqs_disabled();
- cid = __this_cpu_read(pcpu_cid->cid);
- if (!mm_cid_is_lazy_put(cid) ||
- !try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
+ if (!mm)
return;
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
-}
-static inline int mm_cid_pcpu_unset(struct mm_struct *mm)
-{
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, res;
-
- lockdep_assert_irqs_disabled();
- cid = __this_cpu_read(pcpu_cid->cid);
- for (;;) {
- if (mm_cid_is_unset(cid))
- return MM_CID_UNSET;
- /*
- * Attempt transition from valid or lazy-put to unset.
- */
- res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
- if (res == cid)
- break;
- cid = res;
- }
- return cid;
+ /* Preset last_mm_cid */
+ max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
+ t->last_mm_cid = max_cid - 1;
}
-static inline void mm_cid_put(struct mm_struct *mm)
+static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
{
- int cid;
+ struct mm_struct *mm = t->mm;
- lockdep_assert_irqs_disabled();
- cid = mm_cid_pcpu_unset(mm);
- if (cid == MM_CID_UNSET)
- return;
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
+ if (cid >= max_cids)
+ return false;
+ if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ return false;
+ t->mm_cid = t->last_mm_cid = cid;
+ __this_cpu_write(mm->pcpu_cid->cid, cid);
+ return true;
}
-static inline int __mm_cid_try_get(struct task_struct *t, struct mm_struct *mm)
+static inline bool mm_cid_get(struct task_struct *t)
{
- struct cpumask *cidmask = mm_cidmask(mm);
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid, max_nr_cid, allowed_max_nr_cid;
+ struct mm_struct *mm = t->mm;
+ unsigned int max_cids;
- /*
- * After shrinking the number of threads or reducing the number
- * of allowed cpus, reduce the value of max_nr_cid so expansion
- * of cid allocation will preserve cache locality if the number
- * of threads or allowed cpus increase again.
- */
- max_nr_cid = atomic_read(&mm->max_nr_cid);
- while ((allowed_max_nr_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed),
- atomic_read(&mm->mm_users))),
- max_nr_cid > allowed_max_nr_cid) {
- /* atomic_try_cmpxchg loads previous mm->max_nr_cid into max_nr_cid. */
- if (atomic_try_cmpxchg(&mm->max_nr_cid, &max_nr_cid, allowed_max_nr_cid)) {
- max_nr_cid = allowed_max_nr_cid;
- break;
- }
- }
- /* Try to re-use recent cid. This improves cache locality. */
- cid = __this_cpu_read(pcpu_cid->recent_cid);
- if (!mm_cid_is_unset(cid) && cid < max_nr_cid &&
- !cpumask_test_and_set_cpu(cid, cidmask))
- return cid;
- /*
- * Expand cid allocation if the maximum number of concurrency
- * IDs allocated (max_nr_cid) is below the number cpus allowed
- * and number of threads. Expanding cid allocation as much as
- * possible improves cache locality.
- */
- cid = max_nr_cid;
- while (cid < READ_ONCE(mm->nr_cpus_allowed) && cid < atomic_read(&mm->mm_users)) {
- /* atomic_try_cmpxchg loads previous mm->max_nr_cid into cid. */
- if (!atomic_try_cmpxchg(&mm->max_nr_cid, &cid, cid + 1))
- continue;
- if (!cpumask_test_and_set_cpu(cid, cidmask))
- return cid;
- }
- /*
- * Find the first available concurrency id.
- * Retry finding first zero bit if the mask is temporarily
- * filled. This only happens during concurrent remote-clear
- * which owns a cid without holding a rq lock.
- */
- for (;;) {
- cid = cpumask_first_zero(cidmask);
- if (cid < READ_ONCE(mm->nr_cpus_allowed))
- break;
- cpu_relax();
- }
- if (cpumask_test_and_set_cpu(cid, cidmask))
- return -1;
+ max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
- return cid;
-}
+ /* Try to reuse the last CID of this task */
+ if (__mm_cid_get(t, t->last_mm_cid, max_cids))
+ return true;
-/*
- * Save a snapshot of the current runqueue time of this cpu
- * with the per-cpu cid value, allowing to estimate how recently it was used.
- */
-static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm)
-{
- struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq));
+ /* Try to reuse the last CID of this mm on this CPU */
+ if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
+ return true;
- lockdep_assert_rq_held(rq);
- WRITE_ONCE(pcpu_cid->time, rq->clock);
+ /* Try the first zero bit in the cidmask. */
+ return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
}
-static inline int __mm_cid_get(struct rq *rq, struct task_struct *t,
- struct mm_struct *mm)
+static inline void mm_cid_select(struct task_struct *t)
{
- int cid;
-
/*
- * All allocations (even those using the cid_lock) are lock-free. If
- * use_cid_lock is set, hold the cid_lock to perform cid allocation to
- * guarantee forward progress.
+ * mm_cid_get() can fail when the maximum CID, which is determined
+ * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
+ * That's a transient failure as there cannot be more tasks
+ * concurrently on a CPU (or about to be scheduled in) than that.
*/
- if (!READ_ONCE(use_cid_lock)) {
- cid = __mm_cid_try_get(t, mm);
- if (cid >= 0)
- goto end;
- raw_spin_lock(&cid_lock);
- } else {
- raw_spin_lock(&cid_lock);
- cid = __mm_cid_try_get(t, mm);
- if (cid >= 0)
- goto unlock;
- }
-
- /*
- * cid concurrently allocated. Retry while forcing following
- * allocations to use the cid_lock to ensure forward progress.
- */
- WRITE_ONCE(use_cid_lock, 1);
- /*
- * Set use_cid_lock before allocation. Only care about program order
- * because this is only required for forward progress.
- */
- barrier();
- /*
- * Retry until it succeeds. It is guaranteed to eventually succeed once
- * all newcoming allocations observe the use_cid_lock flag set.
- */
- do {
- cid = __mm_cid_try_get(t, mm);
- cpu_relax();
- } while (cid < 0);
- /*
- * Allocate before clearing use_cid_lock. Only care about
- * program order because this is for forward progress.
- */
- barrier();
- WRITE_ONCE(use_cid_lock, 0);
-unlock:
- raw_spin_unlock(&cid_lock);
-end:
- mm_cid_snapshot_time(rq, mm);
-
- return cid;
-}
-
-static inline int mm_cid_get(struct rq *rq, struct task_struct *t,
- struct mm_struct *mm)
-{
- struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
- int cid;
-
- lockdep_assert_rq_held(rq);
- cid = __this_cpu_read(pcpu_cid->cid);
- if (mm_cid_is_valid(cid)) {
- mm_cid_snapshot_time(rq, mm);
- return cid;
- }
- if (mm_cid_is_lazy_put(cid)) {
- if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
- __mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
+ for (;;) {
+ if (mm_cid_get(t))
+ break;
}
- cid = __mm_cid_get(rq, t, mm);
- __this_cpu_write(pcpu_cid->cid, cid);
- __this_cpu_write(pcpu_cid->recent_cid, cid);
-
- return cid;
}
-static inline void switch_mm_cid(struct rq *rq,
- struct task_struct *prev,
- struct task_struct *next)
+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
{
- /*
- * Provide a memory barrier between rq->curr store and load of
- * {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition.
- *
- * Should be adapted if context_switch() is modified.
- */
- if (!next->mm) { // to kernel
- /*
- * user -> kernel transition does not guarantee a barrier, but
- * we can use the fact that it performs an atomic operation in
- * mmgrab().
- */
- if (prev->mm) // from user
- smp_mb__after_mmgrab();
- /*
- * kernel -> kernel transition does not change rq->curr->mm
- * state. It stays NULL.
- */
- } else { // to user
- /*
- * kernel -> user transition does not provide a barrier
- * between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu].
- * Provide it here.
- */
- if (!prev->mm) { // from kernel
- smp_mb();
- } else { // from user
- /*
- * user->user transition relies on an implicit
- * memory barrier in switch_mm() when
- * current->mm changes. If the architecture
- * switch_mm() does not have an implicit memory
- * barrier, it is emitted here. If current->mm
- * is unchanged, no barrier is needed.
- */
- smp_mb__after_switch_mm();
- }
- }
if (prev->mm_cid_active) {
- mm_cid_snapshot_time(rq, prev->mm);
- mm_cid_put_lazy(prev);
- prev->mm_cid = -1;
+ if (prev->mm_cid != MM_CID_UNSET)
+ cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
+ prev->mm_cid = MM_CID_UNSET;
}
+
if (next->mm_cid_active) {
- next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next, next->mm);
+ mm_cid_select(next);
rseq_sched_set_task_mm_cid(next, next->mm_cid);
}
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { }
-static inline void sched_mm_cid_migrate_from(struct task_struct *t) { }
-static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { }
-static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { }
static inline void init_sched_mm_cid(struct task_struct *t) { }
+static inline void mm_cid_select(struct task_struct *t) { }
+static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 02/20] sched/mmcid: Use proper data structures
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
2025-11-19 17:26 ` [patch V5 01/20] sched/mmcid: Revert the complex " Thomas Gleixner
@ 2025-11-19 17:26 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-19 17:26 ` [patch V5 03/20] sched/mmcid: Cacheline align MM CID storage Thomas Gleixner
` (18 subsequent siblings)
20 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:26 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Having a lot of CID functionality specific members in struct task_struct
and struct mm_struct is not really making the code easier to read.
Encapsulate the CID specific parts in data structures and keep them
separate from the stuff they are embedded in.
No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/mm_types.h | 56 +++++++++++----------------------------------
include/linux/rseq_types.h | 42 +++++++++++++++++++++++++++++++++
include/linux/sched.h | 11 +-------
init/init_task.c | 3 ++
kernel/fork.c | 6 ++--
kernel/sched/core.c | 16 ++++++------
kernel/sched/sched.h | 26 ++++++++++----------
7 files changed, 85 insertions(+), 75 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -20,6 +20,7 @@
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
#include <linux/types.h>
+#include <linux/rseq_types.h>
#include <linux/bitmap.h>
#include <asm/mmu.h>
@@ -922,10 +923,6 @@ struct vm_area_struct {
#define vma_policy(vma) NULL
#endif
-struct mm_cid {
- unsigned int cid;
-};
-
/*
* Opaque type representing current mm_struct flag state. Must be accessed via
* mm_flags_xxx() helper functions.
@@ -987,30 +984,9 @@ struct mm_struct {
*/
atomic_t mm_users;
-#ifdef CONFIG_SCHED_MM_CID
- /**
- * @pcpu_cid: Per-cpu current cid.
- *
- * Keep track of the currently allocated mm_cid for each cpu.
- * The per-cpu mm_cid values are serialized by their respective
- * runqueue locks.
- */
- struct mm_cid __percpu *pcpu_cid;
- /**
- * @nr_cpus_allowed: Number of CPUs allowed for mm.
- *
- * Number of CPUs allowed in the union of all mm's
- * threads allowed CPUs.
- */
- unsigned int nr_cpus_allowed;
- /**
- * @cpus_allowed_lock: Lock protecting mm cpus_allowed.
- *
- * Provide mutual exclusion for mm cpus_allowed and
- * mm nr_cpus_allowed updates.
- */
- raw_spinlock_t cpus_allowed_lock;
-#endif
+ /* MM CID related storage */
+ struct mm_mm_cid mm_cid;
+
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* size of all page tables */
#endif
@@ -1352,9 +1328,6 @@ static inline void vma_iter_init(struct
}
#ifdef CONFIG_SCHED_MM_CID
-
-#define MM_CID_UNSET (~0U)
-
/*
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
*/
@@ -1383,20 +1356,20 @@ static inline void mm_init_cid(struct mm
int i;
for_each_possible_cpu(i) {
- struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
+ struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
- pcpu_cid->cid = MM_CID_UNSET;
+ pcpu->cid = MM_CID_UNSET;
}
- mm->nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->cpus_allowed_lock);
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
cpumask_clear(mm_cidmask(mm));
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
- mm->pcpu_cid = alloc_percpu_noprof(struct mm_cid);
- if (!mm->pcpu_cid)
+ mm->mm_cid.pcpu = alloc_percpu_noprof(struct mm_cid_pcpu);
+ if (!mm->mm_cid.pcpu)
return -ENOMEM;
mm_init_cid(mm, p);
return 0;
@@ -1405,8 +1378,8 @@ static inline int mm_alloc_cid_noprof(st
static inline void mm_destroy_cid(struct mm_struct *mm)
{
- free_percpu(mm->pcpu_cid);
- mm->pcpu_cid = NULL;
+ free_percpu(mm->mm_cid.pcpu);
+ mm->mm_cid.pcpu = NULL;
}
static inline unsigned int mm_cid_size(void)
@@ -1421,10 +1394,9 @@ static inline void mm_set_cpus_allowed(s
if (!mm)
return;
/* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
- raw_spin_lock(&mm->cpus_allowed_lock);
+ guard(raw_spinlock)(&mm->mm_cid.lock);
cpumask_or(mm_allowed, mm_allowed, cpumask);
- WRITE_ONCE(mm->nr_cpus_allowed, cpumask_weight(mm_allowed));
- raw_spin_unlock(&mm->cpus_allowed_lock);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -90,4 +90,46 @@ struct rseq_data {
struct rseq_data { };
#endif /* !CONFIG_RSEQ */
+#ifdef CONFIG_SCHED_MM_CID
+
+#define MM_CID_UNSET (~0U)
+
+/**
+ * struct sched_mm_cid - Storage for per task MM CID data
+ * @active: MM CID is active for the task
+ * @cid: The CID associated to the task
+ * @last_cid: The last CID associated to the task
+ */
+struct sched_mm_cid {
+ unsigned int active;
+ unsigned int cid;
+ unsigned int last_cid;
+};
+
+/**
+ * struct mm_cid_pcpu - Storage for per CPU MM_CID data
+ * @cid: The CID associated to the CPU
+ */
+struct mm_cid_pcpu {
+ unsigned int cid;
+};
+
+/**
+ * struct mm_mm_cid - Storage for per MM CID data
+ * @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
+ * is growth only.
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ */
+struct mm_mm_cid {
+ struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int nr_cpus_allowed;
+ raw_spinlock_t lock;
+};
+#else /* CONFIG_SCHED_MM_CID */
+struct mm_mm_cid { };
+struct sched_mm_cid { };
+#endif /* !CONFIG_SCHED_MM_CID */
+
#endif
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1407,14 +1407,7 @@ struct task_struct {
#endif /* CONFIG_NUMA_BALANCING */
struct rseq_data rseq;
-
-#ifdef CONFIG_SCHED_MM_CID
- int mm_cid; /* Current cid in mm */
- int last_mm_cid; /* Most recent cid in mm */
- int migrate_from_cpu;
- int mm_cid_active; /* Whether cid bitmap is active */
- struct callback_head cid_work;
-#endif
+ struct sched_mm_cid mm_cid;
struct tlbflush_unmap_batch tlb_ubc;
@@ -2308,7 +2301,7 @@ void sched_mm_cid_fork(struct task_struc
void sched_mm_cid_exit_signals(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid;
+ return t->mm_cid.cid;
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -223,6 +223,9 @@ struct task_struct init_task __aligned(L
#ifdef CONFIG_SECCOMP_FILTER
.seccomp = { .filter_count = ATOMIC_INIT(0) },
#endif
+#ifdef CONFIG_SCHED_MM_CID
+ .mm_cid = { .cid = MM_CID_UNSET, },
+#endif
};
EXPORT_SYMBOL(init_task);
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -955,9 +955,9 @@ static struct task_struct *dup_task_stru
#endif
#ifdef CONFIG_SCHED_MM_CID
- tsk->mm_cid = MM_CID_UNSET;
- tsk->last_mm_cid = MM_CID_UNSET;
- tsk->mm_cid_active = 0;
+ tsk->mm_cid.cid = MM_CID_UNSET;
+ tsk->mm_cid.last_cid = MM_CID_UNSET;
+ tsk->mm_cid.active = 0;
#endif
return tsk;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10376,14 +10376,14 @@ void sched_mm_cid_exit_signals(struct ta
{
struct mm_struct *mm = t->mm;
- if (!mm || !t->mm_cid_active)
+ if (!mm || !t->mm_cid.active)
return;
guard(preempt)();
- t->mm_cid_active = 0;
- if (t->mm_cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
- t->mm_cid = MM_CID_UNSET;
+ t->mm_cid.active = 0;
+ if (t->mm_cid.cid != MM_CID_UNSET) {
+ cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ t->mm_cid.cid = MM_CID_UNSET;
}
}
@@ -10402,14 +10402,14 @@ void sched_mm_cid_after_execve(struct ta
return;
guard(preempt)();
- t->mm_cid_active = 1;
+ t->mm_cid.active = 1;
mm_cid_select(t);
}
void sched_mm_cid_fork(struct task_struct *t)
{
- WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
- t->mm_cid_active = 1;
+ WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
+ t->mm_cid.active = 1;
}
#endif /* CONFIG_SCHED_MM_CID */
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3549,8 +3549,8 @@ static inline void init_sched_mm_cid(str
return;
/* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->last_mm_cid = max_cid - 1;
+ max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ t->mm_cid.last_cid = max_cid - 1;
}
static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
@@ -3561,8 +3561,8 @@ static inline bool __mm_cid_get(struct t
return false;
if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
return false;
- t->mm_cid = t->last_mm_cid = cid;
- __this_cpu_write(mm->pcpu_cid->cid, cid);
+ t->mm_cid.cid = t->mm_cid.last_cid = cid;
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
return true;
}
@@ -3571,14 +3571,14 @@ static inline bool mm_cid_get(struct tas
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
/* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->last_mm_cid, max_cids))
+ if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
return true;
/* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
+ if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
return true;
/* Try the first zero bit in the cidmask. */
@@ -3601,15 +3601,15 @@ static inline void mm_cid_select(struct
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
{
- if (prev->mm_cid_active) {
- if (prev->mm_cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
- prev->mm_cid = MM_CID_UNSET;
+ if (prev->mm_cid.active) {
+ if (prev->mm_cid.cid != MM_CID_UNSET)
+ cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ prev->mm_cid.cid = MM_CID_UNSET;
}
- if (next->mm_cid_active) {
+ if (next->mm_cid.active) {
mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid);
+ rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
}
}
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Use proper data structures
2025-11-19 17:26 ` [patch V5 02/20] sched/mmcid: Use proper data structures Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 8cea569ca785060b8c5cc7800713ddc3b1548a94
Gitweb: https://git.kernel.org/tip/8cea569ca785060b8c5cc7800713ddc3b1548a94
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:47 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:52 +01:00
sched/mmcid: Use proper data structures
Having a lot of CID functionality specific members in struct task_struct
and struct mm_struct is not really making the code easier to read.
Encapsulate the CID specific parts in data structures and keep them
separate from the stuff they are embedded in.
No functional change.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.131573768@linutronix.de
---
include/linux/mm_types.h | 56 +++++++++----------------------------
include/linux/rseq_types.h | 42 ++++++++++++++++++++++++++++-
include/linux/sched.h | 11 +------
init/init_task.c | 3 ++-
kernel/fork.c | 6 ++--
kernel/sched/core.c | 16 +++++------
kernel/sched/sched.h | 26 ++++++++---------
7 files changed, 85 insertions(+), 75 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 63b8c12..e4818e9 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -20,6 +20,7 @@
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
#include <linux/types.h>
+#include <linux/rseq_types.h>
#include <linux/bitmap.h>
#include <asm/mmu.h>
@@ -922,10 +923,6 @@ struct vm_area_struct {
#define vma_policy(vma) NULL
#endif
-struct mm_cid {
- unsigned int cid;
-};
-
/*
* Opaque type representing current mm_struct flag state. Must be accessed via
* mm_flags_xxx() helper functions.
@@ -987,30 +984,9 @@ struct mm_struct {
*/
atomic_t mm_users;
-#ifdef CONFIG_SCHED_MM_CID
- /**
- * @pcpu_cid: Per-cpu current cid.
- *
- * Keep track of the currently allocated mm_cid for each cpu.
- * The per-cpu mm_cid values are serialized by their respective
- * runqueue locks.
- */
- struct mm_cid __percpu *pcpu_cid;
- /**
- * @nr_cpus_allowed: Number of CPUs allowed for mm.
- *
- * Number of CPUs allowed in the union of all mm's
- * threads allowed CPUs.
- */
- unsigned int nr_cpus_allowed;
- /**
- * @cpus_allowed_lock: Lock protecting mm cpus_allowed.
- *
- * Provide mutual exclusion for mm cpus_allowed and
- * mm nr_cpus_allowed updates.
- */
- raw_spinlock_t cpus_allowed_lock;
-#endif
+ /* MM CID related storage */
+ struct mm_mm_cid mm_cid;
+
#ifdef CONFIG_MMU
atomic_long_t pgtables_bytes; /* size of all page tables */
#endif
@@ -1352,9 +1328,6 @@ static inline void vma_iter_init(struct vma_iterator *vmi,
}
#ifdef CONFIG_SCHED_MM_CID
-
-#define MM_CID_UNSET (~0U)
-
/*
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
*/
@@ -1383,20 +1356,20 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
int i;
for_each_possible_cpu(i) {
- struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
+ struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
- pcpu_cid->cid = MM_CID_UNSET;
+ pcpu->cid = MM_CID_UNSET;
}
- mm->nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->cpus_allowed_lock);
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
cpumask_clear(mm_cidmask(mm));
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
- mm->pcpu_cid = alloc_percpu_noprof(struct mm_cid);
- if (!mm->pcpu_cid)
+ mm->mm_cid.pcpu = alloc_percpu_noprof(struct mm_cid_pcpu);
+ if (!mm->mm_cid.pcpu)
return -ENOMEM;
mm_init_cid(mm, p);
return 0;
@@ -1405,8 +1378,8 @@ static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *
static inline void mm_destroy_cid(struct mm_struct *mm)
{
- free_percpu(mm->pcpu_cid);
- mm->pcpu_cid = NULL;
+ free_percpu(mm->mm_cid.pcpu);
+ mm->mm_cid.pcpu = NULL;
}
static inline unsigned int mm_cid_size(void)
@@ -1421,10 +1394,9 @@ static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumas
if (!mm)
return;
/* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
- raw_spin_lock(&mm->cpus_allowed_lock);
+ guard(raw_spinlock)(&mm->mm_cid.lock);
cpumask_or(mm_allowed, mm_allowed, cpumask);
- WRITE_ONCE(mm->nr_cpus_allowed, cpumask_weight(mm_allowed));
- raw_spin_unlock(&mm->cpus_allowed_lock);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 9c7a341..e444dd2 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -90,4 +90,46 @@ struct rseq_data {
struct rseq_data { };
#endif /* !CONFIG_RSEQ */
+#ifdef CONFIG_SCHED_MM_CID
+
+#define MM_CID_UNSET (~0U)
+
+/**
+ * struct sched_mm_cid - Storage for per task MM CID data
+ * @active: MM CID is active for the task
+ * @cid: The CID associated to the task
+ * @last_cid: The last CID associated to the task
+ */
+struct sched_mm_cid {
+ unsigned int active;
+ unsigned int cid;
+ unsigned int last_cid;
+};
+
+/**
+ * struct mm_cid_pcpu - Storage for per CPU MM_CID data
+ * @cid: The CID associated to the CPU
+ */
+struct mm_cid_pcpu {
+ unsigned int cid;
+};
+
+/**
+ * struct mm_mm_cid - Storage for per MM CID data
+ * @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
+ * is growth only.
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ */
+struct mm_mm_cid {
+ struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int nr_cpus_allowed;
+ raw_spinlock_t lock;
+};
+#else /* CONFIG_SCHED_MM_CID */
+struct mm_mm_cid { };
+struct sched_mm_cid { };
+#endif /* !CONFIG_SCHED_MM_CID */
+
#endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e47abc8..64f080d 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1407,14 +1407,7 @@ struct task_struct {
#endif /* CONFIG_NUMA_BALANCING */
struct rseq_data rseq;
-
-#ifdef CONFIG_SCHED_MM_CID
- int mm_cid; /* Current cid in mm */
- int last_mm_cid; /* Most recent cid in mm */
- int migrate_from_cpu;
- int mm_cid_active; /* Whether cid bitmap is active */
- struct callback_head cid_work;
-#endif
+ struct sched_mm_cid mm_cid;
struct tlbflush_unmap_batch tlb_ubc;
@@ -2308,7 +2301,7 @@ void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit_signals(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid;
+ return t->mm_cid.cid;
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
diff --git a/init/init_task.c b/init/init_task.c
index a55e218..5d12269 100644
--- a/init/init_task.c
+++ b/init/init_task.c
@@ -223,6 +223,9 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
#ifdef CONFIG_SECCOMP_FILTER
.seccomp = { .filter_count = ATOMIC_INIT(0) },
#endif
+#ifdef CONFIG_SCHED_MM_CID
+ .mm_cid = { .cid = MM_CID_UNSET, },
+#endif
};
EXPORT_SYMBOL(init_task);
diff --git a/kernel/fork.c b/kernel/fork.c
index 9d9afe4..74bc7c9 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -955,9 +955,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#endif
#ifdef CONFIG_SCHED_MM_CID
- tsk->mm_cid = MM_CID_UNSET;
- tsk->last_mm_cid = MM_CID_UNSET;
- tsk->mm_cid_active = 0;
+ tsk->mm_cid.cid = MM_CID_UNSET;
+ tsk->mm_cid.last_cid = MM_CID_UNSET;
+ tsk->mm_cid.active = 0;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 11a1735..b1aa7d1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10376,14 +10376,14 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
- if (!mm || !t->mm_cid_active)
+ if (!mm || !t->mm_cid.active)
return;
guard(preempt)();
- t->mm_cid_active = 0;
- if (t->mm_cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid, mm_cidmask(mm));
- t->mm_cid = MM_CID_UNSET;
+ t->mm_cid.active = 0;
+ if (t->mm_cid.cid != MM_CID_UNSET) {
+ cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ t->mm_cid.cid = MM_CID_UNSET;
}
}
@@ -10402,14 +10402,14 @@ void sched_mm_cid_after_execve(struct task_struct *t)
return;
guard(preempt)();
- t->mm_cid_active = 1;
+ t->mm_cid.active = 1;
mm_cid_select(t);
}
void sched_mm_cid_fork(struct task_struct *t)
{
- WARN_ON_ONCE(!t->mm || t->mm_cid != MM_CID_UNSET);
- t->mm_cid_active = 1;
+ WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
+ t->mm_cid.active = 1;
}
#endif /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index bf227c2..a17f04f 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3549,8 +3549,8 @@ static inline void init_sched_mm_cid(struct task_struct *t)
return;
/* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->last_mm_cid = max_cid - 1;
+ max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ t->mm_cid.last_cid = max_cid - 1;
}
static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
@@ -3561,8 +3561,8 @@ static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigne
return false;
if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
return false;
- t->mm_cid = t->last_mm_cid = cid;
- __this_cpu_write(mm->pcpu_cid->cid, cid);
+ t->mm_cid.cid = t->mm_cid.last_cid = cid;
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
return true;
}
@@ -3571,14 +3571,14 @@ static inline bool mm_cid_get(struct task_struct *t)
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
/* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->last_mm_cid, max_cids))
+ if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
return true;
/* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
+ if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
return true;
/* Try the first zero bit in the cidmask. */
@@ -3601,15 +3601,15 @@ static inline void mm_cid_select(struct task_struct *t)
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
{
- if (prev->mm_cid_active) {
- if (prev->mm_cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
- prev->mm_cid = MM_CID_UNSET;
+ if (prev->mm_cid.active) {
+ if (prev->mm_cid.cid != MM_CID_UNSET)
+ cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ prev->mm_cid.cid = MM_CID_UNSET;
}
- if (next->mm_cid_active) {
+ if (next->mm_cid.active) {
mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid);
+ rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
}
}
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 03/20] sched/mmcid: Cacheline align MM CID storage
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
2025-11-19 17:26 ` [patch V5 01/20] sched/mmcid: Revert the complex " Thomas Gleixner
2025-11-19 17:26 ` [patch V5 02/20] sched/mmcid: Use proper data structures Thomas Gleixner
@ 2025-11-19 17:26 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-19 17:26 ` [patch V5 04/20] sched: Fixup whitespace damage Thomas Gleixner
` (17 subsequent siblings)
20 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:26 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Both the per CPU storage and the data in mm_struct are heavily used in
context switch. As they can end up next to other frequently modified data,
they are subject to false sharing.
Make them cache line aligned.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq_types.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -112,7 +112,7 @@ struct sched_mm_cid {
*/
struct mm_cid_pcpu {
unsigned int cid;
-};
+}____cacheline_aligned_in_smp;
/**
* struct mm_mm_cid - Storage for per MM CID data
@@ -126,7 +126,7 @@ struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
unsigned int nr_cpus_allowed;
raw_spinlock_t lock;
-};
+}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
struct sched_mm_cid { };
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Cacheline align MM CID storage
2025-11-19 17:26 ` [patch V5 03/20] sched/mmcid: Cacheline align MM CID storage Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: be4463fa2c7185823d2989562162d578b45a89ae
Gitweb: https://git.kernel.org/tip/be4463fa2c7185823d2989562162d578b45a89ae
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:49 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:53 +01:00
sched/mmcid: Cacheline align MM CID storage
Both the per CPU storage and the data in mm_struct are heavily used in
context switch. As they can end up next to other frequently modified data,
they are subject to false sharing.
Make them cache line aligned.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.194111661@linutronix.de
---
include/linux/rseq_types.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index e444dd2..d7e8071 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -112,7 +112,7 @@ struct sched_mm_cid {
*/
struct mm_cid_pcpu {
unsigned int cid;
-};
+}____cacheline_aligned_in_smp;
/**
* struct mm_mm_cid - Storage for per MM CID data
@@ -126,7 +126,7 @@ struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
unsigned int nr_cpus_allowed;
raw_spinlock_t lock;
-};
+}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
struct sched_mm_cid { };
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 04/20] sched: Fixup whitespace damage
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (2 preceding siblings ...)
2025-11-19 17:26 ` [patch V5 03/20] sched/mmcid: Cacheline align MM CID storage Thomas Gleixner
@ 2025-11-19 17:26 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-19 17:26 ` [patch V5 05/20] sched/mmcid: Move scheduler code out of global header Thomas Gleixner
` (16 subsequent siblings)
20 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:26 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
With whitespace checks enabled in the editor this makes eyes bleed.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
kernel/sched/core.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5278,15 +5278,15 @@ context_switch(struct rq *rq, struct tas
* kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch
*/
- if (!next->mm) { // to kernel
+ if (!next->mm) { // to kernel
enter_lazy_tlb(prev->active_mm, next);
next->active_mm = prev->active_mm;
- if (prev->mm) // from user
+ if (prev->mm) // from user
mmgrab_lazy_tlb(prev->active_mm);
else
prev->active_mm = NULL;
- } else { // to user
+ } else { // to user
membarrier_switch_mm(rq, prev->active_mm, next->mm);
/*
* sys_membarrier() requires an smp_mb() between setting
@@ -5299,7 +5299,7 @@ context_switch(struct rq *rq, struct tas
switch_mm_irqs_off(prev->active_mm, next->mm, next);
lru_gen_use_mm(next->mm);
- if (!prev->mm) { // from kernel
+ if (!prev->mm) { // from kernel
/* will mmdrop_lazy_tlb() in finish_task_switch(). */
rq->prev_mm = prev->active_mm;
prev->active_mm = NULL;
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched: Fixup whitespace damage
2025-11-19 17:26 ` [patch V5 04/20] sched: Fixup whitespace damage Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 925b7847bb7d4eb523a7698b309e8441647796f2
Gitweb: https://git.kernel.org/tip/925b7847bb7d4eb523a7698b309e8441647796f2
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:51 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:53 +01:00
sched: Fixup whitespace damage
With whitespace checks enabled in the editor this makes eyes bleed.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.258651925@linutronix.de
---
kernel/sched/core.c | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b1aa7d1..b667171 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5278,15 +5278,15 @@ context_switch(struct rq *rq, struct task_struct *prev,
* kernel -> user switch + mmdrop_lazy_tlb() active
* user -> user switch
*/
- if (!next->mm) { // to kernel
+ if (!next->mm) { // to kernel
enter_lazy_tlb(prev->active_mm, next);
next->active_mm = prev->active_mm;
- if (prev->mm) // from user
+ if (prev->mm) // from user
mmgrab_lazy_tlb(prev->active_mm);
else
prev->active_mm = NULL;
- } else { // to user
+ } else { // to user
membarrier_switch_mm(rq, prev->active_mm, next->mm);
/*
* sys_membarrier() requires an smp_mb() between setting
@@ -5299,7 +5299,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
switch_mm_irqs_off(prev->active_mm, next->mm, next);
lru_gen_use_mm(next->mm);
- if (!prev->mm) { // from kernel
+ if (!prev->mm) { // from kernel
/* will mmdrop_lazy_tlb() in finish_task_switch(). */
rq->prev_mm = prev->active_mm;
prev->active_mm = NULL;
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 05/20] sched/mmcid: Move scheduler code out of global header
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (3 preceding siblings ...)
2025-11-19 17:26 ` [patch V5 04/20] sched: Fixup whitespace damage Thomas Gleixner
@ 2025-11-19 17:26 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-19 17:26 ` [patch V5 06/20] sched/mmcid: Prevent pointless work in mm_update_cpus_allowed() Thomas Gleixner
` (15 subsequent siblings)
20 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:26 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
This is only used in the scheduler core code, so there is no point to have
it in a global header.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
include/linux/mm_types.h | 13 -------------
kernel/sched/core.c | 20 ++++++++++++++++++--
2 files changed, 18 insertions(+), 15 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1387,27 +1387,14 @@ static inline unsigned int mm_cid_size(v
return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
}
-static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask)
-{
- struct cpumask *mm_allowed = mm_cpus_allowed(mm);
-
- if (!mm)
- return;
- /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
- guard(raw_spinlock)(&mm->mm_cid.lock);
- cpumask_or(mm_allowed, mm_allowed, cpumask);
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
-}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
static inline int mm_alloc_cid(struct mm_struct *mm, struct task_struct *p) { return 0; }
static inline void mm_destroy_cid(struct mm_struct *mm) { }
-
static inline unsigned int mm_cid_size(void)
{
return 0;
}
-static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask) { }
#endif /* CONFIG_SCHED_MM_CID */
struct mmu_gather;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2669,6 +2669,8 @@ int push_cpu_stop(void *arg)
return 0;
}
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const cpumask_t *affmask);
+
/*
* sched_class::set_cpus_allowed must do the below, but is not required to
* actually call this function.
@@ -2728,7 +2730,7 @@ static void
put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, ctx);
- mm_set_cpus_allowed(p->mm, ctx->new_mask);
+ mm_update_cpus_allowed(p->mm, ctx->new_mask);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -10372,6 +10374,18 @@ void call_trace_sched_update_nr_running(
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
*/
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
+{
+ struct cpumask *mm_allowed = mm_cpus_allowed(mm);
+
+ if (!mm)
+ return;
+ /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ cpumask_or(mm_allowed, mm_allowed, affmsk);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
+}
+
void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10411,7 +10425,9 @@ void sched_mm_cid_fork(struct task_struc
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
-#endif /* CONFIG_SCHED_MM_CID */
+#else /* CONFIG_SCHED_MM_CID */
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
+#endif /* !CONFIG_SCHED_MM_CID */
#ifdef CONFIG_SCHED_CLASS_EXT
void sched_deq_and_put_task(struct task_struct *p, int queue_flags,
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Move scheduler code out of global header
2025-11-19 17:26 ` [patch V5 05/20] sched/mmcid: Move scheduler code out of global header Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers,
Yury Norov (NVIDIA), x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: b08ef5fc8fa01ae5285bef5ff783bbb425d1fb08
Gitweb: https://git.kernel.org/tip/b08ef5fc8fa01ae5285bef5ff783bbb425d1fb08
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:53 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:53 +01:00
sched/mmcid: Move scheduler code out of global header
This is only used in the scheduler core code, so there is no point to have
it in a global header.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Link: https://patch.msgid.link/20251119172549.321259077@linutronix.de
---
include/linux/mm_types.h | 13 -------------
kernel/sched/core.c | 20 ++++++++++++++++++--
2 files changed, 18 insertions(+), 15 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index e4818e9..67a7bdf 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1387,27 +1387,14 @@ static inline unsigned int mm_cid_size(void)
return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
}
-static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask)
-{
- struct cpumask *mm_allowed = mm_cpus_allowed(mm);
-
- if (!mm)
- return;
- /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
- guard(raw_spinlock)(&mm->mm_cid.lock);
- cpumask_or(mm_allowed, mm_allowed, cpumask);
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
-}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p) { }
static inline int mm_alloc_cid(struct mm_struct *mm, struct task_struct *p) { return 0; }
static inline void mm_destroy_cid(struct mm_struct *mm) { }
-
static inline unsigned int mm_cid_size(void)
{
return 0;
}
-static inline void mm_set_cpus_allowed(struct mm_struct *mm, const struct cpumask *cpumask) { }
#endif /* CONFIG_SCHED_MM_CID */
struct mmu_gather;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b667171..f5e37c2 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2669,6 +2669,8 @@ out_unlock:
return 0;
}
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const cpumask_t *affmask);
+
/*
* sched_class::set_cpus_allowed must do the below, but is not required to
* actually call this function.
@@ -2728,7 +2730,7 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, ctx);
- mm_set_cpus_allowed(p->mm, ctx->new_mask);
+ mm_update_cpus_allowed(p->mm, ctx->new_mask);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -10372,6 +10374,18 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
*/
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
+{
+ struct cpumask *mm_allowed = mm_cpus_allowed(mm);
+
+ if (!mm)
+ return;
+ /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ cpumask_or(mm_allowed, mm_allowed, affmsk);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
+}
+
void sched_mm_cid_exit_signals(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10411,7 +10425,9 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
-#endif /* CONFIG_SCHED_MM_CID */
+#else /* CONFIG_SCHED_MM_CID */
+static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
+#endif /* !CONFIG_SCHED_MM_CID */
#ifdef CONFIG_SCHED_CLASS_EXT
void sched_deq_and_put_task(struct task_struct *p, int queue_flags,
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 06/20] sched/mmcid: Prevent pointless work in mm_update_cpus_allowed()
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (4 preceding siblings ...)
2025-11-19 17:26 ` [patch V5 05/20] sched/mmcid: Move scheduler code out of global header Thomas Gleixner
@ 2025-11-19 17:26 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-19 17:26 ` [patch V5 07/20] cpumask: Introduce cpumask_weighted_or() Thomas Gleixner
` (14 subsequent siblings)
20 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:26 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
mm_update_cpus_allowed() is not required to be invoked for affinity changes
due to migrate_disable() and migrate_enable().
migrate_disable() restricts the task temporarily to a CPU on which the task
was already allowed to run, so nothing changes. migrate_enable() restores
the actual task affinity mask.
If that mask changed between migrate_disable() and migrate_enable() then
that change was already accounted for.
Move the invocation to the proper place to avoid that.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V2: Remove the nr_cpu_ids optimization which does not really work - PeterZ
---
kernel/sched/core.c | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2684,6 +2684,7 @@ void set_cpus_allowed_common(struct task
cpumask_copy(&p->cpus_mask, ctx->new_mask);
p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
+ mm_update_cpus_allowed(p->mm, ctx->new_mask);
/*
* Swap in a new user_cpus_ptr if SCA_USER flag set
@@ -2730,7 +2731,6 @@ static void
put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, ctx);
- mm_update_cpus_allowed(p->mm, ctx->new_mask);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -10376,12 +10376,17 @@ void call_trace_sched_update_nr_running(
*/
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
- struct cpumask *mm_allowed = mm_cpus_allowed(mm);
+ struct cpumask *mm_allowed;
if (!mm)
return;
- /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
+
+ /*
+ * mm::mm_cid::mm_cpus_allowed is the superset of each threads
+ * allowed CPUs mask which means it can only grow.
+ */
guard(raw_spinlock)(&mm->mm_cid.lock);
+ mm_allowed = mm_cpus_allowed(mm);
cpumask_or(mm_allowed, mm_allowed, affmsk);
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
}
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Prevent pointless work in mm_update_cpus_allowed()
2025-11-19 17:26 ` [patch V5 06/20] sched/mmcid: Prevent pointless work in mm_update_cpus_allowed() Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 0d032a43ebeb9bf255cd7e3dad5f7a6371571648
Gitweb: https://git.kernel.org/tip/0d032a43ebeb9bf255cd7e3dad5f7a6371571648
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:55 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:54 +01:00
sched/mmcid: Prevent pointless work in mm_update_cpus_allowed()
mm_update_cpus_allowed() is not required to be invoked for affinity changes
due to migrate_disable() and migrate_enable().
migrate_disable() restricts the task temporarily to a CPU on which the task
was already allowed to run, so nothing changes. migrate_enable() restores
the actual task affinity mask.
If that mask changed between migrate_disable() and migrate_enable() then
that change was already accounted for.
Move the invocation to the proper place to avoid that.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.385208276@linutronix.de
---
kernel/sched/core.c | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f5e37c2..2ea77e7 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2684,6 +2684,7 @@ void set_cpus_allowed_common(struct task_struct *p, struct affinity_context *ctx
cpumask_copy(&p->cpus_mask, ctx->new_mask);
p->nr_cpus_allowed = cpumask_weight(ctx->new_mask);
+ mm_update_cpus_allowed(p->mm, ctx->new_mask);
/*
* Swap in a new user_cpus_ptr if SCA_USER flag set
@@ -2730,7 +2731,6 @@ __do_set_cpus_allowed(struct task_struct *p, struct affinity_context *ctx)
put_prev_task(rq, p);
p->sched_class->set_cpus_allowed(p, ctx);
- mm_update_cpus_allowed(p->mm, ctx->new_mask);
if (queued)
enqueue_task(rq, p, ENQUEUE_RESTORE | ENQUEUE_NOCLOCK);
@@ -10376,12 +10376,17 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
*/
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
- struct cpumask *mm_allowed = mm_cpus_allowed(mm);
+ struct cpumask *mm_allowed;
if (!mm)
return;
- /* The mm_cpus_allowed is the union of each thread allowed CPUs masks. */
+
+ /*
+ * mm::mm_cid::mm_cpus_allowed is the superset of each threads
+ * allowed CPUs mask which means it can only grow.
+ */
guard(raw_spinlock)(&mm->mm_cid.lock);
+ mm_allowed = mm_cpus_allowed(mm);
cpumask_or(mm_allowed, mm_allowed, affmsk);
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
}
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 07/20] cpumask: Introduce cpumask_weighted_or()
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (5 preceding siblings ...)
2025-11-19 17:26 ` [patch V5 06/20] sched/mmcid: Prevent pointless work in mm_update_cpus_allowed() Thomas Gleixner
@ 2025-11-19 17:26 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-19 17:26 ` [patch V5 08/20] sched/mmcid: Use cpumask_weighted_or() Thomas Gleixner
` (13 subsequent siblings)
20 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:26 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
CID management OR's two cpumasks and then calculates the weight on the
result. That's inefficient as that has to walk the same stuff twice. As
this is done with runqueue lock held, there is a real benefit of speeding
this up. Depending on the system this results in 10-20% less cycles spent
with runqueue lock held for a 4K cpumask.
Provide cpumask_weighted_or() and the corresponding bitmap functions which
return the weight of the OR result right away.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V3: Rename again - Yury
V2: Rename and use the BITMAP_WEIGHT() macro - Yury
---
include/linux/bitmap.h | 15 +++++++++++++++
include/linux/cpumask.h | 16 ++++++++++++++++
lib/bitmap.c | 6 ++++++
3 files changed, 37 insertions(+)
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -45,6 +45,7 @@ struct device;
* bitmap_copy(dst, src, nbits) *dst = *src
* bitmap_and(dst, src1, src2, nbits) *dst = *src1 & *src2
* bitmap_or(dst, src1, src2, nbits) *dst = *src1 | *src2
+ * bitmap_weighted_or(dst, src1, src2, nbits) *dst = *src1 | *src2. Returns Hamming Weight of dst
* bitmap_xor(dst, src1, src2, nbits) *dst = *src1 ^ *src2
* bitmap_andnot(dst, src1, src2, nbits) *dst = *src1 & ~(*src2)
* bitmap_complement(dst, src, nbits) *dst = ~(*src)
@@ -165,6 +166,8 @@ bool __bitmap_and(unsigned long *dst, co
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
@@ -338,6 +341,18 @@ void bitmap_or(unsigned long *dst, const
}
static __always_inline
+unsigned int bitmap_weighted_or(unsigned long *dst, const unsigned long *src1,
+ const unsigned long *src2, unsigned int nbits)
+{
+ if (small_const_nbits(nbits)) {
+ *dst = *src1 | *src2;
+ return hweight_long(*dst & BITMAP_LAST_WORD_MASK(nbits));
+ } else {
+ return __bitmap_weighted_or(dst, src1, src2, nbits);
+ }
+}
+
+static __always_inline
void bitmap_xor(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -729,6 +729,22 @@ void cpumask_or(struct cpumask *dstp, co
}
/**
+ * cpumask_weighted_or - *dstp = *src1p | *src2p and return the weight of the result
+ * @dstp: the cpumask result
+ * @src1p: the first input
+ * @src2p: the second input
+ *
+ * Return: The number of bits set in the resulting cpumask @dstp
+ */
+static __always_inline
+unsigned int cpumask_weighted_or(struct cpumask *dstp, const struct cpumask *src1p,
+ const struct cpumask *src2p)
+{
+ return bitmap_weighted_or(cpumask_bits(dstp), cpumask_bits(src1p),
+ cpumask_bits(src2p), small_cpumask_bits);
+}
+
+/**
* cpumask_xor - *dstp = *src1p ^ *src2p
* @dstp: the cpumask result
* @src1p: the first input
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -355,6 +355,12 @@ unsigned int __bitmap_weight_andnot(cons
}
EXPORT_SYMBOL(__bitmap_weight_andnot);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int bits)
+{
+ return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] | bitmap2[idx]; dst[idx]; }), bits);
+}
+
void __bitmap_set(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start);
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] cpumask: Introduce cpumask_weighted_or()
2025-11-19 17:26 ` [patch V5 07/20] cpumask: Introduce cpumask_weighted_or() Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Yury Norov (NVIDIA),
Mathieu Desnoyers, x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 437cb3ded25038d5280d21de489ce78c745118d5
Gitweb: https://git.kernel.org/tip/437cb3ded25038d5280d21de489ce78c745118d5
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:57 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:54 +01:00
cpumask: Introduce cpumask_weighted_or()
CID management OR's two cpumasks and then calculates the weight on the
result. That's inefficient as that has to walk the same stuff twice. As
this is done with runqueue lock held, there is a real benefit of speeding
this up. Depending on the system this results in 10-20% less cycles spent
with runqueue lock held for a 4K cpumask.
Provide cpumask_weighted_or() and the corresponding bitmap functions which
return the weight of the OR result right away.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.448263340@linutronix.de
---
include/linux/bitmap.h | 15 +++++++++++++++
include/linux/cpumask.h | 16 ++++++++++++++++
lib/bitmap.c | 6 ++++++
3 files changed, 37 insertions(+)
diff --git a/include/linux/bitmap.h b/include/linux/bitmap.h
index 595217b..b0395e4 100644
--- a/include/linux/bitmap.h
+++ b/include/linux/bitmap.h
@@ -45,6 +45,7 @@ struct device;
* bitmap_copy(dst, src, nbits) *dst = *src
* bitmap_and(dst, src1, src2, nbits) *dst = *src1 & *src2
* bitmap_or(dst, src1, src2, nbits) *dst = *src1 | *src2
+ * bitmap_weighted_or(dst, src1, src2, nbits) *dst = *src1 | *src2. Returns Hamming Weight of dst
* bitmap_xor(dst, src1, src2, nbits) *dst = *src1 ^ *src2
* bitmap_andnot(dst, src1, src2, nbits) *dst = *src1 & ~(*src2)
* bitmap_complement(dst, src, nbits) *dst = ~(*src)
@@ -165,6 +166,8 @@ bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int nbits);
void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1,
const unsigned long *bitmap2, unsigned int nbits);
bool __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1,
@@ -338,6 +341,18 @@ void bitmap_or(unsigned long *dst, const unsigned long *src1,
}
static __always_inline
+unsigned int bitmap_weighted_or(unsigned long *dst, const unsigned long *src1,
+ const unsigned long *src2, unsigned int nbits)
+{
+ if (small_const_nbits(nbits)) {
+ *dst = *src1 | *src2;
+ return hweight_long(*dst & BITMAP_LAST_WORD_MASK(nbits));
+ } else {
+ return __bitmap_weighted_or(dst, src1, src2, nbits);
+ }
+}
+
+static __always_inline
void bitmap_xor(unsigned long *dst, const unsigned long *src1,
const unsigned long *src2, unsigned int nbits)
{
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index ff8f41a..feba06e 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -729,6 +729,22 @@ void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
}
/**
+ * cpumask_weighted_or - *dstp = *src1p | *src2p and return the weight of the result
+ * @dstp: the cpumask result
+ * @src1p: the first input
+ * @src2p: the second input
+ *
+ * Return: The number of bits set in the resulting cpumask @dstp
+ */
+static __always_inline
+unsigned int cpumask_weighted_or(struct cpumask *dstp, const struct cpumask *src1p,
+ const struct cpumask *src2p)
+{
+ return bitmap_weighted_or(cpumask_bits(dstp), cpumask_bits(src1p),
+ cpumask_bits(src2p), small_cpumask_bits);
+}
+
+/**
* cpumask_xor - *dstp = *src1p ^ *src2p
* @dstp: the cpumask result
* @src1p: the first input
diff --git a/lib/bitmap.c b/lib/bitmap.c
index b976928..9dc5265 100644
--- a/lib/bitmap.c
+++ b/lib/bitmap.c
@@ -355,6 +355,12 @@ unsigned int __bitmap_weight_andnot(const unsigned long *bitmap1,
}
EXPORT_SYMBOL(__bitmap_weight_andnot);
+unsigned int __bitmap_weighted_or(unsigned long *dst, const unsigned long *bitmap1,
+ const unsigned long *bitmap2, unsigned int bits)
+{
+ return BITMAP_WEIGHT(({dst[idx] = bitmap1[idx] | bitmap2[idx]; dst[idx]; }), bits);
+}
+
void __bitmap_set(unsigned long *map, unsigned int start, int len)
{
unsigned long *p = map + BIT_WORD(start);
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 08/20] sched/mmcid: Use cpumask_weighted_or()
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (6 preceding siblings ...)
2025-11-19 17:26 ` [patch V5 07/20] cpumask: Introduce cpumask_weighted_or() Thomas Gleixner
@ 2025-11-19 17:26 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Thomas Gleixner
` (12 subsequent siblings)
20 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:26 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Use cpumask_weighted_or() instead of cpumask_or() and cpumask_weight() on
the result, which walks the same bitmap twice. Results in 10-20% less
cycles, which reduces the runqueue lock hold time.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
kernel/sched/core.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10377,6 +10377,7 @@ void call_trace_sched_update_nr_running(
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ unsigned int weight;
if (!mm)
return;
@@ -10387,8 +10388,8 @@ static inline void mm_update_cpus_allowe
*/
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
- cpumask_or(mm_allowed, mm_allowed, affmsk);
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
+ weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
void sched_mm_cid_exit_signals(struct task_struct *t)
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Use cpumask_weighted_or()
2025-11-19 17:26 ` [patch V5 08/20] sched/mmcid: Use cpumask_weighted_or() Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers,
Yury Norov (NVIDIA), x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 79c11fb3da8581a2f222b290ce62a153ab1108fc
Gitweb: https://git.kernel.org/tip/79c11fb3da8581a2f222b290ce62a153ab1108fc
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:26:59 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:54 +01:00
sched/mmcid: Use cpumask_weighted_or()
Use cpumask_weighted_or() instead of cpumask_or() and cpumask_weight() on
the result, which walks the same bitmap twice. Results in 10-20% less
cycles, which reduces the runqueue lock hold time.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Link: https://patch.msgid.link/20251119172549.511736272@linutronix.de
---
kernel/sched/core.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 2ea77e7..f6bbfa1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10377,6 +10377,7 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ unsigned int weight;
if (!mm)
return;
@@ -10387,8 +10388,8 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
*/
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
- cpumask_or(mm_allowed, mm_allowed, affmsk);
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, cpumask_weight(mm_allowed));
+ weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
void sched_mm_cid_exit_signals(struct task_struct *t)
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 09/20] cpumask: Cache num_possible_cpus()
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (7 preceding siblings ...)
2025-11-19 17:26 ` [patch V5 08/20] sched/mmcid: Use cpumask_weighted_or() Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
` (3 more replies)
2025-11-19 17:27 ` [patch V5 10/20] sched/mmcid: Convert mm CID mask to a bitmap Thomas Gleixner
` (11 subsequent siblings)
20 siblings, 4 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Reevaluating num_possible_cpus() over and over does not make sense. That
becomes a constant after init as cpu_possible_mask is marked ro_after_init.
Cache the value during initialization and provide that for consumption.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Yury Norov <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
---
V4: Add comment why this is not marked __init ....
V2: New patch
---
include/linux/cpumask.h | 10 ++++++++--
kernel/cpu.c | 19 +++++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
+extern unsigned int __num_possible_cpus;
extern cpumask_t cpus_booted_once_mask;
@@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpum
#define __assign_cpu(cpu, mask, val) \
__assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
-#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
#define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
#define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
#define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
#define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
void set_cpu_online(unsigned int cpu, bool online);
+void set_cpu_possible(unsigned int cpu, bool possible);
/**
* to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
@@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_
{
return raw_atomic_read(&__num_online_cpus);
}
-#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
+
+static __always_inline unsigned int num_possible_cpus(void)
+{
+ return __num_possible_cpus;
+}
+
#define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3108,6 +3108,9 @@ EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
+EXPORT_SYMBOL(__num_possible_cpus);
+
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);
@@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpuma
void init_cpu_possible(const struct cpumask *src)
{
cpumask_copy(&__cpu_possible_mask, src);
+ __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
}
void set_cpu_online(unsigned int cpu, bool online)
@@ -3139,6 +3143,21 @@ void set_cpu_online(unsigned int cpu, bo
}
}
+/*
+ * This should be marked __init, but there is a boatload of call sites
+ * which need to be fixed up to do so. Sigh...
+ */
+void set_cpu_possible(unsigned int cpu, bool possible)
+{
+ if (possible) {
+ if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus++;
+ } else {
+ if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus--;
+ }
+}
+
/*
* Activate the first processor.
*/
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] cpumask: Cache num_possible_cpus()
2025-11-19 17:27 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Thomas Gleixner
@ 2025-11-20 11:20 ` tip-bot2 for Thomas Gleixner
2025-11-21 22:56 ` [patch V5 09/20] " Marek Szyprowski
` (2 subsequent siblings)
3 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:20 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Yury Norov,
Mathieu Desnoyers, Shrikanth Hegde, x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: d0f23ccf6ba9e2cc202e9ad25a427b8e5ea3ca1e
Gitweb: https://git.kernel.org/tip/d0f23ccf6ba9e2cc202e9ad25a427b8e5ea3ca1e
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:01 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:54 +01:00
cpumask: Cache num_possible_cpus()
Reevaluating num_possible_cpus() over and over does not make sense. That
becomes a constant after init as cpu_possible_mask is marked ro_after_init.
Cache the value during initialization and provide that for consumption.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Yury Norov <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
Link: https://patch.msgid.link/20251119172549.578653738@linutronix.de
---
include/linux/cpumask.h | 10 ++++++++--
kernel/cpu.c | 19 +++++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index feba06e..66694ee 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
+extern unsigned int __num_possible_cpus;
extern cpumask_t cpus_booted_once_mask;
@@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpumask *src);
#define __assign_cpu(cpu, mask, val) \
__assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
-#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
#define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
#define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
#define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
#define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
void set_cpu_online(unsigned int cpu, bool online);
+void set_cpu_possible(unsigned int cpu, bool possible);
/**
* to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
@@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_online_cpus(void)
{
return raw_atomic_read(&__num_online_cpus);
}
-#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
+
+static __always_inline unsigned int num_possible_cpus(void)
+{
+ return __num_possible_cpus;
+}
+
#define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
diff --git a/kernel/cpu.c b/kernel/cpu.c
index db9f6c5..18e530d 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3108,6 +3108,9 @@ EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
+EXPORT_SYMBOL(__num_possible_cpus);
+
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);
@@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpumask *src)
void init_cpu_possible(const struct cpumask *src)
{
cpumask_copy(&__cpu_possible_mask, src);
+ __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
}
void set_cpu_online(unsigned int cpu, bool online)
@@ -3140,6 +3144,21 @@ void set_cpu_online(unsigned int cpu, bool online)
}
/*
+ * This should be marked __init, but there is a boatload of call sites
+ * which need to be fixed up to do so. Sigh...
+ */
+void set_cpu_possible(unsigned int cpu, bool possible)
+{
+ if (possible) {
+ if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus++;
+ } else {
+ if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus--;
+ }
+}
+
+/*
* Activate the first processor.
*/
void __init boot_cpu_init(void)
^ permalink raw reply related [flat|nested] 74+ messages in thread* Re: [patch V5 09/20] cpumask: Cache num_possible_cpus()
2025-11-19 17:27 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-21 22:56 ` Marek Szyprowski
2025-11-22 15:36 ` Thomas Gleixner
2025-11-22 18:47 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Paul E. McKenney
2025-11-22 0:27 ` Nathan Chancellor
2025-11-26 4:36 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
3 siblings, 2 replies; 74+ messages in thread
From: Marek Szyprowski @ 2025-11-21 22:56 UTC (permalink / raw)
To: Thomas Gleixner, LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Hi
On 19.11.2025 18:27, Thomas Gleixner wrote:
> Reevaluating num_possible_cpus() over and over does not make sense. That
> becomes a constant after init as cpu_possible_mask is marked ro_after_init.
>
> Cache the value during initialization and provide that for consumption.
>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Reviewed-by: Yury Norov <yury.norov@gmail.com>
> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
> Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
This patch landed recently in linux-next as commit d0f23ccf6ba9
("cpumask: Cache num_possible_cpus()"). I found that it triggers the
following warning during boot on some of my test systems (namely
Raspberry Pi 3 and 4):
kvm [1]: nv: 568 coarse grained trap handlers
kvm [1]: IPA Size Limit: 40 bits
------------[ cut here ]------------
WARNING: CPU: 1 PID: 1 at arch/arm64/kvm/vmid.c:183
kvm_arm_vmid_alloc_init+0x98/0xac
Modules linked in:
CPU: 1 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.18.0-rc4+ #16195 PREEMPT
Hardware name: Raspberry Pi 3 Model B (DT)
pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
pc : kvm_arm_vmid_alloc_init+0x98/0xac
lr : kvm_arm_vmid_alloc_init+0x1c/0xac
...
Call trace:
kvm_arm_vmid_alloc_init+0x98/0xac (P)
kvm_arm_init+0x144/0x153c
do_one_initcall+0x64/0x308
kernel_init_freeable+0x280/0x4fc
kernel_init+0x20/0x1d8
ret_from_fork+0x10/0x20
irq event stamp: 65606
hardirqs last enabled at (65605): [<ffff8000801534e4>]
__up_console_sem+0x6c/0x80
hardirqs last disabled at (65606): [<ffff800081358f7c>] el1_brk64+0x20/0x60
softirqs last enabled at (65262): [<ffff8000800c4b1c>]
handle_softirqs+0x4c4/0x4dc
softirqs last disabled at (65257): [<ffff8000800106a0>]
__do_softirq+0x14/0x20
---[ end trace 0000000000000000 ]---
kvm [1]: Hyp nVHE mode initialized successfully
Reverting it on top of linux-next fixes the issue. Let me know how can I
help debugging it.
> ---
> V4: Add comment why this is not marked __init ....
> V2: New patch
> ---
> include/linux/cpumask.h | 10 ++++++++--
> kernel/cpu.c | 19 +++++++++++++++++++
> 2 files changed, 27 insertions(+), 2 deletions(-)
> --- a/include/linux/cpumask.h
> +++ b/include/linux/cpumask.h
> @@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
> #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
>
> extern atomic_t __num_online_cpus;
> +extern unsigned int __num_possible_cpus;
>
> extern cpumask_t cpus_booted_once_mask;
>
> @@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpum
> #define __assign_cpu(cpu, mask, val) \
> __assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
>
> -#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
> #define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
> #define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
> #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
> #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
>
> void set_cpu_online(unsigned int cpu, bool online);
> +void set_cpu_possible(unsigned int cpu, bool possible);
>
> /**
> * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
> @@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_
> {
> return raw_atomic_read(&__num_online_cpus);
> }
> -#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
> +
> +static __always_inline unsigned int num_possible_cpus(void)
> +{
> + return __num_possible_cpus;
> +}
> +
> #define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
> #define num_present_cpus() cpumask_weight(cpu_present_mask)
> #define num_active_cpus() cpumask_weight(cpu_active_mask)
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -3108,6 +3108,9 @@ EXPORT_SYMBOL(__cpu_dying_mask);
> atomic_t __num_online_cpus __read_mostly;
> EXPORT_SYMBOL(__num_online_cpus);
>
> +unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> +EXPORT_SYMBOL(__num_possible_cpus);
> +
> void init_cpu_present(const struct cpumask *src)
> {
> cpumask_copy(&__cpu_present_mask, src);
> @@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpuma
> void init_cpu_possible(const struct cpumask *src)
> {
> cpumask_copy(&__cpu_possible_mask, src);
> + __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
> }
>
> void set_cpu_online(unsigned int cpu, bool online)
> @@ -3139,6 +3143,21 @@ void set_cpu_online(unsigned int cpu, bo
> }
> }
>
> +/*
> + * This should be marked __init, but there is a boatload of call sites
> + * which need to be fixed up to do so. Sigh...
> + */
> +void set_cpu_possible(unsigned int cpu, bool possible)
> +{
> + if (possible) {
> + if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
> + __num_possible_cpus++;
> + } else {
> + if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
> + __num_possible_cpus--;
> + }
> +}
> +
> /*
> * Activate the first processor.
> */
>
>
>
Best regards
--
Marek Szyprowski, PhD
Samsung R&D Institute Poland
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 09/20] cpumask: Cache num_possible_cpus()
2025-11-21 22:56 ` [patch V5 09/20] " Marek Szyprowski
@ 2025-11-22 15:36 ` Thomas Gleixner
2025-11-22 16:24 ` Marek Szyprowski
2025-11-23 19:03 ` [tip: core/rseq] cpu: Initialize __num_possible_cpus correctly tip-bot2 for Thomas Gleixner
2025-11-22 18:47 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Paul E. McKenney
1 sibling, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-22 15:36 UTC (permalink / raw)
To: Marek Szyprowski, LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde,
Nathan Chancellor
On Fri, Nov 21 2025 at 23:56, Marek Szyprowski wrote:
> Reverting it on top of linux-next fixes the issue. Let me know how can I
> help debugging it.
Can you test the fix below please?
Thanks,
tglx
---
Subject: cpu: Initialize __num_possible_cpus correctly
From: Thomas Gleixner <tglx@linutronix.de>
Date: Sat, 22 Nov 2025 16:19:18 +0100
The variable to cache the number of possible CPUs is initialized to NR_CPUS
at build time, but that's only correct when cpu_possible_mask is
initialized with CPU_BITS_ALL. That's only the case on PARISC.
On x86 and some other architectures this does not matter because they
initialize cpu_possible_mask via init_cpu_possible() which does a proper
weight calculation. Though on architectures which do not, this results
in a completely wrong cached value 'NR_CPUS + actual possible CPUs'.
Initialize it correctly to 0 when CONFIG_INIT_ALL_POSSIBLE=n and move the
NR_CPUS initialization into the PARISC specific section.
Fixes: d0f23ccf6ba9 ("cpumask: Cache num_possible_cpus()")
Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
Reported-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Closes: https://lore.kernel.org/all/89c7106e-a431-443a-9527-3d5fbce77fe1@samsung.com
Closes: https://lore.kernel.org/all/20251122002755.GA2682494@ax162
---
kernel/cpu.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
#ifdef CONFIG_INIT_ALL_POSSIBLE
struct cpumask __cpu_possible_mask __ro_after_init
= {CPU_BITS_ALL};
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
#else
struct cpumask __cpu_possible_mask __ro_after_init;
+unsigned int __num_possible_cpus __ro_after_init;
#endif
EXPORT_SYMBOL(__cpu_possible_mask);
+EXPORT_SYMBOL(__num_possible_cpus);
struct cpumask __cpu_online_mask __read_mostly;
EXPORT_SYMBOL(__cpu_online_mask);
@@ -3108,9 +3111,6 @@ EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
-unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
-EXPORT_SYMBOL(__num_possible_cpus);
-
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 09/20] cpumask: Cache num_possible_cpus()
2025-11-22 15:36 ` Thomas Gleixner
@ 2025-11-22 16:24 ` Marek Szyprowski
2025-11-22 19:09 ` Paul E. McKenney
2025-11-23 19:03 ` [tip: core/rseq] cpu: Initialize __num_possible_cpus correctly tip-bot2 for Thomas Gleixner
1 sibling, 1 reply; 74+ messages in thread
From: Marek Szyprowski @ 2025-11-22 16:24 UTC (permalink / raw)
To: Thomas Gleixner, LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde,
Nathan Chancellor
On 22.11.2025 16:36, Thomas Gleixner wrote:
> On Fri, Nov 21 2025 at 23:56, Marek Szyprowski wrote:
>> Reverting it on top of linux-next fixes the issue. Let me know how can I
>> help debugging it.
> Can you test the fix below please?
This fixes the observed issue. Thanks!
Tested-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Thanks,
>
> tglx
> ---
> Subject: cpu: Initialize __num_possible_cpus correctly
> From: Thomas Gleixner <tglx@linutronix.de>
> Date: Sat, 22 Nov 2025 16:19:18 +0100
>
> The variable to cache the number of possible CPUs is initialized to NR_CPUS
> at build time, but that's only correct when cpu_possible_mask is
> initialized with CPU_BITS_ALL. That's only the case on PARISC.
>
> On x86 and some other architectures this does not matter because they
> initialize cpu_possible_mask via init_cpu_possible() which does a proper
> weight calculation. Though on architectures which do not, this results
> in a completely wrong cached value 'NR_CPUS + actual possible CPUs'.
>
> Initialize it correctly to 0 when CONFIG_INIT_ALL_POSSIBLE=n and move the
> NR_CPUS initialization into the PARISC specific section.
>
> Fixes: d0f23ccf6ba9 ("cpumask: Cache num_possible_cpus()")
> Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
> Reported-by: Nathan Chancellor <nathan@kernel.org>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Closes: https://lore.kernel.org/all/89c7106e-a431-443a-9527-3d5fbce77fe1@samsung.com
> Closes: https://lore.kernel.org/all/20251122002755.GA2682494@ax162
> ---
> kernel/cpu.c | 6 +++---
> 1 file changed, 3 insertions(+), 3 deletions(-)
>
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
> #ifdef CONFIG_INIT_ALL_POSSIBLE
> struct cpumask __cpu_possible_mask __ro_after_init
> = {CPU_BITS_ALL};
> +unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> #else
> struct cpumask __cpu_possible_mask __ro_after_init;
> +unsigned int __num_possible_cpus __ro_after_init;
> #endif
> EXPORT_SYMBOL(__cpu_possible_mask);
> +EXPORT_SYMBOL(__num_possible_cpus);
>
> struct cpumask __cpu_online_mask __read_mostly;
> EXPORT_SYMBOL(__cpu_online_mask);
> @@ -3108,9 +3111,6 @@ EXPORT_SYMBOL(__cpu_dying_mask);
> atomic_t __num_online_cpus __read_mostly;
> EXPORT_SYMBOL(__num_online_cpus);
>
> -unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> -EXPORT_SYMBOL(__num_possible_cpus);
> -
> void init_cpu_present(const struct cpumask *src)
> {
> cpumask_copy(&__cpu_present_mask, src);
>
Best regards
--
Marek Szyprowski, PhD
Samsung R&D Institute Poland
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 09/20] cpumask: Cache num_possible_cpus()
2025-11-22 16:24 ` Marek Szyprowski
@ 2025-11-22 19:09 ` Paul E. McKenney
0 siblings, 0 replies; 74+ messages in thread
From: Paul E. McKenney @ 2025-11-22 19:09 UTC (permalink / raw)
To: Marek Szyprowski
Cc: Thomas Gleixner, LKML, Peter Zijlstra, Gabriele Monaco,
Mathieu Desnoyers, Michael Jeanson, Jens Axboe, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde,
Nathan Chancellor
On Sat, Nov 22, 2025 at 05:24:22PM +0100, Marek Szyprowski wrote:
>
> On 22.11.2025 16:36, Thomas Gleixner wrote:
> > On Fri, Nov 21 2025 at 23:56, Marek Szyprowski wrote:
> >> Reverting it on top of linux-next fixes the issue. Let me know how can I
> >> help debugging it.
> > Can you test the fix below please?
>
> This fixes the observed issue. Thanks!
>
> Tested-by: Marek Szyprowski <m.szyprowski@samsung.com>
And I clearly should have read ahead, but thank you for the quick fix!
Tested-by: Paul E. McKenney <paulmck@kernel.org>
> > Thanks,
> >
> > tglx
> > ---
> > Subject: cpu: Initialize __num_possible_cpus correctly
> > From: Thomas Gleixner <tglx@linutronix.de>
> > Date: Sat, 22 Nov 2025 16:19:18 +0100
> >
> > The variable to cache the number of possible CPUs is initialized to NR_CPUS
> > at build time, but that's only correct when cpu_possible_mask is
> > initialized with CPU_BITS_ALL. That's only the case on PARISC.
> >
> > On x86 and some other architectures this does not matter because they
> > initialize cpu_possible_mask via init_cpu_possible() which does a proper
> > weight calculation. Though on architectures which do not, this results
> > in a completely wrong cached value 'NR_CPUS + actual possible CPUs'.
> >
> > Initialize it correctly to 0 when CONFIG_INIT_ALL_POSSIBLE=n and move the
> > NR_CPUS initialization into the PARISC specific section.
> >
> > Fixes: d0f23ccf6ba9 ("cpumask: Cache num_possible_cpus()")
> > Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
> > Reported-by: Nathan Chancellor <nathan@kernel.org>
> > Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> > Closes: https://lore.kernel.org/all/89c7106e-a431-443a-9527-3d5fbce77fe1@samsung.com
> > Closes: https://lore.kernel.org/all/20251122002755.GA2682494@ax162
> > ---
> > kernel/cpu.c | 6 +++---
> > 1 file changed, 3 insertions(+), 3 deletions(-)
> >
> > --- a/kernel/cpu.c
> > +++ b/kernel/cpu.c
> > @@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
> > #ifdef CONFIG_INIT_ALL_POSSIBLE
> > struct cpumask __cpu_possible_mask __ro_after_init
> > = {CPU_BITS_ALL};
> > +unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> > #else
> > struct cpumask __cpu_possible_mask __ro_after_init;
> > +unsigned int __num_possible_cpus __ro_after_init;
> > #endif
> > EXPORT_SYMBOL(__cpu_possible_mask);
> > +EXPORT_SYMBOL(__num_possible_cpus);
> >
> > struct cpumask __cpu_online_mask __read_mostly;
> > EXPORT_SYMBOL(__cpu_online_mask);
> > @@ -3108,9 +3111,6 @@ EXPORT_SYMBOL(__cpu_dying_mask);
> > atomic_t __num_online_cpus __read_mostly;
> > EXPORT_SYMBOL(__num_online_cpus);
> >
> > -unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> > -EXPORT_SYMBOL(__num_possible_cpus);
> > -
> > void init_cpu_present(const struct cpumask *src)
> > {
> > cpumask_copy(&__cpu_present_mask, src);
> >
> Best regards
> --
> Marek Szyprowski, PhD
> Samsung R&D Institute Poland
>
^ permalink raw reply [flat|nested] 74+ messages in thread
* [tip: core/rseq] cpu: Initialize __num_possible_cpus correctly
2025-11-22 15:36 ` Thomas Gleixner
2025-11-22 16:24 ` Marek Szyprowski
@ 2025-11-23 19:03 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-23 19:03 UTC (permalink / raw)
To: linux-tip-commits
Cc: Marek Szyprowski, Nathan Chancellor, Paul E. McKenney,
Thomas Gleixner, x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 21782b3a5cd40892cb2995aa1ec3e74dd1112f1d
Gitweb: https://git.kernel.org/tip/21782b3a5cd40892cb2995aa1ec3e74dd1112f1d
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Sat, 22 Nov 2025 16:19:18 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Sun, 23 Nov 2025 19:59:30 +01:00
cpu: Initialize __num_possible_cpus correctly
The variable to cache the number of possible CPUs is initialized to NR_CPUS
at build time, but that's only correct when cpu_possible_mask is
initialized with CPU_BITS_ALL. That's only the case on PARISC.
On x86 and some other architectures this does not matter because they
initialize cpu_possible_mask via init_cpu_possible() which does a proper
weight calculation. Though on architectures which do not, this results
in a completely wrong cached value 'NR_CPUS + actual possible CPUs'.
Initialize it correctly to 0 when CONFIG_INIT_ALL_POSSIBLE=n and move the
NR_CPUS initialization into the PARISC specific section.
Fixes: d0f23ccf6ba9 ("cpumask: Cache num_possible_cpus()")
Reported-by: Marek Szyprowski <m.szyprowski@samsung.com>
Reported-by: Nathan Chancellor <nathan@kernel.org>
Reported-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Marek Szyprowski <m.szyprowski@samsung.com>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Link: https://patch.msgid.link/87zf8ehyf7.ffs@tglx
Closes: https://lore.kernel.org/all/89c7106e-a431-443a-9527-3d5fbce77fe1@samsung.com
Closes: https://lore.kernel.org/all/20251122002755.GA2682494@ax162
Closes: https://lore.kernel.org/all/ad2a55f9-9bca-41bf-a6ec-efb23eaa778f@paulmck-laptop
---
kernel/cpu.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/kernel/cpu.c b/kernel/cpu.c
index 18e530d..b674fdf 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
#ifdef CONFIG_INIT_ALL_POSSIBLE
struct cpumask __cpu_possible_mask __ro_after_init
= {CPU_BITS_ALL};
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
#else
struct cpumask __cpu_possible_mask __ro_after_init;
+unsigned int __num_possible_cpus __ro_after_init;
#endif
EXPORT_SYMBOL(__cpu_possible_mask);
+EXPORT_SYMBOL(__num_possible_cpus);
struct cpumask __cpu_online_mask __read_mostly;
EXPORT_SYMBOL(__cpu_online_mask);
@@ -3108,9 +3111,6 @@ EXPORT_SYMBOL(__cpu_dying_mask);
atomic_t __num_online_cpus __read_mostly;
EXPORT_SYMBOL(__num_online_cpus);
-unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
-EXPORT_SYMBOL(__num_possible_cpus);
-
void init_cpu_present(const struct cpumask *src)
{
cpumask_copy(&__cpu_present_mask, src);
^ permalink raw reply related [flat|nested] 74+ messages in thread
* Re: [patch V5 09/20] cpumask: Cache num_possible_cpus()
2025-11-21 22:56 ` [patch V5 09/20] " Marek Szyprowski
2025-11-22 15:36 ` Thomas Gleixner
@ 2025-11-22 18:47 ` Paul E. McKenney
2025-11-22 19:10 ` Thomas Gleixner
1 sibling, 1 reply; 74+ messages in thread
From: Paul E. McKenney @ 2025-11-22 18:47 UTC (permalink / raw)
To: Marek Szyprowski
Cc: Thomas Gleixner, LKML, Peter Zijlstra, Gabriele Monaco,
Mathieu Desnoyers, Michael Jeanson, Jens Axboe, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
On Fri, Nov 21, 2025 at 11:56:44PM +0100, Marek Szyprowski wrote:
> Hi
>
> On 19.11.2025 18:27, Thomas Gleixner wrote:
> > Reevaluating num_possible_cpus() over and over does not make sense. That
> > becomes a constant after init as cpu_possible_mask is marked ro_after_init.
> >
> > Cache the value during initialization and provide that for consumption.
> >
> > Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> > Reviewed-by: Yury Norov <yury.norov@gmail.com>
> > Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
> > Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
>
> This patch landed recently in linux-next as commit d0f23ccf6ba9
> ("cpumask: Cache num_possible_cpus()"). I found that it triggers the
> following warning during boot on some of my test systems (namely
> Raspberry Pi 3 and 4):
I also bisected to this commit on an ARM Neoverse V2. It happens twice
at about the same time, so the splats are interleaved, but they start
with an almost-NULL pointer dereference during either TASKS01 or TRACE02
callback processing. Things go surprisingly normally after that, at
least until a hang on shutdown. This sometimes happens quite late, as in
*minutes* after boot.
I do not see this on x86, nor on any rcutorture scenarios other
than TASKS01 and TRACE02. These are unusual in that they are the
only variants of the common-code tasks-RCU flavors that build with
CONFIG_PROVE_LOCKING=y.
My reproducer is:
tools/testing/selftests/rcutorture/bin/torture.sh --duration 10 --do-none --do-rcutorture --configs-rcutorture "TRACE01 TRACE02"
My current guess is that the snapshot is taken too early, though I would
be more confident of that if it happened on TREE01, in which CPUs come
online quite late.
Thanx, Paul
> kvm [1]: nv: 568 coarse grained trap handlers
> kvm [1]: IPA Size Limit: 40 bits
> ------------[ cut here ]------------
> WARNING: CPU: 1 PID: 1 at arch/arm64/kvm/vmid.c:183
> kvm_arm_vmid_alloc_init+0x98/0xac
> Modules linked in:
> CPU: 1 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.18.0-rc4+ #16195 PREEMPT
> Hardware name: Raspberry Pi 3 Model B (DT)
> pstate: 20000005 (nzCv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> pc : kvm_arm_vmid_alloc_init+0x98/0xac
> lr : kvm_arm_vmid_alloc_init+0x1c/0xac
> ...
> Call trace:
> kvm_arm_vmid_alloc_init+0x98/0xac (P)
> kvm_arm_init+0x144/0x153c
> do_one_initcall+0x64/0x308
> kernel_init_freeable+0x280/0x4fc
> kernel_init+0x20/0x1d8
> ret_from_fork+0x10/0x20
> irq event stamp: 65606
> hardirqs last enabled at (65605): [<ffff8000801534e4>]
> __up_console_sem+0x6c/0x80
> hardirqs last disabled at (65606): [<ffff800081358f7c>] el1_brk64+0x20/0x60
> softirqs last enabled at (65262): [<ffff8000800c4b1c>]
> handle_softirqs+0x4c4/0x4dc
> softirqs last disabled at (65257): [<ffff8000800106a0>]
> __do_softirq+0x14/0x20
> ---[ end trace 0000000000000000 ]---
> kvm [1]: Hyp nVHE mode initialized successfully
>
>
> Reverting it on top of linux-next fixes the issue. Let me know how can I
> help debugging it.
>
>
> > ---
> > V4: Add comment why this is not marked __init ....
> > V2: New patch
> > ---
> > include/linux/cpumask.h | 10 ++++++++--
> > kernel/cpu.c | 19 +++++++++++++++++++
> > 2 files changed, 27 insertions(+), 2 deletions(-)
> > --- a/include/linux/cpumask.h
> > +++ b/include/linux/cpumask.h
> > @@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
> > #define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
> >
> > extern atomic_t __num_online_cpus;
> > +extern unsigned int __num_possible_cpus;
> >
> > extern cpumask_t cpus_booted_once_mask;
> >
> > @@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpum
> > #define __assign_cpu(cpu, mask, val) \
> > __assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
> >
> > -#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
> > #define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
> > #define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
> > #define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
> > #define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
> >
> > void set_cpu_online(unsigned int cpu, bool online);
> > +void set_cpu_possible(unsigned int cpu, bool possible);
> >
> > /**
> > * to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
> > @@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_
> > {
> > return raw_atomic_read(&__num_online_cpus);
> > }
> > -#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
> > +
> > +static __always_inline unsigned int num_possible_cpus(void)
> > +{
> > + return __num_possible_cpus;
> > +}
> > +
> > #define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
> > #define num_present_cpus() cpumask_weight(cpu_present_mask)
> > #define num_active_cpus() cpumask_weight(cpu_active_mask)
> > --- a/kernel/cpu.c
> > +++ b/kernel/cpu.c
> > @@ -3108,6 +3108,9 @@ EXPORT_SYMBOL(__cpu_dying_mask);
> > atomic_t __num_online_cpus __read_mostly;
> > EXPORT_SYMBOL(__num_online_cpus);
> >
> > +unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
> > +EXPORT_SYMBOL(__num_possible_cpus);
> > +
> > void init_cpu_present(const struct cpumask *src)
> > {
> > cpumask_copy(&__cpu_present_mask, src);
> > @@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpuma
> > void init_cpu_possible(const struct cpumask *src)
> > {
> > cpumask_copy(&__cpu_possible_mask, src);
> > + __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
> > }
> >
> > void set_cpu_online(unsigned int cpu, bool online)
> > @@ -3139,6 +3143,21 @@ void set_cpu_online(unsigned int cpu, bo
> > }
> > }
> >
> > +/*
> > + * This should be marked __init, but there is a boatload of call sites
> > + * which need to be fixed up to do so. Sigh...
> > + */
> > +void set_cpu_possible(unsigned int cpu, bool possible)
> > +{
> > + if (possible) {
> > + if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
> > + __num_possible_cpus++;
> > + } else {
> > + if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
> > + __num_possible_cpus--;
> > + }
> > +}
> > +
> > /*
> > * Activate the first processor.
> > */
> >
> >
> >
> Best regards
> --
> Marek Szyprowski, PhD
> Samsung R&D Institute Poland
>
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 09/20] cpumask: Cache num_possible_cpus()
2025-11-22 18:47 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Paul E. McKenney
@ 2025-11-22 19:10 ` Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-22 19:10 UTC (permalink / raw)
To: paulmck, Marek Szyprowski
Cc: LKML, Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Gautham R. Shenoy, Florian Weimer,
Tim Chen, Yury Norov, Shrikanth Hegde
On Sat, Nov 22 2025 at 10:47, Paul E. McKenney wrote:
> On Fri, Nov 21, 2025 at 11:56:44PM +0100, Marek Szyprowski wrote:
>> On 19.11.2025 18:27, Thomas Gleixner wrote:
> My current guess is that the snapshot is taken too early, though I would
> be more confident of that if it happened on TREE01, in which CPUs come
> online quite late.
https://lore.kernel.org/all/87zf8ehyf7.ffs@tglx
^ permalink raw reply [flat|nested] 74+ messages in thread
* Re: [patch V5 09/20] cpumask: Cache num_possible_cpus()
2025-11-19 17:27 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Thomas Gleixner
2025-11-20 11:20 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-21 22:56 ` [patch V5 09/20] " Marek Szyprowski
@ 2025-11-22 0:27 ` Nathan Chancellor
2025-11-26 4:36 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
3 siblings, 0 replies; 74+ messages in thread
From: Nathan Chancellor @ 2025-11-22 0:27 UTC (permalink / raw)
To: Thomas Gleixner
Cc: LKML, Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Hi Thomas,
On Wed, Nov 19, 2025 at 06:27:01PM +0100, Thomas Gleixner wrote:
> Reevaluating num_possible_cpus() over and over does not make sense. That
> becomes a constant after init as cpu_possible_mask is marked ro_after_init.
>
> Cache the value during initialization and provide that for consumption.
>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Reviewed-by: Yury Norov <yury.norov@gmail.com>
> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
> Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
After this change in -next as commit d0f23ccf6ba9 ("cpumask: Cache
num_possible_cpus()"), I am seeing boot failures on MIPS and PowerPC in
QEMU.
For MIPS using malta_defconfig:
$ make -skj"$(nproc)" ARCH=mips CROSS_COMPILE=mips-linux- clean malta_defconfig vmlinux
$ curl -LSs https://github.com/ClangBuiltLinux/boot-utils/releases/download/20241120-044434/mipsel-rootfs.cpio.zst | zstd -d >rootfs.cpio
$ qemu-system-mipsel \
-display none \
-nodefaults \
-cpu 24Kf \
-machine malta \
-kernel vmlinux \
-initrd rootfs.cpio \
-m 512m \
-serial mon:stdio
...
Linux version 6.18.0-rc4-00055-gd0f23ccf6ba9 (nathan@ax162) (mips-linux-gcc (GCC) 15.2.0, GNU ld (GNU Binutils) 2.45) #1 SMP Fri Nov 21 17:16:39 MST 2025
...
Kernel bug detected[#1]:
CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.18.0-rc4-00055-gd0f23ccf6ba9 #1 NONE
Hardware name: mti,malta
$ 0 : 00000000 00000001 00000001 00000008
$ 4 : 00000044 82090000 00000000 00000001
$ 8 : 00000004 808e1b98 80140000 80140000
$12 : 00000121 00000001 00000122 82090080
$16 : 00000000 80a17c7c 80a80000 80b10000
$20 : 00000000 00000000 00000000 00000000
$24 : 00000008 82095780
$28 : 82084000 82087db0 00000000 80122358
Hi : 51eb5fc3
Lo : 28f5a1df
epc : 80122360 mips_smp_ipi_allocate+0x200/0x214
ra : 80122358 mips_smp_ipi_allocate+0x1f8/0x214
Status: 1000a403 KERNEL EXL IE
Cause : 00800034 (ExcCode 0d)
PrId : 00019300 (MIPS 24Kc)
Modules linked in:
Process swapper/0 (pid: 1, threadinfo=(ptrval), task=(ptrval), tls=00000000)
Stack : 00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
00000000 9cc98073 80abb628 80a80000 00000000 80b10000 00000000 80abb650
8200e500 80abb628 80a80000 00000000 80b10000 80110144 8200e500 808f5ae0
...
Call Trace:
[<80122360>] mips_smp_ipi_allocate+0x200/0x214
[<80abb650>] _gp+0x10/0x50
[<80110144>] do_one_initcall+0x64/0x2d0
[<80ab4fc8>] kernel_init_freeable+0x100/0x2f0
[<808e3980>] kernel_init+0x24/0x118
[<80112258>] ret_from_kernel_thread+0x14/0x1c
Code: 9044ac0c 2c420002 38420001 <00020336> 1000ffd1 00000000 0c238c2c 00000000 27bdff80
---[ end trace 0000000000000000 ]---
...
For PowerPC using Debian's configuration:
$ curl -LSs https://github.com/nathanchance/llvm-kernel-testing/raw/61ffff42ede84576a9f8721d06765d8de36a912f/configs/debian/powerpc64le.config >.config
$ make -skj"$(nproc)" ARCH=powerpc CROSS_COMPILE=powerpc64-linux- olddefconfig zImage.epapr
$ curl -LSs https://github.com/ClangBuiltLinux/boot-utils/releases/download/20241120-044434/ppc64le-rootfs.cpio.zst | zstd -d >rootfs.cpio
$ qemu-system-ppc64 \
-display none \
-nodefaults \
-device ipmi-bmc-sim,id=bmc0 \
-device isa-ipmi-bt,bmc=bmc0,irq=10 \
-machine powernv \
-kernel arch/powerpc/boot/zImage.epapr \
-initrd rootfs.cpio \
-m 2G \
-serial mon:stdio
...
[ 0.000000] Linux version 6.18.0-rc4+ (nathan@ax162) (powerpc64-linux-gcc (GCC) 15.2.0, GNU ld (GNU Binutils) 2.45) #1 SMP PREEMPT Fri Nov 21 17:05:23 MST 2025
...
[ 0.615917] BUG: Kernel NULL pointer dereference on read at 0x00000130
[ 0.616333] Faulting instruction address: 0xc000000000328804
[ 0.616885] Oops: Kernel access of bad area, sig: 7 [#1]
[ 0.617185] LE PAGE_SIZE=4K MMU=Radix SMP NR_CPUS=2048 NUMA PowerNV
[ 0.617618] Modules linked in:
[ 0.618051] CPU: 0 UID: 0 PID: 23 Comm: rcu_tasks_kthre Not tainted 6.18.0-rc4+ #1 PREEMPTLAZY
[ 0.618557] Hardware name: IBM PowerNV (emulated by qemu) POWER10 0x801200 opal:v7.1-106-g785a5e307 PowerNV
[ 0.619137] NIP: c000000000328804 LR: c00000000032a9f0 CTR: 0000000000000000
[ 0.619478] REGS: c0000000034ffb00 TRAP: 0300 Not tainted (6.18.0-rc4+)
[ 0.619817] MSR: 9000000000009033 <SF,HV,EE,ME,IR,DR,RI,LE> CR: 84000884 XER: 20040000
[ 0.620396] CFAR: c00000000032a9ec DAR: 0000000000000130 DSISR: 00080000 IRQMASK: 0
[ 0.620396] GPR00: c00000000032a9ac c0000000034ffda0 c000000001cb7100 c000000002764430
[ 0.620396] GPR04: c00000007f33fb90 0000000000000000 1db5beac45697d00 c00000000344aa00
[ 0.620396] GPR08: c000000003238000 0000000000000008 0000000000000001 0000000000000002
[ 0.620396] GPR12: 0000000000000000 c000000002a7d000 c000000000269438 c00000000349bc40
[ 0.620396] GPR16: 0000000000000000 0000000000000000 c00000000290ea80 0000000000000000
[ 0.620396] GPR20: 0000000000000000 0000000000000000 0000000000000000 0000000000000000
[ 0.620396] GPR24: c00000000291e460 c000000000328270 c00000000291e460 c000000001779218
[ 0.620396] GPR28: c000000002764430 0000000000000000 c00000007f33fb90 c0000000034ffdd0
[ 0.623797] NIP [c000000000328804] rcu_tasks_invoke_cbs+0x84/0x320
[ 0.624497] LR [c00000000032a9f0] rcu_tasks_one_gp+0x450/0x680
[ 0.624851] Call Trace:
[ 0.625036] [c0000000034ffda0] [c00000000290ea80] jiffies+0x0/0x80 (unreliable)
[ 0.625517] [c0000000034ffe20] [c00000000032a9ac] rcu_tasks_one_gp+0x40c/0x680
[ 0.625869] [c0000000034fff20] [c00000000032aebc] rcu_tasks_kthread+0x10c/0x140
[ 0.626294] [c0000000034fff90] [c0000000002695bc] kthread+0x18c/0x1a0
[ 0.626608] [c0000000034fffe0] [c00000000000de58] start_kernel_thread+0x14/0x18
[ 0.627031] Code: f9210030 fbe10038 f9210040 57bd083c 393d0001 7c095040 40800028 e90300c0 7d2907b4 814300d0 79291f24 7ca8482a <81250130> 7c2004ac 7c095000 418001a0
[ 0.628259] ---[ end trace 0000000000000000 ]---
...
At the parent change, neither of these problems exist. If there is any
other information I can provide or patches I can test, please let me
know.
Cheers,
Nathan
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] cpumask: Cache num_possible_cpus()
2025-11-19 17:27 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Thomas Gleixner
` (2 preceding siblings ...)
2025-11-22 0:27 ` Nathan Chancellor
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
3 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Yury Norov,
Mathieu Desnoyers, Shrikanth Hegde, x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 35a5c37cb9f1f947dff18e7cfc75a8cfcfd557ca
Gitweb: https://git.kernel.org/tip/35a5c37cb9f1f947dff18e7cfc75a8cfcfd557ca
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:01 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
cpumask: Cache num_possible_cpus()
Reevaluating num_possible_cpus() over and over does not make sense. That
becomes a constant after init as cpu_possible_mask is marked ro_after_init.
Cache the value during initialization and provide that for consumption.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Yury Norov <yury.norov@gmail.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Reviewed-by: Shrikanth Hegde <sshegde@linux.ibm.com>
Link: https://patch.msgid.link/20251119172549.578653738@linutronix.de
---
include/linux/cpumask.h | 10 ++++++++--
kernel/cpu.c | 19 +++++++++++++++++++
2 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h
index feba06e..66694ee 100644
--- a/include/linux/cpumask.h
+++ b/include/linux/cpumask.h
@@ -126,6 +126,7 @@ extern struct cpumask __cpu_dying_mask;
#define cpu_dying_mask ((const struct cpumask *)&__cpu_dying_mask)
extern atomic_t __num_online_cpus;
+extern unsigned int __num_possible_cpus;
extern cpumask_t cpus_booted_once_mask;
@@ -1152,13 +1153,13 @@ void init_cpu_possible(const struct cpumask *src);
#define __assign_cpu(cpu, mask, val) \
__assign_bit(cpumask_check(cpu), cpumask_bits(mask), (val))
-#define set_cpu_possible(cpu, possible) assign_cpu((cpu), &__cpu_possible_mask, (possible))
#define set_cpu_enabled(cpu, enabled) assign_cpu((cpu), &__cpu_enabled_mask, (enabled))
#define set_cpu_present(cpu, present) assign_cpu((cpu), &__cpu_present_mask, (present))
#define set_cpu_active(cpu, active) assign_cpu((cpu), &__cpu_active_mask, (active))
#define set_cpu_dying(cpu, dying) assign_cpu((cpu), &__cpu_dying_mask, (dying))
void set_cpu_online(unsigned int cpu, bool online);
+void set_cpu_possible(unsigned int cpu, bool possible);
/**
* to_cpumask - convert a NR_CPUS bitmap to a struct cpumask *
@@ -1211,7 +1212,12 @@ static __always_inline unsigned int num_online_cpus(void)
{
return raw_atomic_read(&__num_online_cpus);
}
-#define num_possible_cpus() cpumask_weight(cpu_possible_mask)
+
+static __always_inline unsigned int num_possible_cpus(void)
+{
+ return __num_possible_cpus;
+}
+
#define num_enabled_cpus() cpumask_weight(cpu_enabled_mask)
#define num_present_cpus() cpumask_weight(cpu_present_mask)
#define num_active_cpus() cpumask_weight(cpu_active_mask)
diff --git a/kernel/cpu.c b/kernel/cpu.c
index db9f6c5..b674fdf 100644
--- a/kernel/cpu.c
+++ b/kernel/cpu.c
@@ -3085,10 +3085,13 @@ EXPORT_SYMBOL(cpu_all_bits);
#ifdef CONFIG_INIT_ALL_POSSIBLE
struct cpumask __cpu_possible_mask __ro_after_init
= {CPU_BITS_ALL};
+unsigned int __num_possible_cpus __ro_after_init = NR_CPUS;
#else
struct cpumask __cpu_possible_mask __ro_after_init;
+unsigned int __num_possible_cpus __ro_after_init;
#endif
EXPORT_SYMBOL(__cpu_possible_mask);
+EXPORT_SYMBOL(__num_possible_cpus);
struct cpumask __cpu_online_mask __read_mostly;
EXPORT_SYMBOL(__cpu_online_mask);
@@ -3116,6 +3119,7 @@ void init_cpu_present(const struct cpumask *src)
void init_cpu_possible(const struct cpumask *src)
{
cpumask_copy(&__cpu_possible_mask, src);
+ __num_possible_cpus = cpumask_weight(&__cpu_possible_mask);
}
void set_cpu_online(unsigned int cpu, bool online)
@@ -3140,6 +3144,21 @@ void set_cpu_online(unsigned int cpu, bool online)
}
/*
+ * This should be marked __init, but there is a boatload of call sites
+ * which need to be fixed up to do so. Sigh...
+ */
+void set_cpu_possible(unsigned int cpu, bool possible)
+{
+ if (possible) {
+ if (!cpumask_test_and_set_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus++;
+ } else {
+ if (cpumask_test_and_clear_cpu(cpu, &__cpu_possible_mask))
+ __num_possible_cpus--;
+ }
+}
+
+/*
* Activate the first processor.
*/
void __init boot_cpu_init(void)
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 10/20] sched/mmcid: Convert mm CID mask to a bitmap
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (8 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 09/20] cpumask: Cache num_possible_cpus() Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 11/20] signal: Move MMCID exit out of sighand lock Thomas Gleixner
` (10 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
This is truly a bitmap and just conveniently uses a cpumask because the
maximum size of the bitmap is nr_cpu_ids.
But that prevents to do searches for a zero bit in a limited range, which
is helpful to provide an efficient mechanism to consolidate the CID space
when the number of users decreases.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
---
V4: Allocate bitmap_size(nr_possible_cpus()) - Mathieu
V4: Zero the bitmap with nrbits=nr_possible_cpus() - Brown paperbag
V3: Zero the bitmap with length of bitmap_size(nr_possible_cpus()) - Shrikanth
---
include/linux/mm_types.h | 9 +++++----
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 6 +++---
3 files changed, 9 insertions(+), 8 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1342,13 +1342,13 @@ static inline cpumask_t *mm_cpus_allowed
}
/* Accessor for struct mm_struct's cidmask. */
-static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
+static inline unsigned long *mm_cidmask(struct mm_struct *mm)
{
unsigned long cid_bitmap = (unsigned long)mm_cpus_allowed(mm);
/* Skip mm_cpus_allowed */
cid_bitmap += cpumask_size();
- return (struct cpumask *)cid_bitmap;
+ return (unsigned long *)cid_bitmap;
}
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
@@ -1363,7 +1363,7 @@ static inline void mm_init_cid(struct mm
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- cpumask_clear(mm_cidmask(mm));
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
@@ -1384,7 +1384,8 @@ static inline void mm_destroy_cid(struct
static inline unsigned int mm_cid_size(void)
{
- return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
+ /* mm_cpus_allowed(), mm_cidmask(). */
+ return cpumask_size() + bitmap_size(num_possible_cpus());
}
#else /* CONFIG_SCHED_MM_CID */
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10402,7 +10402,7 @@ void sched_mm_cid_exit_signals(struct ta
guard(preempt)();
t->mm_cid.active = 0;
if (t->mm_cid.cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
}
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3559,7 +3559,7 @@ static inline bool __mm_cid_get(struct t
if (cid >= max_cids)
return false;
- if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
return false;
t->mm_cid.cid = t->mm_cid.last_cid = cid;
__this_cpu_write(mm->mm_cid.pcpu->cid, cid);
@@ -3582,7 +3582,7 @@ static inline bool mm_cid_get(struct tas
return true;
/* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
+ return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
}
static inline void mm_cid_select(struct task_struct *t)
@@ -3603,7 +3603,7 @@ static inline void switch_mm_cid(struct
{
if (prev->mm_cid.active) {
if (prev->mm_cid.cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
prev->mm_cid.cid = MM_CID_UNSET;
}
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Convert mm CID mask to a bitmap
2025-11-19 17:27 ` [patch V5 10/20] sched/mmcid: Convert mm CID mask to a bitmap Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers,
Yury Norov (NVIDIA), x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 472931e757fb3dfad1f78ce6f5abd821155433b2
Gitweb: https://git.kernel.org/tip/472931e757fb3dfad1f78ce6f5abd821155433b2
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:03 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:55 +01:00
sched/mmcid: Convert mm CID mask to a bitmap
This is truly a bitmap and just conveniently uses a cpumask because the
maximum size of the bitmap is nr_cpu_ids.
But that prevents to do searches for a zero bit in a limited range, which
is helpful to provide an efficient mechanism to consolidate the CID space
when the number of users decreases.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Link: https://patch.msgid.link/20251119172549.642866767@linutronix.de
---
include/linux/mm_types.h | 9 +++++----
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 6 +++---
3 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 67a7bdf..bafb81b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1342,13 +1342,13 @@ static inline cpumask_t *mm_cpus_allowed(struct mm_struct *mm)
}
/* Accessor for struct mm_struct's cidmask. */
-static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
+static inline unsigned long *mm_cidmask(struct mm_struct *mm)
{
unsigned long cid_bitmap = (unsigned long)mm_cpus_allowed(mm);
/* Skip mm_cpus_allowed */
cid_bitmap += cpumask_size();
- return (struct cpumask *)cid_bitmap;
+ return (unsigned long *)cid_bitmap;
}
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
@@ -1363,7 +1363,7 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- cpumask_clear(mm_cidmask(mm));
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
@@ -1384,7 +1384,8 @@ static inline void mm_destroy_cid(struct mm_struct *mm)
static inline unsigned int mm_cid_size(void)
{
- return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
+ /* mm_cpus_allowed(), mm_cidmask(). */
+ return cpumask_size() + bitmap_size(num_possible_cpus());
}
#else /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f6bbfa1..9a114b6 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10402,7 +10402,7 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
guard(preempt)();
t->mm_cid.active = 0;
if (t->mm_cid.cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index a17f04f..31f2e43 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3559,7 +3559,7 @@ static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigne
if (cid >= max_cids)
return false;
- if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
return false;
t->mm_cid.cid = t->mm_cid.last_cid = cid;
__this_cpu_write(mm->mm_cid.pcpu->cid, cid);
@@ -3582,7 +3582,7 @@ static inline bool mm_cid_get(struct task_struct *t)
return true;
/* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
+ return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
}
static inline void mm_cid_select(struct task_struct *t)
@@ -3603,7 +3603,7 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
{
if (prev->mm_cid.active) {
if (prev->mm_cid.cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
prev->mm_cid.cid = MM_CID_UNSET;
}
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Convert mm CID mask to a bitmap
2025-11-19 17:27 ` [patch V5 10/20] sched/mmcid: Convert mm CID mask to a bitmap Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers,
Yury Norov (NVIDIA), x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 539115f08cf850b9fdc6526b31da0839ff6c1631
Gitweb: https://git.kernel.org/tip/539115f08cf850b9fdc6526b31da0839ff6c1631
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:03 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
sched/mmcid: Convert mm CID mask to a bitmap
This is truly a bitmap and just conveniently uses a cpumask because the
maximum size of the bitmap is nr_cpu_ids.
But that prevents to do searches for a zero bit in a limited range, which
is helpful to provide an efficient mechanism to consolidate the CID space
when the number of users decreases.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Acked-by: Yury Norov (NVIDIA) <yury.norov@gmail.com>
Link: https://patch.msgid.link/20251119172549.642866767@linutronix.de
---
include/linux/mm_types.h | 9 +++++----
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 6 +++---
3 files changed, 9 insertions(+), 8 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 67a7bdf..bafb81b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1342,13 +1342,13 @@ static inline cpumask_t *mm_cpus_allowed(struct mm_struct *mm)
}
/* Accessor for struct mm_struct's cidmask. */
-static inline cpumask_t *mm_cidmask(struct mm_struct *mm)
+static inline unsigned long *mm_cidmask(struct mm_struct *mm)
{
unsigned long cid_bitmap = (unsigned long)mm_cpus_allowed(mm);
/* Skip mm_cpus_allowed */
cid_bitmap += cpumask_size();
- return (struct cpumask *)cid_bitmap;
+ return (unsigned long *)cid_bitmap;
}
static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
@@ -1363,7 +1363,7 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- cpumask_clear(mm_cidmask(mm));
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
@@ -1384,7 +1384,8 @@ static inline void mm_destroy_cid(struct mm_struct *mm)
static inline unsigned int mm_cid_size(void)
{
- return 2 * cpumask_size(); /* mm_cpus_allowed(), mm_cidmask(). */
+ /* mm_cpus_allowed(), mm_cidmask(). */
+ return cpumask_size() + bitmap_size(num_possible_cpus());
}
#else /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f6bbfa1..9a114b6 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10402,7 +10402,7 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
guard(preempt)();
t->mm_cid.active = 0;
if (t->mm_cid.cid != MM_CID_UNSET) {
- cpumask_clear_cpu(t->mm_cid.cid, mm_cidmask(mm));
+ clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
}
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index a17f04f..31f2e43 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3559,7 +3559,7 @@ static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigne
if (cid >= max_cids)
return false;
- if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
return false;
t->mm_cid.cid = t->mm_cid.last_cid = cid;
__this_cpu_write(mm->mm_cid.pcpu->cid, cid);
@@ -3582,7 +3582,7 @@ static inline bool mm_cid_get(struct task_struct *t)
return true;
/* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
+ return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
}
static inline void mm_cid_select(struct task_struct *t)
@@ -3603,7 +3603,7 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
{
if (prev->mm_cid.active) {
if (prev->mm_cid.cid != MM_CID_UNSET)
- cpumask_clear_cpu(prev->mm_cid.cid, mm_cidmask(prev->mm));
+ clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
prev->mm_cid.cid = MM_CID_UNSET;
}
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 11/20] signal: Move MMCID exit out of sighand lock
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (9 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 10/20] sched/mmcid: Convert mm CID mask to a bitmap Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 12/20] sched/mmcid: Move initialization out of line Thomas Gleixner
` (9 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
There is no need anymore to keep this under sighand lock as the current
code and the upcoming replacement are not depending on the exit state of a
task anymore.
That allows to use a mutex in the exit path.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/sched.h | 4 ++--
kernel/exit.c | 1 +
kernel/sched/core.c | 4 ++--
kernel/signal.c | 2 --
4 files changed, 5 insertions(+), 6 deletions(-)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2298,7 +2298,7 @@ static __always_inline void alloc_tag_re
void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
-void sched_mm_cid_exit_signals(struct task_struct *t);
+void sched_mm_cid_exit(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
return t->mm_cid.cid;
@@ -2307,7 +2307,7 @@ static inline int task_mm_cid(struct tas
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
-static inline void sched_mm_cid_exit_signals(struct task_struct *t) { }
+static inline void sched_mm_cid_exit(struct task_struct *t) { }
static inline int task_mm_cid(struct task_struct *t)
{
/*
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -910,6 +910,7 @@ void __noreturn do_exit(long code)
user_events_exit(tsk);
io_uring_files_cancel();
+ sched_mm_cid_exit(tsk);
exit_signals(tsk); /* sets PF_EXITING */
seccomp_filter_release(tsk);
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10392,7 +10392,7 @@ static inline void mm_update_cpus_allowe
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
-void sched_mm_cid_exit_signals(struct task_struct *t)
+void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10410,7 +10410,7 @@ void sched_mm_cid_exit_signals(struct ta
/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- sched_mm_cid_exit_signals(t);
+ sched_mm_cid_exit(t);
}
/* Reactivate MM CID after successful execve() */
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -3125,7 +3125,6 @@ void exit_signals(struct task_struct *ts
cgroup_threadgroup_change_begin(tsk);
if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
return;
@@ -3136,7 +3135,6 @@ void exit_signals(struct task_struct *ts
* From now this task is not visible for group-wide signals,
* see wants_signal(), do_signal_stop().
*/
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] signal: Move MMCID exit out of sighand lock
2025-11-19 17:27 ` [patch V5 11/20] signal: Move MMCID exit out of sighand lock Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 1497ac189541df3c7c344bca723a7597009a99f4
Gitweb: https://git.kernel.org/tip/1497ac189541df3c7c344bca723a7597009a99f4
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:05 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:55 +01:00
signal: Move MMCID exit out of sighand lock
There is no need anymore to keep this under sighand lock as the current
code and the upcoming replacement are not depending on the exit state of a
task anymore.
That allows to use a mutex in the exit path.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.706439391@linutronix.de
---
include/linux/sched.h | 4 ++--
kernel/exit.c | 1 +
kernel/sched/core.c | 4 ++--
kernel/signal.c | 2 --
4 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 64f080d..c411ae0 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2298,7 +2298,7 @@ static __always_inline void alloc_tag_restore(struct alloc_tag *tag, struct allo
void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
-void sched_mm_cid_exit_signals(struct task_struct *t);
+void sched_mm_cid_exit(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
return t->mm_cid.cid;
@@ -2307,7 +2307,7 @@ static inline int task_mm_cid(struct task_struct *t)
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
-static inline void sched_mm_cid_exit_signals(struct task_struct *t) { }
+static inline void sched_mm_cid_exit(struct task_struct *t) { }
static inline int task_mm_cid(struct task_struct *t)
{
/*
diff --git a/kernel/exit.c b/kernel/exit.c
index 9f74e8f..324616f 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -910,6 +910,7 @@ void __noreturn do_exit(long code)
user_events_exit(tsk);
io_uring_files_cancel();
+ sched_mm_cid_exit(tsk);
exit_signals(tsk); /* sets PF_EXITING */
seccomp_filter_release(tsk);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9a114b6..3fdf90a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10392,7 +10392,7 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
-void sched_mm_cid_exit_signals(struct task_struct *t)
+void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10410,7 +10410,7 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- sched_mm_cid_exit_signals(t);
+ sched_mm_cid_exit(t);
}
/* Reactivate MM CID after successful execve() */
diff --git a/kernel/signal.c b/kernel/signal.c
index fe9190d..e42b8bd 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -3125,7 +3125,6 @@ void exit_signals(struct task_struct *tsk)
cgroup_threadgroup_change_begin(tsk);
if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
return;
@@ -3136,7 +3135,6 @@ void exit_signals(struct task_struct *tsk)
* From now this task is not visible for group-wide signals,
* see wants_signal(), do_signal_stop().
*/
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] signal: Move MMCID exit out of sighand lock
2025-11-19 17:27 ` [patch V5 11/20] signal: Move MMCID exit out of sighand lock Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 2b1642b881088bbf73fcb1147c474a198ec46729
Gitweb: https://git.kernel.org/tip/2b1642b881088bbf73fcb1147c474a198ec46729
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:05 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
signal: Move MMCID exit out of sighand lock
There is no need anymore to keep this under sighand lock as the current
code and the upcoming replacement are not depending on the exit state of a
task anymore.
That allows to use a mutex in the exit path.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.706439391@linutronix.de
---
include/linux/sched.h | 4 ++--
kernel/exit.c | 1 +
kernel/sched/core.c | 4 ++--
kernel/signal.c | 2 --
4 files changed, 5 insertions(+), 6 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index 64f080d..c411ae0 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2298,7 +2298,7 @@ static __always_inline void alloc_tag_restore(struct alloc_tag *tag, struct allo
void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
-void sched_mm_cid_exit_signals(struct task_struct *t);
+void sched_mm_cid_exit(struct task_struct *t);
static inline int task_mm_cid(struct task_struct *t)
{
return t->mm_cid.cid;
@@ -2307,7 +2307,7 @@ static inline int task_mm_cid(struct task_struct *t)
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
-static inline void sched_mm_cid_exit_signals(struct task_struct *t) { }
+static inline void sched_mm_cid_exit(struct task_struct *t) { }
static inline int task_mm_cid(struct task_struct *t)
{
/*
diff --git a/kernel/exit.c b/kernel/exit.c
index 9f74e8f..324616f 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -910,6 +910,7 @@ void __noreturn do_exit(long code)
user_events_exit(tsk);
io_uring_files_cancel();
+ sched_mm_cid_exit(tsk);
exit_signals(tsk); /* sets PF_EXITING */
seccomp_filter_release(tsk);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9a114b6..3fdf90a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10392,7 +10392,7 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
}
-void sched_mm_cid_exit_signals(struct task_struct *t)
+void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10410,7 +10410,7 @@ void sched_mm_cid_exit_signals(struct task_struct *t)
/* Deactivate MM CID allocation across execve() */
void sched_mm_cid_before_execve(struct task_struct *t)
{
- sched_mm_cid_exit_signals(t);
+ sched_mm_cid_exit(t);
}
/* Reactivate MM CID after successful execve() */
diff --git a/kernel/signal.c b/kernel/signal.c
index fe9190d..e42b8bd 100644
--- a/kernel/signal.c
+++ b/kernel/signal.c
@@ -3125,7 +3125,6 @@ void exit_signals(struct task_struct *tsk)
cgroup_threadgroup_change_begin(tsk);
if (thread_group_empty(tsk) || (tsk->signal->flags & SIGNAL_GROUP_EXIT)) {
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
return;
@@ -3136,7 +3135,6 @@ void exit_signals(struct task_struct *tsk)
* From now this task is not visible for group-wide signals,
* see wants_signal(), do_signal_stop().
*/
- sched_mm_cid_exit_signals(tsk);
tsk->flags |= PF_EXITING;
cgroup_threadgroup_change_end(tsk);
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 12/20] sched/mmcid: Move initialization out of line
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (10 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 11/20] signal: Move MMCID exit out of sighand lock Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 13/20] sched/mmcid: Provide precomputed maximal value Thomas Gleixner
` (8 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
It's getting bigger soon, so just move it out of line to the rest of the
code.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/mm_types.h | 15 +--------------
kernel/sched/core.c | 14 ++++++++++++++
2 files changed, 15 insertions(+), 14 deletions(-)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1351,20 +1351,7 @@ static inline unsigned long *mm_cidmask(
return (unsigned long *)cid_bitmap;
}
-static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
-{
- int i;
-
- for_each_possible_cpu(i) {
- struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
-
- pcpu->cid = MM_CID_UNSET;
- }
- mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->mm_cid.lock);
- cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- bitmap_zero(mm_cidmask(mm), num_possible_cpus());
-}
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p);
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10431,6 +10431,20 @@ void sched_mm_cid_fork(struct task_struc
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
+ struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
+ cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
+}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Move initialization out of line
2025-11-19 17:27 ` [patch V5 12/20] sched/mmcid: Move initialization out of line Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 61c7810a91fb15cffaa0b44810b8074c5e8a125d
Gitweb: https://git.kernel.org/tip/61c7810a91fb15cffaa0b44810b8074c5e8a125d
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:07 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:55 +01:00
sched/mmcid: Move initialization out of line
It's getting bigger soon, so just move it out of line to the rest of the
code.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.769636491@linutronix.de
---
include/linux/mm_types.h | 15 +--------------
kernel/sched/core.c | 14 ++++++++++++++
2 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index bafb81b..3b7d05e 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1351,20 +1351,7 @@ static inline unsigned long *mm_cidmask(struct mm_struct *mm)
return (unsigned long *)cid_bitmap;
}
-static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
-{
- int i;
-
- for_each_possible_cpu(i) {
- struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
-
- pcpu->cid = MM_CID_UNSET;
- }
- mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->mm_cid.lock);
- cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- bitmap_zero(mm_cidmask(mm), num_possible_cpus());
-}
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p);
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3fdf90a..34b6c31 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10431,6 +10431,20 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
+ struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
+ cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
+}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Move initialization out of line
2025-11-19 17:27 ` [patch V5 12/20] sched/mmcid: Move initialization out of line Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: bf070520e398679cd582b3c3e44107bf22c143ba
Gitweb: https://git.kernel.org/tip/bf070520e398679cd582b3c3e44107bf22c143ba
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:07 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
sched/mmcid: Move initialization out of line
It's getting bigger soon, so just move it out of line to the rest of the
code.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.769636491@linutronix.de
---
include/linux/mm_types.h | 15 +--------------
kernel/sched/core.c | 14 ++++++++++++++
2 files changed, 15 insertions(+), 14 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index bafb81b..3b7d05e 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -1351,20 +1351,7 @@ static inline unsigned long *mm_cidmask(struct mm_struct *mm)
return (unsigned long *)cid_bitmap;
}
-static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
-{
- int i;
-
- for_each_possible_cpu(i) {
- struct mm_cid_pcpu *pcpu = per_cpu_ptr(mm->mm_cid.pcpu, i);
-
- pcpu->cid = MM_CID_UNSET;
- }
- mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
- raw_spin_lock_init(&mm->mm_cid.lock);
- cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
- bitmap_zero(mm_cidmask(mm), num_possible_cpus());
-}
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p);
static inline int mm_alloc_cid_noprof(struct mm_struct *mm, struct task_struct *p)
{
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3fdf90a..34b6c31 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10431,6 +10431,20 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
t->mm_cid.active = 1;
}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
+ struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+
+ mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ raw_spin_lock_init(&mm->mm_cid.lock);
+ cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
+ bitmap_zero(mm_cidmask(mm), num_possible_cpus());
+}
#else /* CONFIG_SCHED_MM_CID */
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 13/20] sched/mmcid: Provide precomputed maximal value
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (11 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 12/20] sched/mmcid: Move initialization out of line Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 14/20] sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex Thomas Gleixner
` (7 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Reading mm::mm_users and mm:::mm_cid::nr_cpus_allowed every time to compute
the maximal CID value is just wasteful as that value is only changing on
fork(), exit() and eventually when the affinity changes.
So it can be easily precomputed at those points and provided in mm::mm_cid
for consumption in the hot path.
But there is an issue with using mm::mm_users for accounting because that
does not necessarily reflect the number of user space tasks as other kernel
code can take temporary references on the MM which skew the picture.
Solve that by adding a users counter to struct mm_mm_cid, which is modified
by fork() and exit() and used for precomputing under mm_mm_cid::lock.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq_types.h | 6 ++++
kernel/fork.c | 1
kernel/sched/core.c | 59 ++++++++++++++++++++++++++++++++-------------
kernel/sched/sched.h | 3 --
4 files changed, 50 insertions(+), 19 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -117,14 +117,20 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @max_cids: The exclusive maximum CID value for allocation and convergence
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
+ * @users: The number of tasks sharing this MM. Separate from mm::mm_users
+ * as that is modified by mmget()/mm_put() by other entities which
+ * do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int max_cids;
unsigned int nr_cpus_allowed;
+ unsigned int users;
raw_spinlock_t lock;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2455,6 +2455,7 @@ static bool need_futex_hash_allocate_def
exit_task_namespaces(p);
bad_fork_cleanup_mm:
if (p->mm) {
+ sched_mm_cid_exit(p);
mm_clear_owner(p->mm, p);
mmput(p->mm);
}
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4485,7 +4485,6 @@ static void __sched_fork(u64 clone_flags
init_numa_balancing(clone_flags, p);
p->wake_entry.u_flags = CSD_TYPE_TTWU;
p->migration_pending = NULL;
- init_sched_mm_cid(p);
}
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -10371,15 +10370,27 @@ void call_trace_sched_update_nr_running(
#ifdef CONFIG_SCHED_MM_CID
/*
- * When a task exits, the MM CID held by the task is not longer required as
- * the task cannot return to user space.
+ * Update the CID range properties when the constraints change. Invoked via
+ * fork(), exit() and affinity changes
*/
+static void mm_update_max_cids(struct mm_struct *mm)
+{
+ struct mm_mm_cid *mc = &mm->mm_cid;
+ unsigned int max_cids;
+
+ lockdep_assert_held(&mm->mm_cid.lock);
+
+ /* Calculate the new maximum constraint */
+ max_cids = min(mc->nr_cpus_allowed, mc->users);
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
unsigned int weight;
- if (!mm)
+ if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
/*
@@ -10389,9 +10400,30 @@ static inline void mm_update_cpus_allowe
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ if (weight == mm->mm_cid.nr_cpus_allowed)
+ return;
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
+ mm_update_max_cids(mm);
+}
+
+void sched_mm_cid_fork(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
+ mm_update_max_cids(mm);
}
+/*
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
+ */
void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10399,12 +10431,14 @@ void sched_mm_cid_exit(struct task_struc
if (!mm || !t->mm_cid.active)
return;
- guard(preempt)();
+ guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
+ mm->mm_cid.users--;
if (t->mm_cid.cid != MM_CID_UNSET) {
clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
+ mm_update_max_cids(mm);
}
/* Deactivate MM CID allocation across execve() */
@@ -10416,22 +10450,11 @@ void sched_mm_cid_before_execve(struct t
/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
-
- if (!mm)
- return;
-
+ sched_mm_cid_fork(t);
guard(preempt)();
- t->mm_cid.active = 1;
mm_cid_select(t);
}
-void sched_mm_cid_fork(struct task_struct *t)
-{
- WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
- t->mm_cid.active = 1;
-}
-
void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
{
struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
@@ -10440,7 +10463,9 @@ void mm_init_cid(struct mm_struct *mm, s
for_each_possible_cpu(cpu)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ mm->mm_cid.max_cids = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3571,7 +3571,7 @@ static inline bool mm_cid_get(struct tas
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
/* Try to reuse the last CID of this task */
if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
@@ -3614,7 +3614,6 @@ static inline void switch_mm_cid(struct
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void init_sched_mm_cid(struct task_struct *t) { }
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Provide precomputed maximal value
2025-11-19 17:27 ` [patch V5 13/20] sched/mmcid: Provide precomputed maximal value Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 3c7a7327102ca35b3081d1a7dde5819d0f5a5e08
Gitweb: https://git.kernel.org/tip/3c7a7327102ca35b3081d1a7dde5819d0f5a5e08
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:09 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:56 +01:00
sched/mmcid: Provide precomputed maximal value
Reading mm::mm_users and mm:::mm_cid::nr_cpus_allowed every time to compute
the maximal CID value is just wasteful as that value is only changing on
fork(), exit() and eventually when the affinity changes.
So it can be easily precomputed at those points and provided in mm::mm_cid
for consumption in the hot path.
But there is an issue with using mm::mm_users for accounting because that
does not necessarily reflect the number of user space tasks as other kernel
code can take temporary references on the MM which skew the picture.
Solve that by adding a users counter to struct mm_mm_cid, which is modified
by fork() and exit() and used for precomputing under mm_mm_cid::lock.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.832764634@linutronix.de
---
include/linux/rseq_types.h | 6 ++++-
kernel/fork.c | 1 +-
kernel/sched/core.c | 59 ++++++++++++++++++++++++++-----------
kernel/sched/sched.h | 3 +--
4 files changed, 50 insertions(+), 19 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index d7e8071..0fab369 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -117,14 +117,20 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @max_cids: The exclusive maximum CID value for allocation and convergence
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
+ * @users: The number of tasks sharing this MM. Separate from mm::mm_users
+ * as that is modified by mmget()/mm_put() by other entities which
+ * do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int max_cids;
unsigned int nr_cpus_allowed;
+ unsigned int users;
raw_spinlock_t lock;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/fork.c b/kernel/fork.c
index 74bc7c9..6c23219 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2455,6 +2455,7 @@ bad_fork_cleanup_namespaces:
exit_task_namespaces(p);
bad_fork_cleanup_mm:
if (p->mm) {
+ sched_mm_cid_exit(p);
mm_clear_owner(p->mm, p);
mmput(p->mm);
}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 34b6c31..f9295c4 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4485,7 +4485,6 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
init_numa_balancing(clone_flags, p);
p->wake_entry.u_flags = CSD_TYPE_TTWU;
p->migration_pending = NULL;
- init_sched_mm_cid(p);
}
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -10371,15 +10370,27 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
#ifdef CONFIG_SCHED_MM_CID
/*
- * When a task exits, the MM CID held by the task is not longer required as
- * the task cannot return to user space.
+ * Update the CID range properties when the constraints change. Invoked via
+ * fork(), exit() and affinity changes
*/
+static void mm_update_max_cids(struct mm_struct *mm)
+{
+ struct mm_mm_cid *mc = &mm->mm_cid;
+ unsigned int max_cids;
+
+ lockdep_assert_held(&mm->mm_cid.lock);
+
+ /* Calculate the new maximum constraint */
+ max_cids = min(mc->nr_cpus_allowed, mc->users);
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
unsigned int weight;
- if (!mm)
+ if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
/*
@@ -10389,9 +10400,30 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ if (weight == mm->mm_cid.nr_cpus_allowed)
+ return;
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
+ mm_update_max_cids(mm);
+}
+
+void sched_mm_cid_fork(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
+ mm_update_max_cids(mm);
}
+/*
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
+ */
void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10399,12 +10431,14 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
- guard(preempt)();
+ guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
+ mm->mm_cid.users--;
if (t->mm_cid.cid != MM_CID_UNSET) {
clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
+ mm_update_max_cids(mm);
}
/* Deactivate MM CID allocation across execve() */
@@ -10416,22 +10450,11 @@ void sched_mm_cid_before_execve(struct task_struct *t)
/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
-
- if (!mm)
- return;
-
+ sched_mm_cid_fork(t);
guard(preempt)();
- t->mm_cid.active = 1;
mm_cid_select(t);
}
-void sched_mm_cid_fork(struct task_struct *t)
-{
- WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
- t->mm_cid.active = 1;
-}
-
void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
{
struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
@@ -10440,7 +10463,9 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
for_each_possible_cpu(cpu)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ mm->mm_cid.max_cids = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 31f2e43..d539fb2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3571,7 +3571,7 @@ static inline bool mm_cid_get(struct task_struct *t)
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
/* Try to reuse the last CID of this task */
if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
@@ -3614,7 +3614,6 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void init_sched_mm_cid(struct task_struct *t) { }
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Provide precomputed maximal value
2025-11-19 17:27 ` [patch V5 13/20] sched/mmcid: Provide precomputed maximal value Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: b0c3d51b54f8a4f4c809432d210c0c983d5cd97e
Gitweb: https://git.kernel.org/tip/b0c3d51b54f8a4f4c809432d210c0c983d5cd97e
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:09 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:40 +01:00
sched/mmcid: Provide precomputed maximal value
Reading mm::mm_users and mm:::mm_cid::nr_cpus_allowed every time to compute
the maximal CID value is just wasteful as that value is only changing on
fork(), exit() and eventually when the affinity changes.
So it can be easily precomputed at those points and provided in mm::mm_cid
for consumption in the hot path.
But there is an issue with using mm::mm_users for accounting because that
does not necessarily reflect the number of user space tasks as other kernel
code can take temporary references on the MM which skew the picture.
Solve that by adding a users counter to struct mm_mm_cid, which is modified
by fork() and exit() and used for precomputing under mm_mm_cid::lock.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.832764634@linutronix.de
---
include/linux/rseq_types.h | 6 ++++-
kernel/fork.c | 1 +-
kernel/sched/core.c | 59 ++++++++++++++++++++++++++-----------
kernel/sched/sched.h | 3 +--
4 files changed, 50 insertions(+), 19 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index d7e8071..0fab369 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -117,14 +117,20 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @max_cids: The exclusive maximum CID value for allocation and convergence
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
+ * @users: The number of tasks sharing this MM. Separate from mm::mm_users
+ * as that is modified by mmget()/mm_put() by other entities which
+ * do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int max_cids;
unsigned int nr_cpus_allowed;
+ unsigned int users;
raw_spinlock_t lock;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
diff --git a/kernel/fork.c b/kernel/fork.c
index 74bc7c9..6c23219 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -2455,6 +2455,7 @@ bad_fork_cleanup_namespaces:
exit_task_namespaces(p);
bad_fork_cleanup_mm:
if (p->mm) {
+ sched_mm_cid_exit(p);
mm_clear_owner(p->mm, p);
mmput(p->mm);
}
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 34b6c31..f9295c4 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4485,7 +4485,6 @@ static void __sched_fork(u64 clone_flags, struct task_struct *p)
init_numa_balancing(clone_flags, p);
p->wake_entry.u_flags = CSD_TYPE_TTWU;
p->migration_pending = NULL;
- init_sched_mm_cid(p);
}
DEFINE_STATIC_KEY_FALSE(sched_numa_balancing);
@@ -10371,15 +10370,27 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
#ifdef CONFIG_SCHED_MM_CID
/*
- * When a task exits, the MM CID held by the task is not longer required as
- * the task cannot return to user space.
+ * Update the CID range properties when the constraints change. Invoked via
+ * fork(), exit() and affinity changes
*/
+static void mm_update_max_cids(struct mm_struct *mm)
+{
+ struct mm_mm_cid *mc = &mm->mm_cid;
+ unsigned int max_cids;
+
+ lockdep_assert_held(&mm->mm_cid.lock);
+
+ /* Calculate the new maximum constraint */
+ max_cids = min(mc->nr_cpus_allowed, mc->users);
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
unsigned int weight;
- if (!mm)
+ if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
/*
@@ -10389,9 +10400,30 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
guard(raw_spinlock)(&mm->mm_cid.lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
+ if (weight == mm->mm_cid.nr_cpus_allowed)
+ return;
WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
+ mm_update_max_cids(mm);
+}
+
+void sched_mm_cid_fork(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+
+ guard(raw_spinlock)(&mm->mm_cid.lock);
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
+ mm_update_max_cids(mm);
}
+/*
+ * When a task exits, the MM CID held by the task is not longer required as
+ * the task cannot return to user space.
+ */
void sched_mm_cid_exit(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
@@ -10399,12 +10431,14 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
- guard(preempt)();
+ guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
+ mm->mm_cid.users--;
if (t->mm_cid.cid != MM_CID_UNSET) {
clear_bit(t->mm_cid.cid, mm_cidmask(mm));
t->mm_cid.cid = MM_CID_UNSET;
}
+ mm_update_max_cids(mm);
}
/* Deactivate MM CID allocation across execve() */
@@ -10416,22 +10450,11 @@ void sched_mm_cid_before_execve(struct task_struct *t)
/* Reactivate MM CID after successful execve() */
void sched_mm_cid_after_execve(struct task_struct *t)
{
- struct mm_struct *mm = t->mm;
-
- if (!mm)
- return;
-
+ sched_mm_cid_fork(t);
guard(preempt)();
- t->mm_cid.active = 1;
mm_cid_select(t);
}
-void sched_mm_cid_fork(struct task_struct *t)
-{
- WARN_ON_ONCE(!t->mm || t->mm_cid.cid != MM_CID_UNSET);
- t->mm_cid.active = 1;
-}
-
void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
{
struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
@@ -10440,7 +10463,9 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
for_each_possible_cpu(cpu)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ mm->mm_cid.max_cids = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
+ mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 31f2e43..d539fb2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3571,7 +3571,7 @@ static inline bool mm_cid_get(struct task_struct *t)
struct mm_struct *mm = t->mm;
unsigned int max_cids;
- max_cids = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
/* Try to reuse the last CID of this task */
if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
@@ -3614,7 +3614,6 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
}
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void init_sched_mm_cid(struct task_struct *t) { }
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 14/20] sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (12 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 13/20] sched/mmcid: Provide precomputed maximal value Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 15/20] sched/mmcid: Introduce per task/CPU ownership infrastructure Thomas Gleixner
` (6 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Prepare for the new CID management scheme which puts the CID ownership
transition into the fork() and exit() slow path by serializing
sched_mm_cid_fork()/exit() with it, so task list and cpu mask walks can be
done in interruptible and preemptible code.
The contention on it is not worse than on other concurrency controls in the
fork()/exit() machinery.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq_types.h | 2 ++
kernel/sched/core.c | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -125,6 +125,7 @@ struct mm_cid_pcpu {
* do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
@@ -132,6 +133,7 @@ struct mm_mm_cid {
unsigned int nr_cpus_allowed;
unsigned int users;
raw_spinlock_t lock;
+ struct mutex mutex;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10370,6 +10370,25 @@ void call_trace_sched_update_nr_running(
#ifdef CONFIG_SCHED_MM_CID
/*
+ * Concurrency IDentifier management
+ *
+ * Serialization rules:
+ *
+ * mm::mm_cid::mutex: Serializes fork() and exit() and therefore
+ * protects mm::mm_cid::users.
+ *
+ * mm::mm_cid::lock: Serializes mm_update_max_cids() and
+ * mm_update_cpus_allowed(). Nests in mm_cid::mutex
+ * and runqueue lock.
+ *
+ * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
+ * and can only be modified with atomic operations.
+ *
+ * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
+ * lock.
+ */
+
+/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
@@ -10412,6 +10431,7 @@ void sched_mm_cid_fork(struct task_struc
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 1;
mm->mm_cid.users++;
@@ -10431,6 +10451,7 @@ void sched_mm_cid_exit(struct task_struc
if (!mm || !t->mm_cid.active)
return;
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
mm->mm_cid.users--;
@@ -10467,6 +10488,7 @@ void mm_init_cid(struct mm_struct *mm, s
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
+ mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex
2025-11-19 17:27 ` [patch V5 14/20] sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 4c42755fe64a411d9f299c1f97e2d114698b6759
Gitweb: https://git.kernel.org/tip/4c42755fe64a411d9f299c1f97e2d114698b6759
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:11 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:56 +01:00
sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex
Prepare for the new CID management scheme which puts the CID ownership
transition into the fork() and exit() slow path by serializing
sched_mm_cid_fork()/exit() with it, so task list and cpu mask walks can be
done in interruptible and preemptible code.
The contention on it is not worse than on other concurrency controls in the
fork()/exit() machinery.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.895826703@linutronix.de
---
include/linux/rseq_types.h | 2 ++
kernel/sched/core.c | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 0fab369..574aba6 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -125,6 +125,7 @@ struct mm_cid_pcpu {
* do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
@@ -132,6 +133,7 @@ struct mm_mm_cid {
unsigned int nr_cpus_allowed;
unsigned int users;
raw_spinlock_t lock;
+ struct mutex mutex;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f9295c4..01903cf 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10370,6 +10370,25 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
#ifdef CONFIG_SCHED_MM_CID
/*
+ * Concurrency IDentifier management
+ *
+ * Serialization rules:
+ *
+ * mm::mm_cid::mutex: Serializes fork() and exit() and therefore
+ * protects mm::mm_cid::users.
+ *
+ * mm::mm_cid::lock: Serializes mm_update_max_cids() and
+ * mm_update_cpus_allowed(). Nests in mm_cid::mutex
+ * and runqueue lock.
+ *
+ * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
+ * and can only be modified with atomic operations.
+ *
+ * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
+ * lock.
+ */
+
+/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
@@ -10412,6 +10431,7 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 1;
mm->mm_cid.users++;
@@ -10431,6 +10451,7 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
mm->mm_cid.users--;
@@ -10467,6 +10488,7 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
+ mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex
2025-11-19 17:27 ` [patch V5 14/20] sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 51dd92c71a38647803478fb81e1812286a8998b1
Gitweb: https://git.kernel.org/tip/51dd92c71a38647803478fb81e1812286a8998b1
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:11 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex
Prepare for the new CID management scheme which puts the CID ownership
transition into the fork() and exit() slow path by serializing
sched_mm_cid_fork()/exit() with it, so task list and cpu mask walks can be
done in interruptible and preemptible code.
The contention on it is not worse than on other concurrency controls in the
fork()/exit() machinery.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172549.895826703@linutronix.de
---
include/linux/rseq_types.h | 2 ++
kernel/sched/core.c | 22 ++++++++++++++++++++++
2 files changed, 24 insertions(+)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 0fab369..574aba6 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -125,6 +125,7 @@ struct mm_cid_pcpu {
* do not actually share the MM.
* @lock: Spinlock to protect all fields except @pcpu. It also protects
* the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
struct mm_cid_pcpu __percpu *pcpu;
@@ -132,6 +133,7 @@ struct mm_mm_cid {
unsigned int nr_cpus_allowed;
unsigned int users;
raw_spinlock_t lock;
+ struct mutex mutex;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f9295c4..01903cf 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10370,6 +10370,25 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
#ifdef CONFIG_SCHED_MM_CID
/*
+ * Concurrency IDentifier management
+ *
+ * Serialization rules:
+ *
+ * mm::mm_cid::mutex: Serializes fork() and exit() and therefore
+ * protects mm::mm_cid::users.
+ *
+ * mm::mm_cid::lock: Serializes mm_update_max_cids() and
+ * mm_update_cpus_allowed(). Nests in mm_cid::mutex
+ * and runqueue lock.
+ *
+ * The mm_cidmask bitmap is not protected by any of the mm::mm_cid locks
+ * and can only be modified with atomic operations.
+ *
+ * The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
+ * lock.
+ */
+
+/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
@@ -10412,6 +10431,7 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 1;
mm->mm_cid.users++;
@@ -10431,6 +10451,7 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
+ guard(mutex)(&mm->mm_cid.mutex);
guard(raw_spinlock)(&mm->mm_cid.lock);
t->mm_cid.active = 0;
mm->mm_cid.users--;
@@ -10467,6 +10488,7 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
+ mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 15/20] sched/mmcid: Introduce per task/CPU ownership infrastructure
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (13 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 14/20] sched/mmcid: Serialize sched_mm_cid_fork()/exit() with a mutex Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 16/20] sched/mmcid: Provide new scheduler CID mechanism Thomas Gleixner
` (5 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not care
about that, there seems to be at least librseq depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes its affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
This can be done differently by implementing a strict CID ownership
mechanism. Either the CIDs are owned by the tasks or by the CPUs. The
latter provides less locality when tasks are heavily migrating, but there
is no justification to optimize for overcommit scenarios and thereby
penalizing everyone else.
Provide the basic infrastructure to implement this:
- Change the UNSET marker to BIT(31) from ~0U
- Add the ONCPU marker as BIT(30)
- Add the TRANSIT marker as BIT(29)
That allows to check for ownership trivially and provides a simple check for
UNSET as well. The TRANSIT marker is required to prevent CID space
exhaustion when switching from per CPU to per task mode.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
V5: Reword change log
V2: Add the TRANSIT(ion) bit
---
include/linux/rseq_types.h | 4 ++-
include/linux/sched.h | 6 ++--
kernel/sched/core.c | 10 +++++++
kernel/sched/sched.h | 59 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 75 insertions(+), 4 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -92,7 +92,9 @@ struct rseq_data { };
#ifdef CONFIG_SCHED_MM_CID
-#define MM_CID_UNSET (~0U)
+#define MM_CID_UNSET BIT(31)
+#define MM_CID_ONCPU BIT(30)
+#define MM_CID_TRANSIT BIT(29)
/**
* struct sched_mm_cid - Storage for per task MM CID data
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2299,16 +2299,16 @@ void sched_mm_cid_before_execve(struct t
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit(struct task_struct *t);
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid.cid;
+ return t->mm_cid.cid & ~(MM_CID_ONCPU | MM_CID_TRANSIT);
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
static inline void sched_mm_cid_exit(struct task_struct *t) { }
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
/*
* Use the processor id as a fall-back when the mm cid feature is
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10386,6 +10386,16 @@ void call_trace_sched_update_nr_running(
*
* The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
* lock.
+ *
+ * CID ownership:
+ *
+ * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
+ * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
+ * MM_CID_ONCPU bit set. During transition from CPU to task ownership mode,
+ * MM_CID_TRANSIT is set on the per task CIDs. When this bit is set the
+ * task needs to drop the CID into the pool when scheduling out. Both bits
+ * (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
+ * actually handed over to user space in the RSEQ memory.
*/
/*
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,6 +3540,65 @@ extern void sched_dynamic_update(int mod
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
+
+static __always_inline bool cid_on_cpu(unsigned int cid)
+{
+ return cid & MM_CID_ONCPU;
+}
+
+static __always_inline bool cid_in_transit(unsigned int cid)
+{
+ return cid & MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cpu_cid_to_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_cpu_cid(unsigned int cid)
+{
+ return cid | MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_transit_cid(unsigned int cid)
+{
+ return cid | MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cid_from_transit_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_TRANSIT;
+}
+
+static __always_inline bool cid_on_task(unsigned int cid)
+{
+ /* True if none of the MM_CID_ONCPU, MM_CID_TRANSIT, MM_CID_UNSET bits is set */
+ return cid < MM_CID_TRANSIT;
+}
+
+static __always_inline void mm_drop_cid(struct mm_struct *mm, unsigned int cid)
+{
+ clear_bit(cid, mm_cidmask(mm));
+}
+
+static __always_inline void mm_unset_cid_on_task(struct task_struct *t)
+{
+ unsigned int cid = t->mm_cid.cid;
+
+ t->mm_cid.cid = MM_CID_UNSET;
+ if (cid_on_task(cid))
+ mm_drop_cid(t->mm, cid);
+}
+
+static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp)
+{
+ /* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */
+ pcp->cid = cpu_cid_to_cid(pcp->cid);
+ mm_drop_cid(mm, pcp->cid);
+}
+
+/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Introduce per task/CPU ownership infrastructure
2025-11-19 17:27 ` [patch V5 15/20] sched/mmcid: Introduce per task/CPU ownership infrastructure Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 1bcc335bb29457d402b4fb033673aa89eca8187b
Gitweb: https://git.kernel.org/tip/1bcc335bb29457d402b4fb033673aa89eca8187b
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:12 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:57 +01:00
sched/mmcid: Introduce per task/CPU ownership infrastructure
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not care
about that, there seems to be at least librseq depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes its affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
This can be done differently by implementing a strict CID ownership
mechanism. Either the CIDs are owned by the tasks or by the CPUs. The
latter provides less locality when tasks are heavily migrating, but there
is no justification to optimize for overcommit scenarios and thereby
penalizing everyone else.
Provide the basic infrastructure to implement this:
- Change the UNSET marker to BIT(31) from ~0U
- Add the ONCPU marker as BIT(30)
- Add the TRANSIT marker as BIT(29)
That allows to check for ownership trivially and provides a simple check for
UNSET as well. The TRANSIT marker is required to prevent CID space
exhaustion when switching from per CPU to per task mode.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20251119172549.960252358@linutronix.de
---
include/linux/rseq_types.h | 4 ++-
include/linux/sched.h | 6 ++--
kernel/sched/core.c | 10 ++++++-
kernel/sched/sched.h | 59 +++++++++++++++++++++++++++++++++++++-
4 files changed, 75 insertions(+), 4 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 574aba6..87854ef 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -92,7 +92,9 @@ struct rseq_data { };
#ifdef CONFIG_SCHED_MM_CID
-#define MM_CID_UNSET (~0U)
+#define MM_CID_UNSET BIT(31)
+#define MM_CID_ONCPU BIT(30)
+#define MM_CID_TRANSIT BIT(29)
/**
* struct sched_mm_cid - Storage for per task MM CID data
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c411ae0..9eec409 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2299,16 +2299,16 @@ void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit(struct task_struct *t);
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid.cid;
+ return t->mm_cid.cid & ~(MM_CID_ONCPU | MM_CID_TRANSIT);
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
static inline void sched_mm_cid_exit(struct task_struct *t) { }
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
/*
* Use the processor id as a fall-back when the mm cid feature is
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 01903cf..55bb9c9 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10386,6 +10386,16 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
*
* The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
* lock.
+ *
+ * CID ownership:
+ *
+ * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
+ * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
+ * MM_CID_ONCPU bit set. During transition from CPU to task ownership mode,
+ * MM_CID_TRANSIT is set on the per task CIDs. When this bit is set the
+ * task needs to drop the CID into the pool when scheduling out. Both bits
+ * (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
+ * actually handed over to user space in the RSEQ memory.
*/
/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d539fb2..4b49284 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,6 +3540,65 @@ extern void sched_dynamic_update(int mode);
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
+
+static __always_inline bool cid_on_cpu(unsigned int cid)
+{
+ return cid & MM_CID_ONCPU;
+}
+
+static __always_inline bool cid_in_transit(unsigned int cid)
+{
+ return cid & MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cpu_cid_to_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_cpu_cid(unsigned int cid)
+{
+ return cid | MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_transit_cid(unsigned int cid)
+{
+ return cid | MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cid_from_transit_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_TRANSIT;
+}
+
+static __always_inline bool cid_on_task(unsigned int cid)
+{
+ /* True if none of the MM_CID_ONCPU, MM_CID_TRANSIT, MM_CID_UNSET bits is set */
+ return cid < MM_CID_TRANSIT;
+}
+
+static __always_inline void mm_drop_cid(struct mm_struct *mm, unsigned int cid)
+{
+ clear_bit(cid, mm_cidmask(mm));
+}
+
+static __always_inline void mm_unset_cid_on_task(struct task_struct *t)
+{
+ unsigned int cid = t->mm_cid.cid;
+
+ t->mm_cid.cid = MM_CID_UNSET;
+ if (cid_on_task(cid))
+ mm_drop_cid(t->mm, cid);
+}
+
+static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp)
+{
+ /* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */
+ pcp->cid = cpu_cid_to_cid(pcp->cid);
+ mm_drop_cid(mm, pcp->cid);
+}
+
+/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Introduce per task/CPU ownership infrastructure
2025-11-19 17:27 ` [patch V5 15/20] sched/mmcid: Introduce per task/CPU ownership infrastructure Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 23343b6b09acb4bf97f34ed60e135000ca57ede1
Gitweb: https://git.kernel.org/tip/23343b6b09acb4bf97f34ed60e135000ca57ede1
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:12 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
sched/mmcid: Introduce per task/CPU ownership infrastructure
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not care
about that, there seems to be at least librseq depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes its affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
This can be done differently by implementing a strict CID ownership
mechanism. Either the CIDs are owned by the tasks or by the CPUs. The
latter provides less locality when tasks are heavily migrating, but there
is no justification to optimize for overcommit scenarios and thereby
penalizing everyone else.
Provide the basic infrastructure to implement this:
- Change the UNSET marker to BIT(31) from ~0U
- Add the ONCPU marker as BIT(30)
- Add the TRANSIT marker as BIT(29)
That allows to check for ownership trivially and provides a simple check for
UNSET as well. The TRANSIT marker is required to prevent CID space
exhaustion when switching from per CPU to per task mode.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://patch.msgid.link/20251119172549.960252358@linutronix.de
---
include/linux/rseq_types.h | 4 ++-
include/linux/sched.h | 6 ++--
kernel/sched/core.c | 10 ++++++-
kernel/sched/sched.h | 59 +++++++++++++++++++++++++++++++++++++-
4 files changed, 75 insertions(+), 4 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 574aba6..87854ef 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -92,7 +92,9 @@ struct rseq_data { };
#ifdef CONFIG_SCHED_MM_CID
-#define MM_CID_UNSET (~0U)
+#define MM_CID_UNSET BIT(31)
+#define MM_CID_ONCPU BIT(30)
+#define MM_CID_TRANSIT BIT(29)
/**
* struct sched_mm_cid - Storage for per task MM CID data
diff --git a/include/linux/sched.h b/include/linux/sched.h
index c411ae0..9eec409 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2299,16 +2299,16 @@ void sched_mm_cid_before_execve(struct task_struct *t);
void sched_mm_cid_after_execve(struct task_struct *t);
void sched_mm_cid_fork(struct task_struct *t);
void sched_mm_cid_exit(struct task_struct *t);
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
- return t->mm_cid.cid;
+ return t->mm_cid.cid & ~(MM_CID_ONCPU | MM_CID_TRANSIT);
}
#else
static inline void sched_mm_cid_before_execve(struct task_struct *t) { }
static inline void sched_mm_cid_after_execve(struct task_struct *t) { }
static inline void sched_mm_cid_fork(struct task_struct *t) { }
static inline void sched_mm_cid_exit(struct task_struct *t) { }
-static inline int task_mm_cid(struct task_struct *t)
+static __always_inline int task_mm_cid(struct task_struct *t)
{
/*
* Use the processor id as a fall-back when the mm cid feature is
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 01903cf..55bb9c9 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10386,6 +10386,16 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
*
* The mm::mm_cid:pcpu per CPU storage is protected by the CPUs runqueue
* lock.
+ *
+ * CID ownership:
+ *
+ * A CID is either owned by a task (stored in task_struct::mm_cid.cid) or
+ * by a CPU (stored in mm::mm_cid.pcpu::cid). CIDs owned by CPUs have the
+ * MM_CID_ONCPU bit set. During transition from CPU to task ownership mode,
+ * MM_CID_TRANSIT is set on the per task CIDs. When this bit is set the
+ * task needs to drop the CID into the pool when scheduling out. Both bits
+ * (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
+ * actually handed over to user space in the RSEQ memory.
*/
/*
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d539fb2..4b49284 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3540,6 +3540,65 @@ extern void sched_dynamic_update(int mode);
extern const char *preempt_modes[];
#ifdef CONFIG_SCHED_MM_CID
+
+static __always_inline bool cid_on_cpu(unsigned int cid)
+{
+ return cid & MM_CID_ONCPU;
+}
+
+static __always_inline bool cid_in_transit(unsigned int cid)
+{
+ return cid & MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cpu_cid_to_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_cpu_cid(unsigned int cid)
+{
+ return cid | MM_CID_ONCPU;
+}
+
+static __always_inline unsigned int cid_to_transit_cid(unsigned int cid)
+{
+ return cid | MM_CID_TRANSIT;
+}
+
+static __always_inline unsigned int cid_from_transit_cid(unsigned int cid)
+{
+ return cid & ~MM_CID_TRANSIT;
+}
+
+static __always_inline bool cid_on_task(unsigned int cid)
+{
+ /* True if none of the MM_CID_ONCPU, MM_CID_TRANSIT, MM_CID_UNSET bits is set */
+ return cid < MM_CID_TRANSIT;
+}
+
+static __always_inline void mm_drop_cid(struct mm_struct *mm, unsigned int cid)
+{
+ clear_bit(cid, mm_cidmask(mm));
+}
+
+static __always_inline void mm_unset_cid_on_task(struct task_struct *t)
+{
+ unsigned int cid = t->mm_cid.cid;
+
+ t->mm_cid.cid = MM_CID_UNSET;
+ if (cid_on_task(cid))
+ mm_drop_cid(t->mm, cid);
+}
+
+static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_cid_pcpu *pcp)
+{
+ /* Clear the ONCPU bit, but do not set UNSET in the per CPU storage */
+ pcp->cid = cpu_cid_to_cid(pcp->cid);
+ mm_drop_cid(mm, pcp->cid);
+}
+
+/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 16/20] sched/mmcid: Provide new scheduler CID mechanism
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (14 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 15/20] sched/mmcid: Introduce per task/CPU ownership infrastructure Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 17/20] sched/mmcid: Provide CID ownership mode fixup functions Thomas Gleixner
` (4 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not
care about that, there seems to be at least some LTTng library
depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes it's affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
Implement the context switch parts of a strict ownership mechanism to
address this.
This removes most of the work from the task which schedules out. Only
during transitioning from per CPU to per task ownership it is required to
drop the CID when leaving the CPU to prevent CID space exhaustion. Other
than that scheduling out is just a single check and branch.
The task which schedules in has to check whether:
1) The ownership mode changed
2) The CID is within the optimal CID space
In stable situations this results in zero work. The only short disruption
is when ownership mode changes or when the associated CID is not in the
optimal CID space. The latter only happens when tasks exit and therefore
the optimal CID space shrinks.
That mechanism is strictly optimized for the common case where no change
happens. The only case where it actually causes a temporary one time spike
is on mode changes when and only when a lot of tasks related to a MM
schedule exactly at the same time and have eventually to compete on
allocating a CID from the bitmap.
In the sysbench test case which triggered the spinlock contention in the
initial CID code, __schedule() drops significantly in perf top on a 128
Core (256 threads) machine when running sysbench with 255 threads, which
fits into the task mode limit of 256 together with the parent thread:
Upstream rseq/perf branch +CID rework
0.42% 0.37% 0.32% [k] __schedule
Increasing the number of threads to 256, which puts the test process into
per CPU mode looks about the same.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V5: Polish changelog and initialize transit - Mathieu
V3: Move cpu_relax() out of the for() construct - Shrikanth
V2: Adjust to the mode transition changes
---
include/linux/rseq.h | 8 +-
include/linux/rseq_types.h | 18 +++--
kernel/sched/core.c | 2
kernel/sched/sched.h | 150 ++++++++++++++++++++++++++++++++++++++++++++-
4 files changed, 168 insertions(+), 10 deletions(-)
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -73,13 +73,13 @@ static __always_inline void rseq_sched_s
}
/*
- * Invoked from __set_task_cpu() when a task migrates to enforce an IDs
- * update.
+ * Invoked from __set_task_cpu() when a task migrates or from
+ * mm_cid_schedin() when the CID changes to enforce an IDs update.
*
* This does not raise TIF_NOTIFY_RESUME as that happens in
* rseq_sched_switch_event().
*/
-static __always_inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu)
+static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
{
t->rseq.event.ids_changed = true;
}
@@ -169,7 +169,7 @@ static inline void rseq_fork(struct task
static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
-static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }
+static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -119,23 +119,31 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @percpu: Set, when CIDs are in per CPU mode
+ * @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
- * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
+ /* Hotpath read mostly members */
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int percpu;
+ unsigned int transit;
unsigned int max_cids;
- unsigned int nr_cpus_allowed;
- unsigned int users;
+
raw_spinlock_t lock;
struct mutex mutex;
+
+ /* Low frequency modified */
+ unsigned int nr_cpus_allowed;
+ unsigned int users;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10495,6 +10495,8 @@ void mm_init_cid(struct mm_struct *mm, s
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
mm->mm_cid.max_cids = 0;
+ mm->mm_cid.percpu = 0;
+ mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2209,7 +2209,7 @@ static inline void __set_task_cpu(struct
smp_wmb();
WRITE_ONCE(task_thread_info(p)->cpu, cpu);
p->wake_cpu = cpu;
- rseq_sched_set_task_cpu(p, cpu);
+ rseq_sched_set_ids_changed(p);
#endif /* CONFIG_SMP */
}
@@ -3598,6 +3598,153 @@ static __always_inline void mm_drop_cid_
mm_drop_cid(mm, pcp->cid);
}
+static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids)
+{
+ unsigned int cid = find_first_zero_bit(mm_cidmask(mm), max_cids);
+
+ if (cid >= max_cids)
+ return MM_CID_UNSET;
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
+ return MM_CID_UNSET;
+ return cid;
+}
+
+static inline unsigned int mm_get_cid(struct mm_struct *mm)
+{
+ unsigned int cid = __mm_get_cid(mm, READ_ONCE(mm->mm_cid.max_cids));
+
+ while (cid == MM_CID_UNSET) {
+ cpu_relax();
+ cid = __mm_get_cid(mm, num_possible_cpus());
+ }
+ return cid;
+}
+
+static inline unsigned int mm_cid_converge(struct mm_struct *mm, unsigned int orig_cid,
+ unsigned int max_cids)
+{
+ unsigned int new_cid, cid = cpu_cid_to_cid(orig_cid);
+
+ /* Is it in the optimal CID space? */
+ if (likely(cid < max_cids))
+ return orig_cid;
+
+ /* Try to find one in the optimal space. Otherwise keep the provided. */
+ new_cid = __mm_get_cid(mm, max_cids);
+ if (new_cid != MM_CID_UNSET) {
+ mm_drop_cid(mm, cid);
+ /* Preserve the ONCPU mode of the original CID */
+ return new_cid | (orig_cid & MM_CID_ONCPU);
+ }
+ return orig_cid;
+}
+
+static __always_inline void mm_cid_update_task_cid(struct task_struct *t, unsigned int cid)
+{
+ if (t->mm_cid.cid != cid) {
+ t->mm_cid.cid = cid;
+ rseq_sched_set_ids_changed(t);
+ }
+}
+
+static __always_inline void mm_cid_update_pcpu_cid(struct mm_struct *mm, unsigned int cid)
+{
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
+}
+
+static __always_inline void mm_cid_from_cpu(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case where both have the ONCPU bit set */
+ if (likely(cid_on_cpu(cpu_cid & tcid))) {
+ if (likely(cpu_cid_to_cid(cpu_cid) < max_cids)) {
+ mm_cid_update_task_cid(t, cpu_cid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ cpu_cid = mm_cid_converge(mm, cpu_cid, max_cids);
+ } else {
+ /* Hand over or drop the task owned CID */
+ if (cid_on_task(tcid)) {
+ if (cid_on_cpu(cpu_cid))
+ mm_unset_cid_on_task(t);
+ else
+ cpu_cid = cid_to_cpu_cid(tcid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_cpu(cpu_cid))
+ cpu_cid = cid_to_cpu_cid(mm_get_cid(mm));
+ }
+ mm_cid_update_pcpu_cid(mm, cpu_cid);
+ mm_cid_update_task_cid(t, cpu_cid);
+}
+
+static __always_inline void mm_cid_from_task(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case, where both have the ONCPU bit clear */
+ if (likely(cid_on_task(tcid | cpu_cid))) {
+ if (likely(tcid < max_cids)) {
+ mm_cid_update_pcpu_cid(mm, tcid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ tcid = mm_cid_converge(mm, tcid, max_cids);
+ } else {
+ /* Hand over or drop the CPU owned CID */
+ if (cid_on_cpu(cpu_cid)) {
+ if (cid_on_task(tcid))
+ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
+ else
+ tcid = cpu_cid_to_cid(cpu_cid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_task(tcid))
+ tcid = mm_get_cid(mm);
+ /* Set the transition mode flag if required */
+ tcid |= READ_ONCE(mm->mm_cid.transit);
+ }
+ mm_cid_update_pcpu_cid(mm, tcid);
+ mm_cid_update_task_cid(t, tcid);
+}
+
+static __always_inline void mm_cid_schedin(struct task_struct *next)
+{
+ struct mm_struct *mm = next->mm;
+ unsigned int cpu_cid;
+
+ if (!next->mm_cid.active)
+ return;
+
+ cpu_cid = __this_cpu_read(mm->mm_cid.pcpu->cid);
+ if (likely(!READ_ONCE(mm->mm_cid.percpu)))
+ mm_cid_from_task(next, cpu_cid);
+ else
+ mm_cid_from_cpu(next, cpu_cid);
+}
+
+static __always_inline void mm_cid_schedout(struct task_struct *prev)
+{
+ /* During mode transitions CIDs are temporary and need to be dropped */
+ if (likely(!cid_in_transit(prev->mm_cid.cid)))
+ return;
+
+ mm_drop_cid(prev->mm, cid_from_transit_cid(prev->mm_cid.cid));
+ prev->mm_cid.cid = MM_CID_UNSET;
+}
+
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next)
+{
+ mm_cid_schedout(prev);
+ mm_cid_schedin(next);
+}
+
/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
@@ -3675,6 +3822,7 @@ static inline void switch_mm_cid(struct
#else /* !CONFIG_SCHED_MM_CID: */
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Provide new scheduler CID mechanism
2025-11-19 17:27 ` [patch V5 16/20] sched/mmcid: Provide new scheduler CID mechanism Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 6fa1e9565d81095ac1c085601a4af8b18a04db1d
Gitweb: https://git.kernel.org/tip/6fa1e9565d81095ac1c085601a4af8b18a04db1d
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:14 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:57 +01:00
sched/mmcid: Provide new scheduler CID mechanism
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not
care about that, there seems to be at least some LTTng library
depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes it's affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
Implement the context switch parts of a strict ownership mechanism to
address this.
This removes most of the work from the task which schedules out. Only
during transitioning from per CPU to per task ownership it is required to
drop the CID when leaving the CPU to prevent CID space exhaustion. Other
than that scheduling out is just a single check and branch.
The task which schedules in has to check whether:
1) The ownership mode changed
2) The CID is within the optimal CID space
In stable situations this results in zero work. The only short disruption
is when ownership mode changes or when the associated CID is not in the
optimal CID space. The latter only happens when tasks exit and therefore
the optimal CID space shrinks.
That mechanism is strictly optimized for the common case where no change
happens. The only case where it actually causes a temporary one time spike
is on mode changes when and only when a lot of tasks related to a MM
schedule exactly at the same time and have eventually to compete on
allocating a CID from the bitmap.
In the sysbench test case which triggered the spinlock contention in the
initial CID code, __schedule() drops significantly in perf top on a 128
Core (256 threads) machine when running sysbench with 255 threads, which
fits into the task mode limit of 256 together with the parent thread:
Upstream rseq/perf branch +CID rework
0.42% 0.37% 0.32% [k] __schedule
Increasing the number of threads to 256, which puts the test process into
per CPU mode looks about the same.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.023984859@linutronix.de
---
include/linux/rseq.h | 8 +-
include/linux/rseq_types.h | 18 ++--
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 150 +++++++++++++++++++++++++++++++++++-
4 files changed, 168 insertions(+), 10 deletions(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index bf8a6bf..4c0e8bd 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -73,13 +73,13 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t)
}
/*
- * Invoked from __set_task_cpu() when a task migrates to enforce an IDs
- * update.
+ * Invoked from __set_task_cpu() when a task migrates or from
+ * mm_cid_schedin() when the CID changes to enforce an IDs update.
*
* This does not raise TIF_NOTIFY_RESUME as that happens in
* rseq_sched_switch_event().
*/
-static __always_inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu)
+static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
{
t->rseq.event.ids_changed = true;
}
@@ -168,7 +168,7 @@ static inline void rseq_fork(struct task_struct *t, u64 clone_flags)
static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
-static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }
+static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 87854ef..66b1482 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -119,23 +119,31 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @percpu: Set, when CIDs are in per CPU mode
+ * @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
- * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
+ /* Hotpath read mostly members */
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int percpu;
+ unsigned int transit;
unsigned int max_cids;
- unsigned int nr_cpus_allowed;
- unsigned int users;
+
raw_spinlock_t lock;
struct mutex mutex;
+
+ /* Low frequency modified */
+ unsigned int nr_cpus_allowed;
+ unsigned int users;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 55bb9c9..659ae56 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10495,6 +10495,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
mm->mm_cid.max_cids = 0;
+ mm->mm_cid.percpu = 0;
+ mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4b49284..82c7978 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2209,7 +2209,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
smp_wmb();
WRITE_ONCE(task_thread_info(p)->cpu, cpu);
p->wake_cpu = cpu;
- rseq_sched_set_task_cpu(p, cpu);
+ rseq_sched_set_ids_changed(p);
#endif /* CONFIG_SMP */
}
@@ -3598,6 +3598,153 @@ static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_c
mm_drop_cid(mm, pcp->cid);
}
+static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids)
+{
+ unsigned int cid = find_first_zero_bit(mm_cidmask(mm), max_cids);
+
+ if (cid >= max_cids)
+ return MM_CID_UNSET;
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
+ return MM_CID_UNSET;
+ return cid;
+}
+
+static inline unsigned int mm_get_cid(struct mm_struct *mm)
+{
+ unsigned int cid = __mm_get_cid(mm, READ_ONCE(mm->mm_cid.max_cids));
+
+ while (cid == MM_CID_UNSET) {
+ cpu_relax();
+ cid = __mm_get_cid(mm, num_possible_cpus());
+ }
+ return cid;
+}
+
+static inline unsigned int mm_cid_converge(struct mm_struct *mm, unsigned int orig_cid,
+ unsigned int max_cids)
+{
+ unsigned int new_cid, cid = cpu_cid_to_cid(orig_cid);
+
+ /* Is it in the optimal CID space? */
+ if (likely(cid < max_cids))
+ return orig_cid;
+
+ /* Try to find one in the optimal space. Otherwise keep the provided. */
+ new_cid = __mm_get_cid(mm, max_cids);
+ if (new_cid != MM_CID_UNSET) {
+ mm_drop_cid(mm, cid);
+ /* Preserve the ONCPU mode of the original CID */
+ return new_cid | (orig_cid & MM_CID_ONCPU);
+ }
+ return orig_cid;
+}
+
+static __always_inline void mm_cid_update_task_cid(struct task_struct *t, unsigned int cid)
+{
+ if (t->mm_cid.cid != cid) {
+ t->mm_cid.cid = cid;
+ rseq_sched_set_ids_changed(t);
+ }
+}
+
+static __always_inline void mm_cid_update_pcpu_cid(struct mm_struct *mm, unsigned int cid)
+{
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
+}
+
+static __always_inline void mm_cid_from_cpu(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case where both have the ONCPU bit set */
+ if (likely(cid_on_cpu(cpu_cid & tcid))) {
+ if (likely(cpu_cid_to_cid(cpu_cid) < max_cids)) {
+ mm_cid_update_task_cid(t, cpu_cid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ cpu_cid = mm_cid_converge(mm, cpu_cid, max_cids);
+ } else {
+ /* Hand over or drop the task owned CID */
+ if (cid_on_task(tcid)) {
+ if (cid_on_cpu(cpu_cid))
+ mm_unset_cid_on_task(t);
+ else
+ cpu_cid = cid_to_cpu_cid(tcid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_cpu(cpu_cid))
+ cpu_cid = cid_to_cpu_cid(mm_get_cid(mm));
+ }
+ mm_cid_update_pcpu_cid(mm, cpu_cid);
+ mm_cid_update_task_cid(t, cpu_cid);
+}
+
+static __always_inline void mm_cid_from_task(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case, where both have the ONCPU bit clear */
+ if (likely(cid_on_task(tcid | cpu_cid))) {
+ if (likely(tcid < max_cids)) {
+ mm_cid_update_pcpu_cid(mm, tcid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ tcid = mm_cid_converge(mm, tcid, max_cids);
+ } else {
+ /* Hand over or drop the CPU owned CID */
+ if (cid_on_cpu(cpu_cid)) {
+ if (cid_on_task(tcid))
+ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
+ else
+ tcid = cpu_cid_to_cid(cpu_cid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_task(tcid))
+ tcid = mm_get_cid(mm);
+ /* Set the transition mode flag if required */
+ tcid |= READ_ONCE(mm->mm_cid.transit);
+ }
+ mm_cid_update_pcpu_cid(mm, tcid);
+ mm_cid_update_task_cid(t, tcid);
+}
+
+static __always_inline void mm_cid_schedin(struct task_struct *next)
+{
+ struct mm_struct *mm = next->mm;
+ unsigned int cpu_cid;
+
+ if (!next->mm_cid.active)
+ return;
+
+ cpu_cid = __this_cpu_read(mm->mm_cid.pcpu->cid);
+ if (likely(!READ_ONCE(mm->mm_cid.percpu)))
+ mm_cid_from_task(next, cpu_cid);
+ else
+ mm_cid_from_cpu(next, cpu_cid);
+}
+
+static __always_inline void mm_cid_schedout(struct task_struct *prev)
+{
+ /* During mode transitions CIDs are temporary and need to be dropped */
+ if (likely(!cid_in_transit(prev->mm_cid.cid)))
+ return;
+
+ mm_drop_cid(prev->mm, cid_from_transit_cid(prev->mm_cid.cid));
+ prev->mm_cid.cid = MM_CID_UNSET;
+}
+
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next)
+{
+ mm_cid_schedout(prev);
+ mm_cid_schedin(next);
+}
+
/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
@@ -3675,6 +3822,7 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
#else /* !CONFIG_SCHED_MM_CID: */
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Provide new scheduler CID mechanism
2025-11-19 17:27 ` [patch V5 16/20] sched/mmcid: Provide new scheduler CID mechanism Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 9a723ed7facff6955da8d64cc9de7066038036c1
Gitweb: https://git.kernel.org/tip/9a723ed7facff6955da8d64cc9de7066038036c1
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:14 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
sched/mmcid: Provide new scheduler CID mechanism
The MM CID management has two fundamental requirements:
1) It has to guarantee that at no given point in time the same CID is
used by concurrent tasks in userspace.
2) The CID space must not exceed the number of possible CPUs in a
system. While most allocators (glibc, tcmalloc, jemalloc) do not
care about that, there seems to be at least some LTTng library
depending on it.
The CID space compaction itself is not a functional correctness
requirement, it is only a useful optimization mechanism to reduce the
memory foot print in unused user space pools.
The optimal CID space is:
min(nr_tasks, nr_cpus_allowed);
Where @nr_tasks is the number of actual user space threads associated to
the mm and @nr_cpus_allowed is the superset of all task affinities. It is
growth only as it would be insane to take a racy snapshot of all task
affinities when the affinity of one task changes just do redo it 2
milliseconds later when the next task changes it's affinity.
That means that as long as the number of tasks is lower or equal than the
number of CPUs allowed, each task owns a CID. If the number of tasks
exceeds the number of CPUs allowed it switches to per CPU mode, where the
CPUs own the CIDs and the tasks borrow them as long as they are scheduled
in.
For transition periods CIDs can go beyond the optimal space as long as they
don't go beyond the number of possible CPUs.
The current upstream implementation adds overhead into task migration to
keep the CID with the task. It also has to do the CID space consolidation
work from a task work in the exit to user space path. As that work is
assigned to a random task related to a MM this can inflict unwanted exit
latencies.
Implement the context switch parts of a strict ownership mechanism to
address this.
This removes most of the work from the task which schedules out. Only
during transitioning from per CPU to per task ownership it is required to
drop the CID when leaving the CPU to prevent CID space exhaustion. Other
than that scheduling out is just a single check and branch.
The task which schedules in has to check whether:
1) The ownership mode changed
2) The CID is within the optimal CID space
In stable situations this results in zero work. The only short disruption
is when ownership mode changes or when the associated CID is not in the
optimal CID space. The latter only happens when tasks exit and therefore
the optimal CID space shrinks.
That mechanism is strictly optimized for the common case where no change
happens. The only case where it actually causes a temporary one time spike
is on mode changes when and only when a lot of tasks related to a MM
schedule exactly at the same time and have eventually to compete on
allocating a CID from the bitmap.
In the sysbench test case which triggered the spinlock contention in the
initial CID code, __schedule() drops significantly in perf top on a 128
Core (256 threads) machine when running sysbench with 255 threads, which
fits into the task mode limit of 256 together with the parent thread:
Upstream rseq/perf branch +CID rework
0.42% 0.37% 0.32% [k] __schedule
Increasing the number of threads to 256, which puts the test process into
per CPU mode looks about the same.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.023984859@linutronix.de
---
include/linux/rseq.h | 8 +-
include/linux/rseq_types.h | 18 ++--
kernel/sched/core.c | 2 +-
kernel/sched/sched.h | 150 +++++++++++++++++++++++++++++++++++-
4 files changed, 168 insertions(+), 10 deletions(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index bf8a6bf..4c0e8bd 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -73,13 +73,13 @@ static __always_inline void rseq_sched_switch_event(struct task_struct *t)
}
/*
- * Invoked from __set_task_cpu() when a task migrates to enforce an IDs
- * update.
+ * Invoked from __set_task_cpu() when a task migrates or from
+ * mm_cid_schedin() when the CID changes to enforce an IDs update.
*
* This does not raise TIF_NOTIFY_RESUME as that happens in
* rseq_sched_switch_event().
*/
-static __always_inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu)
+static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
{
t->rseq.event.ids_changed = true;
}
@@ -168,7 +168,7 @@ static inline void rseq_fork(struct task_struct *t, u64 clone_flags)
static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
-static inline void rseq_sched_set_task_cpu(struct task_struct *t, unsigned int cpu) { }
+static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 87854ef..66b1482 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -119,23 +119,31 @@ struct mm_cid_pcpu {
/**
* struct mm_mm_cid - Storage for per MM CID data
* @pcpu: Per CPU storage for CIDs associated to a CPU
+ * @percpu: Set, when CIDs are in per CPU mode
+ * @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @lock: Spinlock to protect all fields except @pcpu. It also protects
+ * the MM cid cpumask and the MM cidmask bitmap.
+ * @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
- * @mutex: Mutex to serialize forks and exits related to this mm
*/
struct mm_mm_cid {
+ /* Hotpath read mostly members */
struct mm_cid_pcpu __percpu *pcpu;
+ unsigned int percpu;
+ unsigned int transit;
unsigned int max_cids;
- unsigned int nr_cpus_allowed;
- unsigned int users;
+
raw_spinlock_t lock;
struct mutex mutex;
+
+ /* Low frequency modified */
+ unsigned int nr_cpus_allowed;
+ unsigned int users;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 55bb9c9..659ae56 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10495,6 +10495,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
mm->mm_cid.max_cids = 0;
+ mm->mm_cid.percpu = 0;
+ mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 4b49284..82c7978 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -2209,7 +2209,7 @@ static inline void __set_task_cpu(struct task_struct *p, unsigned int cpu)
smp_wmb();
WRITE_ONCE(task_thread_info(p)->cpu, cpu);
p->wake_cpu = cpu;
- rseq_sched_set_task_cpu(p, cpu);
+ rseq_sched_set_ids_changed(p);
#endif /* CONFIG_SMP */
}
@@ -3598,6 +3598,153 @@ static __always_inline void mm_drop_cid_on_cpu(struct mm_struct *mm, struct mm_c
mm_drop_cid(mm, pcp->cid);
}
+static inline unsigned int __mm_get_cid(struct mm_struct *mm, unsigned int max_cids)
+{
+ unsigned int cid = find_first_zero_bit(mm_cidmask(mm), max_cids);
+
+ if (cid >= max_cids)
+ return MM_CID_UNSET;
+ if (test_and_set_bit(cid, mm_cidmask(mm)))
+ return MM_CID_UNSET;
+ return cid;
+}
+
+static inline unsigned int mm_get_cid(struct mm_struct *mm)
+{
+ unsigned int cid = __mm_get_cid(mm, READ_ONCE(mm->mm_cid.max_cids));
+
+ while (cid == MM_CID_UNSET) {
+ cpu_relax();
+ cid = __mm_get_cid(mm, num_possible_cpus());
+ }
+ return cid;
+}
+
+static inline unsigned int mm_cid_converge(struct mm_struct *mm, unsigned int orig_cid,
+ unsigned int max_cids)
+{
+ unsigned int new_cid, cid = cpu_cid_to_cid(orig_cid);
+
+ /* Is it in the optimal CID space? */
+ if (likely(cid < max_cids))
+ return orig_cid;
+
+ /* Try to find one in the optimal space. Otherwise keep the provided. */
+ new_cid = __mm_get_cid(mm, max_cids);
+ if (new_cid != MM_CID_UNSET) {
+ mm_drop_cid(mm, cid);
+ /* Preserve the ONCPU mode of the original CID */
+ return new_cid | (orig_cid & MM_CID_ONCPU);
+ }
+ return orig_cid;
+}
+
+static __always_inline void mm_cid_update_task_cid(struct task_struct *t, unsigned int cid)
+{
+ if (t->mm_cid.cid != cid) {
+ t->mm_cid.cid = cid;
+ rseq_sched_set_ids_changed(t);
+ }
+}
+
+static __always_inline void mm_cid_update_pcpu_cid(struct mm_struct *mm, unsigned int cid)
+{
+ __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
+}
+
+static __always_inline void mm_cid_from_cpu(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case where both have the ONCPU bit set */
+ if (likely(cid_on_cpu(cpu_cid & tcid))) {
+ if (likely(cpu_cid_to_cid(cpu_cid) < max_cids)) {
+ mm_cid_update_task_cid(t, cpu_cid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ cpu_cid = mm_cid_converge(mm, cpu_cid, max_cids);
+ } else {
+ /* Hand over or drop the task owned CID */
+ if (cid_on_task(tcid)) {
+ if (cid_on_cpu(cpu_cid))
+ mm_unset_cid_on_task(t);
+ else
+ cpu_cid = cid_to_cpu_cid(tcid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_cpu(cpu_cid))
+ cpu_cid = cid_to_cpu_cid(mm_get_cid(mm));
+ }
+ mm_cid_update_pcpu_cid(mm, cpu_cid);
+ mm_cid_update_task_cid(t, cpu_cid);
+}
+
+static __always_inline void mm_cid_from_task(struct task_struct *t, unsigned int cpu_cid)
+{
+ unsigned int max_cids, tcid = t->mm_cid.cid;
+ struct mm_struct *mm = t->mm;
+
+ max_cids = READ_ONCE(mm->mm_cid.max_cids);
+ /* Optimize for the common case, where both have the ONCPU bit clear */
+ if (likely(cid_on_task(tcid | cpu_cid))) {
+ if (likely(tcid < max_cids)) {
+ mm_cid_update_pcpu_cid(mm, tcid);
+ return;
+ }
+ /* Try to converge into the optimal CID space */
+ tcid = mm_cid_converge(mm, tcid, max_cids);
+ } else {
+ /* Hand over or drop the CPU owned CID */
+ if (cid_on_cpu(cpu_cid)) {
+ if (cid_on_task(tcid))
+ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
+ else
+ tcid = cpu_cid_to_cid(cpu_cid);
+ }
+ /* Still nothing, allocate a new one */
+ if (!cid_on_task(tcid))
+ tcid = mm_get_cid(mm);
+ /* Set the transition mode flag if required */
+ tcid |= READ_ONCE(mm->mm_cid.transit);
+ }
+ mm_cid_update_pcpu_cid(mm, tcid);
+ mm_cid_update_task_cid(t, tcid);
+}
+
+static __always_inline void mm_cid_schedin(struct task_struct *next)
+{
+ struct mm_struct *mm = next->mm;
+ unsigned int cpu_cid;
+
+ if (!next->mm_cid.active)
+ return;
+
+ cpu_cid = __this_cpu_read(mm->mm_cid.pcpu->cid);
+ if (likely(!READ_ONCE(mm->mm_cid.percpu)))
+ mm_cid_from_task(next, cpu_cid);
+ else
+ mm_cid_from_cpu(next, cpu_cid);
+}
+
+static __always_inline void mm_cid_schedout(struct task_struct *prev)
+{
+ /* During mode transitions CIDs are temporary and need to be dropped */
+ if (likely(!cid_in_transit(prev->mm_cid.cid)))
+ return;
+
+ mm_drop_cid(prev->mm, cid_from_transit_cid(prev->mm_cid.cid));
+ prev->mm_cid.cid = MM_CID_UNSET;
+}
+
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next)
+{
+ mm_cid_schedout(prev);
+ mm_cid_schedin(next);
+}
+
/* Active implementation */
static inline void init_sched_mm_cid(struct task_struct *t)
{
@@ -3675,6 +3822,7 @@ static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *n
#else /* !CONFIG_SCHED_MM_CID: */
static inline void mm_cid_select(struct task_struct *t) { }
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
+static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 17/20] sched/mmcid: Provide CID ownership mode fixup functions
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (15 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 16/20] sched/mmcid: Provide new scheduler CID mechanism Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 18/20] irqwork: Move data struct to a types header Thomas Gleixner
` (3 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
CIDs are either owned by tasks or by CPUs. The ownership mode depends on
the number of tasks related to a MM and the number of CPUs on which these
tasks are theoretically allowed to run on. Theoretically because that
number is the superset of CPU affinities of all tasks which only grows and
never shrinks.
Switching to per CPU mode happens when the user count becomes greater than
the maximum number of CIDs, which is calculated by:
opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
max_cids = min(1.25 * opt_cids, nr_cpu_ids);
The +25% allowance is useful for tight CPU masks in scenarios where only a
few threads are created and destroyed to avoid frequent mode
switches. Though this allowance shrinks, the closer opt_cids becomes to
nr_cpu_ids, which is the (unfortunate) hard ABI limit.
At the point of switching to per CPU mode the new user is not yet visible
in the system, so the task which initiated the fork() runs the fixup
function: mm_cid_fixup_tasks_to_cpu() walks the thread list and either
transfers each tasks owned CID to the CPU the task runs on or drops it into
the CID pool if a task is not on a CPU at that point in time. Tasks which
schedule in before the task walk reaches them do the handover in
mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes it's
guaranteed that no task related to that MM owns a CID anymore.
Switching back to task mode happens when the user count goes below the
threshold which was recorded on the per CPU mode switch:
pcpu_thrs = min(opt_cids - (opt_cids / 4), nr_cpu_ids / 2);
This threshold is updated when a affinity change increases the number of
allowed CPUs for the MM, which might cause a switch back to per task mode.
If the switch back was initiated by a exiting task, then that task runs the
fixup function. If it was initiated by a affinity change, then it's run
either in the deferred update function in context of a workqueue or by a
task which forks a new one or by a task which exits. Whatever happens
first. mm_cid_fixup_cpus_to_task() walks through the possible CPUs and
either transfers the CPU owned CIDs to a related task which runs on the CPU
or drops it into the pool. Tasks which schedule in on a CPU which the walk
did not cover yet do the handover themselves.
This transition from CPU to per task ownership happens in two phases:
1) mm:mm_cid.transit contains MM_CID_TRANSIT. This is OR'ed on the task
CID and denotes that the CID is only temporarily owned by the
task. When it schedules out the task drops the CID back into the
pool if this bit is set.
2) The initiating context walks the per CPU space and after completion
clears mm:mm_cid.transit. After that point the CIDs are strictly
task owned again.
This two phase transition is required to prevent CID space exhaustion
during the transition as a direct transfer of ownership would fail if
two tasks are scheduled in on the same CPU before the fixup freed per
CPU CIDs.
When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
related to that MM is owned by a CPU anymore.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
V2: Fix CPU to task owner mode
---
include/linux/rseq_types.h | 7 -
kernel/sched/core.c | 277 +++++++++++++++++++++++++++++++++++++++++----
2 files changed, 258 insertions(+), 26 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -122,14 +122,15 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
+ * @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
+ * @pcpu_thrs: Threshold for switching back from per CPU mode
+ * @update_deferred: A deferred switch back to per task mode is pending.
*/
struct mm_mm_cid {
/* Hotpath read mostly members */
@@ -144,6 +145,8 @@ struct mm_mm_cid {
/* Low frequency modified */
unsigned int nr_cpus_allowed;
unsigned int users;
+ unsigned int pcpu_thrs;
+ unsigned int update_deferred;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10396,43 +10396,269 @@ void call_trace_sched_update_nr_running(
* task needs to drop the CID into the pool when scheduling out. Both bits
* (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
* actually handed over to user space in the RSEQ memory.
+ *
+ * Mode switching:
+ *
+ * Switching to per CPU mode happens when the user count becomes greater
+ * than the maximum number of CIDs, which is calculated by:
+ *
+ * opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
+ * max_cids = min(1.25 * opt_cids, num_possible_cpus());
+ *
+ * The +25% allowance is useful for tight CPU masks in scenarios where only
+ * a few threads are created and destroyed to avoid frequent mode
+ * switches. Though this allowance shrinks, the closer opt_cids becomes to
+ * num_possible_cpus(), which is the (unfortunate) hard ABI limit.
+ *
+ * At the point of switching to per CPU mode the new user is not yet
+ * visible in the system, so the task which initiated the fork() runs the
+ * fixup function: mm_cid_fixup_tasks_to_cpu() walks the thread list and
+ * either transfers each tasks owned CID to the CPU the task runs on or
+ * drops it into the CID pool if a task is not on a CPU at that point in
+ * time. Tasks which schedule in before the task walk reaches them do the
+ * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes
+ * it's guaranteed that no task related to that MM owns a CID anymore.
+ *
+ * Switching back to task mode happens when the user count goes below the
+ * threshold which was recorded on the per CPU mode switch:
+ *
+ * pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
+ *
+ * This threshold is updated when a affinity change increases the number of
+ * allowed CPUs for the MM, which might cause a switch back to per task
+ * mode.
+ *
+ * If the switch back was initiated by a exiting task, then that task runs
+ * the fixup function. If it was initiated by a affinity change, then it's
+ * run either in the deferred update function in context of a workqueue or
+ * by a task which forks a new one or by a task which exits. Whatever
+ * happens first. mm_cid_fixup_cpus_to_task() walks through the possible
+ * CPUs and either transfers the CPU owned CIDs to a related task which
+ * runs on the CPU or drops it into the pool. Tasks which schedule in on a
+ * CPU which the walk did not cover yet do the handover themself.
+ *
+ * This transition from CPU to per task ownership happens in two phases:
+ *
+ * 1) mm:mm_cid.transit contains MM_CID_TRANSIT This is OR'ed on the task
+ * CID and denotes that the CID is only temporarily owned by the
+ * task. When it schedules out the task drops the CID back into the
+ * pool if this bit is set.
+ *
+ * 2) The initiating context walks the per CPU space and after completion
+ * clears mm:mm_cid.transit. So after that point the CIDs are strictly
+ * task owned again.
+ *
+ * This two phase transition is required to prevent CID space exhaustion
+ * during the transition as a direct transfer of ownership would fail if
+ * two tasks are scheduled in on the same CPU before the fixup freed per
+ * CPU CIDs.
+ *
+ * When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
+ * related to that MM is owned by a CPU anymore.
*/
/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
-static void mm_update_max_cids(struct mm_struct *mm)
+static void __mm_update_max_cids(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids, max_cids;
+
+ /* Calculate the new optimal constraint */
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+
+ /* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
+ max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
+static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids;
+
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+ return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
+}
+
+static bool mm_update_max_cids(struct mm_struct *mm)
{
struct mm_mm_cid *mc = &mm->mm_cid;
- unsigned int max_cids;
lockdep_assert_held(&mm->mm_cid.lock);
- /* Calculate the new maximum constraint */
- max_cids = min(mc->nr_cpus_allowed, mc->users);
- WRITE_ONCE(mc->max_cids, max_cids);
+ /* Clear deferred mode switch flag. A change is handled by the caller */
+ mc->update_deferred = false;
+ __mm_update_max_cids(mc);
+
+ /* Check whether owner mode must be changed */
+ if (!mc->percpu) {
+ /* Enable per CPU mode when the number of users is above max_cids */
+ if (mc->users > mc->max_cids)
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ } else {
+ /* Switch back to per task if user count under threshold */
+ if (mc->users < mc->pcpu_thrs)
+ mc->pcpu_thrs = 0;
+ }
+
+ /* Mode change required? */
+ if (!!mc->percpu == !!mc->pcpu_thrs)
+ return false;
+ /* When switching back to per TASK mode, set the transition flag */
+ if (!mc->pcpu_thrs)
+ WRITE_ONCE(mc->transit, MM_CID_TRANSIT);
+ WRITE_ONCE(mc->percpu, !!mc->pcpu_thrs);
+ return true;
}
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ struct mm_mm_cid *mc;
unsigned int weight;
if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
-
/*
* mm::mm_cid::mm_cpus_allowed is the superset of each threads
* allowed CPUs mask which means it can only grow.
*/
- guard(raw_spinlock)(&mm->mm_cid.lock);
+ mc = &mm->mm_cid;
+ guard(raw_spinlock)(&mc->lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
- if (weight == mm->mm_cid.nr_cpus_allowed)
+ if (weight == mc->nr_cpus_allowed)
+ return;
+
+ WRITE_ONCE(mc->nr_cpus_allowed, weight);
+ __mm_update_max_cids(mc);
+ if (!mc->percpu)
return;
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
- mm_update_max_cids(mm);
+
+ /* Adjust the threshold to the wider set */
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+
+ /* Scheduling of deferred mode switch goes here */
+}
+
+static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_cpu(t->mm_cid.cid)) {
+ unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid);
+
+ t->mm_cid.cid = cid_to_transit_cid(cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+{
+ unsigned int cpu;
+
+ /* Walk the CPUs and fixup all stale CIDs */
+ for_each_possible_cpu(cpu) {
+ struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
+ struct rq *rq = cpu_rq(cpu);
+
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(rq_lock_irq)(rq);
+ /* Is the CID still owned by the CPU? */
+ if (cid_on_cpu(pcp->cid)) {
+ /*
+ * If rq->curr has @mm, transfer it with the
+ * transition bit set. Otherwise drop it.
+ */
+ if (rq->curr->mm == mm && rq->curr->mm_cid.active)
+ mm_cid_transit_to_task(rq->curr, pcp);
+ else
+ mm_drop_cid_on_cpu(mm, pcp);
+
+ } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
+ unsigned int cid = rq->curr->mm_cid.cid;
+
+ /* Ensure it has the transition bit set */
+ if (!cid_in_transit(cid)) {
+ cid = cid_to_transit_cid(cid);
+ rq->curr->mm_cid.cid = cid;
+ pcp->cid = cid;
+ }
+ }
+ }
+ /* Clear the transition bit */
+ WRITE_ONCE(mm->mm_cid.transit, 0);
+}
+
+static inline void mm_cid_transfer_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_task(t->mm_cid.cid)) {
+ t->mm_cid.cid = cid_to_cpu_cid(t->mm_cid.cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
+{
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(task_rq_lock)(t);
+ /* If the task is not active it is not in the users count */
+ if (!t->mm_cid.active)
+ return false;
+ if (cid_on_task(t->mm_cid.cid)) {
+ /* If running on the CPU, transfer the CID, otherwise drop it */
+ if (task_rq(t)->curr == t)
+ mm_cid_transfer_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
+ else
+ mm_unset_cid_on_task(t);
+ }
+ return true;
+}
+
+static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+{
+ struct mm_struct *mm = current->mm;
+ struct task_struct *p, *t;
+ unsigned int users;
+
+ /*
+ * This can obviously race with a concurrent affinity change, which
+ * increases the number of allowed CPUs for this mm, but that does
+ * not affect the mode and only changes the CID constraints. A
+ * possible switch back to per task mode happens either in the
+ * deferred handler function or in the next fork()/exit().
+ *
+ * The caller has already transferred. The newly incoming task is
+ * already accounted for, but not yet visible.
+ */
+ users = mm->mm_cid.users - 2;
+ if (!users)
+ return;
+
+ guard(rcu)();
+ for_other_threads(current, t) {
+ if (mm_cid_fixup_task_to_cpu(t, mm))
+ users--;
+ }
+
+ if (!users)
+ return;
+
+ /* Happens only for VM_CLONE processes. */
+ for_each_process_thread(p, t) {
+ if (t == current || t->mm != mm)
+ continue;
+ if (mm_cid_fixup_task_to_cpu(t, mm)) {
+ if (--users == 0)
+ return;
+ }
+ }
+}
+
+static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
+{
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ return mm_update_max_cids(mm);
}
void sched_mm_cid_fork(struct task_struct *t)
@@ -10442,12 +10668,19 @@ void sched_mm_cid_fork(struct task_struc
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 1;
- mm->mm_cid.users++;
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
+ sched_mm_cid_add_user(t, mm);
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ }
+}
+
+static bool sched_mm_cid_remove_user(struct task_struct *t)
+{
+ t->mm_cid.active = 0;
+ mm_unset_cid_on_task(t);
+ t->mm->mm_cid.users--;
+ return mm_update_max_cids(t->mm);
}
/*
@@ -10462,14 +10695,8 @@ void sched_mm_cid_exit(struct task_struc
return;
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 0;
- mm->mm_cid.users--;
- if (t->mm_cid.cid != MM_CID_UNSET) {
- clear_bit(t->mm_cid.cid, mm_cidmask(mm));
- t->mm_cid.cid = MM_CID_UNSET;
- }
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock)
+ sched_mm_cid_remove_user(t);
}
/* Deactivate MM CID allocation across execve() */
@@ -10499,6 +10726,8 @@ void mm_init_cid(struct mm_struct *mm, s
mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
+ mm->mm_cid.pcpu_thrs = 0;
+ mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Provide CID ownership mode fixup functions
2025-11-19 17:27 ` [patch V5 17/20] sched/mmcid: Provide CID ownership mode fixup functions Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 340af997d25dab0f05c4de8399d656b112592a93
Gitweb: https://git.kernel.org/tip/340af997d25dab0f05c4de8399d656b112592a93
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:16 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:57 +01:00
sched/mmcid: Provide CID ownership mode fixup functions
CIDs are either owned by tasks or by CPUs. The ownership mode depends on
the number of tasks related to a MM and the number of CPUs on which these
tasks are theoretically allowed to run on. Theoretically because that
number is the superset of CPU affinities of all tasks which only grows and
never shrinks.
Switching to per CPU mode happens when the user count becomes greater than
the maximum number of CIDs, which is calculated by:
opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
max_cids = min(1.25 * opt_cids, nr_cpu_ids);
The +25% allowance is useful for tight CPU masks in scenarios where only a
few threads are created and destroyed to avoid frequent mode
switches. Though this allowance shrinks, the closer opt_cids becomes to
nr_cpu_ids, which is the (unfortunate) hard ABI limit.
At the point of switching to per CPU mode the new user is not yet visible
in the system, so the task which initiated the fork() runs the fixup
function: mm_cid_fixup_tasks_to_cpu() walks the thread list and either
transfers each tasks owned CID to the CPU the task runs on or drops it into
the CID pool if a task is not on a CPU at that point in time. Tasks which
schedule in before the task walk reaches them do the handover in
mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes it's
guaranteed that no task related to that MM owns a CID anymore.
Switching back to task mode happens when the user count goes below the
threshold which was recorded on the per CPU mode switch:
pcpu_thrs = min(opt_cids - (opt_cids / 4), nr_cpu_ids / 2);
This threshold is updated when a affinity change increases the number of
allowed CPUs for the MM, which might cause a switch back to per task mode.
If the switch back was initiated by a exiting task, then that task runs the
fixup function. If it was initiated by a affinity change, then it's run
either in the deferred update function in context of a workqueue or by a
task which forks a new one or by a task which exits. Whatever happens
first. mm_cid_fixup_cpus_to_task() walks through the possible CPUs and
either transfers the CPU owned CIDs to a related task which runs on the CPU
or drops it into the pool. Tasks which schedule in on a CPU which the walk
did not cover yet do the handover themselves.
This transition from CPU to per task ownership happens in two phases:
1) mm:mm_cid.transit contains MM_CID_TRANSIT. This is OR'ed on the task
CID and denotes that the CID is only temporarily owned by the
task. When it schedules out the task drops the CID back into the
pool if this bit is set.
2) The initiating context walks the per CPU space and after completion
clears mm:mm_cid.transit. After that point the CIDs are strictly
task owned again.
This two phase transition is required to prevent CID space exhaustion
during the transition as a direct transfer of ownership would fail if
two tasks are scheduled in on the same CPU before the fixup freed per
CPU CIDs.
When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
related to that MM is owned by a CPU anymore.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.088189028@linutronix.de
---
include/linux/rseq_types.h | 7 +-
kernel/sched/core.c | 277 ++++++++++++++++++++++++++++++++----
2 files changed, 258 insertions(+), 26 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 66b1482..a3a4f3f 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -122,14 +122,15 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
+ * @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
+ * @pcpu_thrs: Threshold for switching back from per CPU mode
+ * @update_deferred: A deferred switch back to per task mode is pending.
*/
struct mm_mm_cid {
/* Hotpath read mostly members */
@@ -144,6 +145,8 @@ struct mm_mm_cid {
/* Low frequency modified */
unsigned int nr_cpus_allowed;
unsigned int users;
+ unsigned int pcpu_thrs;
+ unsigned int update_deferred;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 659ae56..f781d59 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10396,43 +10396,269 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
* task needs to drop the CID into the pool when scheduling out. Both bits
* (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
* actually handed over to user space in the RSEQ memory.
+ *
+ * Mode switching:
+ *
+ * Switching to per CPU mode happens when the user count becomes greater
+ * than the maximum number of CIDs, which is calculated by:
+ *
+ * opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
+ * max_cids = min(1.25 * opt_cids, num_possible_cpus());
+ *
+ * The +25% allowance is useful for tight CPU masks in scenarios where only
+ * a few threads are created and destroyed to avoid frequent mode
+ * switches. Though this allowance shrinks, the closer opt_cids becomes to
+ * num_possible_cpus(), which is the (unfortunate) hard ABI limit.
+ *
+ * At the point of switching to per CPU mode the new user is not yet
+ * visible in the system, so the task which initiated the fork() runs the
+ * fixup function: mm_cid_fixup_tasks_to_cpu() walks the thread list and
+ * either transfers each tasks owned CID to the CPU the task runs on or
+ * drops it into the CID pool if a task is not on a CPU at that point in
+ * time. Tasks which schedule in before the task walk reaches them do the
+ * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes
+ * it's guaranteed that no task related to that MM owns a CID anymore.
+ *
+ * Switching back to task mode happens when the user count goes below the
+ * threshold which was recorded on the per CPU mode switch:
+ *
+ * pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
+ *
+ * This threshold is updated when a affinity change increases the number of
+ * allowed CPUs for the MM, which might cause a switch back to per task
+ * mode.
+ *
+ * If the switch back was initiated by a exiting task, then that task runs
+ * the fixup function. If it was initiated by a affinity change, then it's
+ * run either in the deferred update function in context of a workqueue or
+ * by a task which forks a new one or by a task which exits. Whatever
+ * happens first. mm_cid_fixup_cpus_to_task() walks through the possible
+ * CPUs and either transfers the CPU owned CIDs to a related task which
+ * runs on the CPU or drops it into the pool. Tasks which schedule in on a
+ * CPU which the walk did not cover yet do the handover themself.
+ *
+ * This transition from CPU to per task ownership happens in two phases:
+ *
+ * 1) mm:mm_cid.transit contains MM_CID_TRANSIT This is OR'ed on the task
+ * CID and denotes that the CID is only temporarily owned by the
+ * task. When it schedules out the task drops the CID back into the
+ * pool if this bit is set.
+ *
+ * 2) The initiating context walks the per CPU space and after completion
+ * clears mm:mm_cid.transit. So after that point the CIDs are strictly
+ * task owned again.
+ *
+ * This two phase transition is required to prevent CID space exhaustion
+ * during the transition as a direct transfer of ownership would fail if
+ * two tasks are scheduled in on the same CPU before the fixup freed per
+ * CPU CIDs.
+ *
+ * When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
+ * related to that MM is owned by a CPU anymore.
*/
/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
-static void mm_update_max_cids(struct mm_struct *mm)
+static void __mm_update_max_cids(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids, max_cids;
+
+ /* Calculate the new optimal constraint */
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+
+ /* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
+ max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
+static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids;
+
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+ return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
+}
+
+static bool mm_update_max_cids(struct mm_struct *mm)
{
struct mm_mm_cid *mc = &mm->mm_cid;
- unsigned int max_cids;
lockdep_assert_held(&mm->mm_cid.lock);
- /* Calculate the new maximum constraint */
- max_cids = min(mc->nr_cpus_allowed, mc->users);
- WRITE_ONCE(mc->max_cids, max_cids);
+ /* Clear deferred mode switch flag. A change is handled by the caller */
+ mc->update_deferred = false;
+ __mm_update_max_cids(mc);
+
+ /* Check whether owner mode must be changed */
+ if (!mc->percpu) {
+ /* Enable per CPU mode when the number of users is above max_cids */
+ if (mc->users > mc->max_cids)
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ } else {
+ /* Switch back to per task if user count under threshold */
+ if (mc->users < mc->pcpu_thrs)
+ mc->pcpu_thrs = 0;
+ }
+
+ /* Mode change required? */
+ if (!!mc->percpu == !!mc->pcpu_thrs)
+ return false;
+ /* When switching back to per TASK mode, set the transition flag */
+ if (!mc->pcpu_thrs)
+ WRITE_ONCE(mc->transit, MM_CID_TRANSIT);
+ WRITE_ONCE(mc->percpu, !!mc->pcpu_thrs);
+ return true;
}
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ struct mm_mm_cid *mc;
unsigned int weight;
if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
-
/*
* mm::mm_cid::mm_cpus_allowed is the superset of each threads
* allowed CPUs mask which means it can only grow.
*/
- guard(raw_spinlock)(&mm->mm_cid.lock);
+ mc = &mm->mm_cid;
+ guard(raw_spinlock)(&mc->lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
- if (weight == mm->mm_cid.nr_cpus_allowed)
+ if (weight == mc->nr_cpus_allowed)
+ return;
+
+ WRITE_ONCE(mc->nr_cpus_allowed, weight);
+ __mm_update_max_cids(mc);
+ if (!mc->percpu)
return;
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
- mm_update_max_cids(mm);
+
+ /* Adjust the threshold to the wider set */
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+
+ /* Scheduling of deferred mode switch goes here */
+}
+
+static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_cpu(t->mm_cid.cid)) {
+ unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid);
+
+ t->mm_cid.cid = cid_to_transit_cid(cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+{
+ unsigned int cpu;
+
+ /* Walk the CPUs and fixup all stale CIDs */
+ for_each_possible_cpu(cpu) {
+ struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
+ struct rq *rq = cpu_rq(cpu);
+
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(rq_lock_irq)(rq);
+ /* Is the CID still owned by the CPU? */
+ if (cid_on_cpu(pcp->cid)) {
+ /*
+ * If rq->curr has @mm, transfer it with the
+ * transition bit set. Otherwise drop it.
+ */
+ if (rq->curr->mm == mm && rq->curr->mm_cid.active)
+ mm_cid_transit_to_task(rq->curr, pcp);
+ else
+ mm_drop_cid_on_cpu(mm, pcp);
+
+ } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
+ unsigned int cid = rq->curr->mm_cid.cid;
+
+ /* Ensure it has the transition bit set */
+ if (!cid_in_transit(cid)) {
+ cid = cid_to_transit_cid(cid);
+ rq->curr->mm_cid.cid = cid;
+ pcp->cid = cid;
+ }
+ }
+ }
+ /* Clear the transition bit */
+ WRITE_ONCE(mm->mm_cid.transit, 0);
+}
+
+static inline void mm_cid_transfer_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_task(t->mm_cid.cid)) {
+ t->mm_cid.cid = cid_to_cpu_cid(t->mm_cid.cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
+{
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(task_rq_lock)(t);
+ /* If the task is not active it is not in the users count */
+ if (!t->mm_cid.active)
+ return false;
+ if (cid_on_task(t->mm_cid.cid)) {
+ /* If running on the CPU, transfer the CID, otherwise drop it */
+ if (task_rq(t)->curr == t)
+ mm_cid_transfer_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
+ else
+ mm_unset_cid_on_task(t);
+ }
+ return true;
+}
+
+static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+{
+ struct mm_struct *mm = current->mm;
+ struct task_struct *p, *t;
+ unsigned int users;
+
+ /*
+ * This can obviously race with a concurrent affinity change, which
+ * increases the number of allowed CPUs for this mm, but that does
+ * not affect the mode and only changes the CID constraints. A
+ * possible switch back to per task mode happens either in the
+ * deferred handler function or in the next fork()/exit().
+ *
+ * The caller has already transferred. The newly incoming task is
+ * already accounted for, but not yet visible.
+ */
+ users = mm->mm_cid.users - 2;
+ if (!users)
+ return;
+
+ guard(rcu)();
+ for_other_threads(current, t) {
+ if (mm_cid_fixup_task_to_cpu(t, mm))
+ users--;
+ }
+
+ if (!users)
+ return;
+
+ /* Happens only for VM_CLONE processes. */
+ for_each_process_thread(p, t) {
+ if (t == current || t->mm != mm)
+ continue;
+ if (mm_cid_fixup_task_to_cpu(t, mm)) {
+ if (--users == 0)
+ return;
+ }
+ }
+}
+
+static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
+{
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ return mm_update_max_cids(mm);
}
void sched_mm_cid_fork(struct task_struct *t)
@@ -10442,12 +10668,19 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 1;
- mm->mm_cid.users++;
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
+ sched_mm_cid_add_user(t, mm);
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ }
+}
+
+static bool sched_mm_cid_remove_user(struct task_struct *t)
+{
+ t->mm_cid.active = 0;
+ mm_unset_cid_on_task(t);
+ t->mm->mm_cid.users--;
+ return mm_update_max_cids(t->mm);
}
/*
@@ -10462,14 +10695,8 @@ void sched_mm_cid_exit(struct task_struct *t)
return;
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 0;
- mm->mm_cid.users--;
- if (t->mm_cid.cid != MM_CID_UNSET) {
- clear_bit(t->mm_cid.cid, mm_cidmask(mm));
- t->mm_cid.cid = MM_CID_UNSET;
- }
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock)
+ sched_mm_cid_remove_user(t);
}
/* Deactivate MM CID allocation across execve() */
@@ -10499,6 +10726,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
+ mm->mm_cid.pcpu_thrs = 0;
+ mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Provide CID ownership mode fixup functions
2025-11-19 17:27 ` [patch V5 17/20] sched/mmcid: Provide CID ownership mode fixup functions Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: fbd0e71dc370af73f6b316e4de9eed273dd90340
Gitweb: https://git.kernel.org/tip/fbd0e71dc370af73f6b316e4de9eed273dd90340
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:16 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
sched/mmcid: Provide CID ownership mode fixup functions
CIDs are either owned by tasks or by CPUs. The ownership mode depends on
the number of tasks related to a MM and the number of CPUs on which these
tasks are theoretically allowed to run on. Theoretically because that
number is the superset of CPU affinities of all tasks which only grows and
never shrinks.
Switching to per CPU mode happens when the user count becomes greater than
the maximum number of CIDs, which is calculated by:
opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
max_cids = min(1.25 * opt_cids, nr_cpu_ids);
The +25% allowance is useful for tight CPU masks in scenarios where only a
few threads are created and destroyed to avoid frequent mode
switches. Though this allowance shrinks, the closer opt_cids becomes to
nr_cpu_ids, which is the (unfortunate) hard ABI limit.
At the point of switching to per CPU mode the new user is not yet visible
in the system, so the task which initiated the fork() runs the fixup
function: mm_cid_fixup_tasks_to_cpu() walks the thread list and either
transfers each tasks owned CID to the CPU the task runs on or drops it into
the CID pool if a task is not on a CPU at that point in time. Tasks which
schedule in before the task walk reaches them do the handover in
mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes it's
guaranteed that no task related to that MM owns a CID anymore.
Switching back to task mode happens when the user count goes below the
threshold which was recorded on the per CPU mode switch:
pcpu_thrs = min(opt_cids - (opt_cids / 4), nr_cpu_ids / 2);
This threshold is updated when a affinity change increases the number of
allowed CPUs for the MM, which might cause a switch back to per task mode.
If the switch back was initiated by a exiting task, then that task runs the
fixup function. If it was initiated by a affinity change, then it's run
either in the deferred update function in context of a workqueue or by a
task which forks a new one or by a task which exits. Whatever happens
first. mm_cid_fixup_cpus_to_task() walks through the possible CPUs and
either transfers the CPU owned CIDs to a related task which runs on the CPU
or drops it into the pool. Tasks which schedule in on a CPU which the walk
did not cover yet do the handover themselves.
This transition from CPU to per task ownership happens in two phases:
1) mm:mm_cid.transit contains MM_CID_TRANSIT. This is OR'ed on the task
CID and denotes that the CID is only temporarily owned by the
task. When it schedules out the task drops the CID back into the
pool if this bit is set.
2) The initiating context walks the per CPU space and after completion
clears mm:mm_cid.transit. After that point the CIDs are strictly
task owned again.
This two phase transition is required to prevent CID space exhaustion
during the transition as a direct transfer of ownership would fail if
two tasks are scheduled in on the same CPU before the fixup freed per
CPU CIDs.
When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
related to that MM is owned by a CPU anymore.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.088189028@linutronix.de
---
include/linux/rseq_types.h | 7 +-
kernel/sched/core.c | 278 ++++++++++++++++++++++++++++++++----
2 files changed, 259 insertions(+), 26 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 66b1482..a3a4f3f 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -122,14 +122,15 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
- * @lock: Spinlock to protect all fields except @pcpu. It also protects
- * the MM cid cpumask and the MM cidmask bitmap.
+ * @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
* is growth only.
* @users: The number of tasks sharing this MM. Separate from mm::mm_users
* as that is modified by mmget()/mm_put() by other entities which
* do not actually share the MM.
+ * @pcpu_thrs: Threshold for switching back from per CPU mode
+ * @update_deferred: A deferred switch back to per task mode is pending.
*/
struct mm_mm_cid {
/* Hotpath read mostly members */
@@ -144,6 +145,8 @@ struct mm_mm_cid {
/* Low frequency modified */
unsigned int nr_cpus_allowed;
unsigned int users;
+ unsigned int pcpu_thrs;
+ unsigned int update_deferred;
}____cacheline_aligned_in_smp;
#else /* CONFIG_SCHED_MM_CID */
struct mm_mm_cid { };
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 659ae56..eb0d59d 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10396,43 +10396,270 @@ void call_trace_sched_update_nr_running(struct rq *rq, int count)
* task needs to drop the CID into the pool when scheduling out. Both bits
* (ONCPU and TRANSIT) are filtered out by task_cid() when the CID is
* actually handed over to user space in the RSEQ memory.
+ *
+ * Mode switching:
+ *
+ * Switching to per CPU mode happens when the user count becomes greater
+ * than the maximum number of CIDs, which is calculated by:
+ *
+ * opt_cids = min(mm_cid::nr_cpus_allowed, mm_cid::users);
+ * max_cids = min(1.25 * opt_cids, num_possible_cpus());
+ *
+ * The +25% allowance is useful for tight CPU masks in scenarios where only
+ * a few threads are created and destroyed to avoid frequent mode
+ * switches. Though this allowance shrinks, the closer opt_cids becomes to
+ * num_possible_cpus(), which is the (unfortunate) hard ABI limit.
+ *
+ * At the point of switching to per CPU mode the new user is not yet
+ * visible in the system, so the task which initiated the fork() runs the
+ * fixup function: mm_cid_fixup_tasks_to_cpu() walks the thread list and
+ * either transfers each tasks owned CID to the CPU the task runs on or
+ * drops it into the CID pool if a task is not on a CPU at that point in
+ * time. Tasks which schedule in before the task walk reaches them do the
+ * handover in mm_cid_schedin(). When mm_cid_fixup_tasks_to_cpus() completes
+ * it's guaranteed that no task related to that MM owns a CID anymore.
+ *
+ * Switching back to task mode happens when the user count goes below the
+ * threshold which was recorded on the per CPU mode switch:
+ *
+ * pcpu_thrs = min(opt_cids - (opt_cids / 4), num_possible_cpus() / 2);
+ *
+ * This threshold is updated when a affinity change increases the number of
+ * allowed CPUs for the MM, which might cause a switch back to per task
+ * mode.
+ *
+ * If the switch back was initiated by a exiting task, then that task runs
+ * the fixup function. If it was initiated by a affinity change, then it's
+ * run either in the deferred update function in context of a workqueue or
+ * by a task which forks a new one or by a task which exits. Whatever
+ * happens first. mm_cid_fixup_cpus_to_task() walks through the possible
+ * CPUs and either transfers the CPU owned CIDs to a related task which
+ * runs on the CPU or drops it into the pool. Tasks which schedule in on a
+ * CPU which the walk did not cover yet do the handover themself.
+ *
+ * This transition from CPU to per task ownership happens in two phases:
+ *
+ * 1) mm:mm_cid.transit contains MM_CID_TRANSIT This is OR'ed on the task
+ * CID and denotes that the CID is only temporarily owned by the
+ * task. When it schedules out the task drops the CID back into the
+ * pool if this bit is set.
+ *
+ * 2) The initiating context walks the per CPU space and after completion
+ * clears mm:mm_cid.transit. So after that point the CIDs are strictly
+ * task owned again.
+ *
+ * This two phase transition is required to prevent CID space exhaustion
+ * during the transition as a direct transfer of ownership would fail if
+ * two tasks are scheduled in on the same CPU before the fixup freed per
+ * CPU CIDs.
+ *
+ * When mm_cid_fixup_cpus_to_tasks() completes it's guaranteed that no CID
+ * related to that MM is owned by a CPU anymore.
*/
/*
* Update the CID range properties when the constraints change. Invoked via
* fork(), exit() and affinity changes
*/
-static void mm_update_max_cids(struct mm_struct *mm)
+static void __mm_update_max_cids(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids, max_cids;
+
+ /* Calculate the new optimal constraint */
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+
+ /* Adjust the maximum CIDs to +25% limited by the number of possible CPUs */
+ max_cids = min(opt_cids + (opt_cids / 4), num_possible_cpus());
+ WRITE_ONCE(mc->max_cids, max_cids);
+}
+
+static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
+{
+ unsigned int opt_cids;
+
+ opt_cids = min(mc->nr_cpus_allowed, mc->users);
+ /* Has to be at least 1 because 0 indicates PCPU mode off */
+ return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
+}
+
+static bool mm_update_max_cids(struct mm_struct *mm)
{
struct mm_mm_cid *mc = &mm->mm_cid;
- unsigned int max_cids;
lockdep_assert_held(&mm->mm_cid.lock);
- /* Calculate the new maximum constraint */
- max_cids = min(mc->nr_cpus_allowed, mc->users);
- WRITE_ONCE(mc->max_cids, max_cids);
+ /* Clear deferred mode switch flag. A change is handled by the caller */
+ mc->update_deferred = false;
+ __mm_update_max_cids(mc);
+
+ /* Check whether owner mode must be changed */
+ if (!mc->percpu) {
+ /* Enable per CPU mode when the number of users is above max_cids */
+ if (mc->users > mc->max_cids)
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ } else {
+ /* Switch back to per task if user count under threshold */
+ if (mc->users < mc->pcpu_thrs)
+ mc->pcpu_thrs = 0;
+ }
+
+ /* Mode change required? */
+ if (!!mc->percpu == !!mc->pcpu_thrs)
+ return false;
+ /* When switching back to per TASK mode, set the transition flag */
+ if (!mc->pcpu_thrs)
+ WRITE_ONCE(mc->transit, MM_CID_TRANSIT);
+ WRITE_ONCE(mc->percpu, !!mc->pcpu_thrs);
+ return true;
}
static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpumask *affmsk)
{
struct cpumask *mm_allowed;
+ struct mm_mm_cid *mc;
unsigned int weight;
if (!mm || !READ_ONCE(mm->mm_cid.users))
return;
-
/*
* mm::mm_cid::mm_cpus_allowed is the superset of each threads
* allowed CPUs mask which means it can only grow.
*/
- guard(raw_spinlock)(&mm->mm_cid.lock);
+ mc = &mm->mm_cid;
+ guard(raw_spinlock)(&mc->lock);
mm_allowed = mm_cpus_allowed(mm);
weight = cpumask_weighted_or(mm_allowed, mm_allowed, affmsk);
- if (weight == mm->mm_cid.nr_cpus_allowed)
+ if (weight == mc->nr_cpus_allowed)
+ return;
+
+ WRITE_ONCE(mc->nr_cpus_allowed, weight);
+ __mm_update_max_cids(mc);
+ if (!mc->percpu)
return;
- WRITE_ONCE(mm->mm_cid.nr_cpus_allowed, weight);
- mm_update_max_cids(mm);
+
+ /* Adjust the threshold to the wider set */
+ mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+
+ /* Scheduling of deferred mode switch goes here */
+}
+
+static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_cpu(t->mm_cid.cid)) {
+ unsigned int cid = cpu_cid_to_cid(t->mm_cid.cid);
+
+ t->mm_cid.cid = cid_to_transit_cid(cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+{
+ unsigned int cpu;
+
+ /* Walk the CPUs and fixup all stale CIDs */
+ for_each_possible_cpu(cpu) {
+ struct mm_cid_pcpu *pcp = per_cpu_ptr(mm->mm_cid.pcpu, cpu);
+ struct rq *rq = cpu_rq(cpu);
+
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(rq_lock_irq)(rq);
+ /* Is the CID still owned by the CPU? */
+ if (cid_on_cpu(pcp->cid)) {
+ /*
+ * If rq->curr has @mm, transfer it with the
+ * transition bit set. Otherwise drop it.
+ */
+ if (rq->curr->mm == mm && rq->curr->mm_cid.active)
+ mm_cid_transit_to_task(rq->curr, pcp);
+ else
+ mm_drop_cid_on_cpu(mm, pcp);
+
+ } else if (rq->curr->mm == mm && rq->curr->mm_cid.active) {
+ unsigned int cid = rq->curr->mm_cid.cid;
+
+ /* Ensure it has the transition bit set */
+ if (!cid_in_transit(cid)) {
+ cid = cid_to_transit_cid(cid);
+ rq->curr->mm_cid.cid = cid;
+ pcp->cid = cid;
+ }
+ }
+ }
+ /* Clear the transition bit */
+ WRITE_ONCE(mm->mm_cid.transit, 0);
+}
+
+static inline void mm_cid_transfer_to_cpu(struct task_struct *t, struct mm_cid_pcpu *pcp)
+{
+ if (cid_on_task(t->mm_cid.cid)) {
+ t->mm_cid.cid = cid_to_cpu_cid(t->mm_cid.cid);
+ pcp->cid = t->mm_cid.cid;
+ }
+}
+
+static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm)
+{
+ /* Remote access to mm::mm_cid::pcpu requires rq_lock */
+ guard(task_rq_lock)(t);
+ /* If the task is not active it is not in the users count */
+ if (!t->mm_cid.active)
+ return false;
+ if (cid_on_task(t->mm_cid.cid)) {
+ /* If running on the CPU, transfer the CID, otherwise drop it */
+ if (task_rq(t)->curr == t)
+ mm_cid_transfer_to_cpu(t, per_cpu_ptr(mm->mm_cid.pcpu, task_cpu(t)));
+ else
+ mm_unset_cid_on_task(t);
+ }
+ return true;
+}
+
+static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+{
+ struct mm_struct *mm = current->mm;
+ struct task_struct *p, *t;
+ unsigned int users;
+
+ /*
+ * This can obviously race with a concurrent affinity change, which
+ * increases the number of allowed CPUs for this mm, but that does
+ * not affect the mode and only changes the CID constraints. A
+ * possible switch back to per task mode happens either in the
+ * deferred handler function or in the next fork()/exit().
+ *
+ * The caller has already transferred. The newly incoming task is
+ * already accounted for, but not yet visible.
+ */
+ users = mm->mm_cid.users - 2;
+ if (!users)
+ return;
+
+ guard(rcu)();
+ for_other_threads(current, t) {
+ if (mm_cid_fixup_task_to_cpu(t, mm))
+ users--;
+ }
+
+ if (!users)
+ return;
+
+ /* Happens only for VM_CLONE processes. */
+ for_each_process_thread(p, t) {
+ if (t == current || t->mm != mm)
+ continue;
+ if (mm_cid_fixup_task_to_cpu(t, mm)) {
+ if (--users == 0)
+ return;
+ }
+ }
+}
+
+static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
+{
+ t->mm_cid.active = 1;
+ mm->mm_cid.users++;
+ return mm_update_max_cids(mm);
}
void sched_mm_cid_fork(struct task_struct *t)
@@ -10442,12 +10669,19 @@ void sched_mm_cid_fork(struct task_struct *t)
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 1;
- mm->mm_cid.users++;
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = READ_ONCE(mm->mm_cid.max_cids) - 1;
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
+ sched_mm_cid_add_user(t, mm);
+ /* Preset last_cid for mm_cid_select() */
+ t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ }
+}
+
+static bool sched_mm_cid_remove_user(struct task_struct *t)
+{
+ t->mm_cid.active = 0;
+ mm_unset_cid_on_task(t);
+ t->mm->mm_cid.users--;
+ return mm_update_max_cids(t->mm);
}
/*
@@ -10462,14 +10696,8 @@ void sched_mm_cid_exit(struct task_struct *t)
return;
guard(mutex)(&mm->mm_cid.mutex);
- guard(raw_spinlock)(&mm->mm_cid.lock);
- t->mm_cid.active = 0;
- mm->mm_cid.users--;
- if (t->mm_cid.cid != MM_CID_UNSET) {
- clear_bit(t->mm_cid.cid, mm_cidmask(mm));
- t->mm_cid.cid = MM_CID_UNSET;
- }
- mm_update_max_cids(mm);
+ scoped_guard(raw_spinlock, &mm->mm_cid.lock)
+ sched_mm_cid_remove_user(t);
}
/* Deactivate MM CID allocation across execve() */
@@ -10499,6 +10727,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.transit = 0;
mm->mm_cid.nr_cpus_allowed = p->nr_cpus_allowed;
mm->mm_cid.users = 0;
+ mm->mm_cid.pcpu_thrs = 0;
+ mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 18/20] irqwork: Move data struct to a types header
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (16 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 17/20] sched/mmcid: Provide CID ownership mode fixup functions Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 19/20] sched/mmcid: Implement deferred mode change Thomas Gleixner
` (2 subsequent siblings)
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
... to avoid header recursion hell.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/irq_work.h | 9 ++-------
include/linux/irq_work_types.h | 14 ++++++++++++++
2 files changed, 16 insertions(+), 7 deletions(-)
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -2,8 +2,9 @@
#ifndef _LINUX_IRQ_WORK_H
#define _LINUX_IRQ_WORK_H
-#include <linux/smp_types.h>
+#include <linux/irq_work_types.h>
#include <linux/rcuwait.h>
+#include <linux/smp_types.h>
/*
* An entry can be in one of four states:
@@ -14,12 +15,6 @@
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
*/
-struct irq_work {
- struct __call_single_node node;
- void (*func)(struct irq_work *);
- struct rcuwait irqwait;
-};
-
#define __IRQ_WORK_INIT(_func, _flags) (struct irq_work){ \
.node = { .u_flags = (_flags), }, \
.func = (_func), \
--- /dev/null
+++ b/include/linux/irq_work_types.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_IRQ_WORK_TYPES_H
+#define _LINUX_IRQ_WORK_TYPES_H
+
+#include <linux/smp_types.h>
+#include <linux/types.h>
+
+struct irq_work {
+ struct __call_single_node node;
+ void (*func)(struct irq_work *);
+ struct rcuwait irqwait;
+};
+
+#endif
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] irqwork: Move data struct to a types header
2025-11-19 17:27 ` [patch V5 18/20] irqwork: Move data struct to a types header Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 2644779ec144d3e8cce5fed9623b47e70b3e0422
Gitweb: https://git.kernel.org/tip/2644779ec144d3e8cce5fed9623b47e70b3e0422
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:18 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:58 +01:00
irqwork: Move data struct to a types header
... to avoid header recursion hell.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.152813625@linutronix.de
---
include/linux/irq_work.h | 9 ++-------
include/linux/irq_work_types.h | 14 ++++++++++++++
2 files changed, 16 insertions(+), 7 deletions(-)
create mode 100644 include/linux/irq_work_types.h
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h
index 136f298..c5afd05 100644
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -2,8 +2,9 @@
#ifndef _LINUX_IRQ_WORK_H
#define _LINUX_IRQ_WORK_H
-#include <linux/smp_types.h>
+#include <linux/irq_work_types.h>
#include <linux/rcuwait.h>
+#include <linux/smp_types.h>
/*
* An entry can be in one of four states:
@@ -14,12 +15,6 @@
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
*/
-struct irq_work {
- struct __call_single_node node;
- void (*func)(struct irq_work *);
- struct rcuwait irqwait;
-};
-
#define __IRQ_WORK_INIT(_func, _flags) (struct irq_work){ \
.node = { .u_flags = (_flags), }, \
.func = (_func), \
diff --git a/include/linux/irq_work_types.h b/include/linux/irq_work_types.h
new file mode 100644
index 0000000..73abec5
--- /dev/null
+++ b/include/linux/irq_work_types.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_IRQ_WORK_TYPES_H
+#define _LINUX_IRQ_WORK_TYPES_H
+
+#include <linux/smp_types.h>
+#include <linux/types.h>
+
+struct irq_work {
+ struct __call_single_node node;
+ void (*func)(struct irq_work *);
+ struct rcuwait irqwait;
+};
+
+#endif
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] irqwork: Move data struct to a types header
2025-11-19 17:27 ` [patch V5 18/20] irqwork: Move data struct to a types header Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: c809f081fe400cb1b9898f4791c0d33146315161
Gitweb: https://git.kernel.org/tip/c809f081fe400cb1b9898f4791c0d33146315161
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:18 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:41 +01:00
irqwork: Move data struct to a types header
... to avoid header recursion hell.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.152813625@linutronix.de
---
include/linux/irq_work.h | 9 ++-------
include/linux/irq_work_types.h | 14 ++++++++++++++
2 files changed, 16 insertions(+), 7 deletions(-)
create mode 100644 include/linux/irq_work_types.h
diff --git a/include/linux/irq_work.h b/include/linux/irq_work.h
index 136f298..c5afd05 100644
--- a/include/linux/irq_work.h
+++ b/include/linux/irq_work.h
@@ -2,8 +2,9 @@
#ifndef _LINUX_IRQ_WORK_H
#define _LINUX_IRQ_WORK_H
-#include <linux/smp_types.h>
+#include <linux/irq_work_types.h>
#include <linux/rcuwait.h>
+#include <linux/smp_types.h>
/*
* An entry can be in one of four states:
@@ -14,12 +15,6 @@
* busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed
*/
-struct irq_work {
- struct __call_single_node node;
- void (*func)(struct irq_work *);
- struct rcuwait irqwait;
-};
-
#define __IRQ_WORK_INIT(_func, _flags) (struct irq_work){ \
.node = { .u_flags = (_flags), }, \
.func = (_func), \
diff --git a/include/linux/irq_work_types.h b/include/linux/irq_work_types.h
new file mode 100644
index 0000000..73abec5
--- /dev/null
+++ b/include/linux/irq_work_types.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _LINUX_IRQ_WORK_TYPES_H
+#define _LINUX_IRQ_WORK_TYPES_H
+
+#include <linux/smp_types.h>
+#include <linux/types.h>
+
+struct irq_work {
+ struct __call_single_node node;
+ void (*func)(struct irq_work *);
+ struct rcuwait irqwait;
+};
+
+#endif
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 19/20] sched/mmcid: Implement deferred mode change
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (17 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 18/20] irqwork: Move data struct to a types header Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2025-11-19 17:27 ` [patch V5 20/20] sched/mmcid: Switch over to the new mechanism Thomas Gleixner
2026-01-28 0:01 ` [patch V5 00/20] sched: Rewrite MM CID management Ihor Solodrai
20 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
When affinity changes cause an increase of the number of CPUs allowed for
tasks which are related to a MM, that might results in a situation where
the ownership mode can go back from per CPU mode to per task mode.
As affinity changes happen with runqueue lock held there is no way to do
the actual mode change and required fixup right there.
Add the infrastructure to defer it to a workqueue. The scheduled work can
race with a fork() or exit(). Whatever happens first takes care of it.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq_types.h | 8 ++++++
kernel/sched/core.c | 58 +++++++++++++++++++++++++++++++++++++++------
2 files changed, 59 insertions(+), 7 deletions(-)
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -2,7 +2,9 @@
#ifndef _LINUX_RSEQ_TYPES_H
#define _LINUX_RSEQ_TYPES_H
+#include <linux/irq_work_types.h>
#include <linux/types.h>
+#include <linux/workqueue_types.h>
#ifdef CONFIG_RSEQ
struct rseq;
@@ -122,6 +124,8 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @irq_work: irq_work to handle the affinity mode change case
+ * @work: Regular work to handle the affinity mode change case
* @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
@@ -139,6 +143,10 @@ struct mm_mm_cid {
unsigned int transit;
unsigned int max_cids;
+ /* Rarely used. Moves @lock and @mutex into the second cacheline */
+ struct irq_work irq_work;
+ struct work_struct work;
+
raw_spinlock_t lock;
struct mutex mutex;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10538,8 +10538,17 @@ static inline void mm_update_cpus_allowe
/* Adjust the threshold to the wider set */
mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ /* Switch back to per task mode? */
+ if (mc->users >= mc->pcpu_thrs)
+ return;
+
+ /* Don't queue twice */
+ if (mc->update_deferred)
+ return;
- /* Scheduling of deferred mode switch goes here */
+ /* Queue the irq work, which schedules the real work */
+ mc->update_deferred = true;
+ irq_work_queue(&mc->irq_work);
}
static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
@@ -10552,7 +10561,7 @@ static inline void mm_cid_transit_to_tas
}
}
-static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
{
unsigned int cpu;
@@ -10713,14 +10722,47 @@ void sched_mm_cid_after_execve(struct ta
mm_cid_select(t);
}
-void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+static void mm_cid_work_fn(struct work_struct *work)
{
- struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
- int cpu;
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
+
+ /* Make it compile, but not functional yet */
+ if (!IS_ENABLED(CONFIG_NEW_MM_CID))
+ return;
- for_each_possible_cpu(cpu)
- per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ guard(mutex)(&mm->mm_cid.mutex);
+ /* Did the last user task exit already? */
+ if (!mm->mm_cid.users)
+ return;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Have fork() or exit() handled it already? */
+ if (!mm->mm_cid.update_deferred)
+ return;
+ /* This clears mm_cid::update_deferred */
+ if (!mm_update_max_cids(mm))
+ return;
+ /* Affinity changes can only switch back to task mode */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return;
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+}
+
+static void mm_cid_irq_work(struct irq_work *work)
+{
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
+
+ /*
+ * Needs to be unconditional because mm_cid::lock cannot be held
+ * when scheduling work as mm_update_cpus_allowed() nests inside
+ * rq::lock and schedule_work() might end up in wakeup...
+ */
+ schedule_work(&mm->mm_cid.work);
+}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
mm->mm_cid.max_cids = 0;
mm->mm_cid.percpu = 0;
mm->mm_cid.transit = 0;
@@ -10730,6 +10772,8 @@ void mm_init_cid(struct mm_struct *mm, s
mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
+ mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
+ INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Implement deferred mode change
2025-11-19 17:27 ` [patch V5 19/20] sched/mmcid: Implement deferred mode change Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: cba5e581161e379037a94f5a75d1a61bd1ccce3b
Gitweb: https://git.kernel.org/tip/cba5e581161e379037a94f5a75d1a61bd1ccce3b
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:20 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:58 +01:00
sched/mmcid: Implement deferred mode change
When affinity changes cause an increase of the number of CPUs allowed for
tasks which are related to a MM, that might results in a situation where
the ownership mode can go back from per CPU mode to per task mode.
As affinity changes happen with runqueue lock held there is no way to do
the actual mode change and required fixup right there.
Add the infrastructure to defer it to a workqueue. The scheduled work can
race with a fork() or exit(). Whatever happens first takes care of it.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.216484739@linutronix.de
---
include/linux/rseq_types.h | 8 +++++-
kernel/sched/core.c | 58 ++++++++++++++++++++++++++++++++-----
2 files changed, 59 insertions(+), 7 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index a3a4f3f..81fbb88 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -2,7 +2,9 @@
#ifndef _LINUX_RSEQ_TYPES_H
#define _LINUX_RSEQ_TYPES_H
+#include <linux/irq_work_types.h>
#include <linux/types.h>
+#include <linux/workqueue_types.h>
#ifdef CONFIG_RSEQ
struct rseq;
@@ -122,6 +124,8 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @irq_work: irq_work to handle the affinity mode change case
+ * @work: Regular work to handle the affinity mode change case
* @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
@@ -139,6 +143,10 @@ struct mm_mm_cid {
unsigned int transit;
unsigned int max_cids;
+ /* Rarely used. Moves @lock and @mutex into the second cacheline */
+ struct irq_work irq_work;
+ struct work_struct work;
+
raw_spinlock_t lock;
struct mutex mutex;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index f781d59..938757e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10538,8 +10538,17 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
/* Adjust the threshold to the wider set */
mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ /* Switch back to per task mode? */
+ if (mc->users >= mc->pcpu_thrs)
+ return;
+
+ /* Don't queue twice */
+ if (mc->update_deferred)
+ return;
- /* Scheduling of deferred mode switch goes here */
+ /* Queue the irq work, which schedules the real work */
+ mc->update_deferred = true;
+ irq_work_queue(&mc->irq_work);
}
static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
@@ -10552,7 +10561,7 @@ static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_p
}
}
-static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
{
unsigned int cpu;
@@ -10713,14 +10722,47 @@ void sched_mm_cid_after_execve(struct task_struct *t)
mm_cid_select(t);
}
-void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+static void mm_cid_work_fn(struct work_struct *work)
{
- struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
- int cpu;
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- for_each_possible_cpu(cpu)
- per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ /* Make it compile, but not functional yet */
+ if (!IS_ENABLED(CONFIG_NEW_MM_CID))
+ return;
+
+ guard(mutex)(&mm->mm_cid.mutex);
+ /* Did the last user task exit already? */
+ if (!mm->mm_cid.users)
+ return;
+
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Have fork() or exit() handled it already? */
+ if (!mm->mm_cid.update_deferred)
+ return;
+ /* This clears mm_cid::update_deferred */
+ if (!mm_update_max_cids(mm))
+ return;
+ /* Affinity changes can only switch back to task mode */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return;
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+}
+
+static void mm_cid_irq_work(struct irq_work *work)
+{
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
+ /*
+ * Needs to be unconditional because mm_cid::lock cannot be held
+ * when scheduling work as mm_update_cpus_allowed() nests inside
+ * rq::lock and schedule_work() might end up in wakeup...
+ */
+ schedule_work(&mm->mm_cid.work);
+}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
mm->mm_cid.max_cids = 0;
mm->mm_cid.percpu = 0;
mm->mm_cid.transit = 0;
@@ -10730,6 +10772,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
+ mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
+ INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
^ permalink raw reply related [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Implement deferred mode change
2025-11-19 17:27 ` [patch V5 19/20] sched/mmcid: Implement deferred mode change Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 9da6ccbcea3de1fa704202e3346fe6c0226bfc18
Gitweb: https://git.kernel.org/tip/9da6ccbcea3de1fa704202e3346fe6c0226bfc18
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:20 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:42 +01:00
sched/mmcid: Implement deferred mode change
When affinity changes cause an increase of the number of CPUs allowed for
tasks which are related to a MM, that might results in a situation where
the ownership mode can go back from per CPU mode to per task mode.
As affinity changes happen with runqueue lock held there is no way to do
the actual mode change and required fixup right there.
Add the infrastructure to defer it to a workqueue. The scheduled work can
race with a fork() or exit(). Whatever happens first takes care of it.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.216484739@linutronix.de
---
include/linux/rseq_types.h | 8 +++++-
kernel/sched/core.c | 58 ++++++++++++++++++++++++++++++++-----
2 files changed, 59 insertions(+), 7 deletions(-)
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index a3a4f3f..81fbb88 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -2,7 +2,9 @@
#ifndef _LINUX_RSEQ_TYPES_H
#define _LINUX_RSEQ_TYPES_H
+#include <linux/irq_work_types.h>
#include <linux/types.h>
+#include <linux/workqueue_types.h>
#ifdef CONFIG_RSEQ
struct rseq;
@@ -122,6 +124,8 @@ struct mm_cid_pcpu {
* @percpu: Set, when CIDs are in per CPU mode
* @transit: Set to MM_CID_TRANSIT during a mode change transition phase
* @max_cids: The exclusive maximum CID value for allocation and convergence
+ * @irq_work: irq_work to handle the affinity mode change case
+ * @work: Regular work to handle the affinity mode change case
* @lock: Spinlock to protect against affinity setting which can't take @mutex
* @mutex: Mutex to serialize forks and exits related to this mm
* @nr_cpus_allowed: The number of CPUs in the per MM allowed CPUs map. The map
@@ -139,6 +143,10 @@ struct mm_mm_cid {
unsigned int transit;
unsigned int max_cids;
+ /* Rarely used. Moves @lock and @mutex into the second cacheline */
+ struct irq_work irq_work;
+ struct work_struct work;
+
raw_spinlock_t lock;
struct mutex mutex;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index eb0d59d..cbb543a 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10539,8 +10539,17 @@ static inline void mm_update_cpus_allowed(struct mm_struct *mm, const struct cpu
/* Adjust the threshold to the wider set */
mc->pcpu_thrs = mm_cid_calc_pcpu_thrs(mc);
+ /* Switch back to per task mode? */
+ if (mc->users >= mc->pcpu_thrs)
+ return;
+
+ /* Don't queue twice */
+ if (mc->update_deferred)
+ return;
- /* Scheduling of deferred mode switch goes here */
+ /* Queue the irq work, which schedules the real work */
+ mc->update_deferred = true;
+ irq_work_queue(&mc->irq_work);
}
static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_pcpu *pcp)
@@ -10553,7 +10562,7 @@ static inline void mm_cid_transit_to_task(struct task_struct *t, struct mm_cid_p
}
}
-static void __maybe_unused mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
+static void mm_cid_fixup_cpus_to_tasks(struct mm_struct *mm)
{
unsigned int cpu;
@@ -10714,14 +10723,47 @@ void sched_mm_cid_after_execve(struct task_struct *t)
mm_cid_select(t);
}
-void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+static void mm_cid_work_fn(struct work_struct *work)
{
- struct mm_cid_pcpu __percpu *pcpu = mm->mm_cid.pcpu;
- int cpu;
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- for_each_possible_cpu(cpu)
- per_cpu_ptr(pcpu, cpu)->cid = MM_CID_UNSET;
+ /* Make it compile, but not functional yet */
+ if (!IS_ENABLED(CONFIG_NEW_MM_CID))
+ return;
+
+ guard(mutex)(&mm->mm_cid.mutex);
+ /* Did the last user task exit already? */
+ if (!mm->mm_cid.users)
+ return;
+
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Have fork() or exit() handled it already? */
+ if (!mm->mm_cid.update_deferred)
+ return;
+ /* This clears mm_cid::update_deferred */
+ if (!mm_update_max_cids(mm))
+ return;
+ /* Affinity changes can only switch back to task mode */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return;
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+}
+
+static void mm_cid_irq_work(struct irq_work *work)
+{
+ struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.irq_work);
+ /*
+ * Needs to be unconditional because mm_cid::lock cannot be held
+ * when scheduling work as mm_update_cpus_allowed() nests inside
+ * rq::lock and schedule_work() might end up in wakeup...
+ */
+ schedule_work(&mm->mm_cid.work);
+}
+
+void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
+{
mm->mm_cid.max_cids = 0;
mm->mm_cid.percpu = 0;
mm->mm_cid.transit = 0;
@@ -10731,6 +10773,8 @@ void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
mm->mm_cid.update_deferred = 0;
raw_spin_lock_init(&mm->mm_cid.lock);
mutex_init(&mm->mm_cid.mutex);
+ mm->mm_cid.irq_work = IRQ_WORK_INIT_HARD(mm_cid_irq_work);
+ INIT_WORK(&mm->mm_cid.work, mm_cid_work_fn);
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
bitmap_zero(mm_cidmask(mm), num_possible_cpus());
}
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [patch V5 20/20] sched/mmcid: Switch over to the new mechanism
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (18 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 19/20] sched/mmcid: Implement deferred mode change Thomas Gleixner
@ 2025-11-19 17:27 ` Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
` (2 more replies)
2026-01-28 0:01 ` [patch V5 00/20] sched: Rewrite MM CID management Ihor Solodrai
20 siblings, 3 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-19 17:27 UTC (permalink / raw)
To: LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Now that all pieces are in place, change the implementations of
sched_mm_cid_fork() and sched_mm_cid_exit() to adhere to the new strict
ownership scheme and switch context_switch() over to use the new
mm_cid_schedin() functionality.
The common case is that there is no mode change required, which makes
fork() and exit() just update the user count and the constraints.
In case that a new user would exceed the CID space limit the fork() context
handles the transition to per CPU mode with mm::mm_cid::mutex held. exit()
handles the transition back to per task mode when the user count drops
below the switch back threshold. fork() might also be forced to handle a
deferred switch back to per task mode, when a affinity change increased the
number of allowed CPUs enough.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
include/linux/rseq.h | 19 -------
include/linux/rseq_types.h | 8 +--
kernel/fork.c | 1
kernel/sched/core.c | 115 ++++++++++++++++++++++++++++++++++++++-------
kernel/sched/sched.h | 76 -----------------------------
5 files changed, 103 insertions(+), 116 deletions(-)
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -84,24 +84,6 @@ static __always_inline void rseq_sched_s
t->rseq.event.ids_changed = true;
}
-/*
- * Invoked from switch_mm_cid() in context switch when the task gets a MM
- * CID assigned.
- *
- * This does not raise TIF_NOTIFY_RESUME as that happens in
- * rseq_sched_switch_event().
- */
-static __always_inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid)
-{
- /*
- * Requires a comparison as the switch_mm_cid() code does not
- * provide a conditional for it readily. So avoid excessive updates
- * when nothing changes.
- */
- if (t->rseq.ids.mm_cid != cid)
- t->rseq.event.ids_changed = true;
-}
-
/* Enforce a full update after RSEQ registration and when execve() failed */
static inline void rseq_force_update(void)
{
@@ -170,7 +152,6 @@ static inline void rseq_handle_slowpath(
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
-static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -101,18 +101,18 @@ struct rseq_data { };
/**
* struct sched_mm_cid - Storage for per task MM CID data
* @active: MM CID is active for the task
- * @cid: The CID associated to the task
- * @last_cid: The last CID associated to the task
+ * @cid: The CID associated to the task either permanently or
+ * borrowed from the CPU
*/
struct sched_mm_cid {
unsigned int active;
unsigned int cid;
- unsigned int last_cid;
};
/**
* struct mm_cid_pcpu - Storage for per CPU MM_CID data
- * @cid: The CID associated to the CPU
+ * @cid: The CID associated to the CPU either permanently or
+ * while a task with a CID is running
*/
struct mm_cid_pcpu {
unsigned int cid;
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -956,7 +956,6 @@ static struct task_struct *dup_task_stru
#ifdef CONFIG_SCHED_MM_CID
tsk->mm_cid.cid = MM_CID_UNSET;
- tsk->mm_cid.last_cid = MM_CID_UNSET;
tsk->mm_cid.active = 0;
#endif
return tsk;
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5307,7 +5307,7 @@ context_switch(struct rq *rq, struct tas
}
}
- switch_mm_cid(prev, next);
+ mm_cid_switch_to(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -10623,7 +10623,7 @@ static bool mm_cid_fixup_task_to_cpu(str
return true;
}
-static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+static void mm_cid_fixup_tasks_to_cpus(void)
{
struct mm_struct *mm = current->mm;
struct task_struct *p, *t;
@@ -10673,25 +10673,81 @@ static bool sched_mm_cid_add_user(struct
void sched_mm_cid_fork(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
+ bool percpu;
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
- sched_mm_cid_add_user(t, mm);
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
+
+ /* First user ? */
+ if (!mm->mm_cid.users) {
+ sched_mm_cid_add_user(t, mm);
+ t->mm_cid.cid = mm_get_cid(mm);
+ /* Required for execve() */
+ pcp->cid = t->mm_cid.cid;
+ return;
+ }
+
+ if (!sched_mm_cid_add_user(t, mm)) {
+ if (!mm->mm_cid.percpu)
+ t->mm_cid.cid = mm_get_cid(mm);
+ return;
+ }
+
+ /* Handle the mode change and transfer current's CID */
+ percpu = !!mm->mm_cid.percpu;
+ if (!percpu)
+ mm_cid_transit_to_task(current, pcp);
+ else
+ mm_cid_transfer_to_cpu(current, pcp);
+ }
+
+ if (percpu) {
+ mm_cid_fixup_tasks_to_cpus();
+ } else {
+ mm_cid_fixup_cpus_to_tasks(mm);
+ t->mm_cid.cid = mm_get_cid(mm);
}
}
static bool sched_mm_cid_remove_user(struct task_struct *t)
{
t->mm_cid.active = 0;
- mm_unset_cid_on_task(t);
+ scoped_guard(preempt) {
+ /* Clear the transition bit */
+ t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid);
+ mm_unset_cid_on_task(t);
+ }
t->mm->mm_cid.users--;
return mm_update_max_cids(t->mm);
}
+static bool __sched_mm_cid_exit(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ if (!sched_mm_cid_remove_user(t))
+ return false;
+ /*
+ * Contrary to fork() this only deals with a switch back to per
+ * task mode either because the above decreased users or an
+ * affinity change increased the number of allowed CPUs and the
+ * deferred fixup did not run yet.
+ */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return false;
+ /*
+ * A failed fork(2) cleanup never gets here, so @current must have
+ * the same MM as @t. That's true for exit() and the failed
+ * pthread_create() cleanup case.
+ */
+ if (WARN_ON_ONCE(current->mm != mm))
+ return false;
+ return true;
+}
+
/*
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
@@ -10702,10 +10758,43 @@ void sched_mm_cid_exit(struct task_struc
if (!mm || !t->mm_cid.active)
return;
+ /*
+ * Ensure that only one instance is doing MM CID operations within
+ * a MM. The common case is uncontended. The rare fixup case adds
+ * some overhead.
+ */
+ scoped_guard(mutex, &mm->mm_cid.mutex) {
+ /* mm_cid::mutex is sufficient to protect mm_cid::users */
+ if (likely(mm->mm_cid.users > 1)) {
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ if (!__sched_mm_cid_exit(t))
+ return;
+ /* Mode change required. Transfer currents CID */
+ mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+ return;
+ }
+ /* Last user */
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Required across execve() */
+ if (t == current)
+ mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
+ /* Ignore mode change. There is nothing to do. */
+ sched_mm_cid_remove_user(t);
+ }
+ }
- guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock)
- sched_mm_cid_remove_user(t);
+ /*
+ * As this is the last user (execve(), process exit or failed
+ * fork(2)) there is no concurrency anymore.
+ *
+ * Synchronize eventually pending work to ensure that there are no
+ * dangling references left. @t->mm_cid.users is zero so nothing
+ * can queue this work anymore.
+ */
+ irq_work_sync(&mm->mm_cid.irq_work);
+ cancel_work_sync(&mm->mm_cid.work);
}
/* Deactivate MM CID allocation across execve() */
@@ -10718,18 +10807,12 @@ void sched_mm_cid_before_execve(struct t
void sched_mm_cid_after_execve(struct task_struct *t)
{
sched_mm_cid_fork(t);
- guard(preempt)();
- mm_cid_select(t);
}
static void mm_cid_work_fn(struct work_struct *work)
{
struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- /* Make it compile, but not functional yet */
- if (!IS_ENABLED(CONFIG_NEW_MM_CID))
- return;
-
guard(mutex)(&mm->mm_cid.mutex);
/* Did the last user task exit already? */
if (!mm->mm_cid.users)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3745,83 +3745,7 @@ static inline void mm_cid_switch_to(stru
mm_cid_schedin(next);
}
-/* Active implementation */
-static inline void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cid;
-
- if (!mm)
- return;
-
- /* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->mm_cid.last_cid = max_cid - 1;
-}
-
-static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
-{
- struct mm_struct *mm = t->mm;
-
- if (cid >= max_cids)
- return false;
- if (test_and_set_bit(cid, mm_cidmask(mm)))
- return false;
- t->mm_cid.cid = t->mm_cid.last_cid = cid;
- __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
- return true;
-}
-
-static inline bool mm_cid_get(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cids;
-
- max_cids = READ_ONCE(mm->mm_cid.max_cids);
-
- /* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
- return true;
-
- /* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
- return true;
-
- /* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
-}
-
-static inline void mm_cid_select(struct task_struct *t)
-{
- /*
- * mm_cid_get() can fail when the maximum CID, which is determined
- * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
- * That's a transient failure as there cannot be more tasks
- * concurrently on a CPU (or about to be scheduled in) than that.
- */
- for (;;) {
- if (mm_cid_get(t))
- break;
- }
-}
-
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
-{
- if (prev->mm_cid.active) {
- if (prev->mm_cid.cid != MM_CID_UNSET)
- clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
- prev->mm_cid.cid = MM_CID_UNSET;
- }
-
- if (next->mm_cid.active) {
- mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
- }
-}
-
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void mm_cid_select(struct task_struct *t) { }
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Switch over to the new mechanism
2025-11-19 17:27 ` [patch V5 20/20] sched/mmcid: Switch over to the new mechanism Thomas Gleixner
@ 2025-11-20 11:19 ` tip-bot2 for Thomas Gleixner
2025-11-22 0:43 ` [patch V5 20/20] " Nathan Chancellor
2025-11-26 4:36 ` [tip: core/rseq] sched/mmcid: Switch over to the new mechanism tip-bot2 for Thomas Gleixner
2 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-20 11:19 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 2635fb0f0973c57c45f03708d52e827ec99ac78e
Gitweb: https://git.kernel.org/tip/2635fb0f0973c57c45f03708d52e827ec99ac78e
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:22 +01:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Thu, 20 Nov 2025 12:14:58 +01:00
sched/mmcid: Switch over to the new mechanism
Now that all pieces are in place, change the implementations of
sched_mm_cid_fork() and sched_mm_cid_exit() to adhere to the new strict
ownership scheme and switch context_switch() over to use the new
mm_cid_schedin() functionality.
The common case is that there is no mode change required, which makes
fork() and exit() just update the user count and the constraints.
In case that a new user would exceed the CID space limit the fork() context
handles the transition to per CPU mode with mm::mm_cid::mutex held. exit()
handles the transition back to per task mode when the user count drops
below the switch back threshold. fork() might also be forced to handle a
deferred switch back to per task mode, when a affinity change increased the
number of allowed CPUs enough.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.280380631@linutronix.de
---
include/linux/rseq.h | 19 +------
include/linux/rseq_types.h | 8 +--
kernel/fork.c | 1 +-
kernel/sched/core.c | 115 ++++++++++++++++++++++++++++++------
kernel/sched/sched.h | 76 +------------------------
5 files changed, 103 insertions(+), 116 deletions(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index 4c0e8bd..2266f4d 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -84,24 +84,6 @@ static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
t->rseq.event.ids_changed = true;
}
-/*
- * Invoked from switch_mm_cid() in context switch when the task gets a MM
- * CID assigned.
- *
- * This does not raise TIF_NOTIFY_RESUME as that happens in
- * rseq_sched_switch_event().
- */
-static __always_inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid)
-{
- /*
- * Requires a comparison as the switch_mm_cid() code does not
- * provide a conditional for it readily. So avoid excessive updates
- * when nothing changes.
- */
- if (t->rseq.ids.mm_cid != cid)
- t->rseq.event.ids_changed = true;
-}
-
/* Enforce a full update after RSEQ registration and when execve() failed */
static inline void rseq_force_update(void)
{
@@ -169,7 +151,6 @@ static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
-static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 81fbb88..332dc14 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -101,18 +101,18 @@ struct rseq_data { };
/**
* struct sched_mm_cid - Storage for per task MM CID data
* @active: MM CID is active for the task
- * @cid: The CID associated to the task
- * @last_cid: The last CID associated to the task
+ * @cid: The CID associated to the task either permanently or
+ * borrowed from the CPU
*/
struct sched_mm_cid {
unsigned int active;
unsigned int cid;
- unsigned int last_cid;
};
/**
* struct mm_cid_pcpu - Storage for per CPU MM_CID data
- * @cid: The CID associated to the CPU
+ * @cid: The CID associated to the CPU either permanently or
+ * while a task with a CID is running
*/
struct mm_cid_pcpu {
unsigned int cid;
diff --git a/kernel/fork.c b/kernel/fork.c
index 6c23219..8475958 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -956,7 +956,6 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#ifdef CONFIG_SCHED_MM_CID
tsk->mm_cid.cid = MM_CID_UNSET;
- tsk->mm_cid.last_cid = MM_CID_UNSET;
tsk->mm_cid.active = 0;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 938757e..72f368f 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5307,7 +5307,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
}
}
- switch_mm_cid(prev, next);
+ mm_cid_switch_to(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -10623,7 +10623,7 @@ static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm
return true;
}
-static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+static void mm_cid_fixup_tasks_to_cpus(void)
{
struct mm_struct *mm = current->mm;
struct task_struct *p, *t;
@@ -10673,25 +10673,81 @@ static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
void sched_mm_cid_fork(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
+ bool percpu;
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
- sched_mm_cid_add_user(t, mm);
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
+
+ /* First user ? */
+ if (!mm->mm_cid.users) {
+ sched_mm_cid_add_user(t, mm);
+ t->mm_cid.cid = mm_get_cid(mm);
+ /* Required for execve() */
+ pcp->cid = t->mm_cid.cid;
+ return;
+ }
+
+ if (!sched_mm_cid_add_user(t, mm)) {
+ if (!mm->mm_cid.percpu)
+ t->mm_cid.cid = mm_get_cid(mm);
+ return;
+ }
+
+ /* Handle the mode change and transfer current's CID */
+ percpu = !!mm->mm_cid.percpu;
+ if (!percpu)
+ mm_cid_transit_to_task(current, pcp);
+ else
+ mm_cid_transfer_to_cpu(current, pcp);
+ }
+
+ if (percpu) {
+ mm_cid_fixup_tasks_to_cpus();
+ } else {
+ mm_cid_fixup_cpus_to_tasks(mm);
+ t->mm_cid.cid = mm_get_cid(mm);
}
}
static bool sched_mm_cid_remove_user(struct task_struct *t)
{
t->mm_cid.active = 0;
- mm_unset_cid_on_task(t);
+ scoped_guard(preempt) {
+ /* Clear the transition bit */
+ t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid);
+ mm_unset_cid_on_task(t);
+ }
t->mm->mm_cid.users--;
return mm_update_max_cids(t->mm);
}
+static bool __sched_mm_cid_exit(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ if (!sched_mm_cid_remove_user(t))
+ return false;
+ /*
+ * Contrary to fork() this only deals with a switch back to per
+ * task mode either because the above decreased users or an
+ * affinity change increased the number of allowed CPUs and the
+ * deferred fixup did not run yet.
+ */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return false;
+ /*
+ * A failed fork(2) cleanup never gets here, so @current must have
+ * the same MM as @t. That's true for exit() and the failed
+ * pthread_create() cleanup case.
+ */
+ if (WARN_ON_ONCE(current->mm != mm))
+ return false;
+ return true;
+}
+
/*
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
@@ -10702,10 +10758,43 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
+ /*
+ * Ensure that only one instance is doing MM CID operations within
+ * a MM. The common case is uncontended. The rare fixup case adds
+ * some overhead.
+ */
+ scoped_guard(mutex, &mm->mm_cid.mutex) {
+ /* mm_cid::mutex is sufficient to protect mm_cid::users */
+ if (likely(mm->mm_cid.users > 1)) {
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ if (!__sched_mm_cid_exit(t))
+ return;
+ /* Mode change required. Transfer currents CID */
+ mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+ return;
+ }
+ /* Last user */
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Required across execve() */
+ if (t == current)
+ mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
+ /* Ignore mode change. There is nothing to do. */
+ sched_mm_cid_remove_user(t);
+ }
+ }
- guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock)
- sched_mm_cid_remove_user(t);
+ /*
+ * As this is the last user (execve(), process exit or failed
+ * fork(2)) there is no concurrency anymore.
+ *
+ * Synchronize eventually pending work to ensure that there are no
+ * dangling references left. @t->mm_cid.users is zero so nothing
+ * can queue this work anymore.
+ */
+ irq_work_sync(&mm->mm_cid.irq_work);
+ cancel_work_sync(&mm->mm_cid.work);
}
/* Deactivate MM CID allocation across execve() */
@@ -10718,18 +10807,12 @@ void sched_mm_cid_before_execve(struct task_struct *t)
void sched_mm_cid_after_execve(struct task_struct *t)
{
sched_mm_cid_fork(t);
- guard(preempt)();
- mm_cid_select(t);
}
static void mm_cid_work_fn(struct work_struct *work)
{
struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- /* Make it compile, but not functional yet */
- if (!IS_ENABLED(CONFIG_NEW_MM_CID))
- return;
-
guard(mutex)(&mm->mm_cid.mutex);
/* Did the last user task exit already? */
if (!mm->mm_cid.users)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 82c7978..f9d0515 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3745,83 +3745,7 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
mm_cid_schedin(next);
}
-/* Active implementation */
-static inline void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cid;
-
- if (!mm)
- return;
-
- /* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->mm_cid.last_cid = max_cid - 1;
-}
-
-static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
-{
- struct mm_struct *mm = t->mm;
-
- if (cid >= max_cids)
- return false;
- if (test_and_set_bit(cid, mm_cidmask(mm)))
- return false;
- t->mm_cid.cid = t->mm_cid.last_cid = cid;
- __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
- return true;
-}
-
-static inline bool mm_cid_get(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cids;
-
- max_cids = READ_ONCE(mm->mm_cid.max_cids);
-
- /* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
- return true;
-
- /* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
- return true;
-
- /* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
-}
-
-static inline void mm_cid_select(struct task_struct *t)
-{
- /*
- * mm_cid_get() can fail when the maximum CID, which is determined
- * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
- * That's a transient failure as there cannot be more tasks
- * concurrently on a CPU (or about to be scheduled in) than that.
- */
- for (;;) {
- if (mm_cid_get(t))
- break;
- }
-}
-
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
-{
- if (prev->mm_cid.active) {
- if (prev->mm_cid.cid != MM_CID_UNSET)
- clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
- prev->mm_cid.cid = MM_CID_UNSET;
- }
-
- if (next->mm_cid.active) {
- mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
- }
-}
-
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void mm_cid_select(struct task_struct *t) { }
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply related [flat|nested] 74+ messages in thread* Re: [patch V5 20/20] sched/mmcid: Switch over to the new mechanism
2025-11-19 17:27 ` [patch V5 20/20] sched/mmcid: Switch over to the new mechanism Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
@ 2025-11-22 0:43 ` Nathan Chancellor
2025-11-22 15:02 ` Thomas Gleixner
2025-11-26 4:36 ` [tip: core/rseq] sched/mmcid: Switch over to the new mechanism tip-bot2 for Thomas Gleixner
2 siblings, 1 reply; 74+ messages in thread
From: Nathan Chancellor @ 2025-11-22 0:43 UTC (permalink / raw)
To: Thomas Gleixner
Cc: LKML, Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
Hi Thomas,
On Wed, Nov 19, 2025 at 06:27:22PM +0100, Thomas Gleixner wrote:
> Now that all pieces are in place, change the implementations of
> sched_mm_cid_fork() and sched_mm_cid_exit() to adhere to the new strict
> ownership scheme and switch context_switch() over to use the new
> mm_cid_schedin() functionality.
>
> The common case is that there is no mode change required, which makes
> fork() and exit() just update the user count and the constraints.
>
> In case that a new user would exceed the CID space limit the fork() context
> handles the transition to per CPU mode with mm::mm_cid::mutex held. exit()
> handles the transition back to per task mode when the user count drops
> below the switch back threshold. fork() might also be forced to handle a
> deferred switch back to per task mode, when a affinity change increased the
> number of allowed CPUs enough.
>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Our CI started seeing a hang in QEMU after getting to userspace when
booting with a single CPU (our default logic when using TCG instead of
KVM) that I bisected to this change as commit 2635fb0f0973
("sched/mmcid: Switch over to the new mechanism") in -next.
https://github.com/ClangBuiltLinux/continuous-integration2/actions/runs/19557323049/job/56003155890
$ make -skj"$(nproc)" ARCH=x86_64 CROSS_COMPILE=x86_64-linux- clean defconfig bzImage
$ curl -LSs https://github.com/ClangBuiltLinux/boot-utils/releases/download/20241120-044434/x86_64-rootfs.cpio.zst | zstd -d >rootfs.cpio
$ qemu-system-x86_64 \
-display none \
-nodefaults \
-M q35 \
-d unimp,guest_errors \
-append 'console=ttyS0 earlycon=uart8250,io,0x3f8' \
-kernel arch/x86/boot/bzImage \
-initrd rootfs.cpio \
-cpu host \
-enable-kvm \
-m 512m \
-smp 1 \
-serial mon:stdio
[ 0.000000] Linux version 6.18.0-rc4-00066-g2635fb0f0973 (nathan@ax162) (x86_64-linux-gcc (GCC) 15.2.0, GNU ld (GNU Binutils) 2.45) #1 SMP PREEMPT_DYNAMIC Fri Nov 21 17:30:21 MST 2025
...
[ 0.946720] Freeing unused kernel image (initmem) memory: 2812K
[ 0.947864] Write protecting the kernel read-only data: 28672k
[ 0.949257] Freeing unused kernel image (text/rodata gap) memory: 1540K
[ 0.950579] Freeing unused kernel image (rodata/data gap) memory: 760K
[ 0.991764] x86/mm: Checked W+X mappings: passed, no W+X pages found.
[ 0.993000] Run /init as init process
<hangs>
If I change '-smp 1' to '-smp 2', userspace runs properly. At the parent
change, this issue does not exist but it is obviously possible that this
change exposes a bug from earlier in the series, I did not test.
Cheers,
Nathan
# bad: [d724c6f85e80a23ed46b7ebc6e38b527c09d64f5] Add linux-next specific files for 20251121
# good: [fd95357fd8c6778ac7dea6c57a19b8b182b6e91f] Merge tag 'sched_ext-for-6.18-rc6-fixes-2' of git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext
git bisect start 'd724c6f85e80a23ed46b7ebc6e38b527c09d64f5' 'fd95357fd8c6778ac7dea6c57a19b8b182b6e91f'
# good: [d8d0238c7ea639aa8c9f8c76d089e684e283ea87] Merge branch 'libcrypto-next' of https://git.kernel.org/pub/scm/linux/kernel/git/ebiggers/linux.git
git bisect good d8d0238c7ea639aa8c9f8c76d089e684e283ea87
# good: [84abc3686f608efd4914a5b4d651307e32ea4000] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/axboe/linux.git
git bisect good 84abc3686f608efd4914a5b4d651307e32ea4000
# bad: [c709a6758d17f1f2bccadb18cc82bccecdf216ff] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/pdx86/platform-drivers-x86.git
git bisect bad c709a6758d17f1f2bccadb18cc82bccecdf216ff
# skip: [8d00858e9326dafe5e27072e3cee5c23b60e7281] Merge branch 'master' of https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git
git bisect skip 8d00858e9326dafe5e27072e3cee5c23b60e7281
# good: [8be00d1ba3a1305469f6747d0285c4496a6855ee] KVM: arm64: GICv3: Completely disable trapping on vcpu exit
git bisect good 8be00d1ba3a1305469f6747d0285c4496a6855ee
# good: [4daaead949b331c271f6df4162f39589088fe27b] Merge branch 'for-next' of https://git.kernel.org/pub/scm/linux/kernel/git/broonie/spi.git
git bisect good 4daaead949b331c271f6df4162f39589088fe27b
# bad: [b1a2fa6d34e27225c2b3f79840f198227af1f53d] Merge branch into tip/master: 'x86/sev'
git bisect bad b1a2fa6d34e27225c2b3f79840f198227af1f53d
# bad: [468f14a4c5c9def982ae9a74344fa6f905a9b725] Merge branch into tip/master: 'locking/core'
git bisect bad 468f14a4c5c9def982ae9a74344fa6f905a9b725
# bad: [705d7cad382be9b3b52b89b220e77b17cf24322a] Merge branch 'sched/core' into core/rseq, to resolve conflict
git bisect bad 705d7cad382be9b3b52b89b220e77b17cf24322a
# good: [472931e757fb3dfad1f78ce6f5abd821155433b2] sched/mmcid: Convert mm CID mask to a bitmap
git bisect good 472931e757fb3dfad1f78ce6f5abd821155433b2
# good: [7f829bde94b1c97b1804fa5860e066ea49dbfca3] sched/core: Optimize core cookie matching check
git bisect good 7f829bde94b1c97b1804fa5860e066ea49dbfca3
# good: [d206fbad9328ddb68ebabd7cf7413392acd38081] sched/fair: Revert max_newidle_lb_cost bump
git bisect good d206fbad9328ddb68ebabd7cf7413392acd38081
# good: [340af997d25dab0f05c4de8399d656b112592a93] sched/mmcid: Provide CID ownership mode fixup functions
git bisect good 340af997d25dab0f05c4de8399d656b112592a93
# bad: [2635fb0f0973c57c45f03708d52e827ec99ac78e] sched/mmcid: Switch over to the new mechanism
git bisect bad 2635fb0f0973c57c45f03708d52e827ec99ac78e
# good: [2644779ec144d3e8cce5fed9623b47e70b3e0422] irqwork: Move data struct to a types header
git bisect good 2644779ec144d3e8cce5fed9623b47e70b3e0422
# good: [cba5e581161e379037a94f5a75d1a61bd1ccce3b] sched/mmcid: Implement deferred mode change
git bisect good cba5e581161e379037a94f5a75d1a61bd1ccce3b
# first bad commit: [2635fb0f0973c57c45f03708d52e827ec99ac78e] sched/mmcid: Switch over to the new mechanism
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 20/20] sched/mmcid: Switch over to the new mechanism
2025-11-22 0:43 ` [patch V5 20/20] " Nathan Chancellor
@ 2025-11-22 15:02 ` Thomas Gleixner
2025-11-22 16:54 ` Shrikanth Hegde
2025-11-23 19:03 ` [tip: core/rseq] sched/mmcid: Ensure that per CPU threshold is > 0 tip-bot2 for Thomas Gleixner
0 siblings, 2 replies; 74+ messages in thread
From: Thomas Gleixner @ 2025-11-22 15:02 UTC (permalink / raw)
To: Nathan Chancellor
Cc: LKML, Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde
On Fri, Nov 21 2025 at 17:43, Nathan Chancellor wrote:
> Our CI started seeing a hang in QEMU after getting to userspace when
> booting with a single CPU (our default logic when using TCG instead of
> KVM) that I bisected to this change as commit 2635fb0f0973
> ("sched/mmcid: Switch over to the new mechanism") in -next.
>
> If I change '-smp 1' to '-smp 2', userspace runs properly. At the parent
> change, this issue does not exist but it is obviously possible that this
> change exposes a bug from earlier in the series, I did not test.
Duh. Never tested with smp 1 :(
Fix is below. Now let me stare at that num_possible_cpus() fallout.
Thanks,
tglx
----
Subject: sched/mmcid: Ensure that per CPU threshold is > 0
From: Thomas Gleixner <tglx@linutronix.de>
Date: Sat, 22 Nov 2025 15:54:39 +0100
When num_possible_cpus() == 1 then the calculation for the threshold to
switch back from per CPU mode to per task mode results in 0, which
indicates that the per CPU mode is disabled.
Ensure that the threshold is > 0 to prevent that.
Fixes: 340af997d25d ("sched/mmcid: Provide CID ownership mode fixup functions")
Reported-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Closes: https://lore.kernel.org/all/20251122004358.GB2682494@ax162
---
kernel/sched/core.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10364,7 +10364,8 @@ static inline unsigned int mm_cid_calc_p
unsigned int opt_cids;
opt_cids = min(mc->nr_cpus_allowed, mc->users);
- return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
+ /* Has to be at least 1 because 0 indicates PCPU mode off */
+ return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
}
static bool mm_update_max_cids(struct mm_struct *mm)
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 20/20] sched/mmcid: Switch over to the new mechanism
2025-11-22 15:02 ` Thomas Gleixner
@ 2025-11-22 16:54 ` Shrikanth Hegde
2025-11-23 19:03 ` [tip: core/rseq] sched/mmcid: Ensure that per CPU threshold is > 0 tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: Shrikanth Hegde @ 2025-11-22 16:54 UTC (permalink / raw)
To: Thomas Gleixner, Nathan Chancellor
Cc: LKML, Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov
On 11/22/25 8:32 PM, Thomas Gleixner wrote:
> On Fri, Nov 21 2025 at 17:43, Nathan Chancellor wrote:
>> Our CI started seeing a hang in QEMU after getting to userspace when
>> booting with a single CPU (our default logic when using TCG instead of
>> KVM) that I bisected to this change as commit 2635fb0f0973
>> ("sched/mmcid: Switch over to the new mechanism") in -next.
>>
>> If I change '-smp 1' to '-smp 2', userspace runs properly. At the parent
>> change, this issue does not exist but it is obviously possible that this
>> change exposes a bug from earlier in the series, I did not test.
>
> Duh. Never tested with smp 1 :(
>
> Fix is below. Now let me stare at that num_possible_cpus() fallout.
>
> Thanks,
>
> tglx
> ----
> Subject: sched/mmcid: Ensure that per CPU threshold is > 0
> From: Thomas Gleixner <tglx@linutronix.de>
> Date: Sat, 22 Nov 2025 15:54:39 +0100
>
> When num_possible_cpus() == 1 then the calculation for the threshold to
> switch back from per CPU mode to per task mode results in 0, which
> indicates that the per CPU mode is disabled.
>
> Ensure that the threshold is > 0 to prevent that.
>
> Fixes: 340af997d25d ("sched/mmcid: Provide CID ownership mode fixup functions")
> Reported-by: Nathan Chancellor <nathan@kernel.org>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
> Closes: https://lore.kernel.org/all/20251122004358.GB2682494@ax162
> ---
> kernel/sched/core.c | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
>
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -10364,7 +10364,8 @@ static inline unsigned int mm_cid_calc_p
> unsigned int opt_cids;
>
> opt_cids = min(mc->nr_cpus_allowed, mc->users);
> - return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
> + /* Has to be at least 1 because 0 indicates PCPU mode off */
> + return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
> }
>
> static bool mm_update_max_cids(struct mm_struct *mm)
Having both patches (this patch and num_possible_cpus fix) patch was
necessary for successful boot of smp=1 on qemu.
^ permalink raw reply [flat|nested] 74+ messages in thread* [tip: core/rseq] sched/mmcid: Ensure that per CPU threshold is > 0
2025-11-22 15:02 ` Thomas Gleixner
2025-11-22 16:54 ` Shrikanth Hegde
@ 2025-11-23 19:03 ` tip-bot2 for Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-23 19:03 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Nathan Chancellor, Thomas Gleixner, x86, linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 14b2b17cca541bf46341beb912cf4420ff27a0c7
Gitweb: https://git.kernel.org/tip/14b2b17cca541bf46341beb912cf4420ff27a0c7
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Sat, 22 Nov 2025 15:54:39 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Sun, 23 Nov 2025 19:59:30 +01:00
sched/mmcid: Ensure that per CPU threshold is > 0
When num_possible_cpus() == 1 then the calculation for the threshold to
switch back from per CPU mode to per task mode results in 0, which
indicates that the per CPU mode is disabled.
Ensure that the threshold is > 0 to prevent that.
Fixes: 340af997d25d ("sched/mmcid: Provide CID ownership mode fixup functions")
Reported-by: Nathan Chancellor <nathan@kernel.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Link: https://patch.msgid.link/873466jekm.ffs@tglx
Closes: https://lore.kernel.org/all/20251122004358.GB2682494@ax162
---
kernel/sched/core.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 5821968..58b4dbe 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10364,7 +10364,8 @@ static inline unsigned int mm_cid_calc_pcpu_thrs(struct mm_mm_cid *mc)
unsigned int opt_cids;
opt_cids = min(mc->nr_cpus_allowed, mc->users);
- return min(opt_cids - opt_cids / 4, num_possible_cpus() / 2);
+ /* Has to be at least 1 because 0 indicates PCPU mode off */
+ return max(min(opt_cids - opt_cids / 4, num_possible_cpus() / 2), 1);
}
static bool mm_update_max_cids(struct mm_struct *mm)
^ permalink raw reply related [flat|nested] 74+ messages in thread
* [tip: core/rseq] sched/mmcid: Switch over to the new mechanism
2025-11-19 17:27 ` [patch V5 20/20] sched/mmcid: Switch over to the new mechanism Thomas Gleixner
2025-11-20 11:19 ` [tip: core/rseq] " tip-bot2 for Thomas Gleixner
2025-11-22 0:43 ` [patch V5 20/20] " Nathan Chancellor
@ 2025-11-26 4:36 ` tip-bot2 for Thomas Gleixner
2 siblings, 0 replies; 74+ messages in thread
From: tip-bot2 for Thomas Gleixner @ 2025-11-26 4:36 UTC (permalink / raw)
To: linux-tip-commits
Cc: Thomas Gleixner, Peter Zijlstra (Intel), Mathieu Desnoyers, x86,
linux-kernel
The following commit has been merged into the core/rseq branch of tip:
Commit-ID: 653fda7ae73d8033dedb65537acac0c2c287dc3f
Gitweb: https://git.kernel.org/tip/653fda7ae73d8033dedb65537acac0c2c287dc3f
Author: Thomas Gleixner <tglx@linutronix.de>
AuthorDate: Wed, 19 Nov 2025 18:27:22 +01:00
Committer: Thomas Gleixner <tglx@linutronix.de>
CommitterDate: Tue, 25 Nov 2025 19:45:42 +01:00
sched/mmcid: Switch over to the new mechanism
Now that all pieces are in place, change the implementations of
sched_mm_cid_fork() and sched_mm_cid_exit() to adhere to the new strict
ownership scheme and switch context_switch() over to use the new
mm_cid_schedin() functionality.
The common case is that there is no mode change required, which makes
fork() and exit() just update the user count and the constraints.
In case that a new user would exceed the CID space limit the fork() context
handles the transition to per CPU mode with mm::mm_cid::mutex held. exit()
handles the transition back to per task mode when the user count drops
below the switch back threshold. fork() might also be forced to handle a
deferred switch back to per task mode, when a affinity change increased the
number of allowed CPUs enough.
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Link: https://patch.msgid.link/20251119172550.280380631@linutronix.de
---
include/linux/rseq.h | 19 +------
include/linux/rseq_types.h | 8 +--
kernel/fork.c | 1 +-
kernel/sched/core.c | 115 ++++++++++++++++++++++++++++++------
kernel/sched/sched.h | 76 +------------------------
5 files changed, 103 insertions(+), 116 deletions(-)
diff --git a/include/linux/rseq.h b/include/linux/rseq.h
index 4c0e8bd..2266f4d 100644
--- a/include/linux/rseq.h
+++ b/include/linux/rseq.h
@@ -84,24 +84,6 @@ static __always_inline void rseq_sched_set_ids_changed(struct task_struct *t)
t->rseq.event.ids_changed = true;
}
-/*
- * Invoked from switch_mm_cid() in context switch when the task gets a MM
- * CID assigned.
- *
- * This does not raise TIF_NOTIFY_RESUME as that happens in
- * rseq_sched_switch_event().
- */
-static __always_inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid)
-{
- /*
- * Requires a comparison as the switch_mm_cid() code does not
- * provide a conditional for it readily. So avoid excessive updates
- * when nothing changes.
- */
- if (t->rseq.ids.mm_cid != cid)
- t->rseq.event.ids_changed = true;
-}
-
/* Enforce a full update after RSEQ registration and when execve() failed */
static inline void rseq_force_update(void)
{
@@ -169,7 +151,6 @@ static inline void rseq_handle_slowpath(struct pt_regs *regs) { }
static inline void rseq_signal_deliver(struct ksignal *ksig, struct pt_regs *regs) { }
static inline void rseq_sched_switch_event(struct task_struct *t) { }
static inline void rseq_sched_set_ids_changed(struct task_struct *t) { }
-static inline void rseq_sched_set_task_mm_cid(struct task_struct *t, unsigned int cid) { }
static inline void rseq_force_update(void) { }
static inline void rseq_virt_userspace_exit(void) { }
static inline void rseq_fork(struct task_struct *t, u64 clone_flags) { }
diff --git a/include/linux/rseq_types.h b/include/linux/rseq_types.h
index 81fbb88..332dc14 100644
--- a/include/linux/rseq_types.h
+++ b/include/linux/rseq_types.h
@@ -101,18 +101,18 @@ struct rseq_data { };
/**
* struct sched_mm_cid - Storage for per task MM CID data
* @active: MM CID is active for the task
- * @cid: The CID associated to the task
- * @last_cid: The last CID associated to the task
+ * @cid: The CID associated to the task either permanently or
+ * borrowed from the CPU
*/
struct sched_mm_cid {
unsigned int active;
unsigned int cid;
- unsigned int last_cid;
};
/**
* struct mm_cid_pcpu - Storage for per CPU MM_CID data
- * @cid: The CID associated to the CPU
+ * @cid: The CID associated to the CPU either permanently or
+ * while a task with a CID is running
*/
struct mm_cid_pcpu {
unsigned int cid;
diff --git a/kernel/fork.c b/kernel/fork.c
index 6c23219..8475958 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -956,7 +956,6 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
#ifdef CONFIG_SCHED_MM_CID
tsk->mm_cid.cid = MM_CID_UNSET;
- tsk->mm_cid.last_cid = MM_CID_UNSET;
tsk->mm_cid.active = 0;
#endif
return tsk;
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index cbb543a..62235f1 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5307,7 +5307,7 @@ context_switch(struct rq *rq, struct task_struct *prev,
}
}
- switch_mm_cid(prev, next);
+ mm_cid_switch_to(prev, next);
/*
* Tell rseq that the task was scheduled in. Must be after
@@ -10624,7 +10624,7 @@ static bool mm_cid_fixup_task_to_cpu(struct task_struct *t, struct mm_struct *mm
return true;
}
-static void __maybe_unused mm_cid_fixup_tasks_to_cpus(void)
+static void mm_cid_fixup_tasks_to_cpus(void)
{
struct mm_struct *mm = current->mm;
struct task_struct *p, *t;
@@ -10674,25 +10674,81 @@ static bool sched_mm_cid_add_user(struct task_struct *t, struct mm_struct *mm)
void sched_mm_cid_fork(struct task_struct *t)
{
struct mm_struct *mm = t->mm;
+ bool percpu;
WARN_ON_ONCE(!mm || t->mm_cid.cid != MM_CID_UNSET);
guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock) {
- sched_mm_cid_add_user(t, mm);
- /* Preset last_cid for mm_cid_select() */
- t->mm_cid.last_cid = mm->mm_cid.max_cids - 1;
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ struct mm_cid_pcpu *pcp = this_cpu_ptr(mm->mm_cid.pcpu);
+
+ /* First user ? */
+ if (!mm->mm_cid.users) {
+ sched_mm_cid_add_user(t, mm);
+ t->mm_cid.cid = mm_get_cid(mm);
+ /* Required for execve() */
+ pcp->cid = t->mm_cid.cid;
+ return;
+ }
+
+ if (!sched_mm_cid_add_user(t, mm)) {
+ if (!mm->mm_cid.percpu)
+ t->mm_cid.cid = mm_get_cid(mm);
+ return;
+ }
+
+ /* Handle the mode change and transfer current's CID */
+ percpu = !!mm->mm_cid.percpu;
+ if (!percpu)
+ mm_cid_transit_to_task(current, pcp);
+ else
+ mm_cid_transfer_to_cpu(current, pcp);
+ }
+
+ if (percpu) {
+ mm_cid_fixup_tasks_to_cpus();
+ } else {
+ mm_cid_fixup_cpus_to_tasks(mm);
+ t->mm_cid.cid = mm_get_cid(mm);
}
}
static bool sched_mm_cid_remove_user(struct task_struct *t)
{
t->mm_cid.active = 0;
- mm_unset_cid_on_task(t);
+ scoped_guard(preempt) {
+ /* Clear the transition bit */
+ t->mm_cid.cid = cid_from_transit_cid(t->mm_cid.cid);
+ mm_unset_cid_on_task(t);
+ }
t->mm->mm_cid.users--;
return mm_update_max_cids(t->mm);
}
+static bool __sched_mm_cid_exit(struct task_struct *t)
+{
+ struct mm_struct *mm = t->mm;
+
+ if (!sched_mm_cid_remove_user(t))
+ return false;
+ /*
+ * Contrary to fork() this only deals with a switch back to per
+ * task mode either because the above decreased users or an
+ * affinity change increased the number of allowed CPUs and the
+ * deferred fixup did not run yet.
+ */
+ if (WARN_ON_ONCE(mm->mm_cid.percpu))
+ return false;
+ /*
+ * A failed fork(2) cleanup never gets here, so @current must have
+ * the same MM as @t. That's true for exit() and the failed
+ * pthread_create() cleanup case.
+ */
+ if (WARN_ON_ONCE(current->mm != mm))
+ return false;
+ return true;
+}
+
/*
* When a task exits, the MM CID held by the task is not longer required as
* the task cannot return to user space.
@@ -10703,10 +10759,43 @@ void sched_mm_cid_exit(struct task_struct *t)
if (!mm || !t->mm_cid.active)
return;
+ /*
+ * Ensure that only one instance is doing MM CID operations within
+ * a MM. The common case is uncontended. The rare fixup case adds
+ * some overhead.
+ */
+ scoped_guard(mutex, &mm->mm_cid.mutex) {
+ /* mm_cid::mutex is sufficient to protect mm_cid::users */
+ if (likely(mm->mm_cid.users > 1)) {
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ if (!__sched_mm_cid_exit(t))
+ return;
+ /* Mode change required. Transfer currents CID */
+ mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
+ }
+ mm_cid_fixup_cpus_to_tasks(mm);
+ return;
+ }
+ /* Last user */
+ scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
+ /* Required across execve() */
+ if (t == current)
+ mm_cid_transit_to_task(t, this_cpu_ptr(mm->mm_cid.pcpu));
+ /* Ignore mode change. There is nothing to do. */
+ sched_mm_cid_remove_user(t);
+ }
+ }
- guard(mutex)(&mm->mm_cid.mutex);
- scoped_guard(raw_spinlock, &mm->mm_cid.lock)
- sched_mm_cid_remove_user(t);
+ /*
+ * As this is the last user (execve(), process exit or failed
+ * fork(2)) there is no concurrency anymore.
+ *
+ * Synchronize eventually pending work to ensure that there are no
+ * dangling references left. @t->mm_cid.users is zero so nothing
+ * can queue this work anymore.
+ */
+ irq_work_sync(&mm->mm_cid.irq_work);
+ cancel_work_sync(&mm->mm_cid.work);
}
/* Deactivate MM CID allocation across execve() */
@@ -10719,18 +10808,12 @@ void sched_mm_cid_before_execve(struct task_struct *t)
void sched_mm_cid_after_execve(struct task_struct *t)
{
sched_mm_cid_fork(t);
- guard(preempt)();
- mm_cid_select(t);
}
static void mm_cid_work_fn(struct work_struct *work)
{
struct mm_struct *mm = container_of(work, struct mm_struct, mm_cid.work);
- /* Make it compile, but not functional yet */
- if (!IS_ENABLED(CONFIG_NEW_MM_CID))
- return;
-
guard(mutex)(&mm->mm_cid.mutex);
/* Did the last user task exit already? */
if (!mm->mm_cid.users)
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 82c7978..f9d0515 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -3745,83 +3745,7 @@ static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct
mm_cid_schedin(next);
}
-/* Active implementation */
-static inline void init_sched_mm_cid(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cid;
-
- if (!mm)
- return;
-
- /* Preset last_mm_cid */
- max_cid = min_t(int, READ_ONCE(mm->mm_cid.nr_cpus_allowed), atomic_read(&mm->mm_users));
- t->mm_cid.last_cid = max_cid - 1;
-}
-
-static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
-{
- struct mm_struct *mm = t->mm;
-
- if (cid >= max_cids)
- return false;
- if (test_and_set_bit(cid, mm_cidmask(mm)))
- return false;
- t->mm_cid.cid = t->mm_cid.last_cid = cid;
- __this_cpu_write(mm->mm_cid.pcpu->cid, cid);
- return true;
-}
-
-static inline bool mm_cid_get(struct task_struct *t)
-{
- struct mm_struct *mm = t->mm;
- unsigned int max_cids;
-
- max_cids = READ_ONCE(mm->mm_cid.max_cids);
-
- /* Try to reuse the last CID of this task */
- if (__mm_cid_get(t, t->mm_cid.last_cid, max_cids))
- return true;
-
- /* Try to reuse the last CID of this mm on this CPU */
- if (__mm_cid_get(t, __this_cpu_read(mm->mm_cid.pcpu->cid), max_cids))
- return true;
-
- /* Try the first zero bit in the cidmask. */
- return __mm_cid_get(t, find_first_zero_bit(mm_cidmask(mm), num_possible_cpus()), max_cids);
-}
-
-static inline void mm_cid_select(struct task_struct *t)
-{
- /*
- * mm_cid_get() can fail when the maximum CID, which is determined
- * by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
- * That's a transient failure as there cannot be more tasks
- * concurrently on a CPU (or about to be scheduled in) than that.
- */
- for (;;) {
- if (mm_cid_get(t))
- break;
- }
-}
-
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
-{
- if (prev->mm_cid.active) {
- if (prev->mm_cid.cid != MM_CID_UNSET)
- clear_bit(prev->mm_cid.cid, mm_cidmask(prev->mm));
- prev->mm_cid.cid = MM_CID_UNSET;
- }
-
- if (next->mm_cid.active) {
- mm_cid_select(next);
- rseq_sched_set_task_mm_cid(next, next->mm_cid.cid);
- }
-}
-
#else /* !CONFIG_SCHED_MM_CID: */
-static inline void mm_cid_select(struct task_struct *t) { }
-static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
static inline void mm_cid_switch_to(struct task_struct *prev, struct task_struct *next) { }
#endif /* !CONFIG_SCHED_MM_CID */
^ permalink raw reply related [flat|nested] 74+ messages in thread
* Re: [patch V5 00/20] sched: Rewrite MM CID management
2025-11-19 17:26 [patch V5 00/20] sched: Rewrite MM CID management Thomas Gleixner
` (19 preceding siblings ...)
2025-11-19 17:27 ` [patch V5 20/20] sched/mmcid: Switch over to the new mechanism Thomas Gleixner
@ 2026-01-28 0:01 ` Ihor Solodrai
2026-01-28 8:46 ` Peter Zijlstra
2026-01-28 11:57 ` Thomas Gleixner
20 siblings, 2 replies; 74+ messages in thread
From: Ihor Solodrai @ 2026-01-28 0:01 UTC (permalink / raw)
To: Thomas Gleixner, LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde, bpf,
sched-ext, Kernel Team, Alexei Starovoitov, Andrii Nakryiko,
Daniel Borkmann, Puranjay Mohan, Tejun Heo
On 11/19/25 9:26 AM, Thomas Gleixner wrote:
> This is a follow up on the V4 series which can be found here:
>
> https://lore.kernel.org/20251104075053.700034556@linutronix.de
>
> The V1 cover letter contains a detailed analyisis of the issues:
>
> https://lore.kernel.org/20251015164952.694882104@linutronix.de
>
> TLDR: The CID management is way to complex and adds significant overhead
> into scheduler hotpaths.
>
> The series rewrites MM CID management in a more simplistic way which
> focusses on low overhead in the scheduler while maintaining per task CIDs
> as long as the number of threads is not exceeding the number of possible
> CPUs.
Hello Thomas, everyone.
BPF CI caught a deadlock on current bpf-next tip (35538dba51b4).
Job: https://github.com/kernel-patches/bpf/actions/runs/21417415035/job/61670254640
It appears to be related to this series. Pasting a splat below.
Any ideas what might be going on?
Thanks!
[ 45.009755] watchdog: CPU2: Watchdog detected hard LOCKUP on cpu 2
[ 45.009763] Modules linked in: bpf_testmod(OE)
[ 45.009769] irq event stamp: 685710
[ 45.009771] hardirqs last enabled at (685709): [<ffffffffb5bfa8b8>] _raw_spin_unlock_irq+0x28/0x50
[ 45.009786] hardirqs last disabled at (685710): [<ffffffffb5bfa651>] _raw_spin_lock_irqsave+0x51/0x60
[ 45.009789] softirqs last enabled at (685650): [<ffffffffb3345e2a>] fpu_clone+0xda/0x4f0
[ 45.009795] softirqs last disabled at (685648): [<ffffffffb3345dd2>] fpu_clone+0x82/0x4f0
[ 45.009803] CPU: 2 UID: 0 PID: 126 Comm: test_progs Tainted: G OE 6.19.0-rc5-g748c6d52700a-dirty #1 PREEMPT(full)
[ 45.009808] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 45.009810] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 45.009813] RIP: 0010:queued_spin_lock_slowpath+0x6cc/0xac0
[ 45.009820] Code: 0c 24 8b 03 66 85 c0 74 38 48 b8 00 00 00 00 00 fc ff df 48 89 da 49 89 de 48 c1 ea 03 41 83 e6 07 48 01 c2 41 83 c6 03 f3 90 <0f> b6 02 41 38 c6 7c 08 84 c0 0f 85 90 02 00 00 8b 03 66 85 c0 75
[ 45.009823] RSP: 0018:ffffc9000128f750 EFLAGS: 00000002
[ 45.009828] RAX: 0000000000100101 RBX: ffff8881520ba000 RCX: 0000000000000000
[ 45.009830] RDX: ffffed102a417400 RSI: 0000000000000002 RDI: ffff8881520ba002
[ 45.009832] RBP: 1ffff92000251eec R08: ffffffffb5bfb6c9 R09: ffffed102a417400
[ 45.009834] R10: ffffed102a417401 R11: 0000000000000004 R12: ffff88815213b100
[ 45.009836] R13: 00000000000c0000 R14: 0000000000000003 R15: 0000000000000002
[ 45.009838] FS: 00007f6ab3e0de00(0000) GS:ffff8881998dd000(0000) knlGS:0000000000000000
[ 45.009841] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 45.009843] CR2: 00007f6ab2873d58 CR3: 0000000103357005 CR4: 0000000000770ef0
[ 45.009845] PKRU: 55555554
[ 45.009846] Call Trace:
[ 45.009850] <TASK>
[ 45.009855] ? __pfx_queued_spin_lock_slowpath+0x10/0x10
[ 45.009862] do_raw_spin_lock+0x1d9/0x270
[ 45.009868] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 45.009871] ? __pfx___might_resched+0x10/0x10
[ 45.009878] task_rq_lock+0xcf/0x3c0
[ 45.009884] mm_cid_fixup_task_to_cpu+0xb0/0x460
[ 45.009888] ? __pfx_mm_cid_fixup_task_to_cpu+0x10/0x10
[ 45.009892] ? lock_acquire+0x14e/0x2b0
[ 45.009896] ? mark_held_locks+0x40/0x70
[ 45.009901] sched_mm_cid_fork+0x6da/0xc20
[ 45.009905] ? __pfx_sched_mm_cid_fork+0x10/0x10
[ 45.009908] ? copy_process+0x217b/0x6950
[ 45.009913] copy_process+0x2bce/0x6950
[ 45.009919] ? __pfx_copy_process+0x10/0x10
[ 45.009921] ? find_held_lock+0x2b/0x80
[ 45.009926] ? _copy_from_user+0x53/0xa0
[ 45.009933] kernel_clone+0xce/0x600
[ 45.009937] ? __pfx_kernel_clone+0x10/0x10
[ 45.009942] ? __lock_acquire+0x481/0x2590
[ 45.009947] __do_sys_clone3+0x16e/0x1b0
[ 45.009950] ? __pfx___do_sys_clone3+0x10/0x10
[ 45.009952] ? lock_acquire+0x14e/0x2b0
[ 45.009955] ? __might_fault+0x9b/0x140
[ 45.009963] ? _copy_to_user+0x5c/0x70
[ 45.009967] ? __x64_sys_rt_sigprocmask+0x258/0x400
[ 45.009974] ? do_user_addr_fault+0x4c2/0xa40
[ 45.009978] ? lockdep_hardirqs_on_prepare+0xd7/0x180
[ 45.009982] do_syscall_64+0x6b/0x3a0
[ 45.009988] entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 45.009992] RIP: 0033:0x7f6ab430fc5d
[ 45.009996] Code: 79 14 0e 00 c3 0f 1f 84 00 00 00 00 00 f3 0f 1e fa b8 ea ff ff ff 48 85 ff 74 28 48 85 d2 74 23 49 89 c8 b8 b3 01 00 00 0f 05 <48> 85 c0 7c 14 74 01 c3 31 ed 4c 89 c7 ff d2 48 89 c7 b8 3c 00 00
[ 45.009998] RSP: 002b:00007fffb282a148 EFLAGS: 00000202 ORIG_RAX: 00000000000001b3
[ 45.010002] RAX: ffffffffffffffda RBX: 00007f6ab4282720 RCX: 00007f6ab430fc5d
[ 45.010004] RDX: 00007f6ab4282720 RSI: 0000000000000058 RDI: 00007fffb282a1a0
[ 45.010005] RBP: 00007fffb282a180 R08: 00007f6ab28736c0 R09: 00007fffb282a2a7
[ 45.010007] R10: 0000000000000008 R11: 0000000000000202 R12: 00007f6ab28736c0
[ 45.010009] R13: ffffffffffffff08 R14: 0000000000000000 R15: 00007fffb282a1a0
[ 45.010015] </TASK>
[ 45.010018] Kernel panic - not syncing: Hard LOCKUP
[ 45.010020] CPU: 2 UID: 0 PID: 126 Comm: test_progs Tainted: G OE 6.19.0-rc5-g748c6d52700a-dirty #1 PREEMPT(full)
[ 45.010025] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 45.010026] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 45.010027] Call Trace:
[ 45.010029] <NMI>
[ 45.010031] dump_stack_lvl+0x5d/0x80
[ 45.010036] vpanic+0x133/0x3f0
[ 45.010042] panic+0xce/0xce
[ 45.010045] ? __pfx_panic+0x10/0x10
[ 45.010050] ? __show_trace_log_lvl+0x2ee/0x323
[ 45.010053] ? entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 45.010057] ? nmi_panic+0x91/0x130
[ 45.010061] nmi_panic.cold+0x14/0x14
[ 45.010065] ? __pfx_nmi_panic+0x10/0x10
[ 45.010070] watchdog_hardlockup_check.cold+0x12a/0x1c5
[ 45.010076] __perf_event_overflow+0x2fe/0xeb0
[ 45.010082] ? __pfx___perf_event_overflow+0x10/0x10
[ 45.010085] ? __pfx_x86_perf_event_set_period+0x10/0x10
[ 45.010091] handle_pmi_common+0x405/0x920
[ 45.010096] ? __pfx_handle_pmi_common+0x10/0x10
[ 45.010109] ? __pfx_intel_bts_interrupt+0x10/0x10
[ 45.010115] intel_pmu_handle_irq+0x1c5/0x5d0
[ 45.010119] ? lock_acquire+0x1e9/0x2b0
[ 45.010122] ? nmi_handle.part.0+0x2f/0x370
[ 45.010127] perf_event_nmi_handler+0x3e/0x70
[ 45.010130] nmi_handle.part.0+0x13f/0x370
[ 45.010134] ? trace_rcu_watching+0x105/0x150
[ 45.010140] default_do_nmi+0x3b/0x110
[ 45.010144] ? irqentry_nmi_enter+0x6f/0x80
[ 45.010147] exc_nmi+0xe3/0x110
[ 45.010151] end_repeat_nmi+0xf/0x53
[ 45.010154] RIP: 0010:queued_spin_lock_slowpath+0x6cc/0xac0
[ 45.010157] Code: 0c 24 8b 03 66 85 c0 74 38 48 b8 00 00 00 00 00 fc ff df 48 89 da 49 89 de 48 c1 ea 03 41 83 e6 07 48 01 c2 41 83 c6 03 f3 90 <0f> b6 02 41 38 c6 7c 08 84 c0 0f 85 90 02 00 00 8b 03 66 85 c0 75
[ 45.010159] RSP: 0018:ffffc9000128f750 EFLAGS: 00000002
[ 45.010162] RAX: 0000000000100101 RBX: ffff8881520ba000 RCX: 0000000000000000
[ 45.010164] RDX: ffffed102a417400 RSI: 0000000000000002 RDI: ffff8881520ba002
[ 45.010165] RBP: 1ffff92000251eec R08: ffffffffb5bfb6c9 R09: ffffed102a417400
[ 45.010167] R10: ffffed102a417401 R11: 0000000000000004 R12: ffff88815213b100
[ 45.010169] R13: 00000000000c0000 R14: 0000000000000003 R15: 0000000000000002
[ 45.010172] ? queued_spin_lock_slowpath+0x559/0xac0
[ 45.010177] ? queued_spin_lock_slowpath+0x6cc/0xac0
[ 45.010181] ? queued_spin_lock_slowpath+0x6cc/0xac0
[ 45.010185] </NMI>
[ 45.010186] <TASK>
[ 45.010187] ? __pfx_queued_spin_lock_slowpath+0x10/0x10
[ 45.010194] do_raw_spin_lock+0x1d9/0x270
[ 45.010198] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 45.010201] ? __pfx___might_resched+0x10/0x10
[ 45.010206] task_rq_lock+0xcf/0x3c0
[ 45.010211] mm_cid_fixup_task_to_cpu+0xb0/0x460
[ 45.010215] ? __pfx_mm_cid_fixup_task_to_cpu+0x10/0x10
[ 45.010219] ? lock_acquire+0x14e/0x2b0
[ 45.010223] ? mark_held_locks+0x40/0x70
[ 45.010228] sched_mm_cid_fork+0x6da/0xc20
[ 45.010232] ? __pfx_sched_mm_cid_fork+0x10/0x10
[ 45.010234] ? copy_process+0x217b/0x6950
[ 45.010238] copy_process+0x2bce/0x6950
[ 45.010245] ? __pfx_copy_process+0x10/0x10
[ 45.010247] ? find_held_lock+0x2b/0x80
[ 45.010251] ? _copy_from_user+0x53/0xa0
[ 45.010256] kernel_clone+0xce/0x600
[ 45.010259] ? __pfx_kernel_clone+0x10/0x10
[ 45.010264] ? __lock_acquire+0x481/0x2590
[ 45.010269] __do_sys_clone3+0x16e/0x1b0
[ 45.010272] ? __pfx___do_sys_clone3+0x10/0x10
[ 45.010274] ? lock_acquire+0x14e/0x2b0
[ 45.010277] ? __might_fault+0x9b/0x140
[ 45.010284] ? _copy_to_user+0x5c/0x70
[ 45.010288] ? __x64_sys_rt_sigprocmask+0x258/0x400
[ 45.010293] ? do_user_addr_fault+0x4c2/0xa40
[ 45.010296] ? lockdep_hardirqs_on_prepare+0xd7/0x180
[ 45.010300] do_syscall_64+0x6b/0x3a0
[ 45.010305] entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 45.010307] RIP: 0033:0x7f6ab430fc5d
[ 45.010309] Code: 79 14 0e 00 c3 0f 1f 84 00 00 00 00 00 f3 0f 1e fa b8 ea ff ff ff 48 85 ff 74 28 48 85 d2 74 23 49 89 c8 b8 b3 01 00 00 0f 05 <48> 85 c0 7c 14 74 01 c3 31 ed 4c 89 c7 ff d2 48 89 c7 b8 3c 00 00
[ 45.010311] RSP: 002b:00007fffb282a148 EFLAGS: 00000202 ORIG_RAX: 00000000000001b3
[ 45.010314] RAX: ffffffffffffffda RBX: 00007f6ab4282720 RCX: 00007f6ab430fc5d
[ 45.010316] RDX: 00007f6ab4282720 RSI: 0000000000000058 RDI: 00007fffb282a1a0
[ 45.010317] RBP: 00007fffb282a180 R08: 00007f6ab28736c0 R09: 00007fffb282a2a7
[ 45.010319] R10: 0000000000000008 R11: 0000000000000202 R12: 00007f6ab28736c0
[ 45.010320] R13: ffffffffffffff08 R14: 0000000000000000 R15: 00007fffb282a1a0
[ 45.010326] </TASK>
[ 46.053092]
[ 46.053095] ================================
[ 46.053096] WARNING: inconsistent lock state
[ 46.053098] 6.19.0-rc5-g748c6d52700a-dirty #1 Tainted: G OE
[ 46.053101] --------------------------------
[ 46.053102] inconsistent {INITIAL USE} -> {IN-NMI} usage.
[ 46.053103] test_progs/126 [HC1[1]:SC0[0]:HE0:SE1] takes:
[ 46.053107] ffffffffb6eace78 (&nmi_desc[NMI_LOCAL].lock){....}-{2:2}, at: __register_nmi_handler+0x83/0x350
[ 46.053119] {INITIAL USE} state was registered at:
[ 46.053120] lock_acquire+0x14e/0x2b0
[ 46.053123] _raw_spin_lock_irqsave+0x39/0x60
[ 46.053127] __register_nmi_handler+0x83/0x350
[ 46.053130] init_hw_perf_events+0x1d0/0x850
[ 46.053135] do_one_initcall+0xd0/0x3a0
[ 46.053138] kernel_init_freeable+0x34c/0x580
[ 46.053141] kernel_init+0x1c/0x150
[ 46.053145] ret_from_fork+0x48c/0x590
[ 46.053149] ret_from_fork_asm+0x1a/0x30
[ 46.053151] irq event stamp: 685710
[ 46.053153] hardirqs last enabled at (685709): [<ffffffffb5bfa8b8>] _raw_spin_unlock_irq+0x28/0x50
[ 46.053156] hardirqs last disabled at (685710): [<ffffffffb5bfa651>] _raw_spin_lock_irqsave+0x51/0x60
[ 46.053159] softirqs last enabled at (685650): [<ffffffffb3345e2a>] fpu_clone+0xda/0x4f0
[ 46.053163] softirqs last disabled at (685648): [<ffffffffb3345dd2>] fpu_clone+0x82/0x4f0
[ 46.053166]
[ 46.053166] other info that might help us debug this:
[ 46.053168] Possible unsafe locking scenario:
[ 46.053168]
[ 46.053168] CPU0
[ 46.053169] ----
[ 46.053170] lock(&nmi_desc[NMI_LOCAL].lock);
[ 46.053172] <Interrupt>
[ 46.053173] lock(&nmi_desc[NMI_LOCAL].lock);
[ 46.053174]
[ 46.053174] *** DEADLOCK ***
[ 46.053174]
[ 46.053175] 5 locks held by test_progs/126:
[ 46.053177] #0: ffffffffb6f49790 (scx_fork_rwsem){.+.+}-{0:0}, at: sched_fork+0xf9/0x6b0
[ 46.053184] #1: ffff88810c4930e8 (&mm->mm_cid.mutex){+.+.}-{4:4}, at: sched_mm_cid_fork+0xdf/0xc20
[ 46.053190] #2: ffffffffb7671a80 (rcu_read_lock){....}-{1:3}, at: sched_mm_cid_fork+0x692/0xc20
[ 46.053195] #3: ffff888110548a90 (&p->pi_lock){-.-.}-{2:2}, at: task_rq_lock+0x6c/0x3c0
[ 46.053201] #4: ffff8881520ba018 (&rq->__lock){-.-.}-{2:2}, at: task_rq_lock+0xcf/0x3c0
[ 46.053207]
[ 46.053207] stack backtrace:
[ 46.053209] CPU: 2 UID: 0 PID: 126 Comm: test_progs Tainted: G OE 6.19.0-rc5-g748c6d52700a-dirty #1 PREEMPT(full)
[ 46.053214] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 46.053215] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 46.053217] Call Trace:
[ 46.053220] <NMI>
[ 46.053223] dump_stack_lvl+0x5d/0x80
[ 46.053227] print_usage_bug.part.0+0x22b/0x2c0
[ 46.053231] lock_acquire+0x272/0x2b0
[ 46.053235] ? __register_nmi_handler+0x83/0x350
[ 46.053240] _raw_spin_lock_irqsave+0x39/0x60
[ 46.053242] ? __register_nmi_handler+0x83/0x350
[ 46.053246] __register_nmi_handler+0x83/0x350
[ 46.053250] native_stop_other_cpus+0x31c/0x460
[ 46.053255] ? __pfx_native_stop_other_cpus+0x10/0x10
[ 46.053260] vpanic+0x1c5/0x3f0
[ 46.053265] panic+0xce/0xce
[ 46.053268] ? __pfx_panic+0x10/0x10
[ 46.053272] ? __show_trace_log_lvl+0x2ee/0x323
[ 46.053276] ? entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 46.053279] ? nmi_panic+0x91/0x130
[ 46.053283] nmi_panic.cold+0x14/0x14
[ 46.053287] ? __pfx_nmi_panic+0x10/0x10
[ 46.053291] watchdog_hardlockup_check.cold+0x12a/0x1c5
[ 46.053296] __perf_event_overflow+0x2fe/0xeb0
[ 46.053300] ? __pfx___perf_event_overflow+0x10/0x10
[ 46.053303] ? __pfx_x86_perf_event_set_period+0x10/0x10
[ 46.053308] handle_pmi_common+0x405/0x920
[ 46.053312] ? __pfx_handle_pmi_common+0x10/0x10
[ 46.053322] ? __pfx_intel_bts_interrupt+0x10/0x10
[ 46.053327] intel_pmu_handle_irq+0x1c5/0x5d0
[ 46.053330] ? lock_acquire+0x1e9/0x2b0
[ 46.053334] ? nmi_handle.part.0+0x2f/0x370
[ 46.053337] perf_event_nmi_handler+0x3e/0x70
[ 46.053340] nmi_handle.part.0+0x13f/0x370
[ 46.053343] ? trace_rcu_watching+0x105/0x150
[ 46.053348] default_do_nmi+0x3b/0x110
[ 46.053351] ? irqentry_nmi_enter+0x6f/0x80
[ 46.053355] exc_nmi+0xe3/0x110
[ 46.053358] end_repeat_nmi+0xf/0x53
[ 46.053361] RIP: 0010:queued_spin_lock_slowpath+0x6cc/0xac0
[ 46.053365] Code: 0c 24 8b 03 66 85 c0 74 38 48 b8 00 00 00 00 00 fc ff df 48 89 da 49 89 de 48 c1 ea 03 41 83 e6 07 48 01 c2 41 83 c6 03 f3 90 <0f> b6 02 41 38 c6 7c 08 84 c0 0f 85 90 02 00 00 8b 03 66 85 c0 75
[ 46.053367] RSP: 0018:ffffc9000128f750 EFLAGS: 00000002
[ 46.053370] RAX: 0000000000100101 RBX: ffff8881520ba000 RCX: 0000000000000000
[ 46.053372] RDX: ffffed102a417400 RSI: 0000000000000002 RDI: ffff8881520ba002
[ 46.053374] RBP: 1ffff92000251eec R08: ffffffffb5bfb6c9 R09: ffffed102a417400
[ 46.053376] R10: ffffed102a417401 R11: 0000000000000004 R12: ffff88815213b100
[ 46.053378] R13: 00000000000c0000 R14: 0000000000000003 R15: 0000000000000002
[ 46.053380] ? queued_spin_lock_slowpath+0x559/0xac0
[ 46.053385] ? queued_spin_lock_slowpath+0x6cc/0xac0
[ 46.053389] ? queued_spin_lock_slowpath+0x6cc/0xac0
[ 46.053392] </NMI>
[ 46.053393] <TASK>
[ 46.053394] ? __pfx_queued_spin_lock_slowpath+0x10/0x10
[ 46.053400] do_raw_spin_lock+0x1d9/0x270
[ 46.053404] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 46.053407] ? __pfx___might_resched+0x10/0x10
[ 46.053411] task_rq_lock+0xcf/0x3c0
[ 46.053416] mm_cid_fixup_task_to_cpu+0xb0/0x460
[ 46.053420] ? __pfx_mm_cid_fixup_task_to_cpu+0x10/0x10
[ 46.053423] ? lock_acquire+0x14e/0x2b0
[ 46.053427] ? mark_held_locks+0x40/0x70
[ 46.053431] sched_mm_cid_fork+0x6da/0xc20
[ 46.053435] ? __pfx_sched_mm_cid_fork+0x10/0x10
[ 46.053437] ? copy_process+0x217b/0x6950
[ 46.053441] copy_process+0x2bce/0x6950
[ 46.053446] ? __pfx_copy_process+0x10/0x10
[ 46.053448] ? find_held_lock+0x2b/0x80
[ 46.053452] ? _copy_from_user+0x53/0xa0
[ 46.053457] kernel_clone+0xce/0x600
[ 46.053460] ? __pfx_kernel_clone+0x10/0x10
[ 46.053465] ? __lock_acquire+0x481/0x2590
[ 46.053469] __do_sys_clone3+0x16e/0x1b0
[ 46.053472] ? __pfx___do_sys_clone3+0x10/0x10
[ 46.053474] ? lock_acquire+0x14e/0x2b0
[ 46.053477] ? __might_fault+0x9b/0x140
[ 46.053483] ? _copy_to_user+0x5c/0x70
[ 46.053486] ? __x64_sys_rt_sigprocmask+0x258/0x400
[ 46.053491] ? do_user_addr_fault+0x4c2/0xa40
[ 46.053495] ? lockdep_hardirqs_on_prepare+0xd7/0x180
[ 46.053498] do_syscall_64+0x6b/0x3a0
[ 46.053503] entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 46.053506] RIP: 0033:0x7f6ab430fc5d
[ 46.053509] Code: 79 14 0e 00 c3 0f 1f 84 00 00 00 00 00 f3 0f 1e fa b8 ea ff ff ff 48 85 ff 74 28 48 85 d2 74 23 49 89 c8 b8 b3 01 00 00 0f 05 <48> 85 c0 7c 14 74 01 c3 31 ed 4c 89 c7 ff d2 48 89 c7 b8 3c 00 00
[ 46.053511] RSP: 002b:00007fffb282a148 EFLAGS: 00000202 ORIG_RAX: 00000000000001b3
[ 46.053514] RAX: ffffffffffffffda RBX: 00007f6ab4282720 RCX: 00007f6ab430fc5d
[ 46.053516] RDX: 00007f6ab4282720 RSI: 0000000000000058 RDI: 00007fffb282a1a0
[ 46.053517] RBP: 00007fffb282a180 R08: 00007f6ab28736c0 R09: 00007fffb282a2a7
[ 46.053519] R10: 0000000000000008 R11: 0000000000000202 R12: 00007f6ab28736c0
[ 46.053521] R13: ffffffffffffff08 R14: 0000000000000000 R15: 00007fffb282a1a0
[ 46.053525] </TASK>
[ 46.053527] Shutting down cpus with NMI
[ 46.053722] Kernel Offset: 0x32000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
>
> [...]
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 00/20] sched: Rewrite MM CID management
2026-01-28 0:01 ` [patch V5 00/20] sched: Rewrite MM CID management Ihor Solodrai
@ 2026-01-28 8:46 ` Peter Zijlstra
2026-01-28 11:57 ` Thomas Gleixner
1 sibling, 0 replies; 74+ messages in thread
From: Peter Zijlstra @ 2026-01-28 8:46 UTC (permalink / raw)
To: Ihor Solodrai
Cc: Thomas Gleixner, LKML, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde, bpf,
sched-ext, Kernel Team, Alexei Starovoitov, Andrii Nakryiko,
Daniel Borkmann, Puranjay Mohan, Tejun Heo
On Tue, Jan 27, 2026 at 04:01:11PM -0800, Ihor Solodrai wrote:
> On 11/19/25 9:26 AM, Thomas Gleixner wrote:
> > This is a follow up on the V4 series which can be found here:
> >
> > https://lore.kernel.org/20251104075053.700034556@linutronix.de
> >
> > The V1 cover letter contains a detailed analyisis of the issues:
> >
> > https://lore.kernel.org/20251015164952.694882104@linutronix.de
> >
> > TLDR: The CID management is way to complex and adds significant overhead
> > into scheduler hotpaths.
> >
> > The series rewrites MM CID management in a more simplistic way which
> > focusses on low overhead in the scheduler while maintaining per task CIDs
> > as long as the number of threads is not exceeding the number of possible
> > CPUs.
>
> Hello Thomas, everyone.
>
> BPF CI caught a deadlock on current bpf-next tip (35538dba51b4).
> Job: https://github.com/kernel-patches/bpf/actions/runs/21417415035/job/61670254640
>
> It appears to be related to this series. Pasting a splat below.
>
> Any ideas what might be going on?
That splat is only CPU2, that's not typically very useful in a lockup
scenario.
^ permalink raw reply [flat|nested] 74+ messages in thread
* Re: [patch V5 00/20] sched: Rewrite MM CID management
2026-01-28 0:01 ` [patch V5 00/20] sched: Rewrite MM CID management Ihor Solodrai
2026-01-28 8:46 ` Peter Zijlstra
@ 2026-01-28 11:57 ` Thomas Gleixner
2026-01-28 12:58 ` Shrikanth Hegde
1 sibling, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2026-01-28 11:57 UTC (permalink / raw)
To: Ihor Solodrai, LKML
Cc: Peter Zijlstra, Gabriele Monaco, Mathieu Desnoyers,
Michael Jeanson, Jens Axboe, Paul E. McKenney, Gautham R. Shenoy,
Florian Weimer, Tim Chen, Yury Norov, Shrikanth Hegde, bpf,
sched-ext, Kernel Team, Alexei Starovoitov, Andrii Nakryiko,
Daniel Borkmann, Puranjay Mohan, Tejun Heo
On Tue, Jan 27 2026 at 16:01, Ihor Solodrai wrote:
> BPF CI caught a deadlock on current bpf-next tip (35538dba51b4).
> Job: https://github.com/kernel-patches/bpf/actions/runs/21417415035/job/61670254640
>
> It appears to be related to this series. Pasting a splat below.
The deadlock splat is completely unrelated as it is a consequence of the
panic which is triggered by the watchdog:
> [ 45.009755] watchdog: CPU2: Watchdog detected hard LOCKUP on cpu 2
...
> [ 46.053170] lock(&nmi_desc[NMI_LOCAL].lock);
> [ 46.053172] <Interrupt>
> [ 46.053173] lock(&nmi_desc[NMI_LOCAL].lock);
...
> Any ideas what might be going on?
Without a full backtrace of all CPUs it's hard to tell because it's
unclear what is holding the runqueue lock of CPU2 long enough to trigger
the hard lockup watchdog.
I'm pretty sure the CID changes are unrelated, that new code just happen
to show up as the messenger which gets stuck on the lock forever.
> [ 46.053209] CPU: 2 UID: 0 PID: 126 Comm: test_progs Tainted: G OE 6.19.0-rc5-g748c6d52700a-dirty #1 PREEMPT(full)
> [ 46.053214] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
> [ 46.053215] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
> [ 46.053217] Call Trace:
> [ 46.053220] <NMI>
> [ 46.053223] dump_stack_lvl+0x5d/0x80
> [ 46.053227] print_usage_bug.part.0+0x22b/0x2c0
> [ 46.053231] lock_acquire+0x272/0x2b0
> [ 46.053235] ? __register_nmi_handler+0x83/0x350
> [ 46.053240] _raw_spin_lock_irqsave+0x39/0x60
> [ 46.053242] ? __register_nmi_handler+0x83/0x350
> [ 46.053246] __register_nmi_handler+0x83/0x350
> [ 46.053250] native_stop_other_cpus+0x31c/0x460
> [ 46.053255] ? __pfx_native_stop_other_cpus+0x10/0x10
> [ 46.053260] vpanic+0x1c5/0x3f0
vpanic() really should disable lockdep here before taking that lock in
NMI context. The resulting lockdep splat is not really useful.
Thanks.
tglx
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 00/20] sched: Rewrite MM CID management
2026-01-28 11:57 ` Thomas Gleixner
@ 2026-01-28 12:58 ` Shrikanth Hegde
2026-01-28 13:56 ` Thomas Gleixner
0 siblings, 1 reply; 74+ messages in thread
From: Shrikanth Hegde @ 2026-01-28 12:58 UTC (permalink / raw)
To: Thomas Gleixner, Peter Zijlstra, Ihor Solodrai, LKML
Cc: Gabriele Monaco, Mathieu Desnoyers, Michael Jeanson, Jens Axboe,
Paul E. McKenney, Gautham R. Shenoy, Florian Weimer, Tim Chen,
Yury Norov, bpf, sched-ext, Kernel Team, Alexei Starovoitov,
Andrii Nakryiko, Daniel Borkmann, Puranjay Mohan, Tejun Heo
On 1/28/26 5:27 PM, Thomas Gleixner wrote:
> On Tue, Jan 27 2026 at 16:01, Ihor Solodrai wrote:
>> BPF CI caught a deadlock on current bpf-next tip (35538dba51b4).
>> Job: https://github.com/kernel-patches/bpf/actions/runs/21417415035/job/61670254640
>>
>> It appears to be related to this series. Pasting a splat below.
>
> The deadlock splat is completely unrelated as it is a consequence of the
> panic which is triggered by the watchdog:
>
>> [ 45.009755] watchdog: CPU2: Watchdog detected hard LOCKUP on cpu 2
>
> ...
>
>> [ 46.053170] lock(&nmi_desc[NMI_LOCAL].lock);
>> [ 46.053172] <Interrupt>
>> [ 46.053173] lock(&nmi_desc[NMI_LOCAL].lock);
>
> ...
>
>> Any ideas what might be going on?
>
> Without a full backtrace of all CPUs it's hard to tell because it's
> unclear what is holding the runqueue lock of CPU2 long enough to trigger
> the hard lockup watchdog.
>
> I'm pretty sure the CID changes are unrelated, that new code just happen
> to show up as the messenger which gets stuck on the lock forever.
>
>> [ 46.053209] CPU: 2 UID: 0 PID: 126 Comm: test_progs Tainted: G OE 6.19.0-rc5-g748c6d52700a-dirty #1 PREEMPT(full)
>> [ 46.053214] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
>> [ 46.053215] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
>> [ 46.053217] Call Trace:
>> [ 46.053220] <NMI>
>> [ 46.053223] dump_stack_lvl+0x5d/0x80
>> [ 46.053227] print_usage_bug.part.0+0x22b/0x2c0
>> [ 46.053231] lock_acquire+0x272/0x2b0
>> [ 46.053235] ? __register_nmi_handler+0x83/0x350
>> [ 46.053240] _raw_spin_lock_irqsave+0x39/0x60
>> [ 46.053242] ? __register_nmi_handler+0x83/0x350
>> [ 46.053246] __register_nmi_handler+0x83/0x350
>> [ 46.053250] native_stop_other_cpus+0x31c/0x460
>> [ 46.053255] ? __pfx_native_stop_other_cpus+0x10/0x10
>> [ 46.053260] vpanic+0x1c5/0x3f0
>
> vpanic() really should disable lockdep here before taking that lock in
> NMI context. The resulting lockdep splat is not really useful.
>
> Thanks.
>
> tglx
Hi Thomas, Peter.
I remember running into this panic, once. But it wasn't consistent and i
couldn't hit it again. And it had vcpu overcommit, and fair bit of steal time.
The trace was like below from different CPUs.
------------------------
watchdog: CPU 23 self-detected hard LOCKUP @ mm_get_cid+0xe8/0x188
watchdog: CPU 23 TB:1434903268401795, last heartbeat TB:1434897252302837 (11750ms ago)
NIP [c0000000001b7134] mm_get_cid+0xe8/0x188
LR [c0000000001b7154] mm_get_cid+0x108/0x188
Call Trace:
[c000000004c37db0] [c000000001145d84] cpuidle_enter_state+0xf8/0x6a4 (unreliable)
[c000000004c37e00] [c0000000001b95ac] mm_cid_switch_to+0x3c4/0x52c
[c000000004c37e60] [c000000001147264] __schedule+0x47c/0x700
[c000000004c37ee0] [c000000001147a70] schedule_idle+0x3c/0x64
[c000000004c37f10] [c0000000001f6d70] do_idle+0x160/0x1b0
[c000000004c37f60] [c0000000001f7084] cpu_startup_entry+0x48/0x50
[c000000004c37f90] [c00000000005f570] start_secondary+0x284/0x288
[c000000004c37fe0] [c00000000000e158] start_secondary_prolog+0x10/0x14
watchdog: CPU 11 self-detected hard LOCKUP @ plpar_hcall_norets_notrace+0x18/0x2c
watchdog: CPU 11 TB:1434903340004919, last heartbeat TB:1434897249749892 (11895ms ago)
NIP [c0000000000f84fc] plpar_hcall_norets_notrace+0x18/0x2c
LR [c000000001152588] queued_spin_lock_slowpath+0xd88/0x15d0
Call Trace:
[c00000056b69fb10] [c00000056b69fba0] 0xc00000056b69fba0 (unreliable)
[c00000056b69fc30] [c000000001153ce0] _raw_spin_lock+0x80/0xa0
[c00000056b69fc50] [c0000000001b9a34] raw_spin_rq_lock_nested+0x3c/0xf8
[c00000056b69fc80] [c0000000001b9bb8] mm_cid_fixup_cpus_to_tasks+0xc8/0x28c
[c00000056b69fd00] [c0000000001bff34] sched_mm_cid_exit+0x108/0x22c
[c00000056b69fd40] [c000000000167b08] do_exit+0xf4/0x5d0
[c00000056b69fdf0] [c00000000016800c] make_task_dead+0x0/0x178
[c00000056b69fe10] [c0000000000316c8] system_call_exception+0x128/0x390
[c00000056b69fe50] [c00000000000cedc] system_call_vectored_common+0x15c/0x2ec
watchdog: CPU 65 self-detected hard LOCKUP @ queued_spin_lock_slowpath+0x10ec/0x15d0
watchdog: CPU 65 TB:1434905824977447, last heartbeat TB:1434899309522065 (12725ms ago)
NIP [c0000000011528ec] queued_spin_lock_slowpath+0x10ec/0x15d0
LR [c000000001152d0c] queued_spin_lock_slowpath+0x150c/0x15d0
Call Trace:
[c000000777e27a60] [0000000000000009] 0x9 (unreliable)
[c000000777e27b80] [c000000001153ce0] _raw_spin_lock+0x80/0xa0
[c000000777e27ba0] [c0000000001b9a34] raw_spin_rq_lock_nested+0x3c/0xf8
[c000000777e27bd0] [c0000000001babb8] ___task_rq_lock+0x64/0x140
[c000000777e27c20] [c0000000001c8294] wake_up_new_task+0x180/0x484
[c000000777e27ca0] [c00000000015bea4] kernel_clone+0x120/0x5bc
[c000000777e27d30] [c00000000015c4c0] __do_sys_clone+0x88/0xc8
[c000000777e27e10] [c0000000000316c8] system_call_exception+0x128/0x390
[c000000777e27e50] [c00000000000cedc] system_call_vectored_common+0x15c/0x2ec
I am wondering if it this loop in mm_get_cid, which may not be getting a cid
for a long time? Is that possible?
static inline unsigned int mm_get_cid(struct mm_struct *mm)
{
unsigned int cid = __mm_get_cid(mm, READ_ONCE(mm->mm_cid.max_cids));
while (cid == MM_CID_UNSET) {
cpu_relax();
cid = __mm_get_cid(mm, num_possible_cpus());
}
return cid;
}
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 00/20] sched: Rewrite MM CID management
2026-01-28 12:58 ` Shrikanth Hegde
@ 2026-01-28 13:56 ` Thomas Gleixner
2026-01-28 22:24 ` Thomas Gleixner
0 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2026-01-28 13:56 UTC (permalink / raw)
To: Shrikanth Hegde, Peter Zijlstra, Ihor Solodrai, LKML
Cc: Gabriele Monaco, Mathieu Desnoyers, Michael Jeanson, Jens Axboe,
Paul E. McKenney, Gautham R. Shenoy, Florian Weimer, Tim Chen,
Yury Norov, bpf, sched-ext, Kernel Team, Alexei Starovoitov,
Andrii Nakryiko, Daniel Borkmann, Puranjay Mohan, Tejun Heo
On Wed, Jan 28 2026 at 18:28, Shrikanth Hegde wrote:
> On 1/28/26 5:27 PM, Thomas Gleixner wrote:
> watchdog: CPU 23 self-detected hard LOCKUP @ mm_get_cid+0xe8/0x188
> watchdog: CPU 23 TB:1434903268401795, last heartbeat TB:1434897252302837 (11750ms ago)
> NIP [c0000000001b7134] mm_get_cid+0xe8/0x188
> LR [c0000000001b7154] mm_get_cid+0x108/0x188
> Call Trace:
> [c000000004c37db0] [c000000001145d84] cpuidle_enter_state+0xf8/0x6a4 (unreliable)
> [c000000004c37e00] [c0000000001b95ac] mm_cid_switch_to+0x3c4/0x52c
> [c000000004c37e60] [c000000001147264] __schedule+0x47c/0x700
So if the above spins in mm_get_cid() then the below is just a consequence.
> watchdog: CPU 11 self-detected hard LOCKUP @ plpar_hcall_norets_notrace+0x18/0x2c
> watchdog: CPU 11 TB:1434903340004919, last heartbeat TB:1434897249749892 (11895ms ago)
> NIP [c0000000000f84fc] plpar_hcall_norets_notrace+0x18/0x2c
> LR [c000000001152588] queued_spin_lock_slowpath+0xd88/0x15d0
> Call Trace:
> [c00000056b69fb10] [c00000056b69fba0] 0xc00000056b69fba0 (unreliable)
> [c00000056b69fc30] [c000000001153ce0] _raw_spin_lock+0x80/0xa0
> [c00000056b69fc50] [c0000000001b9a34] raw_spin_rq_lock_nested+0x3c/0xf8
> [c00000056b69fc80] [c0000000001b9bb8] mm_cid_fixup_cpus_to_tasks+0xc8/0x28c
> [c00000056b69fd00] [c0000000001bff34] sched_mm_cid_exit+0x108/0x22c
> [c00000056b69fd40] [c000000000167b08] do_exit+0xf4/0x5d0
> [c00000056b69fdf0] [c00000000016800c] make_task_dead+0x0/0x178
> [c00000056b69fe10] [c0000000000316c8] system_call_exception+0x128/0x390
> [c00000056b69fe50] [c00000000000cedc] system_call_vectored_common+0x15c/0x2ec
> I am wondering if it this loop in mm_get_cid, which may not be getting a cid
> for a long time? Is that possible?
It shouldn't be possible by design, but it seems there is a corner case
lurking somewhere which hasn't been covered. Let me stare at the logic
in the transition functions once more. That's where CPU11 comes from:
> [c00000056b69fc80] [c0000000001b9bb8] mm_cid_fixup_cpus_to_tasks+0xc8/0x28c
The exiting it initiated a transition back from per CPU to per task mode
and that seems to make things unhappy for mysterious reasons.
Thanks,
tglx
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 00/20] sched: Rewrite MM CID management
2026-01-28 13:56 ` Thomas Gleixner
@ 2026-01-28 22:24 ` Thomas Gleixner
2026-01-28 22:33 ` Ihor Solodrai
0 siblings, 1 reply; 74+ messages in thread
From: Thomas Gleixner @ 2026-01-28 22:24 UTC (permalink / raw)
To: Shrikanth Hegde, Peter Zijlstra, Ihor Solodrai, LKML
Cc: Gabriele Monaco, Mathieu Desnoyers, Michael Jeanson, Jens Axboe,
Paul E. McKenney, Gautham R. Shenoy, Florian Weimer, Tim Chen,
Yury Norov, bpf, sched-ext, Kernel Team, Alexei Starovoitov,
Andrii Nakryiko, Daniel Borkmann, Puranjay Mohan, Tejun Heo
On Wed, Jan 28 2026 at 14:56, Thomas Gleixner wrote:
> On Wed, Jan 28 2026 at 18:28, Shrikanth Hegde wrote:
>> On 1/28/26 5:27 PM, Thomas Gleixner wrote:
>> watchdog: CPU 23 self-detected hard LOCKUP @ mm_get_cid+0xe8/0x188
>> watchdog: CPU 23 TB:1434903268401795, last heartbeat TB:1434897252302837 (11750ms ago)
>> NIP [c0000000001b7134] mm_get_cid+0xe8/0x188
>> LR [c0000000001b7154] mm_get_cid+0x108/0x188
>> Call Trace:
>> [c000000004c37db0] [c000000001145d84] cpuidle_enter_state+0xf8/0x6a4 (unreliable)
>> [c000000004c37e00] [c0000000001b95ac] mm_cid_switch_to+0x3c4/0x52c
>> [c000000004c37e60] [c000000001147264] __schedule+0x47c/0x700
>
> So if the above spins in mm_get_cid() then the below is just a consequence.
>
>> watchdog: CPU 11 self-detected hard LOCKUP @ plpar_hcall_norets_notrace+0x18/0x2c
>> watchdog: CPU 11 TB:1434903340004919, last heartbeat TB:1434897249749892 (11895ms ago)
>> NIP [c0000000000f84fc] plpar_hcall_norets_notrace+0x18/0x2c
>> LR [c000000001152588] queued_spin_lock_slowpath+0xd88/0x15d0
>> Call Trace:
>> [c00000056b69fb10] [c00000056b69fba0] 0xc00000056b69fba0 (unreliable)
>> [c00000056b69fc30] [c000000001153ce0] _raw_spin_lock+0x80/0xa0
>> [c00000056b69fc50] [c0000000001b9a34] raw_spin_rq_lock_nested+0x3c/0xf8
>> [c00000056b69fc80] [c0000000001b9bb8] mm_cid_fixup_cpus_to_tasks+0xc8/0x28c
>> [c00000056b69fd00] [c0000000001bff34] sched_mm_cid_exit+0x108/0x22c
>> [c00000056b69fd40] [c000000000167b08] do_exit+0xf4/0x5d0
>> [c00000056b69fdf0] [c00000000016800c] make_task_dead+0x0/0x178
>> [c00000056b69fe10] [c0000000000316c8] system_call_exception+0x128/0x390
>> [c00000056b69fe50] [c00000000000cedc] system_call_vectored_common+0x15c/0x2ec
>
>> I am wondering if it this loop in mm_get_cid, which may not be getting a cid
>> for a long time? Is that possible?
>
> It shouldn't be possible by design, but it seems there is a corner case
> lurking somewhere which hasn't been covered. Let me stare at the logic
> in the transition functions once more. That's where CPU11 comes from:
>
>> [c00000056b69fc80] [c0000000001b9bb8] mm_cid_fixup_cpus_to_tasks+0xc8/0x28c
>
> The exiting it initiated a transition back from per CPU to per task mode
> and that seems to make things unhappy for mysterious reasons.
I stared at it for a while and found the below stupidity. But when I
actually sat down after a while away from the keyboard and tried to
write a concise changelog explaining the root cause I failed to come up
with a coherent explanation why this would prevent the above scenario,
which hints at a situation of MMCID exhaustion.
@Ihor: Is the BPF CI fallout reproducible? If so, can you please provide
it?
Thanks,
tglx
---
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -10664,8 +10664,14 @@ void sched_mm_cid_exit(struct task_struc
scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
if (!__sched_mm_cid_exit(t))
return;
- /* Mode change required. Transfer currents CID */
- mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
+ /*
+ * Mode change. The task has the CID unset
+ * already. The CPU CID is still valid and
+ * does not have MM_CID_TRANSIT set as the
+ * mode change has just taken effect under
+ * mm::mm_cid::lock. Drop it.
+ */
+ mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
}
mm_cid_fixup_cpus_to_tasks(mm);
return;
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 00/20] sched: Rewrite MM CID management
2026-01-28 22:24 ` Thomas Gleixner
@ 2026-01-28 22:33 ` Ihor Solodrai
2026-01-28 23:08 ` Ihor Solodrai
0 siblings, 1 reply; 74+ messages in thread
From: Ihor Solodrai @ 2026-01-28 22:33 UTC (permalink / raw)
To: Thomas Gleixner, Shrikanth Hegde, Peter Zijlstra, LKML
Cc: Gabriele Monaco, Mathieu Desnoyers, Michael Jeanson, Jens Axboe,
Paul E. McKenney, Gautham R. Shenoy, Florian Weimer, Tim Chen,
Yury Norov, bpf, sched-ext, Kernel Team, Alexei Starovoitov,
Andrii Nakryiko, Daniel Borkmann, Puranjay Mohan, Tejun Heo
On 1/28/26 2:24 PM, Thomas Gleixner wrote:
> On Wed, Jan 28 2026 at 14:56, Thomas Gleixner wrote:
>> On Wed, Jan 28 2026 at 18:28, Shrikanth Hegde wrote:
>>> On 1/28/26 5:27 PM, Thomas Gleixner wrote:
>>> watchdog: CPU 23 self-detected hard LOCKUP @ mm_get_cid+0xe8/0x188
>>> watchdog: CPU 23 TB:1434903268401795, last heartbeat TB:1434897252302837 (11750ms ago)
>>> NIP [c0000000001b7134] mm_get_cid+0xe8/0x188
>>> LR [c0000000001b7154] mm_get_cid+0x108/0x188
>>> Call Trace:
>>> [c000000004c37db0] [c000000001145d84] cpuidle_enter_state+0xf8/0x6a4 (unreliable)
>>> [c000000004c37e00] [c0000000001b95ac] mm_cid_switch_to+0x3c4/0x52c
>>> [c000000004c37e60] [c000000001147264] __schedule+0x47c/0x700
>>
>> So if the above spins in mm_get_cid() then the below is just a consequence.
>>
>>> watchdog: CPU 11 self-detected hard LOCKUP @ plpar_hcall_norets_notrace+0x18/0x2c
>>> watchdog: CPU 11 TB:1434903340004919, last heartbeat TB:1434897249749892 (11895ms ago)
>>> NIP [c0000000000f84fc] plpar_hcall_norets_notrace+0x18/0x2c
>>> LR [c000000001152588] queued_spin_lock_slowpath+0xd88/0x15d0
>>> Call Trace:
>>> [c00000056b69fb10] [c00000056b69fba0] 0xc00000056b69fba0 (unreliable)
>>> [c00000056b69fc30] [c000000001153ce0] _raw_spin_lock+0x80/0xa0
>>> [c00000056b69fc50] [c0000000001b9a34] raw_spin_rq_lock_nested+0x3c/0xf8
>>> [c00000056b69fc80] [c0000000001b9bb8] mm_cid_fixup_cpus_to_tasks+0xc8/0x28c
>>> [c00000056b69fd00] [c0000000001bff34] sched_mm_cid_exit+0x108/0x22c
>>> [c00000056b69fd40] [c000000000167b08] do_exit+0xf4/0x5d0
>>> [c00000056b69fdf0] [c00000000016800c] make_task_dead+0x0/0x178
>>> [c00000056b69fe10] [c0000000000316c8] system_call_exception+0x128/0x390
>>> [c00000056b69fe50] [c00000000000cedc] system_call_vectored_common+0x15c/0x2ec
>>
>>> I am wondering if it this loop in mm_get_cid, which may not be getting a cid
>>> for a long time? Is that possible?
>>
>> It shouldn't be possible by design, but it seems there is a corner case
>> lurking somewhere which hasn't been covered. Let me stare at the logic
>> in the transition functions once more. That's where CPU11 comes from:
>>
>>> [c00000056b69fc80] [c0000000001b9bb8] mm_cid_fixup_cpus_to_tasks+0xc8/0x28c
>>
>> The exiting it initiated a transition back from per CPU to per task mode
>> and that seems to make things unhappy for mysterious reasons.
>
> I stared at it for a while and found the below stupidity. But when I
> actually sat down after a while away from the keyboard and tried to
> write a concise changelog explaining the root cause I failed to come up
> with a coherent explanation why this would prevent the above scenario,
> which hints at a situation of MMCID exhaustion.
>
> @Ihor: Is the BPF CI fallout reproducible? If so, can you please provide
> it?
Not reliably, unfortunately. I saw it at least twice (out of 100+
runs) this week.
I added `hardlockup_all_cpu_backtrace=1` to get more logs. If there
is anything else I could set up (kconfigs, debug switches) that may be
helpful, let me know.
We have a steady stream of jobs running, so if it's not a one-off it's
likely to happen again. I'll share if we get anything.
Thank you for investigating!
>
> Thanks,
>
> tglx
> ---
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -10664,8 +10664,14 @@ void sched_mm_cid_exit(struct task_struc
> scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
> if (!__sched_mm_cid_exit(t))
> return;
> - /* Mode change required. Transfer currents CID */
> - mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
> + /*
> + * Mode change. The task has the CID unset
> + * already. The CPU CID is still valid and
> + * does not have MM_CID_TRANSIT set as the
> + * mode change has just taken effect under
> + * mm::mm_cid::lock. Drop it.
> + */
> + mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
> }
> mm_cid_fixup_cpus_to_tasks(mm);
> return;
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 00/20] sched: Rewrite MM CID management
2026-01-28 22:33 ` Ihor Solodrai
@ 2026-01-28 23:08 ` Ihor Solodrai
2026-01-29 17:06 ` Thomas Gleixner
0 siblings, 1 reply; 74+ messages in thread
From: Ihor Solodrai @ 2026-01-28 23:08 UTC (permalink / raw)
To: Thomas Gleixner, Shrikanth Hegde, Peter Zijlstra, LKML
Cc: Gabriele Monaco, Mathieu Desnoyers, Michael Jeanson, Jens Axboe,
Paul E. McKenney, Gautham R. Shenoy, Florian Weimer, Tim Chen,
Yury Norov, bpf, sched-ext, Kernel Team, Alexei Starovoitov,
Andrii Nakryiko, Daniel Borkmann, Puranjay Mohan, Tejun Heo
On 1/28/26 2:33 PM, Ihor Solodrai wrote:
> [...]
>
> We have a steady stream of jobs running, so if it's not a one-off it's
> likely to happen again. I'll share if we get anything.
Here is another one, with backtraces of other CPUs:
[ 59.133878] watchdog: CPU2: Watchdog detected hard LOCKUP on cpu 2
[ 59.133886] Modules linked in: bpf_testmod(OE)
[ 59.133892] irq event stamp: 687092
[ 59.133893] hardirqs last enabled at (687091): [<ffffffff8fbfbf78>] _raw_spin_unlock_irq+0x28/0x50
[ 59.133908] hardirqs last disabled at (687092): [<ffffffff8fbfbd11>] _raw_spin_lock_irqsave+0x51/0x60
[ 59.133912] softirqs last enabled at (687006): [<ffffffff8d345e2a>] fpu_clone+0xda/0x4f0
[ 59.133918] softirqs last disabled at (687004): [<ffffffff8d345dd2>] fpu_clone+0x82/0x4f0
[ 59.133925] CPU: 2 UID: 0 PID: 127 Comm: test_progs Tainted: G OE 6.19.0-rc5-gbe9790cb9e63-dirty #1 PREEMPT(full)
[ 59.133930] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 59.133932] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 59.133935] RIP: 0010:queued_spin_lock_slowpath+0x3a9/0xac0
[ 59.133943] Code: 00 00 85 c0 74 3d 0f b6 03 84 c0 74 36 48 b8 00 00 00 00 00 fc ff df 49 89 dc 49 89 dd 49 c1 ec 03 41 83 e5 07 49 01 c4 f3 90 <41> 0f b6 04 24 44 38 e8 7f 08 84 c0 0f 85 9f 05 00 00 0f b6 03 84
[ 59.133945] RSP: 0018:ffffc900012df750 EFLAGS: 00000002
[ 59.133950] RAX: 0000000000000001 RBX: ffff8881520ba000 RCX: 0000000000000001
[ 59.133952] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff8881520ba000
[ 59.133954] RBP: 1ffff9200025beec R08: ffffffff8fbfcb69 R09: ffffed102a417400
[ 59.133956] R10: ffffed102a417401 R11: 0000000000000004 R12: ffffed102a417400
[ 59.133958] R13: 0000000000000000 R14: dffffc0000000000 R15: ffff8881520ba000
[ 59.133960] FS: 00007f7230740e00(0000) GS:ffff8881bf8db000(0000) knlGS:0000000000000000
[ 59.133964] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 59.133966] CR2: 00007f722f1a6d58 CR3: 000000010ed2f001 CR4: 0000000000770ef0
[ 59.133968] PKRU: 55555554
[ 59.133969] Call Trace:
[ 59.133973] <TASK>
[ 59.133977] ? __pfx_queued_spin_lock_slowpath+0x10/0x10
[ 59.133985] do_raw_spin_lock+0x1d9/0x270
[ 59.133991] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 59.133994] ? __pfx___might_resched+0x10/0x10
[ 59.134001] task_rq_lock+0xcf/0x3c0
[ 59.134007] mm_cid_fixup_task_to_cpu+0xb0/0x460
[ 59.134011] ? __pfx_mm_cid_fixup_task_to_cpu+0x10/0x10
[ 59.134015] ? lock_acquire+0x14e/0x2b0
[ 59.134020] ? mark_held_locks+0x40/0x70
[ 59.134025] sched_mm_cid_fork+0x6da/0xc20
[ 59.134030] ? __pfx_sched_mm_cid_fork+0x10/0x10
[ 59.134032] ? copy_process+0x217b/0x6950
[ 59.134037] copy_process+0x2bce/0x6950
[ 59.134044] ? __pfx_copy_process+0x10/0x10
[ 59.134046] ? find_held_lock+0x2b/0x80
[ 59.134051] ? _copy_from_user+0x53/0xa0
[ 59.134058] kernel_clone+0xce/0x600
[ 59.134061] ? __pfx_kernel_clone+0x10/0x10
[ 59.134066] ? __lock_acquire+0x481/0x2590
[ 59.134071] __do_sys_clone3+0x16e/0x1b0
[ 59.134074] ? __pfx___do_sys_clone3+0x10/0x10
[ 59.134077] ? lock_acquire+0x14e/0x2b0
[ 59.134080] ? __might_fault+0x9b/0x140
[ 59.134089] ? _copy_to_user+0x5c/0x70
[ 59.134092] ? __x64_sys_rt_sigprocmask+0x258/0x400
[ 59.134099] ? do_user_addr_fault+0x4c2/0xa40
[ 59.134103] ? lockdep_hardirqs_on_prepare+0xd7/0x180
[ 59.134107] do_syscall_64+0x6b/0x3a0
[ 59.134111] entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 59.134116] RIP: 0033:0x7f7230c42c5d
[ 59.134120] Code: 79 14 0e 00 c3 0f 1f 84 00 00 00 00 00 f3 0f 1e fa b8 ea ff ff ff 48 85 ff 74 28 48 85 d2 74 23 49 89 c8 b8 b3 01 00 00 0f 05 <48> 85 c0 7c 14 74 01 c3 31 ed 4c 89 c7 ff d2 48 89 c7 b8 3c 00 00
[ 59.134122] RSP: 002b:00007ffe90d4e1f8 EFLAGS: 00000202 ORIG_RAX: 00000000000001b3
[ 59.134126] RAX: ffffffffffffffda RBX: 00007f7230bb5720 RCX: 00007f7230c42c5d
[ 59.134128] RDX: 00007f7230bb5720 RSI: 0000000000000058 RDI: 00007ffe90d4e250
[ 59.134129] RBP: 00007ffe90d4e230 R08: 00007f722f1a66c0 R09: 00007ffe90d4e357
[ 59.134131] R10: 0000000000000008 R11: 0000000000000202 R12: 00007f722f1a66c0
[ 59.134133] R13: ffffffffffffff08 R14: 0000000000000000 R15: 00007ffe90d4e250
[ 59.134139] </TASK>
[ 59.134141] Sending NMI from CPU 2 to CPUs 0-1,3:
[ 59.134168] NMI backtrace for cpu 3
[ 59.134176] CPU: 3 UID: 0 PID: 67 Comm: kworker/3:1 Tainted: G OE 6.19.0-rc5-gbe9790cb9e63-dirty #1 PREEMPT(full)
[ 59.134181] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 59.134183] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 59.134186] Workqueue: events drain_vmap_area_work
[ 59.134194] RIP: 0010:smp_call_function_many_cond+0x772/0xe60
[ 59.134200] Code: 38 c8 7c 08 84 c9 0f 85 92 05 00 00 8b 43 08 a8 01 74 2e 48 89 f1 49 89 f5 48 c1 e9 03 41 83 e5 07 4c 01 f1 41 83 c5 03 f3 90 <0f> b6 01 41 38 c5 7c 08 84 c0 0f 85 c1 04 00 00 8b 43 08 a8 01 75
[ 59.134203] RSP: 0018:ffffc90000587948 EFLAGS: 00000202
[ 59.134206] RAX: 0000000000000011 RBX: ffff8881520c1ac0 RCX: ffffed102a418359
[ 59.134208] RDX: 0000000000000001 RSI: ffff8881520c1ac8 RDI: ffffffff90713be8
[ 59.134210] RBP: ffffed102a437680 R08: ffff8881521bb408 R09: 0000000000000000
[ 59.134212] R10: 1ffff1102a437681 R11: ffff888103aa8bb0 R12: ffff8881521bb408
[ 59.134213] R13: 0000000000000003 R14: dffffc0000000000 R15: ffff8881521bb400
[ 59.134215] FS: 0000000000000000(0000) GS:ffff8881bf95b000(0000) knlGS:0000000000000000
[ 59.134219] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 59.134221] CR2: 00007fc9ae7762a0 CR3: 000000010c435001 CR4: 0000000000770ef0
[ 59.134223] PKRU: 55555554
[ 59.134224] Call Trace:
[ 59.134226] <TASK>
[ 59.134230] ? __pfx_do_flush_tlb_all+0x10/0x10
[ 59.134238] ? __pfx_smp_call_function_many_cond+0x10/0x10
[ 59.134242] ? __pfx___apply_to_page_range+0x10/0x10
[ 59.134245] ? mark_held_locks+0x40/0x70
[ 59.134250] on_each_cpu_cond_mask+0x24/0x40
[ 59.134254] flush_tlb_kernel_range+0x402/0x6b0
[ 59.134259] ? __kasan_release_vmalloc+0xd6/0x110
[ 59.134265] purge_vmap_node+0x1db/0x9c0
[ 59.134270] ? __pfx_smp_call_function_many_cond+0x10/0x10
[ 59.134275] ? __pfx_purge_vmap_node+0x10/0x10
[ 59.134280] __purge_vmap_area_lazy+0x6ea/0xac0
[ 59.134286] drain_vmap_area_work+0x27/0x40
[ 59.134289] process_one_work+0x800/0x13e0
[ 59.134296] ? __pfx_process_one_work+0x10/0x10
[ 59.134298] ? lock_acquire+0x14e/0x2b0
[ 59.134302] ? lock_is_held_type+0x87/0xf0
[ 59.134307] ? assign_work+0x156/0x390
[ 59.134313] worker_thread+0x5c8/0xfa0
[ 59.134319] ? __pfx_worker_thread+0x10/0x10
[ 59.134322] kthread+0x3bd/0x780
[ 59.134327] ? do_raw_spin_lock+0x128/0x270
[ 59.134332] ? __pfx_kthread+0x10/0x10
[ 59.134335] ? __pfx_kthread+0x10/0x10
[ 59.134340] ? ret_from_fork+0x6e/0x590
[ 59.134344] ? lock_release+0xd4/0x2c0
[ 59.134348] ? __pfx_kthread+0x10/0x10
[ 59.134351] ret_from_fork+0x48c/0x590
[ 59.134355] ? __pfx_ret_from_fork+0x10/0x10
[ 59.134359] ? __pfx_kthread+0x10/0x10
[ 59.134363] ret_from_fork_asm+0x1a/0x30
[ 59.134371] </TASK>
[ 59.134374] NMI backtrace for cpu 1
[ 59.134380] CPU: 1 UID: 0 PID: 0 Comm: swapper/1 Tainted: G OE 6.19.0-rc5-gbe9790cb9e63-dirty #1 PREEMPT(full)
[ 59.134385] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 59.134386] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 59.134388] RIP: 0010:_find_first_zero_bit+0x50/0x90
[ 59.134394] Code: 48 39 c1 73 25 48 89 fa 48 c1 ea 03 80 3c 32 00 75 26 48 8b 17 48 83 f2 ff 74 dd f3 48 0f bc d2 48 01 d1 48 39 c8 48 0f 47 c1 <48> 83 c4 18 c3 cc cc cc cc c3 cc cc cc cc 48 89 44 24 10 48 89 4c
[ 59.134396] RSP: 0018:ffffc9000014fd58 EFLAGS: 00000046
[ 59.134400] RAX: 0000000000000004 RBX: ffff888100d3a440 RCX: 0000000000000004
[ 59.134402] RDX: 0000000000000004 RSI: dffffc0000000000 RDI: ffff88810e9d22a0
[ 59.134403] RBP: ffffc9000014fe60 R08: ffff88810e9d1840 R09: ffff8881396e0000
[ 59.134405] R10: 0000000080000000 R11: 0000000000000004 R12: ffff88810e9d1840
[ 59.134407] R13: ffff8881520ba000 R14: ffff88810e9d22a0 R15: ffff8881396e0000
[ 59.134409] FS: 0000000000000000(0000) GS:ffff8881bf85b000(0000) knlGS:0000000000000000
[ 59.134413] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 59.134414] CR2: 00007f72301a8d58 CR3: 000000010ed2f005 CR4: 0000000000770ef0
[ 59.134416] PKRU: 55555554
[ 59.134417] Call Trace:
[ 59.134420] <TASK>
[ 59.134423] __schedule+0x3312/0x4390
[ 59.134430] ? __pfx___schedule+0x10/0x10
[ 59.134434] ? trace_rcu_watching+0x105/0x150
[ 59.134440] schedule_idle+0x59/0x90
[ 59.134443] do_idle+0x26b/0x4d0
[ 59.134449] ? __pfx_do_idle+0x10/0x10
[ 59.134452] ? do_idle+0x278/0x4d0
[ 59.134456] cpu_startup_entry+0x53/0x70
[ 59.134459] start_secondary+0x1b9/0x230
[ 59.134463] common_startup_64+0x12c/0x138
[ 59.134472] </TASK>
[ 59.134474] NMI backtrace for cpu 0 skipped: idling at default_idle+0xf/0x20
[ 59.135160] Kernel panic - not syncing: Hard LOCKUP
[ 59.135163] CPU: 2 UID: 0 PID: 127 Comm: test_progs Tainted: G OE 6.19.0-rc5-gbe9790cb9e63-dirty #1 PREEMPT(full)
[ 59.135167] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 59.135169] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 59.135170] Call Trace:
[ 59.135173] <NMI>
[ 59.135174] dump_stack_lvl+0x5d/0x80
[ 59.135179] vpanic+0x133/0x3f0
[ 59.135185] panic+0xce/0xce
[ 59.135188] ? __pfx_panic+0x10/0x10
[ 59.135193] ? _printk+0xc7/0x100
[ 59.135198] ? nmi_panic+0x91/0x130
[ 59.135202] nmi_panic.cold+0x14/0x14
[ 59.135206] ? __pfx_nmi_panic+0x10/0x10
[ 59.135209] ? __pfx_nmi_raise_cpu_backtrace+0x10/0x10
[ 59.135214] watchdog_hardlockup_check.cold+0x12a/0x1c5
[ 59.135220] __perf_event_overflow+0x2fe/0xeb0
[ 59.135226] ? __pfx___perf_event_overflow+0x10/0x10
[ 59.135229] ? __pfx_x86_perf_event_set_period+0x10/0x10
[ 59.135235] handle_pmi_common+0x405/0x920
[ 59.135240] ? __pfx_handle_pmi_common+0x10/0x10
[ 59.135253] ? __pfx_intel_bts_interrupt+0x10/0x10
[ 59.135259] intel_pmu_handle_irq+0x1c5/0x5d0
[ 59.135263] ? lock_acquire+0x1e9/0x2b0
[ 59.135266] ? nmi_handle.part.0+0x2f/0x370
[ 59.135271] perf_event_nmi_handler+0x3e/0x70
[ 59.135275] nmi_handle.part.0+0x13f/0x370
[ 59.135278] ? trace_rcu_watching+0x105/0x150
[ 59.135283] default_do_nmi+0x3b/0x110
[ 59.135287] ? irqentry_nmi_enter+0x6f/0x80
[ 59.135291] exc_nmi+0xe3/0x110
[ 59.135294] end_repeat_nmi+0xf/0x53
[ 59.135297] RIP: 0010:queued_spin_lock_slowpath+0x3a9/0xac0
[ 59.135301] Code: 00 00 85 c0 74 3d 0f b6 03 84 c0 74 36 48 b8 00 00 00 00 00 fc ff df 49 89 dc 49 89 dd 49 c1 ec 03 41 83 e5 07 49 01 c4 f3 90 <41> 0f b6 04 24 44 38 e8 7f 08 84 c0 0f 85 9f 05 00 00 0f b6 03 84
[ 59.135303] RSP: 0018:ffffc900012df750 EFLAGS: 00000002
[ 59.135305] RAX: 0000000000000001 RBX: ffff8881520ba000 RCX: 0000000000000001
[ 59.135307] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff8881520ba000
[ 59.135309] RBP: 1ffff9200025beec R08: ffffffff8fbfcb69 R09: ffffed102a417400
[ 59.135311] R10: ffffed102a417401 R11: 0000000000000004 R12: ffffed102a417400
[ 59.135313] R13: 0000000000000000 R14: dffffc0000000000 R15: ffff8881520ba000
[ 59.135316] ? queued_spin_lock_slowpath+0x339/0xac0
[ 59.135321] ? queued_spin_lock_slowpath+0x3a9/0xac0
[ 59.135325] ? queued_spin_lock_slowpath+0x3a9/0xac0
[ 59.135329] </NMI>
[ 59.135330] <TASK>
[ 59.135332] ? __pfx_queued_spin_lock_slowpath+0x10/0x10
[ 59.135338] do_raw_spin_lock+0x1d9/0x270
[ 59.135342] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 59.135346] ? __pfx___might_resched+0x10/0x10
[ 59.135350] task_rq_lock+0xcf/0x3c0
[ 59.135355] mm_cid_fixup_task_to_cpu+0xb0/0x460
[ 59.135359] ? __pfx_mm_cid_fixup_task_to_cpu+0x10/0x10
[ 59.135364] ? lock_acquire+0x14e/0x2b0
[ 59.135368] ? mark_held_locks+0x40/0x70
[ 59.135372] sched_mm_cid_fork+0x6da/0xc20
[ 59.135376] ? __pfx_sched_mm_cid_fork+0x10/0x10
[ 59.135379] ? copy_process+0x217b/0x6950
[ 59.135383] copy_process+0x2bce/0x6950
[ 59.135389] ? __pfx_copy_process+0x10/0x10
[ 59.135391] ? find_held_lock+0x2b/0x80
[ 59.135396] ? _copy_from_user+0x53/0xa0
[ 59.135401] kernel_clone+0xce/0x600
[ 59.135404] ? __pfx_kernel_clone+0x10/0x10
[ 59.135409] ? __lock_acquire+0x481/0x2590
[ 59.135414] __do_sys_clone3+0x16e/0x1b0
[ 59.135417] ? __pfx___do_sys_clone3+0x10/0x10
[ 59.135419] ? lock_acquire+0x14e/0x2b0
[ 59.135422] ? __might_fault+0x9b/0x140
[ 59.135429] ? _copy_to_user+0x5c/0x70
[ 59.135432] ? __x64_sys_rt_sigprocmask+0x258/0x400
[ 59.135438] ? do_user_addr_fault+0x4c2/0xa40
[ 59.135441] ? lockdep_hardirqs_on_prepare+0xd7/0x180
[ 59.135445] do_syscall_64+0x6b/0x3a0
[ 59.135448] entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 59.135451] RIP: 0033:0x7f7230c42c5d
[ 59.135453] Code: 79 14 0e 00 c3 0f 1f 84 00 00 00 00 00 f3 0f 1e fa b8 ea ff ff ff 48 85 ff 74 28 48 85 d2 74 23 49 89 c8 b8 b3 01 00 00 0f 05 <48> 85 c0 7c 14 74 01 c3 31 ed 4c 89 c7 ff d2 48 89 c7 b8 3c 00 00
[ 59.135455] RSP: 002b:00007ffe90d4e1f8 EFLAGS: 00000202 ORIG_RAX: 00000000000001b3
[ 59.135458] RAX: ffffffffffffffda RBX: 00007f7230bb5720 RCX: 00007f7230c42c5d
[ 59.135459] RDX: 00007f7230bb5720 RSI: 0000000000000058 RDI: 00007ffe90d4e250
[ 59.135461] RBP: 00007ffe90d4e230 R08: 00007f722f1a66c0 R09: 00007ffe90d4e357
[ 59.135462] R10: 0000000000000008 R11: 0000000000000202 R12: 00007f722f1a66c0
[ 59.135464] R13: ffffffffffffff08 R14: 0000000000000000 R15: 00007ffe90d4e250
[ 59.135470] </TASK>
[ 60.170882]
[ 60.170886] ================================
[ 60.170888] WARNING: inconsistent lock state
[ 60.170890] 6.19.0-rc5-gbe9790cb9e63-dirty #1 Tainted: G OE
[ 60.170893] --------------------------------
[ 60.170894] inconsistent {INITIAL USE} -> {IN-NMI} usage.
[ 60.170895] test_progs/127 [HC1[1]:SC0[0]:HE0:SE1] takes:
[ 60.170899] ffffffff90eace78 (&nmi_desc[NMI_LOCAL].lock){....}-{2:2}, at: __register_nmi_handler+0x83/0x350
[ 60.170912] {INITIAL USE} state was registered at:
[ 60.170913] lock_acquire+0x14e/0x2b0
[ 60.170918] _raw_spin_lock_irqsave+0x39/0x60
[ 60.170921] __register_nmi_handler+0x83/0x350
[ 60.170924] init_hw_perf_events+0x1d0/0x850
[ 60.170929] do_one_initcall+0xd0/0x3a0
[ 60.170934] kernel_init_freeable+0x34c/0x580
[ 60.170937] kernel_init+0x1c/0x150
[ 60.170939] ret_from_fork+0x48c/0x590
[ 60.170942] ret_from_fork_asm+0x1a/0x30
[ 60.170945] irq event stamp: 687092
[ 60.170946] hardirqs last enabled at (687091): [<ffffffff8fbfbf78>] _raw_spin_unlock_irq+0x28/0x50
[ 60.170950] hardirqs last disabled at (687092): [<ffffffff8fbfbd11>] _raw_spin_lock_irqsave+0x51/0x60
[ 60.170952] softirqs last enabled at (687006): [<ffffffff8d345e2a>] fpu_clone+0xda/0x4f0
[ 60.170956] softirqs last disabled at (687004): [<ffffffff8d345dd2>] fpu_clone+0x82/0x4f0
[ 60.170959]
[ 60.170959] other info that might help us debug this:
[ 60.170961] Possible unsafe locking scenario:
[ 60.170961]
[ 60.170962] CPU0
[ 60.170963] ----
[ 60.170963] lock(&nmi_desc[NMI_LOCAL].lock);
[ 60.170965] <Interrupt>
[ 60.170966] lock(&nmi_desc[NMI_LOCAL].lock);
[ 60.170968]
[ 60.170968] *** DEADLOCK ***
[ 60.170968]
[ 60.170969] 5 locks held by test_progs/127:
[ 60.170970] #0: ffffffff90f49790 (scx_fork_rwsem){.+.+}-{0:0}, at: sched_fork+0xf9/0x6b0
[ 60.170978] #1: ffff88810e9d1968 (&mm->mm_cid.mutex){+.+.}-{4:4}, at: sched_mm_cid_fork+0xdf/0xc20
[ 60.170983] #2: ffffffff91671a80 (rcu_read_lock){....}-{1:3}, at: sched_mm_cid_fork+0x692/0xc20
[ 60.170989] #3: ffff88810cfbaed0 (&p->pi_lock){-.-.}-{2:2}, at: task_rq_lock+0x6c/0x3c0
[ 60.170995] #4: ffff8881520ba018 (&rq->__lock){-.-.}-{2:2}, at: task_rq_lock+0xcf/0x3c0
[ 60.171001]
[ 60.171001] stack backtrace:
[ 60.171004] CPU: 2 UID: 0 PID: 127 Comm: test_progs Tainted: G OE 6.19.0-rc5-gbe9790cb9e63-dirty #1 PREEMPT(full)
[ 60.171009] Tainted: [O]=OOT_MODULE, [E]=UNSIGNED_MODULE
[ 60.171011] Hardware name: QEMU Ubuntu 24.04 PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 60.171013] Call Trace:
[ 60.171016] <NMI>
[ 60.171020] dump_stack_lvl+0x5d/0x80
[ 60.171024] print_usage_bug.part.0+0x22b/0x2c0
[ 60.171029] lock_acquire+0x272/0x2b0
[ 60.171032] ? __register_nmi_handler+0x83/0x350
[ 60.171037] _raw_spin_lock_irqsave+0x39/0x60
[ 60.171040] ? __register_nmi_handler+0x83/0x350
[ 60.171043] __register_nmi_handler+0x83/0x350
[ 60.171048] native_stop_other_cpus+0x31c/0x460
[ 60.171052] ? __pfx_native_stop_other_cpus+0x10/0x10
[ 60.171057] vpanic+0x1c5/0x3f0
[ 60.171060] panic+0xce/0xce
[ 60.171064] ? __pfx_panic+0x10/0x10
[ 60.171068] ? _printk+0xc7/0x100
[ 60.171072] ? nmi_panic+0x91/0x130
[ 60.171075] nmi_panic.cold+0x14/0x14
[ 60.171078] ? __pfx_nmi_panic+0x10/0x10
[ 60.171081] ? __pfx_nmi_raise_cpu_backtrace+0x10/0x10
[ 60.171085] watchdog_hardlockup_check.cold+0x12a/0x1c5
[ 60.171090] __perf_event_overflow+0x2fe/0xeb0
[ 60.171094] ? __pfx___perf_event_overflow+0x10/0x10
[ 60.171097] ? __pfx_x86_perf_event_set_period+0x10/0x10
[ 60.171102] handle_pmi_common+0x405/0x920
[ 60.171105] ? __pfx_handle_pmi_common+0x10/0x10
[ 60.171115] ? __pfx_intel_bts_interrupt+0x10/0x10
[ 60.171120] intel_pmu_handle_irq+0x1c5/0x5d0
[ 60.171123] ? lock_acquire+0x1e9/0x2b0
[ 60.171127] ? nmi_handle.part.0+0x2f/0x370
[ 60.171130] perf_event_nmi_handler+0x3e/0x70
[ 60.171133] nmi_handle.part.0+0x13f/0x370
[ 60.171135] ? trace_rcu_watching+0x105/0x150
[ 60.171141] default_do_nmi+0x3b/0x110
[ 60.171144] ? irqentry_nmi_enter+0x6f/0x80
[ 60.171147] exc_nmi+0xe3/0x110
[ 60.171150] end_repeat_nmi+0xf/0x53
[ 60.171154] RIP: 0010:queued_spin_lock_slowpath+0x3a9/0xac0
[ 60.171158] Code: 00 00 85 c0 74 3d 0f b6 03 84 c0 74 36 48 b8 00 00 00 00 00 fc ff df 49 89 dc 49 89 dd 49 c1 ec 03 41 83 e5 07 49 01 c4 f3 90 <41> 0f b6 04 24 44 38 e8 7f 08 84 c0 0f 85 9f 05 00 00 0f b6 03 84
[ 60.171160] RSP: 0018:ffffc900012df750 EFLAGS: 00000002
[ 60.171163] RAX: 0000000000000001 RBX: ffff8881520ba000 RCX: 0000000000000001
[ 60.171165] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff8881520ba000
[ 60.171167] RBP: 1ffff9200025beec R08: ffffffff8fbfcb69 R09: ffffed102a417400
[ 60.171168] R10: ffffed102a417401 R11: 0000000000000004 R12: ffffed102a417400
[ 60.171170] R13: 0000000000000000 R14: dffffc0000000000 R15: ffff8881520ba000
[ 60.171173] ? queued_spin_lock_slowpath+0x339/0xac0
[ 60.171178] ? queued_spin_lock_slowpath+0x3a9/0xac0
[ 60.171181] ? queued_spin_lock_slowpath+0x3a9/0xac0
[ 60.171184] </NMI>
[ 60.171185] <TASK>
[ 60.171187] ? __pfx_queued_spin_lock_slowpath+0x10/0x10
[ 60.171192] do_raw_spin_lock+0x1d9/0x270
[ 60.171197] ? __pfx_do_raw_spin_lock+0x10/0x10
[ 60.171200] ? __pfx___might_resched+0x10/0x10
[ 60.171204] task_rq_lock+0xcf/0x3c0
[ 60.171209] mm_cid_fixup_task_to_cpu+0xb0/0x460
[ 60.171212] ? __pfx_mm_cid_fixup_task_to_cpu+0x10/0x10
[ 60.171216] ? lock_acquire+0x14e/0x2b0
[ 60.171220] ? mark_held_locks+0x40/0x70
[ 60.171224] sched_mm_cid_fork+0x6da/0xc20
[ 60.171227] ? __pfx_sched_mm_cid_fork+0x10/0x10
[ 60.171230] ? copy_process+0x217b/0x6950
[ 60.171233] copy_process+0x2bce/0x6950
[ 60.171238] ? __pfx_copy_process+0x10/0x10
[ 60.171241] ? find_held_lock+0x2b/0x80
[ 60.171245] ? _copy_from_user+0x53/0xa0
[ 60.171251] kernel_clone+0xce/0x600
[ 60.171254] ? __pfx_kernel_clone+0x10/0x10
[ 60.171258] ? __lock_acquire+0x481/0x2590
[ 60.171262] __do_sys_clone3+0x16e/0x1b0
[ 60.171265] ? __pfx___do_sys_clone3+0x10/0x10
[ 60.171267] ? lock_acquire+0x14e/0x2b0
[ 60.171270] ? __might_fault+0x9b/0x140
[ 60.171276] ? _copy_to_user+0x5c/0x70
[ 60.171280] ? __x64_sys_rt_sigprocmask+0x258/0x400
[ 60.171285] ? do_user_addr_fault+0x4c2/0xa40
[ 60.171289] ? lockdep_hardirqs_on_prepare+0xd7/0x180
[ 60.171292] do_syscall_64+0x6b/0x3a0
[ 60.171295] entry_SYSCALL_64_after_hwframe+0x76/0x7e
[ 60.171298] RIP: 0033:0x7f7230c42c5d
[ 60.171300] Code: 79 14 0e 00 c3 0f 1f 84 00 00 00 00 00 f3 0f 1e fa b8 ea ff ff ff 48 85 ff 74 28 48 85 d2 74 23 49 89 c8 b8 b3 01 00 00 0f 05 <48> 85 c0 7c 14 74 01 c3 31 ed 4c 89 c7 ff d2 48 89 c7 b8 3c 00 00
[ 60.171302] RSP: 002b:00007ffe90d4e1f8 EFLAGS: 00000202 ORIG_RAX: 00000000000001b3
[ 60.171305] RAX: ffffffffffffffda RBX: 00007f7230bb5720 RCX: 00007f7230c42c5d
[ 60.171307] RDX: 00007f7230bb5720 RSI: 0000000000000058 RDI: 00007ffe90d4e250
[ 60.171309] RBP: 00007ffe90d4e230 R08: 00007f722f1a66c0 R09: 00007ffe90d4e357
[ 60.171310] R10: 0000000000000008 R11: 0000000000000202 R12: 00007f722f1a66c0
[ 60.171312] R13: ffffffffffffff08 R14: 0000000000000000 R15: 00007ffe90d4e250
[ 60.171316] </TASK>
[ 60.171319] Shutting down cpus with NMI
[ 60.171381] Kernel Offset: 0xc000000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff)
>
> Thank you for investigating!
>
>
>>
>> Thanks,
>>
>> tglx
>> ---
>> --- a/kernel/sched/core.c
>> +++ b/kernel/sched/core.c
>> @@ -10664,8 +10664,14 @@ void sched_mm_cid_exit(struct task_struc
>> scoped_guard(raw_spinlock_irq, &mm->mm_cid.lock) {
>> if (!__sched_mm_cid_exit(t))
>> return;
>> - /* Mode change required. Transfer currents CID */
>> - mm_cid_transit_to_task(current, this_cpu_ptr(mm->mm_cid.pcpu));
>> + /*
>> + * Mode change. The task has the CID unset
>> + * already. The CPU CID is still valid and
>> + * does not have MM_CID_TRANSIT set as the
>> + * mode change has just taken effect under
>> + * mm::mm_cid::lock. Drop it.
>> + */
>> + mm_drop_cid_on_cpu(mm, this_cpu_ptr(mm->mm_cid.pcpu));
>> }
>> mm_cid_fixup_cpus_to_tasks(mm);
>> return;
>
^ permalink raw reply [flat|nested] 74+ messages in thread* Re: [patch V5 00/20] sched: Rewrite MM CID management
2026-01-28 23:08 ` Ihor Solodrai
@ 2026-01-29 17:06 ` Thomas Gleixner
0 siblings, 0 replies; 74+ messages in thread
From: Thomas Gleixner @ 2026-01-29 17:06 UTC (permalink / raw)
To: Ihor Solodrai, Shrikanth Hegde, Peter Zijlstra, LKML
Cc: Gabriele Monaco, Mathieu Desnoyers, Michael Jeanson, Jens Axboe,
Paul E. McKenney, Gautham R. Shenoy, Florian Weimer, Tim Chen,
Yury Norov, bpf, sched-ext, Kernel Team, Alexei Starovoitov,
Andrii Nakryiko, Daniel Borkmann, Puranjay Mohan, Tejun Heo
On Wed, Jan 28 2026 at 15:08, Ihor Solodrai wrote:
> On 1/28/26 2:33 PM, Ihor Solodrai wrote:
>> [...]
>>
>> We have a steady stream of jobs running, so if it's not a one-off it's
>> likely to happen again. I'll share if we get anything.
>
> Here is another one, with backtraces of other CPUs:
>
> [ 59.133925] CPU: 2 UID: 0 PID: 127 Comm: test_progs Tainted: G OE 6.19.0-rc5-gbe9790cb9e63-dirty #1 PREEMPT(full)
> [ 59.133935] RIP: 0010:queued_spin_lock_slowpath+0x3a9/0xac0
> [ 59.133985] do_raw_spin_lock+0x1d9/0x270
> [ 59.134001] task_rq_lock+0xcf/0x3c0
> [ 59.134007] mm_cid_fixup_task_to_cpu+0xb0/0x460
> [ 59.134025] sched_mm_cid_fork+0x6da/0xc20
Compared to Shrikanth's splat this is the reverse situation, i.e. fork()
reached the point where it needs to switch to per CPU mode and the fixup
function is stuck on a runqueue lock.
> [ 59.134176] CPU: 3 UID: 0 PID: 67 Comm: kworker/3:1 Tainted: G OE 6.19.0-rc5-gbe9790cb9e63-dirty #1 PREEMPT(full)
> [ 59.134186] Workqueue: events drain_vmap_area_work
> [ 59.134194] RIP: 0010:smp_call_function_many_cond+0x772/0xe60
> [ 59.134250] on_each_cpu_cond_mask+0x24/0x40
> [ 59.134254] flush_tlb_kernel_range+0x402/0x6b0
CPU3 is unrelated as it does not hold runqueue lock.
> [ 59.134374] NMI backtrace for cpu 1
> [ 59.134388] RIP: 0010:_find_first_zero_bit+0x50/0x90
> [ 59.134423] __schedule+0x3312/0x4390
> [ 59.134430] ? __pfx___schedule+0x10/0x10
> [ 59.134434] ? trace_rcu_watching+0x105/0x150
> [ 59.134440] schedule_idle+0x59/0x90
CPU1 holds runqueue lock and find_first_zero_bit() suggests that this
comes from mm_get_cid(), but w/o decoding the return address it's hard
to tell for sure.
> [ 59.134474] NMI backtrace for cpu 0 skipped: idling at default_idle+0xf/0x20
CPU0 is idle and not involved at all.
So the situation is:
test_prog creates the 4th child, which exceeds the number of CPUs, so
it switches to per CPU mode.
At this point each task of test_prog has a CID associated. Let's
assume thread creation order assignment for simplicity.
T0 (main thread) CID0 runs fork()
T1 (1st child) CID1
T2 (2nd child) CID2
T3 (3rd child) CID3
T4 (4th child) --- is about to be forked and causes the
mode switch
T0 sets mm_cid::percpu = true
transfers the CID from T0 to CPU2
Starts the fixup which walks through the threads
During that T1 - T3 are free to schedule in and out before the fixup
caught up with them. Now I played through all possible permutations with
a python script and came up with the following snafu:
T1 schedules in on CPU3 and observes percpu == true, so it transfers
it's CID to CPU3
T1 is migrated to CPU1 and schedule in observes percpu == true, but
CPU1 does not have a CID associated and T1 transferred it's own to
CPU3
So it has to allocate one with CPU1 runqueue lock held, but the
pool is empty, so it keeps looping.
Now T0 reaches T1 in the thread walk and tries to lock the corresponding
runqueue lock, which is held. ---> Livelock
So this side needs the same MM_CID_TRANSIT treatment as the other side,
which brings me back to the splat Shrikanth observed.
I used the same script to run through all possible permutations on that
side too, but nothing showed up there and the yesterday finding is
harmless because that only creates slightly inconsistent state as the
task is already marked CID inactive. But the CID has the MM_CID_TRANSIT
bit set, so the CID is dropped back into the pool when the exiting task
schedules out via preemption or the final schedule().
So I scratched my head some more and stared at the code with two things
in mind:
1) It seems to be hard to reproduce
2) It happened on a weakly ordered architecture
and indeed there is a opportunity to get this wrong:
The mode switch does:
WRITE_ONCE(mm->mm_cid.transit, MM_CID_TRANSIT);
WRITE_ONCE(mm->mm_cid.percpu, ....);
sched_in() does:
if (!READ_ONCE(mm->mm_cid.percpu))
...
cid |= READ_ONCE(mm->mm_cid.transit);
so it can observe percpu == false and transit == 0 even if the fixup
function has not yet completed. As a consequence the task will not drop
the CID when scheduling out before the fixup is completed, which means
the CID space can be exhausted and the next task scheduling in will loop
in mm_get_cid() and the fixup thread can livelock on the held runqueue
lock as above.
I'll send out a series to address all of that later this evening when
tests have completed and changelogs are polished.
Thanks,
tglx
^ permalink raw reply [flat|nested] 74+ messages in thread