* linux-next: manual merge of the paulmck tree with the tip-fixes and tip trees
@ 2026-07-23 14:38 Mark Brown
2026-07-23 16:03 ` Paul E. McKenney
0 siblings, 1 reply; 2+ messages in thread
From: Mark Brown @ 2026-07-23 14:38 UTC (permalink / raw)
To: Paul E. McKenney
Cc: Chuyi Zhou, Linux Kernel Mailing List, Linux Next Mailing List,
Thomas Gleixner, Usama Arif
[-- Attachment #1: Type: text/plain, Size: 15572 bytes --]
Hi all,
Today's linux-next merge of the paulmck tree got a conflict in:
kernel/smp.c
between commits:
35551efb155e3 ("smp: Make CSD lock acquisition atomic for debug mode")
ec9f57e6fefbc ("smp: Use release stores for csd_lock_record() state")
from the tip-fixes and tip trees and commit:
c2d939be90724 ("smp: Make CSD lock acquisition atomic for debug mode")
from the paulmck tree.
I fixed it up (see below) and can carry the fix as necessary. This
is now fixed as far as linux-next is concerned, but any non trivial
conflicts should be mentioned to your upstream maintainer when your tree
is submitted for merging. You may also want to consider cooperating
with the maintainer of the conflicting tree to minimise any particularly
complex conflicts.
diff --combined kernel/smp.c
index b696bcc60c08f,b9448fe3b84ff..0000000000000
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@@ -16,7 -16,6 +16,7 @@@
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/gfp.h>
+#include <linux/slab.h>
#include <linux/smp.h>
#include <linux/cpu.h>
#include <linux/sched.h>
@@@ -64,14 -63,7 +64,14 @@@ int smpcfd_prepare_cpu(unsigned int cpu
free_cpumask_var(cfd->cpumask);
return -ENOMEM;
}
- cfd->csd = alloc_percpu(call_single_data_t);
+
+ /*
+ * Allocate the per-CPU CSD the first time a CPU comes up. It is
+ * not freed when the CPU is offlined, so csd_lock_wait() can access
+ * it even when the CPU was offlined after preemption was re-enabled.
+ */
+ if (!cfd->csd)
+ cfd->csd = alloc_percpu(call_single_data_t);
if (!cfd->csd) {
free_cpumask_var(cfd->cpumask);
free_cpumask_var(cfd->cpumask_ipi);
@@@ -87,6 -79,7 +87,6 @@@ int smpcfd_dead_cpu(unsigned int cpu
free_cpumask_var(cfd->cpumask);
free_cpumask_var(cfd->cpumask_ipi);
- free_percpu(cfd->csd);
return 0;
}
@@@ -189,22 -182,16 +189,22 @@@ static atomic_t csd_bug_count = ATOMIC_
static void __csd_lock_record(call_single_data_t *csd)
{
if (!csd) {
- smp_mb(); /* NULL cur_csd after unlock. */
- __this_cpu_write(cur_csd, NULL);
+ /*
+ * Pairs with smp_load_acquire() of cur_csd in
+ * csd_lock_wait_toolong(): orders any preceding CSD
+ * callback/unlock before a remote reader observes NULL.
+ */
+ smp_store_release(this_cpu_ptr(&cur_csd), NULL);
return;
}
__this_cpu_write(cur_csd_func, csd->func);
__this_cpu_write(cur_csd_info, csd->info);
- smp_wmb(); /* func and info before csd. */
- __this_cpu_write(cur_csd, csd);
- smp_mb(); /* Update cur_csd before function call. */
- /* Or before unlock, as the case may be. */
+ /*
+ * Pairs with smp_load_acquire() of cur_csd in
+ * csd_lock_wait_toolong(): publishes cur_csd_func and
+ * cur_csd_info before the non-NULL pointer becomes visible.
+ */
+ smp_store_release(this_cpu_ptr(&cur_csd), csd);
}
static __always_inline void csd_lock_record(call_single_data_t *csd)
@@@ -285,13 -272,7 +285,13 @@@ static bool csd_lock_wait_toolong(call_
cpux = 0;
else
cpux = cpu;
- cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */
+ /*
+ * Pairs with smp_store_release() of cur_csd in __csd_lock_record():
+ * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info
+ * are the matching publication; a NULL value is ordered after any
+ * preceding CSD callback/unlock on the remote CPU.
+ */
+ cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux));
/* How long since this CSD lock was stuck. */
ts_delta = ts2 - ts0;
pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n",
@@@ -342,8 -323,6 +342,8 @@@ static void __csd_lock_wait(call_single
int bug_id = 0;
u64 ts0, ts1;
+ guard(preempt)();
+
ts1 = ts0 = ktime_get_mono_fast_ns();
for (;;) {
if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages))
@@@ -381,15 -360,14 +381,15 @@@ static __always_inline void csd_lock(ca
{
if (IS_ENABLED(CONFIG_CSD_LOCK_WAIT_DEBUG) &&
static_branch_unlikely(&csdlock_debug_enabled)) {
- unsigned int flags;
for (;;) {
+ unsigned int flags;
+
__csd_lock_wait(csd);
flags = READ_ONCE(csd->node.u_flags);
+
if (!(flags & CSD_FLAG_LOCK) &&
- try_cmpxchg_acquire(&csd->node.u_flags, &flags,
- flags | CSD_FLAG_LOCK))
+ try_cmpxchg_acquire(&csd->node.u_flags, &flags, flags | CSD_FLAG_LOCK))
break;
}
} else {
@@@ -680,9 -658,17 +680,9 @@@ void flush_smp_call_function_queue(void
local_irq_restore(flags);
}
-/**
- * smp_call_function_single - Run a function on a specific CPU
- * @cpu: Specific target CPU for this function.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait until function has completed on other CPUs.
- *
- * Returns: %0 on success, else a negative status code.
- */
-int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
- int wait)
+static int __smp_call_function_single(int cpu, smp_call_func_t func,
+ void *info, const struct cpumask *mask,
+ bool wait)
{
call_single_data_t *csd;
call_single_data_t csd_stack = {
@@@ -699,14 -685,6 +699,14 @@@
*/
this_cpu = get_cpu();
+ if (mask) {
+ /* Try for same CPU (cheapest) */
+ if (!cpumask_test_cpu(this_cpu, mask))
+ cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(this_cpu));
+ else
+ cpu = this_cpu;
+ }
+
/*
* Can deadlock when called with interrupts disabled.
* We allow cpu's that are not yet online though, as no one else can
@@@ -739,32 -717,13 +739,32 @@@
err = generic_exec_single(cpu, csd);
+ /*
+ * @csd is stack-allocated when @wait is true. No concurrent access
+ * except from the IPI completion path, so we can re-enable preemption
+ * early to reduce latency.
+ */
+ put_cpu();
+
if (wait)
csd_lock_wait(csd);
- put_cpu();
-
return err;
}
+
+/**
+ * smp_call_function_single - Run a function on a specific CPU
+ * @cpu: Specific target CPU for this function.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait until function has completed on other CPUs.
+ *
+ * Returns: %0 on success, else a negative status code.
+ */
+int smp_call_function_single(int cpu, smp_call_func_t func, void *info, bool wait)
+{
+ return __smp_call_function_single(cpu, func, info, NULL, wait);
+}
EXPORT_SYMBOL(smp_call_function_single);
/**
@@@ -815,10 -774,10 +815,10 @@@ EXPORT_SYMBOL_GPL(smp_call_function_sin
/**
* smp_call_function_any - Run a function on any of the given cpus
- * @mask: The mask of cpus it can run on.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait until function has completed.
+ * @mask: The mask of cpus it can run on.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait until function has completed.
*
* Selection preference:
* 1) current cpu if in @mask
@@@ -829,54 -788,20 +829,54 @@@
int smp_call_function_any(const struct cpumask *mask,
smp_call_func_t func, void *info, int wait)
{
- unsigned int cpu;
- int ret;
-
- /* Try for same CPU (cheapest) */
- cpu = get_cpu();
- if (!cpumask_test_cpu(cpu, mask))
- cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(cpu));
-
- ret = smp_call_function_single(cpu, func, info, wait);
- put_cpu();
- return ret;
+ return __smp_call_function_single(-1, func, info, mask, wait);
}
EXPORT_SYMBOL_GPL(smp_call_function_any);
+static DEFINE_STATIC_KEY_FALSE(ipi_mask_inlined);
+
+#ifdef CONFIG_PREEMPTION
+
+int smp_task_ipi_mask_alloc(struct task_struct *task)
+{
+ if (static_branch_unlikely(&ipi_mask_inlined))
+ return 0;
+
+ ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr =
+ kmalloc(cpumask_size(), GFP_KERNEL);
+ if (!ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr)
+ return -ENOMEM;
+
+ return 0;
+}
+
+void smp_task_ipi_mask_free(struct task_struct *task)
+{
+ if (static_branch_unlikely(&ipi_mask_inlined))
+ return;
+
+ kfree(ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr);
+}
+
+static cpumask_t *smp_task_ipi_mask(struct task_struct *cur)
+{
+ /*
+ * If cpumask_size() is smaller than or equal to the pointer
+ * size, it stashes the cpumask in the pointer itself to
+ * avoid extra memory allocations.
+ */
+ if (static_branch_unlikely(&ipi_mask_inlined))
+ return (cpumask_t *)&ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_val;
+
+ return ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_ptr;
+}
+#else
+static cpumask_t *smp_task_ipi_mask(struct task_struct *cur)
+{
+ return NULL;
+}
+#endif
+
/*
* Flags to be used as scf_flags argument of smp_call_function_many_cond().
*
@@@ -891,20 -816,13 +891,20 @@@ static void smp_call_function_many_cond
unsigned int scf_flags,
smp_cond_func_t cond_func)
{
- int cpu, last_cpu, this_cpu = smp_processor_id();
- struct call_function_data *cfd;
+ struct cpumask *cpumask, *task_mask;
bool wait = scf_flags & SCF_WAIT;
- int nr_cpus = 0;
+ struct call_function_data *cfd;
+ int cpu, last_cpu, this_cpu;
bool run_remote = false;
+ int nr_cpus = 0;
- lockdep_assert_preemption_disabled();
+ this_cpu = get_cpu();
+ cfd = this_cpu_ptr(&cfd_data);
+ task_mask = smp_task_ipi_mask(current);
+ if (task_mask)
+ cpumask = task_mask;
+ else
+ cpumask = cfd->cpumask;
/*
* Can deadlock when called with interrupts disabled.
@@@ -926,15 -844,16 +926,15 @@@
/* Check if we need remote execution, i.e., any CPU excluding this one. */
if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) {
- cfd = this_cpu_ptr(&cfd_data);
- cpumask_and(cfd->cpumask, mask, cpu_online_mask);
- __cpumask_clear_cpu(this_cpu, cfd->cpumask);
+ cpumask_and(cpumask, mask, cpu_online_mask);
+ __cpumask_clear_cpu(this_cpu, cpumask);
cpumask_clear(cfd->cpumask_ipi);
- for_each_cpu(cpu, cfd->cpumask) {
+ for_each_cpu(cpu, cpumask) {
call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
if (cond_func && !cond_func(cpu, info)) {
- __cpumask_clear_cpu(cpu, cfd->cpumask);
+ __cpumask_clear_cpu(cpu, cpumask);
continue;
}
@@@ -984,18 -903,8 +984,18 @@@
local_irq_restore(flags);
}
+ /*
+ * The IPI work has been queued and dispatched. On PREEMPT kernels,
+ * tasks created through dup_task_struct() have task-local wait masks.
+ * The boot init_task can fall back to cfd->cpumask when the mask is
+ * not inlined, but other tasks still use task-local masks and cannot
+ * overwrite it. On !PREEMPT kernels, preempt_enable() cannot schedule
+ * another task, so the per-CPU mask remains protected.
+ */
+ put_cpu();
+
if (run_remote && wait) {
- for_each_cpu(cpu, cfd->cpumask) {
+ for_each_cpu(cpu, cpumask) {
call_single_data_t *csd;
csd = per_cpu_ptr(cfd->csd, cpu);
@@@ -1006,14 -915,15 +1006,14 @@@
/**
* smp_call_function_many() - Run a function on a set of CPUs.
- * @mask: The set of cpus to run on (only runs on online subset).
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait (atomically) until function has completed
- * on other CPUs.
+ * @mask: The set of cpus to run on (only runs on online subset).
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait (atomically) until function has completed
+ * on other CPUs.
*
* You must not call this function with disabled interrupts or from a
- * hardware interrupt handler or from a bottom half handler. Preemption
- * must be disabled when calling this function.
+ * hardware interrupt handler or from a bottom half handler.
*
* @func is not called on the local CPU even if @mask contains it. Consider
* using on_each_cpu_cond_mask() instead if this is not desirable.
@@@ -1027,10 -937,10 +1027,10 @@@ EXPORT_SYMBOL(smp_call_function_many)
/**
* smp_call_function() - Run a function on all other CPUs.
- * @func: The function to run. This must be fast and non-blocking.
- * @info: An arbitrary pointer to pass to the function.
- * @wait: If true, wait (atomically) until function has completed
- * on other CPUs.
+ * @func: The function to run. This must be fast and non-blocking.
+ * @info: An arbitrary pointer to pass to the function.
+ * @wait: If true, wait (atomically) until function has completed
+ * on other CPUs.
*
* If @wait is true, then returns once @func has returned; otherwise
* it returns just before the target cpu calls @func.
@@@ -1040,8 -950,9 +1040,8 @@@
*/
void smp_call_function(smp_call_func_t func, void *info, int wait)
{
- preempt_disable();
- smp_call_function_many(cpu_online_mask, func, info, wait);
- preempt_enable();
+ smp_call_function_many_cond(cpu_online_mask, func, info,
+ wait ? SCF_WAIT : 0, NULL);
}
EXPORT_SYMBOL(smp_call_function);
@@@ -1107,9 -1018,6 +1107,9 @@@ EXPORT_SYMBOL(nr_cpu_ids)
void __init setup_nr_cpu_ids(void)
{
set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1);
+
+ if (IS_ENABLED(CONFIG_PREEMPTION) && cpumask_size() <= sizeof(unsigned long))
+ static_branch_enable(&ipi_mask_inlined);
}
/* Called by boot processor to activate the rest. */
@@@ -1146,14 -1054,12 +1146,14 @@@ void __init smp_init(void
* @func: The function to run on all applicable CPUs.
* This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to both functions.
- * @wait: If true, wait (atomically) until function has
- * completed on other CPUs.
+ * @wait: If true, wait until function has completed on other CPUs.
* @mask: The set of cpus to run on (only runs on online subset).
*
- * Preemption is disabled to protect against CPUs going offline but not online.
- * CPUs going online during the call will not be seen or sent an IPI.
+ * Target CPU selection and work queueing are done with preemption
+ * disabled. This protects against CPUs going offline, but not against
+ * CPUs coming online concurrently; newly online CPUs are not guaranteed
+ * to be seen or sent an IPI. If @wait is true, the final wait for remote
+ * completion happens after that preemption-disabled section.
*
* You must not call this function with disabled interrupts or
* from a hardware interrupt handler or from a bottom half handler.
@@@ -1166,7 -1072,9 +1166,7 @@@ void on_each_cpu_cond_mask(smp_cond_fun
if (wait)
scf_flags |= SCF_WAIT;
- preempt_disable();
smp_call_function_many_cond(mask, func, info, scf_flags, cond_func);
- preempt_enable();
}
EXPORT_SYMBOL(on_each_cpu_cond_mask);
[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 488 bytes --]
^ permalink raw reply [flat|nested] 2+ messages in thread
* Re: linux-next: manual merge of the paulmck tree with the tip-fixes and tip trees
2026-07-23 14:38 linux-next: manual merge of the paulmck tree with the tip-fixes and tip trees Mark Brown
@ 2026-07-23 16:03 ` Paul E. McKenney
0 siblings, 0 replies; 2+ messages in thread
From: Paul E. McKenney @ 2026-07-23 16:03 UTC (permalink / raw)
To: Mark Brown
Cc: Chuyi Zhou, Linux Kernel Mailing List, Linux Next Mailing List,
Thomas Gleixner, Usama Arif
On Thu, Jul 23, 2026 at 03:38:00PM +0100, Mark Brown wrote:
> Hi all,
>
> Today's linux-next merge of the paulmck tree got a conflict in:
>
> kernel/smp.c
>
> between commits:
>
> 35551efb155e3 ("smp: Make CSD lock acquisition atomic for debug mode")
> ec9f57e6fefbc ("smp: Use release stores for csd_lock_record() state")
>
> from the tip-fixes and tip trees and commit:
>
> c2d939be90724 ("smp: Make CSD lock acquisition atomic for debug mode")
>
> from the paulmck tree.
>
> I fixed it up (see below) and can carry the fix as necessary. This
> is now fixed as far as linux-next is concerned, but any non trivial
> conflicts should be mentioned to your upstream maintainer when your tree
> is submitted for merging. You may also want to consider cooperating
> with the maintainer of the conflicting tree to minimise any particularly
> complex conflicts.
Excellent! I have dropped this from my tree in favor of tip-fixes.
Thanx, Paul
> diff --combined kernel/smp.c
> index b696bcc60c08f,b9448fe3b84ff..0000000000000
> --- a/kernel/smp.c
> +++ b/kernel/smp.c
> @@@ -16,7 -16,6 +16,7 @@@
> #include <linux/init.h>
> #include <linux/interrupt.h>
> #include <linux/gfp.h>
> +#include <linux/slab.h>
> #include <linux/smp.h>
> #include <linux/cpu.h>
> #include <linux/sched.h>
> @@@ -64,14 -63,7 +64,14 @@@ int smpcfd_prepare_cpu(unsigned int cpu
> free_cpumask_var(cfd->cpumask);
> return -ENOMEM;
> }
> - cfd->csd = alloc_percpu(call_single_data_t);
> +
> + /*
> + * Allocate the per-CPU CSD the first time a CPU comes up. It is
> + * not freed when the CPU is offlined, so csd_lock_wait() can access
> + * it even when the CPU was offlined after preemption was re-enabled.
> + */
> + if (!cfd->csd)
> + cfd->csd = alloc_percpu(call_single_data_t);
> if (!cfd->csd) {
> free_cpumask_var(cfd->cpumask);
> free_cpumask_var(cfd->cpumask_ipi);
> @@@ -87,6 -79,7 +87,6 @@@ int smpcfd_dead_cpu(unsigned int cpu
>
> free_cpumask_var(cfd->cpumask);
> free_cpumask_var(cfd->cpumask_ipi);
> - free_percpu(cfd->csd);
> return 0;
> }
>
> @@@ -189,22 -182,16 +189,22 @@@ static atomic_t csd_bug_count = ATOMIC_
> static void __csd_lock_record(call_single_data_t *csd)
> {
> if (!csd) {
> - smp_mb(); /* NULL cur_csd after unlock. */
> - __this_cpu_write(cur_csd, NULL);
> + /*
> + * Pairs with smp_load_acquire() of cur_csd in
> + * csd_lock_wait_toolong(): orders any preceding CSD
> + * callback/unlock before a remote reader observes NULL.
> + */
> + smp_store_release(this_cpu_ptr(&cur_csd), NULL);
> return;
> }
> __this_cpu_write(cur_csd_func, csd->func);
> __this_cpu_write(cur_csd_info, csd->info);
> - smp_wmb(); /* func and info before csd. */
> - __this_cpu_write(cur_csd, csd);
> - smp_mb(); /* Update cur_csd before function call. */
> - /* Or before unlock, as the case may be. */
> + /*
> + * Pairs with smp_load_acquire() of cur_csd in
> + * csd_lock_wait_toolong(): publishes cur_csd_func and
> + * cur_csd_info before the non-NULL pointer becomes visible.
> + */
> + smp_store_release(this_cpu_ptr(&cur_csd), csd);
> }
>
> static __always_inline void csd_lock_record(call_single_data_t *csd)
> @@@ -285,13 -272,7 +285,13 @@@ static bool csd_lock_wait_toolong(call_
> cpux = 0;
> else
> cpux = cpu;
> - cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux)); /* Before func and info. */
> + /*
> + * Pairs with smp_store_release() of cur_csd in __csd_lock_record():
> + * a non-NULL cur_csd here implies cur_csd_func and cur_csd_info
> + * are the matching publication; a NULL value is ordered after any
> + * preceding CSD callback/unlock on the remote CPU.
> + */
> + cpu_cur_csd = smp_load_acquire(&per_cpu(cur_csd, cpux));
> /* How long since this CSD lock was stuck. */
> ts_delta = ts2 - ts0;
> pr_alert("csd: %s non-responsive CSD lock (#%d) on CPU#%d, waiting %lld ns for CPU#%02d %pS(%ps).\n",
> @@@ -342,8 -323,6 +342,8 @@@ static void __csd_lock_wait(call_single
> int bug_id = 0;
> u64 ts0, ts1;
>
> + guard(preempt)();
> +
> ts1 = ts0 = ktime_get_mono_fast_ns();
> for (;;) {
> if (csd_lock_wait_toolong(csd, ts0, &ts1, &bug_id, &nmessages))
> @@@ -381,15 -360,14 +381,15 @@@ static __always_inline void csd_lock(ca
> {
> if (IS_ENABLED(CONFIG_CSD_LOCK_WAIT_DEBUG) &&
> static_branch_unlikely(&csdlock_debug_enabled)) {
> - unsigned int flags;
>
> for (;;) {
> + unsigned int flags;
> +
> __csd_lock_wait(csd);
> flags = READ_ONCE(csd->node.u_flags);
> +
> if (!(flags & CSD_FLAG_LOCK) &&
> - try_cmpxchg_acquire(&csd->node.u_flags, &flags,
> - flags | CSD_FLAG_LOCK))
> + try_cmpxchg_acquire(&csd->node.u_flags, &flags, flags | CSD_FLAG_LOCK))
> break;
> }
> } else {
> @@@ -680,9 -658,17 +680,9 @@@ void flush_smp_call_function_queue(void
> local_irq_restore(flags);
> }
>
> -/**
> - * smp_call_function_single - Run a function on a specific CPU
> - * @cpu: Specific target CPU for this function.
> - * @func: The function to run. This must be fast and non-blocking.
> - * @info: An arbitrary pointer to pass to the function.
> - * @wait: If true, wait until function has completed on other CPUs.
> - *
> - * Returns: %0 on success, else a negative status code.
> - */
> -int smp_call_function_single(int cpu, smp_call_func_t func, void *info,
> - int wait)
> +static int __smp_call_function_single(int cpu, smp_call_func_t func,
> + void *info, const struct cpumask *mask,
> + bool wait)
> {
> call_single_data_t *csd;
> call_single_data_t csd_stack = {
> @@@ -699,14 -685,6 +699,14 @@@
> */
> this_cpu = get_cpu();
>
> + if (mask) {
> + /* Try for same CPU (cheapest) */
> + if (!cpumask_test_cpu(this_cpu, mask))
> + cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(this_cpu));
> + else
> + cpu = this_cpu;
> + }
> +
> /*
> * Can deadlock when called with interrupts disabled.
> * We allow cpu's that are not yet online though, as no one else can
> @@@ -739,32 -717,13 +739,32 @@@
>
> err = generic_exec_single(cpu, csd);
>
> + /*
> + * @csd is stack-allocated when @wait is true. No concurrent access
> + * except from the IPI completion path, so we can re-enable preemption
> + * early to reduce latency.
> + */
> + put_cpu();
> +
> if (wait)
> csd_lock_wait(csd);
>
> - put_cpu();
> -
> return err;
> }
> +
> +/**
> + * smp_call_function_single - Run a function on a specific CPU
> + * @cpu: Specific target CPU for this function.
> + * @func: The function to run. This must be fast and non-blocking.
> + * @info: An arbitrary pointer to pass to the function.
> + * @wait: If true, wait until function has completed on other CPUs.
> + *
> + * Returns: %0 on success, else a negative status code.
> + */
> +int smp_call_function_single(int cpu, smp_call_func_t func, void *info, bool wait)
> +{
> + return __smp_call_function_single(cpu, func, info, NULL, wait);
> +}
> EXPORT_SYMBOL(smp_call_function_single);
>
> /**
> @@@ -815,10 -774,10 +815,10 @@@ EXPORT_SYMBOL_GPL(smp_call_function_sin
>
> /**
> * smp_call_function_any - Run a function on any of the given cpus
> - * @mask: The mask of cpus it can run on.
> - * @func: The function to run. This must be fast and non-blocking.
> - * @info: An arbitrary pointer to pass to the function.
> - * @wait: If true, wait until function has completed.
> + * @mask: The mask of cpus it can run on.
> + * @func: The function to run. This must be fast and non-blocking.
> + * @info: An arbitrary pointer to pass to the function.
> + * @wait: If true, wait until function has completed.
> *
> * Selection preference:
> * 1) current cpu if in @mask
> @@@ -829,54 -788,20 +829,54 @@@
> int smp_call_function_any(const struct cpumask *mask,
> smp_call_func_t func, void *info, int wait)
> {
> - unsigned int cpu;
> - int ret;
> -
> - /* Try for same CPU (cheapest) */
> - cpu = get_cpu();
> - if (!cpumask_test_cpu(cpu, mask))
> - cpu = sched_numa_find_nth_cpu(mask, 0, cpu_to_node(cpu));
> -
> - ret = smp_call_function_single(cpu, func, info, wait);
> - put_cpu();
> - return ret;
> + return __smp_call_function_single(-1, func, info, mask, wait);
> }
> EXPORT_SYMBOL_GPL(smp_call_function_any);
>
> +static DEFINE_STATIC_KEY_FALSE(ipi_mask_inlined);
> +
> +#ifdef CONFIG_PREEMPTION
> +
> +int smp_task_ipi_mask_alloc(struct task_struct *task)
> +{
> + if (static_branch_unlikely(&ipi_mask_inlined))
> + return 0;
> +
> + ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr =
> + kmalloc(cpumask_size(), GFP_KERNEL);
> + if (!ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr)
> + return -ENOMEM;
> +
> + return 0;
> +}
> +
> +void smp_task_ipi_mask_free(struct task_struct *task)
> +{
> + if (static_branch_unlikely(&ipi_mask_inlined))
> + return;
> +
> + kfree(ACCESS_PRIVATE(task, ipi_mask).ipi_mask_ptr);
> +}
> +
> +static cpumask_t *smp_task_ipi_mask(struct task_struct *cur)
> +{
> + /*
> + * If cpumask_size() is smaller than or equal to the pointer
> + * size, it stashes the cpumask in the pointer itself to
> + * avoid extra memory allocations.
> + */
> + if (static_branch_unlikely(&ipi_mask_inlined))
> + return (cpumask_t *)&ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_val;
> +
> + return ACCESS_PRIVATE(cur, ipi_mask).ipi_mask_ptr;
> +}
> +#else
> +static cpumask_t *smp_task_ipi_mask(struct task_struct *cur)
> +{
> + return NULL;
> +}
> +#endif
> +
> /*
> * Flags to be used as scf_flags argument of smp_call_function_many_cond().
> *
> @@@ -891,20 -816,13 +891,20 @@@ static void smp_call_function_many_cond
> unsigned int scf_flags,
> smp_cond_func_t cond_func)
> {
> - int cpu, last_cpu, this_cpu = smp_processor_id();
> - struct call_function_data *cfd;
> + struct cpumask *cpumask, *task_mask;
> bool wait = scf_flags & SCF_WAIT;
> - int nr_cpus = 0;
> + struct call_function_data *cfd;
> + int cpu, last_cpu, this_cpu;
> bool run_remote = false;
> + int nr_cpus = 0;
>
> - lockdep_assert_preemption_disabled();
> + this_cpu = get_cpu();
> + cfd = this_cpu_ptr(&cfd_data);
> + task_mask = smp_task_ipi_mask(current);
> + if (task_mask)
> + cpumask = task_mask;
> + else
> + cpumask = cfd->cpumask;
>
> /*
> * Can deadlock when called with interrupts disabled.
> @@@ -926,15 -844,16 +926,15 @@@
>
> /* Check if we need remote execution, i.e., any CPU excluding this one. */
> if (cpumask_any_and_but(mask, cpu_online_mask, this_cpu) < nr_cpu_ids) {
> - cfd = this_cpu_ptr(&cfd_data);
> - cpumask_and(cfd->cpumask, mask, cpu_online_mask);
> - __cpumask_clear_cpu(this_cpu, cfd->cpumask);
> + cpumask_and(cpumask, mask, cpu_online_mask);
> + __cpumask_clear_cpu(this_cpu, cpumask);
>
> cpumask_clear(cfd->cpumask_ipi);
> - for_each_cpu(cpu, cfd->cpumask) {
> + for_each_cpu(cpu, cpumask) {
> call_single_data_t *csd = per_cpu_ptr(cfd->csd, cpu);
>
> if (cond_func && !cond_func(cpu, info)) {
> - __cpumask_clear_cpu(cpu, cfd->cpumask);
> + __cpumask_clear_cpu(cpu, cpumask);
> continue;
> }
>
> @@@ -984,18 -903,8 +984,18 @@@
> local_irq_restore(flags);
> }
>
> + /*
> + * The IPI work has been queued and dispatched. On PREEMPT kernels,
> + * tasks created through dup_task_struct() have task-local wait masks.
> + * The boot init_task can fall back to cfd->cpumask when the mask is
> + * not inlined, but other tasks still use task-local masks and cannot
> + * overwrite it. On !PREEMPT kernels, preempt_enable() cannot schedule
> + * another task, so the per-CPU mask remains protected.
> + */
> + put_cpu();
> +
> if (run_remote && wait) {
> - for_each_cpu(cpu, cfd->cpumask) {
> + for_each_cpu(cpu, cpumask) {
> call_single_data_t *csd;
>
> csd = per_cpu_ptr(cfd->csd, cpu);
> @@@ -1006,14 -915,15 +1006,14 @@@
>
> /**
> * smp_call_function_many() - Run a function on a set of CPUs.
> - * @mask: The set of cpus to run on (only runs on online subset).
> - * @func: The function to run. This must be fast and non-blocking.
> - * @info: An arbitrary pointer to pass to the function.
> - * @wait: If true, wait (atomically) until function has completed
> - * on other CPUs.
> + * @mask: The set of cpus to run on (only runs on online subset).
> + * @func: The function to run. This must be fast and non-blocking.
> + * @info: An arbitrary pointer to pass to the function.
> + * @wait: If true, wait (atomically) until function has completed
> + * on other CPUs.
> *
> * You must not call this function with disabled interrupts or from a
> - * hardware interrupt handler or from a bottom half handler. Preemption
> - * must be disabled when calling this function.
> + * hardware interrupt handler or from a bottom half handler.
> *
> * @func is not called on the local CPU even if @mask contains it. Consider
> * using on_each_cpu_cond_mask() instead if this is not desirable.
> @@@ -1027,10 -937,10 +1027,10 @@@ EXPORT_SYMBOL(smp_call_function_many)
>
> /**
> * smp_call_function() - Run a function on all other CPUs.
> - * @func: The function to run. This must be fast and non-blocking.
> - * @info: An arbitrary pointer to pass to the function.
> - * @wait: If true, wait (atomically) until function has completed
> - * on other CPUs.
> + * @func: The function to run. This must be fast and non-blocking.
> + * @info: An arbitrary pointer to pass to the function.
> + * @wait: If true, wait (atomically) until function has completed
> + * on other CPUs.
> *
> * If @wait is true, then returns once @func has returned; otherwise
> * it returns just before the target cpu calls @func.
> @@@ -1040,8 -950,9 +1040,8 @@@
> */
> void smp_call_function(smp_call_func_t func, void *info, int wait)
> {
> - preempt_disable();
> - smp_call_function_many(cpu_online_mask, func, info, wait);
> - preempt_enable();
> + smp_call_function_many_cond(cpu_online_mask, func, info,
> + wait ? SCF_WAIT : 0, NULL);
> }
> EXPORT_SYMBOL(smp_call_function);
>
> @@@ -1107,9 -1018,6 +1107,9 @@@ EXPORT_SYMBOL(nr_cpu_ids)
> void __init setup_nr_cpu_ids(void)
> {
> set_nr_cpu_ids(find_last_bit(cpumask_bits(cpu_possible_mask), NR_CPUS) + 1);
> +
> + if (IS_ENABLED(CONFIG_PREEMPTION) && cpumask_size() <= sizeof(unsigned long))
> + static_branch_enable(&ipi_mask_inlined);
> }
>
> /* Called by boot processor to activate the rest. */
> @@@ -1146,14 -1054,12 +1146,14 @@@ void __init smp_init(void
> * @func: The function to run on all applicable CPUs.
> * This must be fast and non-blocking.
> * @info: An arbitrary pointer to pass to both functions.
> - * @wait: If true, wait (atomically) until function has
> - * completed on other CPUs.
> + * @wait: If true, wait until function has completed on other CPUs.
> * @mask: The set of cpus to run on (only runs on online subset).
> *
> - * Preemption is disabled to protect against CPUs going offline but not online.
> - * CPUs going online during the call will not be seen or sent an IPI.
> + * Target CPU selection and work queueing are done with preemption
> + * disabled. This protects against CPUs going offline, but not against
> + * CPUs coming online concurrently; newly online CPUs are not guaranteed
> + * to be seen or sent an IPI. If @wait is true, the final wait for remote
> + * completion happens after that preemption-disabled section.
> *
> * You must not call this function with disabled interrupts or
> * from a hardware interrupt handler or from a bottom half handler.
> @@@ -1166,7 -1072,9 +1166,7 @@@ void on_each_cpu_cond_mask(smp_cond_fun
> if (wait)
> scf_flags |= SCF_WAIT;
>
> - preempt_disable();
> smp_call_function_many_cond(mask, func, info, scf_flags, cond_func);
> - preempt_enable();
> }
> EXPORT_SYMBOL(on_each_cpu_cond_mask);
>
^ permalink raw reply [flat|nested] 2+ messages in thread
end of thread, other threads:[~2026-07-23 16:04 UTC | newest]
Thread overview: 2+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2026-07-23 14:38 linux-next: manual merge of the paulmck tree with the tip-fixes and tip trees Mark Brown
2026-07-23 16:03 ` Paul E. McKenney
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox