[PATCH] RISC-V: Enable HOTPLUG_PARALLEL for secondary CPUs

[PATCH] RISC-V: Enable HOTPLUG_PARALLEL for secondary CPUs

[Anup Patel]

The core kernel already supports parallel bringup of secondary
CPUs (aka HOTPLUG_PARALLEL). The x86 and MIPS architectures
already use HOTPLUG_PARALLEL and ARM is also moving toward it.

On RISC-V, there is no arch specific global data accessed in the
RISC-V secondary CPU bringup path so enabling HOTPLUG_PARALLEL for
RISC-V would only requires:
1) Providing RISC-V specific arch_cpuhp_kick_ap_alive()
2) Calling cpuhp_ap_sync_alive() from smp_callin()

This patch is tested natively with OpenSBI on QEMU RV64 virt machine
with 64 cores and also tested with KVM RISC-V guest with 32 VCPUs.

Signed-off-by: Anup Patel <apatel@ventanamicro.com>
---
 arch/riscv/Kconfig          |  2 +-
 arch/riscv/kernel/smpboot.c | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index a4b233a0659e..d5800d6f9a15 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -196,7 +196,7 @@ config RISCV
 	select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
 	select HAVE_STACKPROTECTOR
 	select HAVE_SYSCALL_TRACEPOINTS
-	select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU
+	select HOTPLUG_PARALLEL if HOTPLUG_CPU
 	select IRQ_DOMAIN
 	select IRQ_FORCED_THREADING
 	select KASAN_VMALLOC if KASAN
diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
index 601a321e0f17..d85916a3660c 100644
--- a/arch/riscv/kernel/smpboot.c
+++ b/arch/riscv/kernel/smpboot.c
@@ -39,7 +39,9 @@
 
 #include "head.h"
 
+#ifndef CONFIG_HOTPLUG_PARALLEL
 static DECLARE_COMPLETION(cpu_running);
+#endif
 
 void __init smp_prepare_cpus(unsigned int max_cpus)
 {
@@ -179,6 +181,12 @@ static int start_secondary_cpu(int cpu, struct task_struct *tidle)
 	return -EOPNOTSUPP;
 }
 
+#ifdef CONFIG_HOTPLUG_PARALLEL
+int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle)
+{
+	return start_secondary_cpu(cpu, tidle);
+}
+#else
 int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 {
 	int ret = 0;
@@ -199,6 +207,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
 
 	return ret;
 }
+#endif
 
 void __init smp_cpus_done(unsigned int max_cpus)
 {
@@ -225,6 +234,10 @@ asmlinkage __visible void smp_callin(void)
 	mmgrab(mm);
 	current->active_mm = mm;
 
+#ifdef CONFIG_HOTPLUG_PARALLEL
+	cpuhp_ap_sync_alive();
+#endif
+
 	store_cpu_topology(curr_cpuid);
 	notify_cpu_starting(curr_cpuid);
 
@@ -243,7 +256,9 @@ asmlinkage __visible void smp_callin(void)
 	 */
 	local_flush_icache_all();
 	local_flush_tlb_all();
+#ifndef CONFIG_HOTPLUG_PARALLEL
 	complete(&cpu_running);
+#endif
 	/*
 	 * Disable preemption before enabling interrupts, so we don't try to
 	 * schedule a CPU that hasn't actually started yet.
-- 
2.43.0

Re: [PATCH] RISC-V: Enable HOTPLUG_PARALLEL for secondary CPUs

[Anup Patel]

On Fri, Sep 5, 2025 at 5:55 PM Anup Patel <apatel@ventanamicro.com> wrote:
>
> The core kernel already supports parallel bringup of secondary
> CPUs (aka HOTPLUG_PARALLEL). The x86 and MIPS architectures
> already use HOTPLUG_PARALLEL and ARM is also moving toward it.
>
> On RISC-V, there is no arch specific global data accessed in the
> RISC-V secondary CPU bringup path so enabling HOTPLUG_PARALLEL for
> RISC-V would only requires:
> 1) Providing RISC-V specific arch_cpuhp_kick_ap_alive()
> 2) Calling cpuhp_ap_sync_alive() from smp_callin()
>
> This patch is tested natively with OpenSBI on QEMU RV64 virt machine
> with 64 cores and also tested with KVM RISC-V guest with 32 VCPUs.
>
> Signed-off-by: Anup Patel <apatel@ventanamicro.com>

Friendly ping ? Any comments on this one ?

Regards,
Anup

> ---
>  arch/riscv/Kconfig          |  2 +-
>  arch/riscv/kernel/smpboot.c | 15 +++++++++++++++
>  2 files changed, 16 insertions(+), 1 deletion(-)
>
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index a4b233a0659e..d5800d6f9a15 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -196,7 +196,7 @@ config RISCV
>         select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
>         select HAVE_STACKPROTECTOR
>         select HAVE_SYSCALL_TRACEPOINTS
> -       select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU
> +       select HOTPLUG_PARALLEL if HOTPLUG_CPU
>         select IRQ_DOMAIN
>         select IRQ_FORCED_THREADING
>         select KASAN_VMALLOC if KASAN
> diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
> index 601a321e0f17..d85916a3660c 100644
> --- a/arch/riscv/kernel/smpboot.c
> +++ b/arch/riscv/kernel/smpboot.c
> @@ -39,7 +39,9 @@
>
>  #include "head.h"
>
> +#ifndef CONFIG_HOTPLUG_PARALLEL
>  static DECLARE_COMPLETION(cpu_running);
> +#endif
>
>  void __init smp_prepare_cpus(unsigned int max_cpus)
>  {
> @@ -179,6 +181,12 @@ static int start_secondary_cpu(int cpu, struct task_struct *tidle)
>         return -EOPNOTSUPP;
>  }
>
> +#ifdef CONFIG_HOTPLUG_PARALLEL
> +int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle)
> +{
> +       return start_secondary_cpu(cpu, tidle);
> +}
> +#else
>  int __cpu_up(unsigned int cpu, struct task_struct *tidle)
>  {
>         int ret = 0;
> @@ -199,6 +207,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
>
>         return ret;
>  }
> +#endif
>
>  void __init smp_cpus_done(unsigned int max_cpus)
>  {
> @@ -225,6 +234,10 @@ asmlinkage __visible void smp_callin(void)
>         mmgrab(mm);
>         current->active_mm = mm;
>
> +#ifdef CONFIG_HOTPLUG_PARALLEL
> +       cpuhp_ap_sync_alive();
> +#endif
> +
>         store_cpu_topology(curr_cpuid);
>         notify_cpu_starting(curr_cpuid);
>
> @@ -243,7 +256,9 @@ asmlinkage __visible void smp_callin(void)
>          */
>         local_flush_icache_all();
>         local_flush_tlb_all();
> +#ifndef CONFIG_HOTPLUG_PARALLEL
>         complete(&cpu_running);
> +#endif
>         /*
>          * Disable preemption before enabling interrupts, so we don't try to
>          * schedule a CPU that hasn't actually started yet.
> --
> 2.43.0
>

Re: [PATCH] RISC-V: Enable HOTPLUG_PARALLEL for secondary CPUs

[Atish Patra]

On 9/5/25 5:25 AM, Anup Patel wrote:
> The core kernel already supports parallel bringup of secondary
> CPUs (aka HOTPLUG_PARALLEL). The x86 and MIPS architectures
> already use HOTPLUG_PARALLEL and ARM is also moving toward it.
>
> On RISC-V, there is no arch specific global data accessed in the
> RISC-V secondary CPU bringup path so enabling HOTPLUG_PARALLEL for
> RISC-V would only requires:
> 1) Providing RISC-V specific arch_cpuhp_kick_ap_alive()
> 2) Calling cpuhp_ap_sync_alive() from smp_callin()
>
> This patch is tested natively with OpenSBI on QEMU RV64 virt machine
> with 64 cores and also tested with KVM RISC-V guest with 32 VCPUs.
>
> Signed-off-by: Anup Patel <apatel@ventanamicro.com>
> ---
>   arch/riscv/Kconfig          |  2 +-
>   arch/riscv/kernel/smpboot.c | 15 +++++++++++++++
>   2 files changed, 16 insertions(+), 1 deletion(-)
>
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index a4b233a0659e..d5800d6f9a15 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -196,7 +196,7 @@ config RISCV
>   	select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
>   	select HAVE_STACKPROTECTOR
>   	select HAVE_SYSCALL_TRACEPOINTS
> -	select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU
> +	select HOTPLUG_PARALLEL if HOTPLUG_CPU
>   	select IRQ_DOMAIN
>   	select IRQ_FORCED_THREADING
>   	select KASAN_VMALLOC if KASAN
> diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
> index 601a321e0f17..d85916a3660c 100644
> --- a/arch/riscv/kernel/smpboot.c
> +++ b/arch/riscv/kernel/smpboot.c
> @@ -39,7 +39,9 @@
>   
>   #include "head.h"
>   
> +#ifndef CONFIG_HOTPLUG_PARALLEL
>   static DECLARE_COMPLETION(cpu_running);
> +#endif
>   
>   void __init smp_prepare_cpus(unsigned int max_cpus)
>   {
> @@ -179,6 +181,12 @@ static int start_secondary_cpu(int cpu, struct task_struct *tidle)
>   	return -EOPNOTSUPP;
>   }
>   
> +#ifdef CONFIG_HOTPLUG_PARALLEL
> +int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle)
> +{
> +	return start_secondary_cpu(cpu, tidle);
> +}
> +#else
>   int __cpu_up(unsigned int cpu, struct task_struct *tidle)
>   {
>   	int ret = 0;
> @@ -199,6 +207,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
>   
>   	return ret;
>   }
> +#endif
>   
>   void __init smp_cpus_done(unsigned int max_cpus)
>   {
> @@ -225,6 +234,10 @@ asmlinkage __visible void smp_callin(void)
>   	mmgrab(mm);
>   	current->active_mm = mm;
>   
> +#ifdef CONFIG_HOTPLUG_PARALLEL
> +	cpuhp_ap_sync_alive();
> +#endif
> +
>   	store_cpu_topology(curr_cpuid);
>   	notify_cpu_starting(curr_cpuid);
>   
> @@ -243,7 +256,9 @@ asmlinkage __visible void smp_callin(void)
>   	 */
>   	local_flush_icache_all();
>   	local_flush_tlb_all();
> +#ifndef CONFIG_HOTPLUG_PARALLEL
>   	complete(&cpu_running);
> +#endif

LGTM.

Reviewed-by: Atish Patra <atishp@rivosinc.com>

Have you tried with 128 harts ? I was not able to boot 128 harts in Qemu 
with NR_CPUS=256.
This is unrelated to this patch though. I can reproduce the issue on 
upstream with 6.18-rc3.


>   	/*
>   	 * Disable preemption before enabling interrupts, so we don't try to
>   	 * schedule a CPU that hasn't actually started yet.

Re: [PATCH] RISC-V: Enable HOTPLUG_PARALLEL for secondary CPUs

[Anup Patel]

On Tue, Oct 28, 2025 at 2:05 PM Atish Patra <atish.patra@linux.dev> wrote:
>
>
> On 9/5/25 5:25 AM, Anup Patel wrote:
> > The core kernel already supports parallel bringup of secondary
> > CPUs (aka HOTPLUG_PARALLEL). The x86 and MIPS architectures
> > already use HOTPLUG_PARALLEL and ARM is also moving toward it.
> >
> > On RISC-V, there is no arch specific global data accessed in the
> > RISC-V secondary CPU bringup path so enabling HOTPLUG_PARALLEL for
> > RISC-V would only requires:
> > 1) Providing RISC-V specific arch_cpuhp_kick_ap_alive()
> > 2) Calling cpuhp_ap_sync_alive() from smp_callin()
> >
> > This patch is tested natively with OpenSBI on QEMU RV64 virt machine
> > with 64 cores and also tested with KVM RISC-V guest with 32 VCPUs.
> >
> > Signed-off-by: Anup Patel <apatel@ventanamicro.com>
> > ---
> >   arch/riscv/Kconfig          |  2 +-
> >   arch/riscv/kernel/smpboot.c | 15 +++++++++++++++
> >   2 files changed, 16 insertions(+), 1 deletion(-)
> >
> > diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> > index a4b233a0659e..d5800d6f9a15 100644
> > --- a/arch/riscv/Kconfig
> > +++ b/arch/riscv/Kconfig
> > @@ -196,7 +196,7 @@ config RISCV
> >       select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
> >       select HAVE_STACKPROTECTOR
> >       select HAVE_SYSCALL_TRACEPOINTS
> > -     select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU
> > +     select HOTPLUG_PARALLEL if HOTPLUG_CPU
> >       select IRQ_DOMAIN
> >       select IRQ_FORCED_THREADING
> >       select KASAN_VMALLOC if KASAN
> > diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
> > index 601a321e0f17..d85916a3660c 100644
> > --- a/arch/riscv/kernel/smpboot.c
> > +++ b/arch/riscv/kernel/smpboot.c
> > @@ -39,7 +39,9 @@
> >
> >   #include "head.h"
> >
> > +#ifndef CONFIG_HOTPLUG_PARALLEL
> >   static DECLARE_COMPLETION(cpu_running);
> > +#endif
> >
> >   void __init smp_prepare_cpus(unsigned int max_cpus)
> >   {
> > @@ -179,6 +181,12 @@ static int start_secondary_cpu(int cpu, struct task_struct *tidle)
> >       return -EOPNOTSUPP;
> >   }
> >
> > +#ifdef CONFIG_HOTPLUG_PARALLEL
> > +int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle)
> > +{
> > +     return start_secondary_cpu(cpu, tidle);
> > +}
> > +#else
> >   int __cpu_up(unsigned int cpu, struct task_struct *tidle)
> >   {
> >       int ret = 0;
> > @@ -199,6 +207,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
> >
> >       return ret;
> >   }
> > +#endif
> >
> >   void __init smp_cpus_done(unsigned int max_cpus)
> >   {
> > @@ -225,6 +234,10 @@ asmlinkage __visible void smp_callin(void)
> >       mmgrab(mm);
> >       current->active_mm = mm;
> >
> > +#ifdef CONFIG_HOTPLUG_PARALLEL
> > +     cpuhp_ap_sync_alive();
> > +#endif
> > +
> >       store_cpu_topology(curr_cpuid);
> >       notify_cpu_starting(curr_cpuid);
> >
> > @@ -243,7 +256,9 @@ asmlinkage __visible void smp_callin(void)
> >        */
> >       local_flush_icache_all();
> >       local_flush_tlb_all();
> > +#ifndef CONFIG_HOTPLUG_PARALLEL
> >       complete(&cpu_running);
> > +#endif
>
> LGTM.
>
> Reviewed-by: Atish Patra <atishp@rivosinc.com>
>
> Have you tried with 128 harts ? I was not able to boot 128 harts in Qemu
> with NR_CPUS=256.
> This is unrelated to this patch though. I can reproduce the issue on
> upstream with 6.18-rc3.

I have tried upto 96 harts and that works fine.

For 128 harts, the memory used by OpenSBI goes beyond 2MB so
OpenSBI can't run from the first 2MB of DRAM (0x80000000) as the
first booting stage. Try with U-Boot SPL loading OpenSBI from FIT image.

Regards,
Anup

>
>
> >       /*
> >        * Disable preemption before enabling interrupts, so we don't try to
> >        * schedule a CPU that hasn't actually started yet.

Re: [PATCH] RISC-V: Enable HOTPLUG_PARALLEL for secondary CPUs

[Atish Patra]

On 10/28/25 1:48 AM, Anup Patel wrote:
> On Tue, Oct 28, 2025 at 2:05 PM Atish Patra <atish.patra@linux.dev> wrote:
>>
>> On 9/5/25 5:25 AM, Anup Patel wrote:
>>> The core kernel already supports parallel bringup of secondary
>>> CPUs (aka HOTPLUG_PARALLEL). The x86 and MIPS architectures
>>> already use HOTPLUG_PARALLEL and ARM is also moving toward it.
>>>
>>> On RISC-V, there is no arch specific global data accessed in the
>>> RISC-V secondary CPU bringup path so enabling HOTPLUG_PARALLEL for
>>> RISC-V would only requires:
>>> 1) Providing RISC-V specific arch_cpuhp_kick_ap_alive()
>>> 2) Calling cpuhp_ap_sync_alive() from smp_callin()
>>>
>>> This patch is tested natively with OpenSBI on QEMU RV64 virt machine
>>> with 64 cores and also tested with KVM RISC-V guest with 32 VCPUs.
>>>
>>> Signed-off-by: Anup Patel <apatel@ventanamicro.com>
>>> ---
>>>    arch/riscv/Kconfig          |  2 +-
>>>    arch/riscv/kernel/smpboot.c | 15 +++++++++++++++
>>>    2 files changed, 16 insertions(+), 1 deletion(-)
>>>
>>> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
>>> index a4b233a0659e..d5800d6f9a15 100644
>>> --- a/arch/riscv/Kconfig
>>> +++ b/arch/riscv/Kconfig
>>> @@ -196,7 +196,7 @@ config RISCV
>>>        select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
>>>        select HAVE_STACKPROTECTOR
>>>        select HAVE_SYSCALL_TRACEPOINTS
>>> -     select HOTPLUG_CORE_SYNC_DEAD if HOTPLUG_CPU
>>> +     select HOTPLUG_PARALLEL if HOTPLUG_CPU
>>>        select IRQ_DOMAIN
>>>        select IRQ_FORCED_THREADING
>>>        select KASAN_VMALLOC if KASAN
>>> diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
>>> index 601a321e0f17..d85916a3660c 100644
>>> --- a/arch/riscv/kernel/smpboot.c
>>> +++ b/arch/riscv/kernel/smpboot.c
>>> @@ -39,7 +39,9 @@
>>>
>>>    #include "head.h"
>>>
>>> +#ifndef CONFIG_HOTPLUG_PARALLEL
>>>    static DECLARE_COMPLETION(cpu_running);
>>> +#endif
>>>
>>>    void __init smp_prepare_cpus(unsigned int max_cpus)
>>>    {
>>> @@ -179,6 +181,12 @@ static int start_secondary_cpu(int cpu, struct task_struct *tidle)
>>>        return -EOPNOTSUPP;
>>>    }
>>>
>>> +#ifdef CONFIG_HOTPLUG_PARALLEL
>>> +int arch_cpuhp_kick_ap_alive(unsigned int cpu, struct task_struct *tidle)
>>> +{
>>> +     return start_secondary_cpu(cpu, tidle);
>>> +}
>>> +#else
>>>    int __cpu_up(unsigned int cpu, struct task_struct *tidle)
>>>    {
>>>        int ret = 0;
>>> @@ -199,6 +207,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle)
>>>
>>>        return ret;
>>>    }
>>> +#endif
>>>
>>>    void __init smp_cpus_done(unsigned int max_cpus)
>>>    {
>>> @@ -225,6 +234,10 @@ asmlinkage __visible void smp_callin(void)
>>>        mmgrab(mm);
>>>        current->active_mm = mm;
>>>
>>> +#ifdef CONFIG_HOTPLUG_PARALLEL
>>> +     cpuhp_ap_sync_alive();
>>> +#endif
>>> +
>>>        store_cpu_topology(curr_cpuid);
>>>        notify_cpu_starting(curr_cpuid);
>>>
>>> @@ -243,7 +256,9 @@ asmlinkage __visible void smp_callin(void)
>>>         */
>>>        local_flush_icache_all();
>>>        local_flush_tlb_all();
>>> +#ifndef CONFIG_HOTPLUG_PARALLEL
>>>        complete(&cpu_running);
>>> +#endif
>> LGTM.
>>
>> Reviewed-by: Atish Patra <atishp@rivosinc.com>
>>
>> Have you tried with 128 harts ? I was not able to boot 128 harts in Qemu
>> with NR_CPUS=256.
>> This is unrelated to this patch though. I can reproduce the issue on
>> upstream with 6.18-rc3.
> I have tried upto 96 harts and that works fine.

Yeah. I had tried upto 92 as well.

> For 128 harts, the memory used by OpenSBI goes beyond 2MB so
> OpenSBI can't run from the first 2MB of DRAM (0x80000000) as the
> first booting stage. Try with U-Boot SPL loading OpenSBI from FIT image.

Ahh yes. That makes sense. I will try with U-Boot SPL.

> Regards,
> Anup
>
>>
>>>        /*
>>>         * Disable preemption before enabling interrupts, so we don't try to
>>>         * schedule a CPU that hasn't actually started yet.

Re: [PATCH] RISC-V: Enable HOTPLUG_PARALLEL for secondary CPUs

[patchwork-bot+linux-riscv]

Hello:

This patch was applied to riscv/linux.git (for-next)
by Paul Walmsley <pjw@kernel.org>:

On Fri,  5 Sep 2025 17:55:12 +0530 you wrote:
> The core kernel already supports parallel bringup of secondary
> CPUs (aka HOTPLUG_PARALLEL). The x86 and MIPS architectures
> already use HOTPLUG_PARALLEL and ARM is also moving toward it.
> 
> On RISC-V, there is no arch specific global data accessed in the
> RISC-V secondary CPU bringup path so enabling HOTPLUG_PARALLEL for
> RISC-V would only requires:
> 1) Providing RISC-V specific arch_cpuhp_kick_ap_alive()
> 2) Calling cpuhp_ap_sync_alive() from smp_callin()
> 
> [...]

Here is the summary with links:
  - RISC-V: Enable HOTPLUG_PARALLEL for secondary CPUs
    https://git.kernel.org/riscv/c/dd2fbacc2ec5

You are awesome, thank you!
-- 
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html