[PATCH v3 0/2] riscv: Idle thread using Zawrs extension

[PATCH v3 0/2] riscv: Idle thread using Zawrs extension

[Xu Lu]

This is the third version of idle thread based on Zawrs extension. We
noticed that Zawrs is supported in v6.11 now and rebased our code on
it. Below is the original description.

This patch series introduces a new implementation of idle thread using
Zawrs extension.

The Zawrs[0] extension introduces two new instructions named WRS.STO and
WRS.NTO in RISC-V. When software registers a reservation set using LR
instruction, a subsequent WRS.STO or WRS.NTO instruction will cause the
hart to stall in a low-power state until a store happens to the
reservation set or an interrupt becomes pending. The difference between
these two instructions is that WRS.STO will terminate stall after an
implementation-defined timeout while WRS.NTO won't.

This patch series implements idle thread using WRS.NTO instruction.
Besides, we found there is no need to send a real IPI to wake up an idle
CPU. Instead, we write IPI information to the reservation set of an idle
CPU to wake it up and let it handle IPI quickly, without going through
tranditional interrupt handling routine.

[0] https://github.com/riscv/riscv-zawrs/blob/main/zawrs.adoc

Xu Lu (2):
  riscv: process: Introduce idle thread using Zawrs extension
  riscv: Use Zawrs to accelerate IPI to idle cpu

 arch/riscv/Kconfig                 |  10 +++
 arch/riscv/include/asm/cpuidle.h   |  11 +---
 arch/riscv/include/asm/processor.h |  32 +++++++++
 arch/riscv/include/asm/smp.h       |  14 ++++
 arch/riscv/kernel/cpu.c            |   5 ++
 arch/riscv/kernel/process.c        | 102 ++++++++++++++++++++++++++++-
 arch/riscv/kernel/smp.c            |  51 +++++++++++----
 7 files changed, 203 insertions(+), 22 deletions(-)

-- 
2.20.1

[PATCH v3 1/2] riscv: process: Introduce idle thread using Zawrs extension

[Xu Lu]

The Zawrs extension introduces a new instruction WRS.NTO, which will
register a reservation set and causes the hart to temporarily stall
execution in a low-power state until a store occurs to the reservation
set or an interrupt is observed.

This commit implements new version of idle thread for RISC-V via Zawrs
extension.

Signed-off-by: Xu Lu <luxu.kernel@bytedance.com>
Reviewed-by: Hangjing Li <lihangjing@bytedance.com>
Reviewed-by: Liang Deng <dengliang.1214@bytedance.com>
Reviewed-by: Wen Chai <chaiwen.cc@bytedance.com>
---
 arch/riscv/Kconfig                 | 10 ++++++++
 arch/riscv/include/asm/cpuidle.h   | 11 +-------
 arch/riscv/include/asm/processor.h | 18 +++++++++++++
 arch/riscv/kernel/cpu.c            |  5 ++++
 arch/riscv/kernel/process.c        | 41 +++++++++++++++++++++++++++++-
 5 files changed, 74 insertions(+), 11 deletions(-)

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 939ea7f6a228..56cf6000d286 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -23,6 +23,7 @@ config RISCV
 	select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
 	select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
 	select ARCH_HAS_BINFMT_FLAT
+	select ARCH_HAS_CPU_FINALIZE_INIT
 	select ARCH_HAS_CURRENT_STACK_POINTER
 	select ARCH_HAS_DEBUG_VIRTUAL if MMU
 	select ARCH_HAS_DEBUG_VM_PGTABLE
@@ -1153,6 +1154,15 @@ endmenu # "Power management options"
 
 menu "CPU Power Management"
 
+config RISCV_ZAWRS_IDLE
+	bool "Idle thread using ZAWRS extensions"
+	depends on RISCV_ISA_ZAWRS
+	default y
+	help
+		Adds support to implement idle thread using ZAWRS extension.
+
+		If you don't know what to do here, say Y.
+
 source "drivers/cpuidle/Kconfig"
 
 source "drivers/cpufreq/Kconfig"
diff --git a/arch/riscv/include/asm/cpuidle.h b/arch/riscv/include/asm/cpuidle.h
index 71fdc607d4bc..94c9ecb46571 100644
--- a/arch/riscv/include/asm/cpuidle.h
+++ b/arch/riscv/include/asm/cpuidle.h
@@ -10,15 +10,6 @@
 #include <asm/barrier.h>
 #include <asm/processor.h>
 
-static inline void cpu_do_idle(void)
-{
-	/*
-	 * Add mb() here to ensure that all
-	 * IO/MEM accesses are completed prior
-	 * to entering WFI.
-	 */
-	mb();
-	wait_for_interrupt();
-}
+void cpu_do_idle(void);
 
 #endif
diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index efa1b3519b23..d0dcdb7e7392 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -12,6 +12,7 @@
 
 #include <vdso/processor.h>
 
+#include <asm/insn-def.h>
 #include <asm/ptrace.h>
 
 #define arch_get_mmap_end(addr, len, flags)			\
@@ -148,6 +149,21 @@ static inline void wait_for_interrupt(void)
 	__asm__ __volatile__ ("wfi");
 }
 
+static inline void wrs_nto(unsigned long *addr)
+{
+	int val;
+
+	__asm__ __volatile__(
+#ifdef CONFIG_64BIT
+			"lr.d %[p], %[v]\n\t"
+#else
+			"lr.w %[p], %[v]\n\t"
+#endif
+			ZAWRS_WRS_NTO "\n\t"
+			: [p] "=&r" (val), [v] "+A" (*addr)
+			: : "memory");
+}
+
 extern phys_addr_t dma32_phys_limit;
 
 struct device_node;
@@ -177,6 +193,8 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val);
 #define RISCV_SET_ICACHE_FLUSH_CTX(arg1, arg2)	riscv_set_icache_flush_ctx(arg1, arg2)
 extern int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long per_thread);
 
+extern void select_idle_routine(void);
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* _ASM_RISCV_PROCESSOR_H */
diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
index f6b13e9f5e6c..97a7144fa6cd 100644
--- a/arch/riscv/kernel/cpu.c
+++ b/arch/riscv/kernel/cpu.c
@@ -23,6 +23,11 @@ bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
 	return phys_id == cpuid_to_hartid_map(cpu);
 }
 
+void __init arch_cpu_finalize_init(void)
+{
+	select_idle_routine();
+}
+
 /*
  * Returns the hart ID of the given device tree node, or -ENODEV if the node
  * isn't an enabled and valid RISC-V hart node.
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index e4bc61c4e58a..77769965609e 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -15,6 +15,7 @@
 #include <linux/tick.h>
 #include <linux/ptrace.h>
 #include <linux/uaccess.h>
+#include <linux/static_call.h>
 
 #include <asm/unistd.h>
 #include <asm/processor.h>
@@ -35,11 +36,49 @@ EXPORT_SYMBOL(__stack_chk_guard);
 
 extern asmlinkage void ret_from_fork(void);
 
-void noinstr arch_cpu_idle(void)
+static __cpuidle void default_idle(void)
+{
+	/*
+	 * Add mb() here to ensure that all
+	 * IO/MEM accesses are completed prior
+	 * to entering WFI.
+	 */
+	mb();
+	wait_for_interrupt();
+}
+
+static __cpuidle void wrs_idle(void)
+{
+	/*
+	 * Add mb() here to ensure that all
+	 * IO/MEM accesses are completed prior
+	 * to entering WRS.NTO.
+	 */
+	mb();
+	wrs_nto(&current_thread_info()->flags);
+}
+
+DEFINE_STATIC_CALL_NULL(riscv_idle, default_idle);
+
+void __cpuidle cpu_do_idle(void)
+{
+	static_call(riscv_idle)();
+}
+
+void __cpuidle arch_cpu_idle(void)
 {
 	cpu_do_idle();
 }
 
+void __init select_idle_routine(void)
+{
+	if (IS_ENABLED(CONFIG_RISCV_ZAWRS_IDLE) &&
+			riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS))
+		static_call_update(riscv_idle, wrs_idle);
+	else
+		static_call_update(riscv_idle, default_idle);
+}
+
 int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
 {
 	if (!unaligned_ctl_available())
-- 
2.20.1

Re: [PATCH v3 1/2] riscv: process: Introduce idle thread using Zawrs extension

[Andrew Jones]

On Wed, Sep 25, 2024 at 09:15:46PM GMT, Xu Lu wrote:
> The Zawrs extension introduces a new instruction WRS.NTO, which will
> register a reservation set and causes the hart to temporarily stall
> execution in a low-power state until a store occurs to the reservation
> set or an interrupt is observed.
> 
> This commit implements new version of idle thread for RISC-V via Zawrs
> extension.
> 
> Signed-off-by: Xu Lu <luxu.kernel@bytedance.com>
> Reviewed-by: Hangjing Li <lihangjing@bytedance.com>
> Reviewed-by: Liang Deng <dengliang.1214@bytedance.com>
> Reviewed-by: Wen Chai <chaiwen.cc@bytedance.com>
> ---
>  arch/riscv/Kconfig                 | 10 ++++++++
>  arch/riscv/include/asm/cpuidle.h   | 11 +-------
>  arch/riscv/include/asm/processor.h | 18 +++++++++++++
>  arch/riscv/kernel/cpu.c            |  5 ++++
>  arch/riscv/kernel/process.c        | 41 +++++++++++++++++++++++++++++-
>  5 files changed, 74 insertions(+), 11 deletions(-)
> 
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index 939ea7f6a228..56cf6000d286 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -23,6 +23,7 @@ config RISCV
>  	select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
>  	select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
>  	select ARCH_HAS_BINFMT_FLAT
> +	select ARCH_HAS_CPU_FINALIZE_INIT
>  	select ARCH_HAS_CURRENT_STACK_POINTER
>  	select ARCH_HAS_DEBUG_VIRTUAL if MMU
>  	select ARCH_HAS_DEBUG_VM_PGTABLE
> @@ -1153,6 +1154,15 @@ endmenu # "Power management options"
>  
>  menu "CPU Power Management"
>  
> +config RISCV_ZAWRS_IDLE
> +	bool "Idle thread using ZAWRS extensions"
> +	depends on RISCV_ISA_ZAWRS
> +	default y
> +	help
> +		Adds support to implement idle thread using ZAWRS extension.
> +
> +		If you don't know what to do here, say Y.
> +
>  source "drivers/cpuidle/Kconfig"
>  
>  source "drivers/cpufreq/Kconfig"
> diff --git a/arch/riscv/include/asm/cpuidle.h b/arch/riscv/include/asm/cpuidle.h
> index 71fdc607d4bc..94c9ecb46571 100644
> --- a/arch/riscv/include/asm/cpuidle.h
> +++ b/arch/riscv/include/asm/cpuidle.h
> @@ -10,15 +10,6 @@
>  #include <asm/barrier.h>
>  #include <asm/processor.h>
>  
> -static inline void cpu_do_idle(void)
> -{
> -	/*
> -	 * Add mb() here to ensure that all
> -	 * IO/MEM accesses are completed prior
> -	 * to entering WFI.
> -	 */
> -	mb();
> -	wait_for_interrupt();
> -}
> +void cpu_do_idle(void);
>  
>  #endif
> diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
> index efa1b3519b23..d0dcdb7e7392 100644
> --- a/arch/riscv/include/asm/processor.h
> +++ b/arch/riscv/include/asm/processor.h
> @@ -12,6 +12,7 @@
>  
>  #include <vdso/processor.h>
>  
> +#include <asm/insn-def.h>
>  #include <asm/ptrace.h>
>  
>  #define arch_get_mmap_end(addr, len, flags)			\
> @@ -148,6 +149,21 @@ static inline void wait_for_interrupt(void)
>  	__asm__ __volatile__ ("wfi");
>  }
>  
> +static inline void wrs_nto(unsigned long *addr)
> +{
> +	int val;
> +
> +	__asm__ __volatile__(
> +#ifdef CONFIG_64BIT
> +			"lr.d %[p], %[v]\n\t"
> +#else
> +			"lr.w %[p], %[v]\n\t"
> +#endif

val is always 32-bit since it's an int. We should always use lr.w.

> +			ZAWRS_WRS_NTO "\n\t"
> +			: [p] "=&r" (val), [v] "+A" (*addr)

What do 'p' and 'v' represent? If they are pointer and value then they're
backwards. I would just spell them out [val] and [addr].

> +			: : "memory");
> +}
> +
>  extern phys_addr_t dma32_phys_limit;
>  
>  struct device_node;
> @@ -177,6 +193,8 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val);
>  #define RISCV_SET_ICACHE_FLUSH_CTX(arg1, arg2)	riscv_set_icache_flush_ctx(arg1, arg2)
>  extern int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long per_thread);
>  
> +extern void select_idle_routine(void);
> +
>  #endif /* __ASSEMBLY__ */
>  
>  #endif /* _ASM_RISCV_PROCESSOR_H */
> diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
> index f6b13e9f5e6c..97a7144fa6cd 100644
> --- a/arch/riscv/kernel/cpu.c
> +++ b/arch/riscv/kernel/cpu.c
> @@ -23,6 +23,11 @@ bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
>  	return phys_id == cpuid_to_hartid_map(cpu);
>  }
>  
> +void __init arch_cpu_finalize_init(void)
> +{
> +	select_idle_routine();
> +}

Is there a reason we need to do this at arch_cpu_finalize_init() time?
This seems like the type of thing we have typically done at the bottom of
setup_arch().

> +
>  /*
>   * Returns the hart ID of the given device tree node, or -ENODEV if the node
>   * isn't an enabled and valid RISC-V hart node.
> diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
> index e4bc61c4e58a..77769965609e 100644
> --- a/arch/riscv/kernel/process.c
> +++ b/arch/riscv/kernel/process.c
> @@ -15,6 +15,7 @@
>  #include <linux/tick.h>
>  #include <linux/ptrace.h>
>  #include <linux/uaccess.h>
> +#include <linux/static_call.h>
>  
>  #include <asm/unistd.h>
>  #include <asm/processor.h>
> @@ -35,11 +36,49 @@ EXPORT_SYMBOL(__stack_chk_guard);
>  
>  extern asmlinkage void ret_from_fork(void);
>  
> -void noinstr arch_cpu_idle(void)
> +static __cpuidle void default_idle(void)
> +{
> +	/*
> +	 * Add mb() here to ensure that all
> +	 * IO/MEM accesses are completed prior
> +	 * to entering WFI.
> +	 */
> +	mb();
> +	wait_for_interrupt();
> +}
> +
> +static __cpuidle void wrs_idle(void)
> +{
> +	/*
> +	 * Add mb() here to ensure that all
> +	 * IO/MEM accesses are completed prior
> +	 * to entering WRS.NTO.
> +	 */
> +	mb();
> +	wrs_nto(&current_thread_info()->flags);
> +}
> +
> +DEFINE_STATIC_CALL_NULL(riscv_idle, default_idle);
> +
> +void __cpuidle cpu_do_idle(void)
> +{
> +	static_call(riscv_idle)();
> +}
> +
> +void __cpuidle arch_cpu_idle(void)

Switching the section of this from '.noinstr.text' to 'cpuidle.text'
should probably be a separate patch.

>  {
>  	cpu_do_idle();
>  }
>  
> +void __init select_idle_routine(void)
> +{
> +	if (IS_ENABLED(CONFIG_RISCV_ZAWRS_IDLE) &&
> +			riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS))
> +		static_call_update(riscv_idle, wrs_idle);
> +	else
> +		static_call_update(riscv_idle, default_idle);

Do we need this 'else'? Can't we set the default at DEFINE_STATIC_CALL*
time?

> +}
> +
>  int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
>  {
>  	if (!unaligned_ctl_available())
> -- 
> 2.20.1
>

Thanks,
drew

Re: [PATCH v3 1/2] riscv: process: Introduce idle thread using Zawrs extension

[Andreas Schwab]

On Sep 25 2024, Andrew Jones wrote:

> On Wed, Sep 25, 2024 at 09:15:46PM GMT, Xu Lu wrote:
>> @@ -148,6 +149,21 @@ static inline void wait_for_interrupt(void)
>>  	__asm__ __volatile__ ("wfi");
>>  }
>>  
>> +static inline void wrs_nto(unsigned long *addr)
>> +{
>> +	int val;
>> +
>> +	__asm__ __volatile__(
>> +#ifdef CONFIG_64BIT
>> +			"lr.d %[p], %[v]\n\t"
>> +#else
>> +			"lr.w %[p], %[v]\n\t"
>> +#endif
>
> val is always 32-bit since it's an int. We should always use lr.w.

Shouldn't val be unsigned long like the pointer that is being read?

-- 
Andreas Schwab, SUSE Labs, schwab@suse.de
GPG Key fingerprint = 0196 BAD8 1CE9 1970 F4BE  1748 E4D4 88E3 0EEA B9D7
"And now for something completely different."

Re: [External] Re: [PATCH v3 1/2] riscv: process: Introduce idle thread using Zawrs extension

[Xu Lu]

Hi Andreas,

Thanks a lot for your reply.

On Wed, Sep 25, 2024 at 10:10 PM Andreas Schwab <schwab@suse.de> wrote:
>
> On Sep 25 2024, Andrew Jones wrote:
>
> > On Wed, Sep 25, 2024 at 09:15:46PM GMT, Xu Lu wrote:
> >> @@ -148,6 +149,21 @@ static inline void wait_for_interrupt(void)
> >>      __asm__ __volatile__ ("wfi");
> >>  }
> >>
> >> +static inline void wrs_nto(unsigned long *addr)
> >> +{
> >> +    int val;
> >> +
> >> +    __asm__ __volatile__(
> >> +#ifdef CONFIG_64BIT
> >> +                    "lr.d %[p], %[v]\n\t"
> >> +#else
> >> +                    "lr.w %[p], %[v]\n\t"
> >> +#endif
> >
> > val is always 32-bit since it's an int. We should always use lr.w.
>
> Shouldn't val be unsigned long like the pointer that is being read?

Yes. As I replied Andrew, the 'int val' is a mistake here. The val can
be an unsigned long (thread_info->flags) when CONFIG_SMP is disabled.
I will update the declaration of val from 'int' to 'unsigned long'.

>
> --
> Andreas Schwab, SUSE Labs, schwab@suse.de
> GPG Key fingerprint = 0196 BAD8 1CE9 1970 F4BE  1748 E4D4 88E3 0EEA B9D7
> "And now for something completely different."

Thanks,

Xu Lu.

Re: [External] Re: [PATCH v3 1/2] riscv: process: Introduce idle thread using Zawrs extension

[Xu Lu]

Hi Andrew,

Thanks a lot for your reply.

On Wed, Sep 25, 2024 at 9:54 PM Andrew Jones <ajones@ventanamicro.com> wrote:
>
> On Wed, Sep 25, 2024 at 09:15:46PM GMT, Xu Lu wrote:
> > The Zawrs extension introduces a new instruction WRS.NTO, which will
> > register a reservation set and causes the hart to temporarily stall
> > execution in a low-power state until a store occurs to the reservation
> > set or an interrupt is observed.
> >
> > This commit implements new version of idle thread for RISC-V via Zawrs
> > extension.
> >
> > Signed-off-by: Xu Lu <luxu.kernel@bytedance.com>
> > Reviewed-by: Hangjing Li <lihangjing@bytedance.com>
> > Reviewed-by: Liang Deng <dengliang.1214@bytedance.com>
> > Reviewed-by: Wen Chai <chaiwen.cc@bytedance.com>
> > ---
> >  arch/riscv/Kconfig                 | 10 ++++++++
> >  arch/riscv/include/asm/cpuidle.h   | 11 +-------
> >  arch/riscv/include/asm/processor.h | 18 +++++++++++++
> >  arch/riscv/kernel/cpu.c            |  5 ++++
> >  arch/riscv/kernel/process.c        | 41 +++++++++++++++++++++++++++++-
> >  5 files changed, 74 insertions(+), 11 deletions(-)
> >
> > diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> > index 939ea7f6a228..56cf6000d286 100644
> > --- a/arch/riscv/Kconfig
> > +++ b/arch/riscv/Kconfig
> > @@ -23,6 +23,7 @@ config RISCV
> >       select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
> >       select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
> >       select ARCH_HAS_BINFMT_FLAT
> > +     select ARCH_HAS_CPU_FINALIZE_INIT
> >       select ARCH_HAS_CURRENT_STACK_POINTER
> >       select ARCH_HAS_DEBUG_VIRTUAL if MMU
> >       select ARCH_HAS_DEBUG_VM_PGTABLE
> > @@ -1153,6 +1154,15 @@ endmenu # "Power management options"
> >
> >  menu "CPU Power Management"
> >
> > +config RISCV_ZAWRS_IDLE
> > +     bool "Idle thread using ZAWRS extensions"
> > +     depends on RISCV_ISA_ZAWRS
> > +     default y
> > +     help
> > +             Adds support to implement idle thread using ZAWRS extension.
> > +
> > +             If you don't know what to do here, say Y.
> > +
> >  source "drivers/cpuidle/Kconfig"
> >
> >  source "drivers/cpufreq/Kconfig"
> > diff --git a/arch/riscv/include/asm/cpuidle.h b/arch/riscv/include/asm/cpuidle.h
> > index 71fdc607d4bc..94c9ecb46571 100644
> > --- a/arch/riscv/include/asm/cpuidle.h
> > +++ b/arch/riscv/include/asm/cpuidle.h
> > @@ -10,15 +10,6 @@
> >  #include <asm/barrier.h>
> >  #include <asm/processor.h>
> >
> > -static inline void cpu_do_idle(void)
> > -{
> > -     /*
> > -      * Add mb() here to ensure that all
> > -      * IO/MEM accesses are completed prior
> > -      * to entering WFI.
> > -      */
> > -     mb();
> > -     wait_for_interrupt();
> > -}
> > +void cpu_do_idle(void);
> >
> >  #endif
> > diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
> > index efa1b3519b23..d0dcdb7e7392 100644
> > --- a/arch/riscv/include/asm/processor.h
> > +++ b/arch/riscv/include/asm/processor.h
> > @@ -12,6 +12,7 @@
> >
> >  #include <vdso/processor.h>
> >
> > +#include <asm/insn-def.h>
> >  #include <asm/ptrace.h>
> >
> >  #define arch_get_mmap_end(addr, len, flags)                  \
> > @@ -148,6 +149,21 @@ static inline void wait_for_interrupt(void)
> >       __asm__ __volatile__ ("wfi");
> >  }
> >
> > +static inline void wrs_nto(unsigned long *addr)
> > +{
> > +     int val;
> > +
> > +     __asm__ __volatile__(
> > +#ifdef CONFIG_64BIT
> > +                     "lr.d %[p], %[v]\n\t"
> > +#else
> > +                     "lr.w %[p], %[v]\n\t"
> > +#endif
>
> val is always 32-bit since it's an int. We should always use lr.w.

The 'int val' is a mistake here. The val can be an unsigned long
(thread_info->flags) when CONFIG_SMP is disabled. So the lr.d is
necessary. I will update the declaration of val from 'int' to
'unsigned long'.

>
> > +                     ZAWRS_WRS_NTO "\n\t"
> > +                     : [p] "=&r" (val), [v] "+A" (*addr)
>
> What do 'p' and 'v' represent? If they are pointer and value then they're
> backwards. I would just spell them out [val] and [addr].

Great insight. I will refine the name here to make it more readable.

>
> > +                     : : "memory");
> > +}
> > +
> >  extern phys_addr_t dma32_phys_limit;
> >
> >  struct device_node;
> > @@ -177,6 +193,8 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val);
> >  #define RISCV_SET_ICACHE_FLUSH_CTX(arg1, arg2)       riscv_set_icache_flush_ctx(arg1, arg2)
> >  extern int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long per_thread);
> >
> > +extern void select_idle_routine(void);
> > +
> >  #endif /* __ASSEMBLY__ */
> >
> >  #endif /* _ASM_RISCV_PROCESSOR_H */
> > diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
> > index f6b13e9f5e6c..97a7144fa6cd 100644
> > --- a/arch/riscv/kernel/cpu.c
> > +++ b/arch/riscv/kernel/cpu.c
> > @@ -23,6 +23,11 @@ bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
> >       return phys_id == cpuid_to_hartid_map(cpu);
> >  }
> >
> > +void __init arch_cpu_finalize_init(void)
> > +{
> > +     select_idle_routine();
> > +}
>
> Is there a reason we need to do this at arch_cpu_finalize_init() time?
> This seems like the type of thing we have typically done at the bottom of
> setup_arch().

Actually, there is no special reason here. Just imitated the placement
of x86. It works well too if we put it at the end of setup_arch().

>
> > +
> >  /*
> >   * Returns the hart ID of the given device tree node, or -ENODEV if the node
> >   * isn't an enabled and valid RISC-V hart node.
> > diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
> > index e4bc61c4e58a..77769965609e 100644
> > --- a/arch/riscv/kernel/process.c
> > +++ b/arch/riscv/kernel/process.c
> > @@ -15,6 +15,7 @@
> >  #include <linux/tick.h>
> >  #include <linux/ptrace.h>
> >  #include <linux/uaccess.h>
> > +#include <linux/static_call.h>
> >
> >  #include <asm/unistd.h>
> >  #include <asm/processor.h>
> > @@ -35,11 +36,49 @@ EXPORT_SYMBOL(__stack_chk_guard);
> >
> >  extern asmlinkage void ret_from_fork(void);
> >
> > -void noinstr arch_cpu_idle(void)
> > +static __cpuidle void default_idle(void)
> > +{
> > +     /*
> > +      * Add mb() here to ensure that all
> > +      * IO/MEM accesses are completed prior
> > +      * to entering WFI.
> > +      */
> > +     mb();
> > +     wait_for_interrupt();
> > +}
> > +
> > +static __cpuidle void wrs_idle(void)
> > +{
> > +     /*
> > +      * Add mb() here to ensure that all
> > +      * IO/MEM accesses are completed prior
> > +      * to entering WRS.NTO.
> > +      */
> > +     mb();
> > +     wrs_nto(&current_thread_info()->flags);
> > +}
> > +
> > +DEFINE_STATIC_CALL_NULL(riscv_idle, default_idle);
> > +
> > +void __cpuidle cpu_do_idle(void)
> > +{
> > +     static_call(riscv_idle)();
> > +}
> > +
> > +void __cpuidle arch_cpu_idle(void)
>
> Switching the section of this from '.noinstr.text' to 'cpuidle.text'
> should probably be a separate patch.
>
> >  {
> >       cpu_do_idle();
> >  }
> >
> > +void __init select_idle_routine(void)
> > +{
> > +     if (IS_ENABLED(CONFIG_RISCV_ZAWRS_IDLE) &&
> > +                     riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS))
> > +             static_call_update(riscv_idle, wrs_idle);
> > +     else
> > +             static_call_update(riscv_idle, default_idle);
>
> Do we need this 'else'? Can't we set the default at DEFINE_STATIC_CALL*
> time?

Yes, the 'else' branch can be canceled if we set the default idle
function to 'wfi' one using DEFINE_STATIC_CALL. Just not sure which
code looks better.

>
> > +}
> > +
> >  int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
> >  {
> >       if (!unaligned_ctl_available())
> > --
> > 2.20.1
> >
>
> Thanks,
> drew

Best regards,

Xu Lu.

Re: [PATCH v3 1/2] riscv: process: Introduce idle thread using Zawrs extension

[Guo Ren]

On Wed, Sep 25, 2024 at 9:16 PM Xu Lu <luxu.kernel@bytedance.com> wrote:
>
> The Zawrs extension introduces a new instruction WRS.NTO, which will
> register a reservation set and causes the hart to temporarily stall
> execution in a low-power state until a store occurs to the reservation
> set or an interrupt is observed.
>
> This commit implements new version of idle thread for RISC-V via Zawrs
> extension.
>
> Signed-off-by: Xu Lu <luxu.kernel@bytedance.com>
> Reviewed-by: Hangjing Li <lihangjing@bytedance.com>
> Reviewed-by: Liang Deng <dengliang.1214@bytedance.com>
> Reviewed-by: Wen Chai <chaiwen.cc@bytedance.com>
> ---
>  arch/riscv/Kconfig                 | 10 ++++++++
>  arch/riscv/include/asm/cpuidle.h   | 11 +-------
>  arch/riscv/include/asm/processor.h | 18 +++++++++++++
>  arch/riscv/kernel/cpu.c            |  5 ++++
>  arch/riscv/kernel/process.c        | 41 +++++++++++++++++++++++++++++-
>  5 files changed, 74 insertions(+), 11 deletions(-)
>
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index 939ea7f6a228..56cf6000d286 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -23,6 +23,7 @@ config RISCV
>         select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2
>         select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE
>         select ARCH_HAS_BINFMT_FLAT
> +       select ARCH_HAS_CPU_FINALIZE_INIT
>         select ARCH_HAS_CURRENT_STACK_POINTER
>         select ARCH_HAS_DEBUG_VIRTUAL if MMU
>         select ARCH_HAS_DEBUG_VM_PGTABLE
> @@ -1153,6 +1154,15 @@ endmenu # "Power management options"
>
>  menu "CPU Power Management"
>
> +config RISCV_ZAWRS_IDLE
> +       bool "Idle thread using ZAWRS extensions"
> +       depends on RISCV_ISA_ZAWRS
> +       default y
> +       help
> +               Adds support to implement idle thread using ZAWRS extension.
> +
> +               If you don't know what to do here, say Y.
> +
>  source "drivers/cpuidle/Kconfig"
>
>  source "drivers/cpufreq/Kconfig"
> diff --git a/arch/riscv/include/asm/cpuidle.h b/arch/riscv/include/asm/cpuidle.h
> index 71fdc607d4bc..94c9ecb46571 100644
> --- a/arch/riscv/include/asm/cpuidle.h
> +++ b/arch/riscv/include/asm/cpuidle.h
> @@ -10,15 +10,6 @@
>  #include <asm/barrier.h>
>  #include <asm/processor.h>
>
> -static inline void cpu_do_idle(void)
> -{
> -       /*
> -        * Add mb() here to ensure that all
> -        * IO/MEM accesses are completed prior
> -        * to entering WFI.
> -        */
> -       mb();
> -       wait_for_interrupt();
> -}
> +void cpu_do_idle(void);
>
>  #endif
> diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
> index efa1b3519b23..d0dcdb7e7392 100644
> --- a/arch/riscv/include/asm/processor.h
> +++ b/arch/riscv/include/asm/processor.h
> @@ -12,6 +12,7 @@
>
>  #include <vdso/processor.h>
>
> +#include <asm/insn-def.h>
>  #include <asm/ptrace.h>
>
>  #define arch_get_mmap_end(addr, len, flags)                    \
> @@ -148,6 +149,21 @@ static inline void wait_for_interrupt(void)
>         __asm__ __volatile__ ("wfi");
>  }
>
> +static inline void wrs_nto(unsigned long *addr)
> +{
> +       int val;
> +
> +       __asm__ __volatile__(
> +#ifdef CONFIG_64BIT
> +                       "lr.d %[p], %[v]\n\t"
> +#else
> +                       "lr.w %[p], %[v]\n\t"
> +#endif
> +                       ZAWRS_WRS_NTO "\n\t"
> +                       : [p] "=&r" (val), [v] "+A" (*addr)
> +                       : : "memory");
> +}
> +
>  extern phys_addr_t dma32_phys_limit;
>
>  struct device_node;
> @@ -177,6 +193,8 @@ extern int set_unalign_ctl(struct task_struct *tsk, unsigned int val);
>  #define RISCV_SET_ICACHE_FLUSH_CTX(arg1, arg2) riscv_set_icache_flush_ctx(arg1, arg2)
>  extern int riscv_set_icache_flush_ctx(unsigned long ctx, unsigned long per_thread);
>
> +extern void select_idle_routine(void);
> +
>  #endif /* __ASSEMBLY__ */
>
>  #endif /* _ASM_RISCV_PROCESSOR_H */
> diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c
> index f6b13e9f5e6c..97a7144fa6cd 100644
> --- a/arch/riscv/kernel/cpu.c
> +++ b/arch/riscv/kernel/cpu.c
> @@ -23,6 +23,11 @@ bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
>         return phys_id == cpuid_to_hartid_map(cpu);
>  }
>
> +void __init arch_cpu_finalize_init(void)
> +{
> +       select_idle_routine();
> +}
> +
>  /*
>   * Returns the hart ID of the given device tree node, or -ENODEV if the node
>   * isn't an enabled and valid RISC-V hart node.
> diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
> index e4bc61c4e58a..77769965609e 100644
> --- a/arch/riscv/kernel/process.c
> +++ b/arch/riscv/kernel/process.c
> @@ -15,6 +15,7 @@
>  #include <linux/tick.h>
>  #include <linux/ptrace.h>
>  #include <linux/uaccess.h>
> +#include <linux/static_call.h>
>
>  #include <asm/unistd.h>
>  #include <asm/processor.h>
> @@ -35,11 +36,49 @@ EXPORT_SYMBOL(__stack_chk_guard);
>
>  extern asmlinkage void ret_from_fork(void);
>
> -void noinstr arch_cpu_idle(void)
> +static __cpuidle void default_idle(void)
> +{
> +       /*
> +        * Add mb() here to ensure that all
> +        * IO/MEM accesses are completed prior
> +        * to entering WFI.
> +        */
> +       mb();
> +       wait_for_interrupt();
> +}
> +
> +static __cpuidle void wrs_idle(void)
> +{
> +       /*
> +        * Add mb() here to ensure that all
> +        * IO/MEM accesses are completed prior
> +        * to entering WRS.NTO.
> +        */
> +       mb();
> +       wrs_nto(&current_thread_info()->flags);
> +}
> +
> +DEFINE_STATIC_CALL_NULL(riscv_idle, default_idle);
> +
> +void __cpuidle cpu_do_idle(void)
> +{
> +       static_call(riscv_idle)();
> +}
> +
> +void __cpuidle arch_cpu_idle(void)
>  {
>         cpu_do_idle();
>  }
>
> +void __init select_idle_routine(void)
> +{
> +       if (IS_ENABLED(CONFIG_RISCV_ZAWRS_IDLE) &&
> +                       riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS))
> +               static_call_update(riscv_idle, wrs_idle);
> +       else
> +               static_call_update(riscv_idle, default_idle);
The hardware implementations of "WFI & WRS" are different.
- The WFI could enter a more profound power consumption-saving needed
to keep IRQ alive, but the WRS doesn't.
- WRS is designed for cond_load, which needs to keep cache coherency alive.

So, we couldn't simply & statically use an extension check to choose
WRS or WFI for idle.

> +}
> +
>  int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
>  {
>         if (!unaligned_ctl_available())
> --
> 2.20.1
>


-- 
Best Regards
 Guo Ren

Re: [External] Re: [PATCH v3 1/2] riscv: process: Introduce idle thread using Zawrs extension

[Xu Lu]

Hi Guo Ren,

> The hardware implementations of "WFI & WRS" are different.
> - The WFI could enter a more profound power consumption-saving needed
> to keep IRQ alive, but the WRS doesn't.
> - WRS is designed for cond_load, which needs to keep cache coherency alive.
>
> So, we couldn't simply & statically use an extension check to choose
> WRS or WFI for idle.

Great comment.

I think maintaining cache coherence is not a big power problem for
WRS. In a naive implementation, the wrs instruction only needs to
maintain a valid cache line in L1 and wake up when other cpu wants to
make it invalid. This behavior won't incur too much power consumption.

And there does exist a power consumption gap if some vendors implement
WFI instruction to flush and power down the entire L1/L2 cache (for
example, provide CPUPWRCTLR register to control WFI behavior like
ARM), while WRS can not do the same thing. Whether WFI will power down
cache is not mentioned in RISC-V spec and is somehow vendor-customized
(Please remind me if I missed anything). So it is hard to use a
unified method to detect or control the cache power down behavior of
WFI instruction. And that is why we provide a CONFIG to let the user
choose whether to enable WRS IDLE or not.

>
> > +}
> > +
> >  int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
> >  {
> >         if (!unaligned_ctl_available())
> > --
> > 2.20.1
> >
>
>
> --
> Best Regards
>  Guo Ren

Best Regards,

Xu Lu

[PATCH v3 2/2] riscv: Use Zawrs to accelerate IPI to idle cpu

[Xu Lu]

When sending IPI to a cpu which has entered idle state using Zawrs
extension, there is no need to send a physical software interrupt.
Instead, we can write the IPI information to the address reserved by
target cpu, which will wake it from WRS.NTO. Then the target cpu can
handle the IPI directly without falling into traditional interrupt
handling routine.

Signed-off-by: Xu Lu <luxu.kernel@bytedance.com>
---
 arch/riscv/include/asm/processor.h | 14 +++++++
 arch/riscv/include/asm/smp.h       | 14 +++++++
 arch/riscv/kernel/process.c        | 65 +++++++++++++++++++++++++++++-
 arch/riscv/kernel/smp.c            | 51 ++++++++++++++++++-----
 4 files changed, 131 insertions(+), 13 deletions(-)

diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h
index d0dcdb7e7392..0dbc9390c3b2 100644
--- a/arch/riscv/include/asm/processor.h
+++ b/arch/riscv/include/asm/processor.h
@@ -164,6 +164,20 @@ static inline void wrs_nto(unsigned long *addr)
 			: : "memory");
 }
 
+static inline void wrs_nto_if(int *addr, int val)
+{
+	int prev;
+
+	__asm__ __volatile__(
+			"lr.w %[p], %[a]\n\t"
+			"bne %[p], %[v], 1f\n\t"
+			ZAWRS_WRS_NTO "\n\t"
+			"1:\n\t"
+			: [p] "=&r" (prev), [a] "+A" (*addr)
+			: [v] "r" (val)
+			: "memory");
+}
+
 extern phys_addr_t dma32_phys_limit;
 
 struct device_node;
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index 7ac80e9f2288..8f2dfbf20e89 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -19,6 +19,20 @@ extern unsigned long boot_cpu_hartid;
 
 #include <linux/jump_label.h>
 
+enum ipi_message_type {
+	IPI_RESCHEDULE,
+	IPI_CALL_FUNC,
+	IPI_CPU_STOP,
+	IPI_CPU_CRASH_STOP,
+	IPI_IRQ_WORK,
+	IPI_TIMER,
+	IPI_MAX
+};
+
+int ipi_virq_base_get(void);
+
+irqreturn_t handle_IPI(int irq, void *data);
+
 /*
  * Mapping between linux logical cpu index and hartid.
  */
diff --git a/arch/riscv/kernel/process.c b/arch/riscv/kernel/process.c
index 77769965609e..975b3f28e8c8 100644
--- a/arch/riscv/kernel/process.c
+++ b/arch/riscv/kernel/process.c
@@ -16,6 +16,7 @@
 #include <linux/ptrace.h>
 #include <linux/uaccess.h>
 #include <linux/static_call.h>
+#include <linux/hardirq.h>
 
 #include <asm/unistd.h>
 #include <asm/processor.h>
@@ -27,6 +28,7 @@
 #include <asm/cpuidle.h>
 #include <asm/vector.h>
 #include <asm/cpufeature.h>
+#include <asm/smp.h>
 
 #if defined(CONFIG_STACKPROTECTOR) && !defined(CONFIG_STACKPROTECTOR_PER_TASK)
 #include <linux/stackprotector.h>
@@ -36,6 +38,8 @@ EXPORT_SYMBOL(__stack_chk_guard);
 
 extern asmlinkage void ret_from_fork(void);
 
+DEFINE_PER_CPU(atomic_t, idle_ipi_mask);
+
 static __cpuidle void default_idle(void)
 {
 	/*
@@ -47,6 +51,16 @@ static __cpuidle void default_idle(void)
 	wait_for_interrupt();
 }
 
+static __cpuidle void default_idle_enter(void)
+{
+	/* Do nothing */
+}
+
+static __cpuidle void default_idle_exit(void)
+{
+	/* Do nothing */
+}
+
 static __cpuidle void wrs_idle(void)
 {
 	/*
@@ -55,10 +69,42 @@ static __cpuidle void wrs_idle(void)
 	 * to entering WRS.NTO.
 	 */
 	mb();
+#ifdef CONFIG_SMP
+	wrs_nto_if(&this_cpu_ptr(&idle_ipi_mask)->counter, BIT(IPI_MAX));
+#else
 	wrs_nto(&current_thread_info()->flags);
+#endif
+}
+
+static __cpuidle void wrs_idle_enter(void)
+{
+#ifdef CONFIG_SMP
+	atomic_set(this_cpu_ptr(&idle_ipi_mask), BIT(IPI_MAX));
+#endif
+}
+
+static __cpuidle void wrs_idle_exit(void)
+{
+#ifdef CONFIG_SMP
+	int pending;
+	unsigned long flags;
+	enum ipi_message_type ipi;
+
+	local_irq_save(flags);
+	pending = atomic_xchg_relaxed(this_cpu_ptr(&idle_ipi_mask), 0);
+	for (ipi = IPI_RESCHEDULE; ipi < IPI_MAX; ipi++)
+		if (pending & BIT(ipi)) {
+			irq_enter();
+			handle_IPI(ipi_virq_base_get() + ipi, NULL);
+			irq_exit();
+		}
+	local_irq_restore(flags);
+#endif
 }
 
 DEFINE_STATIC_CALL_NULL(riscv_idle, default_idle);
+DEFINE_STATIC_CALL_NULL(riscv_idle_enter, default_idle_enter);
+DEFINE_STATIC_CALL_NULL(riscv_idle_exit, default_idle_exit);
 
 void __cpuidle cpu_do_idle(void)
 {
@@ -70,13 +116,28 @@ void __cpuidle arch_cpu_idle(void)
 	cpu_do_idle();
 }
 
+void __cpuidle arch_cpu_idle_enter(void)
+{
+	static_call(riscv_idle_enter)();
+}
+
+void __cpuidle arch_cpu_idle_exit(void)
+{
+	static_call(riscv_idle_exit)();
+}
+
 void __init select_idle_routine(void)
 {
 	if (IS_ENABLED(CONFIG_RISCV_ZAWRS_IDLE) &&
-			riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS))
+			riscv_has_extension_likely(RISCV_ISA_EXT_ZAWRS)) {
 		static_call_update(riscv_idle, wrs_idle);
-	else
+		static_call_update(riscv_idle_enter, wrs_idle_enter);
+		static_call_update(riscv_idle_exit, wrs_idle_exit);
+	} else {
 		static_call_update(riscv_idle, default_idle);
+		static_call_update(riscv_idle_enter, default_idle_enter);
+		static_call_update(riscv_idle_exit, default_idle_exit);
+	}
 }
 
 int set_unalign_ctl(struct task_struct *tsk, unsigned int val)
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index 8e6eb64459af..6e7d41ed4144 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -26,16 +26,6 @@
 #include <asm/cacheflush.h>
 #include <asm/cpu_ops.h>
 
-enum ipi_message_type {
-	IPI_RESCHEDULE,
-	IPI_CALL_FUNC,
-	IPI_CPU_STOP,
-	IPI_CPU_CRASH_STOP,
-	IPI_IRQ_WORK,
-	IPI_TIMER,
-	IPI_MAX
-};
-
 unsigned long __cpuid_to_hartid_map[NR_CPUS] __ro_after_init = {
 	[0 ... NR_CPUS-1] = INVALID_HARTID
 };
@@ -94,13 +84,47 @@ static inline void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
 }
 #endif
 
+#ifdef CONFIG_RISCV_ZAWRS_IDLE
+DECLARE_PER_CPU(atomic_t, idle_ipi_mask);
+#endif
+
 static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op)
 {
+#ifdef CONFIG_RISCV_ZAWRS_IDLE
+	int cpu, val;
+
+	asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", 0, RISCV_ISA_EXT_ZAWRS, 1)
+			: : : : no_zawrs);
+
+	for_each_cpu(cpu, mask) {
+		val = atomic_fetch_or_relaxed(BIT(op), per_cpu_ptr(&idle_ipi_mask, cpu));
+		if (likely(!(val & BIT(IPI_MAX))))
+			__ipi_send_mask(ipi_desc[op], cpumask_of(cpu));
+	}
+
+	return;
+
+no_zawrs:
+#endif
 	__ipi_send_mask(ipi_desc[op], mask);
 }
 
 static void send_ipi_single(int cpu, enum ipi_message_type op)
 {
+#ifdef CONFIG_RISCV_ZAWRS_IDLE
+	int val;
+
+	asm goto(ALTERNATIVE("j %l[no_zawrs]", "nop", 0, RISCV_ISA_EXT_ZAWRS, 1)
+			: : : : no_zawrs);
+
+	val = atomic_fetch_or_relaxed(BIT(op), per_cpu_ptr(&idle_ipi_mask, cpu));
+	if (likely(!(val & BIT(IPI_MAX))))
+		__ipi_send_mask(ipi_desc[op], cpumask_of(cpu));
+
+	return;
+
+no_zawrs:
+#endif
 	__ipi_send_mask(ipi_desc[op], cpumask_of(cpu));
 }
 
@@ -111,7 +135,7 @@ void arch_irq_work_raise(void)
 }
 #endif
 
-static irqreturn_t handle_IPI(int irq, void *data)
+irqreturn_t handle_IPI(int irq, void *data)
 {
 	int ipi = irq - ipi_virq_base;
 
@@ -323,3 +347,8 @@ void arch_smp_send_reschedule(int cpu)
 	send_ipi_single(cpu, IPI_RESCHEDULE);
 }
 EXPORT_SYMBOL_GPL(arch_smp_send_reschedule);
+
+int ipi_virq_base_get(void)
+{
+	return ipi_virq_base;
+}
-- 
2.20.1