[PATCH v4] Enable haltpoll for arm64

[PATCH v4] Enable haltpoll for arm64

[Mihai Carabas]

This patchset enables the usage of haltpoll governer on arm64. This is
specifically interesting for KVM guests by reducing the IPC latencies.

Here are some benchmarks without/with haltpoll for a KVM guest:

a) without haltpoll:
perf bench sched pipe
# Running 'sched/pipe' benchmark:
# Executed 1000000 pipe operations between two processes

     Total time: 8.138 [sec]

            8.138094 usecs/op
             122878 ops/sec

b) with haltpoll:
perf bench sched pipe
# Running 'sched/pipe' benchmark:
# Executed 1000000 pipe operations between two processes

     Total time: 5.003 [sec]

            5.003085 usecs/op
             199876 ops/sec

v4 changes from v3:
- change 7/8 per Rafael input: drop the parens and use ret for the final check
- add 8/8 which renames the guard for building poll_state

v3 changes from v2:
- fix 1/7 per Petr Mladek - remove ARCH_HAS_CPU_RELAX from arch/x86/Kconfig
- add Ack-by from Rafael Wysocki on 2/7

v2 changes from v1:
- added patch 7 where we change cpu_relax with smp_cond_load_relaxed per PeterZ
  (this improves by 50% at least the CPU cycles consumed in the tests above:
  10,716,881,137 now vs 14,503,014,257 before)
- removed the ifdef from patch 1 per RafaelW


Joao Martins (6):
  x86: Move ARCH_HAS_CPU_RELAX to arch
  x86/kvm: Move haltpoll_want() to be arch defined
  governors/haltpoll: Drop kvm_para_available() check
  arm64: Select ARCH_HAS_CPU_RELAX
  arm64: Define TIF_POLLING_NRFLAG
  cpuidle-haltpoll: ARM64 support

Mihai Carabas (2):
  cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed
  cpuidle: replace with HAS_CPU_RELAX with HAS_WANTS_IDLE_POLL

 arch/Kconfig                            |  3 +++
 arch/arm64/Kconfig                      |  1 +
 arch/arm64/include/asm/thread_info.h    |  6 ++++++
 arch/x86/Kconfig                        |  4 +---
 arch/x86/include/asm/cpuidle_haltpoll.h |  1 +
 arch/x86/kernel/kvm.c                   | 10 ++++++++++
 drivers/acpi/processor_idle.c           |  4 ++--
 drivers/cpuidle/Kconfig                 |  4 ++--
 drivers/cpuidle/Makefile                |  2 +-
 drivers/cpuidle/cpuidle-haltpoll.c      |  8 ++------
 drivers/cpuidle/governors/haltpoll.c    |  5 +----
 drivers/cpuidle/poll_state.c            | 15 ++++++++++-----
 include/linux/cpuidle.h                 |  2 +-
 include/linux/cpuidle_haltpoll.h        |  5 +++++
 14 files changed, 46 insertions(+), 24 deletions(-)

-- 
1.8.3.1

[PATCH v4 1/8] x86: Move ARCH_HAS_CPU_RELAX to arch

[Mihai Carabas]

From: Joao Martins <joao.m.martins@oracle.com>

ARM64 is going to use it for haltpoll support (for poll-state)
so move the definition to be arch-agnostic and allow architectures
to override it.

Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
---
 arch/Kconfig     | 3 +++
 arch/x86/Kconfig | 4 +---
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/arch/Kconfig b/arch/Kconfig
index a5af0edd3eb8..5b2e8a88853c 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -1363,6 +1363,9 @@ config RELR
 config ARCH_HAS_MEM_ENCRYPT
 	bool
 
+config ARCH_HAS_CPU_RELAX
+	bool
+
 config ARCH_HAS_CC_PLATFORM
 	bool
 
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 5edec175b9bf..8c4312133832 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -73,6 +73,7 @@ config X86
 	select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
 	select ARCH_HAS_CPU_FINALIZE_INIT
 	select ARCH_HAS_CPU_PASID		if IOMMU_SVA
+	select ARCH_HAS_CPU_RELAX
 	select ARCH_HAS_CURRENT_STACK_POINTER
 	select ARCH_HAS_DEBUG_VIRTUAL
 	select ARCH_HAS_DEBUG_VM_PGTABLE	if !X86_PAE
@@ -367,9 +368,6 @@ config ARCH_MAY_HAVE_PC_FDC
 config GENERIC_CALIBRATE_DELAY
 	def_bool y
 
-config ARCH_HAS_CPU_RELAX
-	def_bool y
-
 config ARCH_HIBERNATION_POSSIBLE
 	def_bool y
 
-- 
1.8.3.1

[PATCH v4 2/8] x86/kvm: Move haltpoll_want() to be arch defined

[Mihai Carabas]

From: Joao Martins <joao.m.martins@oracle.com>

Right now, kvm_para_has_hint(KVM_HINTS_REALTIME) is x86 only, and so in the
pursuit of making cpuidle-haltpoll arch independent, move the check for
haltpoll enablement to be defined per architecture. Same thing for
boot_option_idle_override. To that end, add a arch_haltpoll_want() and move the
check there.

Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
Signed-off-by: Ankur Arora <ankur.a.arora@oracle.com>
Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
Acked-by: Rafael J. Wysocki <rafael@kernel.org>
---
 arch/x86/include/asm/cpuidle_haltpoll.h |  1 +
 arch/x86/kernel/kvm.c                   | 10 ++++++++++
 drivers/cpuidle/cpuidle-haltpoll.c      |  8 ++------
 include/linux/cpuidle_haltpoll.h        |  5 +++++
 4 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/arch/x86/include/asm/cpuidle_haltpoll.h b/arch/x86/include/asm/cpuidle_haltpoll.h
index c8b39c6716ff..2c5a53ce266f 100644
--- a/arch/x86/include/asm/cpuidle_haltpoll.h
+++ b/arch/x86/include/asm/cpuidle_haltpoll.h
@@ -4,5 +4,6 @@
 
 void arch_haltpoll_enable(unsigned int cpu);
 void arch_haltpoll_disable(unsigned int cpu);
+bool arch_haltpoll_want(void);
 
 #endif
diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c
index 428ee74002e1..259212eb478d 100644
--- a/arch/x86/kernel/kvm.c
+++ b/arch/x86/kernel/kvm.c
@@ -1150,4 +1150,14 @@ void arch_haltpoll_disable(unsigned int cpu)
 	smp_call_function_single(cpu, kvm_enable_host_haltpoll, NULL, 1);
 }
 EXPORT_SYMBOL_GPL(arch_haltpoll_disable);
+
+bool arch_haltpoll_want(void)
+{
+	/* Do not load haltpoll if idle= is passed */
+	if (boot_option_idle_override != IDLE_NO_OVERRIDE)
+		return false;
+
+	return kvm_para_has_hint(KVM_HINTS_REALTIME);
+}
+EXPORT_SYMBOL_GPL(arch_haltpoll_want);
 #endif
diff --git a/drivers/cpuidle/cpuidle-haltpoll.c b/drivers/cpuidle/cpuidle-haltpoll.c
index d8515d5c0853..d68550270802 100644
--- a/drivers/cpuidle/cpuidle-haltpoll.c
+++ b/drivers/cpuidle/cpuidle-haltpoll.c
@@ -95,7 +95,7 @@ static void haltpoll_uninit(void)
 
 static bool haltpoll_want(void)
 {
-	return kvm_para_has_hint(KVM_HINTS_REALTIME) || force;
+	return (kvm_para_available() && arch_haltpoll_want()) || force;
 }
 
 static int __init haltpoll_init(void)
@@ -103,11 +103,7 @@ static int __init haltpoll_init(void)
 	int ret;
 	struct cpuidle_driver *drv = &haltpoll_driver;
 
-	/* Do not load haltpoll if idle= is passed */
-	if (boot_option_idle_override != IDLE_NO_OVERRIDE)
-		return -ENODEV;
-
-	if (!kvm_para_available() || !haltpoll_want())
+	if (!haltpoll_want())
 		return -ENODEV;
 
 	cpuidle_poll_state_init(drv);
diff --git a/include/linux/cpuidle_haltpoll.h b/include/linux/cpuidle_haltpoll.h
index d50c1e0411a2..bae68a6603e3 100644
--- a/include/linux/cpuidle_haltpoll.h
+++ b/include/linux/cpuidle_haltpoll.h
@@ -12,5 +12,10 @@ static inline void arch_haltpoll_enable(unsigned int cpu)
 static inline void arch_haltpoll_disable(unsigned int cpu)
 {
 }
+
+static inline bool arch_haltpoll_want(void)
+{
+	return false;
+}
 #endif
 #endif
-- 
1.8.3.1

[PATCH v4 3/8] governors/haltpoll: Drop kvm_para_available() check

[Mihai Carabas]

From: Joao Martins <joao.m.martins@oracle.com>

This is duplicated already in the haltpoll idle driver,
and there's no need to re-check KVM guest availability in
the governor.

Either guests uses the module which explicitly selects this
governor, and given that it has the lowest rating of all governors
(menu=20,teo=19,ladder=10/25,haltpoll=9) means that unless it's
the only one compiled in, it won't be selected.

Dropping such check also allows to test haltpoll in baremetal.

Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
Acked-by: Rafael J. Wysocki <rafael@kernel.org>
---
 drivers/cpuidle/governors/haltpoll.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/drivers/cpuidle/governors/haltpoll.c b/drivers/cpuidle/governors/haltpoll.c
index 1dff3a52917d..c9b69651d377 100644
--- a/drivers/cpuidle/governors/haltpoll.c
+++ b/drivers/cpuidle/governors/haltpoll.c
@@ -143,10 +143,7 @@ static int haltpoll_enable_device(struct cpuidle_driver *drv,
 
 static int __init init_haltpoll(void)
 {
-	if (kvm_para_available())
-		return cpuidle_register_governor(&haltpoll_governor);
-
-	return 0;
+	return cpuidle_register_governor(&haltpoll_governor);
 }
 
 postcore_initcall(init_haltpoll);
-- 
1.8.3.1

[PATCH v4 4/8] arm64: Select ARCH_HAS_CPU_RELAX

[Mihai Carabas]

From: Joao Martins <joao.m.martins@oracle.com>

ARCH_HAS_CPU_RELAX controls the build of poll-state, so select it from ARM64
kconfig.

Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
---
 arch/arm64/Kconfig | 1 +
 1 file changed, 1 insertion(+)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index aa7c1d435139..bc628a3165eb 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -107,6 +107,7 @@ config ARM64
 	select ARCH_WANT_LD_ORPHAN_WARN
 	select ARCH_WANTS_NO_INSTR
 	select ARCH_WANTS_THP_SWAP if ARM64_4K_PAGES
+	select ARCH_HAS_CPU_RELAX
 	select ARCH_HAS_UBSAN_SANITIZE_ALL
 	select ARM_AMBA
 	select ARM_ARCH_TIMER
-- 
1.8.3.1

[PATCH v4 5/8] arm64: Define TIF_POLLING_NRFLAG

[Mihai Carabas]

From: Joao Martins <joao.m.martins@oracle.com>

The default idle method for arm64 is WFI and it therefore
unconditionally requires the reschedule interrupt when idle.

Commit 842514849a61 ("arm64: Remove TIF_POLLING_NRFLAG") had
reverted it because WFI was the only idle method. ARM64 support
for haltpoll means that poll_idle() polls for TIF_POLLING_NRFLAG,
so define on arm64 *only if* haltpoll is built, using the same bit.

Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
---
 arch/arm64/include/asm/thread_info.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/arch/arm64/include/asm/thread_info.h b/arch/arm64/include/asm/thread_info.h
index e72a3bf9e563..72273a2168fa 100644
--- a/arch/arm64/include/asm/thread_info.h
+++ b/arch/arm64/include/asm/thread_info.h
@@ -69,6 +69,9 @@ struct thread_info {
 #define TIF_SYSCALL_TRACEPOINT	10	/* syscall tracepoint for ftrace */
 #define TIF_SECCOMP		11	/* syscall secure computing */
 #define TIF_SYSCALL_EMU		12	/* syscall emulation active */
+#if IS_ENABLED(CONFIG_HALTPOLL_CPUIDLE) || IS_ENABLED(CONFIG_HALTPOLL_CPUIDLE_MODULE)
+#define TIF_POLLING_NRFLAG      16      /* poll_idle() polls TIF_NEED_RESCHED */
+#endif
 #define TIF_MEMDIE		18	/* is terminating due to OOM killer */
 #define TIF_FREEZE		19
 #define TIF_RESTORE_SIGMASK	20
@@ -91,6 +94,9 @@ struct thread_info {
 #define _TIF_SYSCALL_TRACEPOINT	(1 << TIF_SYSCALL_TRACEPOINT)
 #define _TIF_SECCOMP		(1 << TIF_SECCOMP)
 #define _TIF_SYSCALL_EMU	(1 << TIF_SYSCALL_EMU)
+#if IS_ENABLED(CONFIG_HALTPOLL_CPUIDLE) || IS_ENABLED(CONFIG_HALTPOLL_CPUIDLE_MODULE)
+#define _TIF_POLLING_NRFLAG	(1 << TIF_POLLING_NRFLAG)
+#endif
 #define _TIF_UPROBE		(1 << TIF_UPROBE)
 #define _TIF_SINGLESTEP		(1 << TIF_SINGLESTEP)
 #define _TIF_32BIT		(1 << TIF_32BIT)
-- 
1.8.3.1

[PATCH v4 6/8] cpuidle-haltpoll: ARM64 support

[Mihai Carabas]

From: Joao Martins <joao.m.martins@oracle.com>

To test whether it's a guest or not for the default cases, the haltpoll
driver uses the kvm_para* helpers to find out if it's a guest or not.

ARM64 doesn't have or defined any of these, so it remains disabled on
the default. Although it allows to be force-loaded.

Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
---
 drivers/cpuidle/Kconfig | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig
index cac5997dca50..067927eda466 100644
--- a/drivers/cpuidle/Kconfig
+++ b/drivers/cpuidle/Kconfig
@@ -35,7 +35,7 @@ config CPU_IDLE_GOV_TEO
 
 config CPU_IDLE_GOV_HALTPOLL
 	bool "Haltpoll governor (for virtualized systems)"
-	depends on KVM_GUEST
+	depends on (X86 && KVM_GUEST) || ARM64
 	help
 	  This governor implements haltpoll idle state selection, to be
 	  used in conjunction with the haltpoll cpuidle driver, allowing
@@ -73,7 +73,7 @@ endmenu
 
 config HALTPOLL_CPUIDLE
 	tristate "Halt poll cpuidle driver"
-	depends on X86 && KVM_GUEST
+	depends on (X86 && KVM_GUEST) || ARM64
 	select CPU_IDLE_GOV_HALTPOLL
 	default y
 	help
-- 
1.8.3.1

RE: [PATCH v4 6/8] cpuidle-haltpoll: ARM64 support

[Tomohiro Misono (Fujitsu)]

Hi,
> Subject: [PATCH v4 6/8] cpuidle-haltpoll: ARM64 support
> 
> From: Joao Martins <joao.m.martins@oracle.com>
> 
> To test whether it's a guest or not for the default cases, the haltpoll
> driver uses the kvm_para* helpers to find out if it's a guest or not.
> 
> ARM64 doesn't have or defined any of these, so it remains disabled on
> the default. Although it allows to be force-loaded.
> 
> Signed-off-by: Joao Martins <joao.m.martins@oracle.com>
> Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
> ---
>  drivers/cpuidle/Kconfig | 4 ++--
>  1 file changed, 2 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig
> index cac5997dca50..067927eda466 100644
> --- a/drivers/cpuidle/Kconfig
> +++ b/drivers/cpuidle/Kconfig
> @@ -35,7 +35,7 @@ config CPU_IDLE_GOV_TEO
> 
>  config CPU_IDLE_GOV_HALTPOLL
>  	bool "Haltpoll governor (for virtualized systems)"
> -	depends on KVM_GUEST
> +	depends on (X86 && KVM_GUEST) || ARM64
>  	help
>  	  This governor implements haltpoll idle state selection, to be
>  	  used in conjunction with the haltpoll cpuidle driver, allowing
> @@ -73,7 +73,7 @@ endmenu
> 
>  config HALTPOLL_CPUIDLE
>  	tristate "Halt poll cpuidle driver"

I noticed that to build as a module, arch_cpu_idle needs to
be exported in arch/arm64/kernel/idle.c like x86.

Regards,
Tomohiro

> -	depends on X86 && KVM_GUEST
> +	depends on (X86 && KVM_GUEST) || ARM64
>  	select CPU_IDLE_GOV_HALTPOLL
>  	default y
>  	help
> --
> 1.8.3.1
> 
> 
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

[PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed

[Mihai Carabas]

cpu_relax on ARM64 does a simple "yield". Thus we replace it with
smp_cond_load_relaxed which basically does a "wfe".

Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
---
 drivers/cpuidle/poll_state.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
index 9b6d90a72601..1e45be906e72 100644
--- a/drivers/cpuidle/poll_state.c
+++ b/drivers/cpuidle/poll_state.c
@@ -13,6 +13,7 @@
 static int __cpuidle poll_idle(struct cpuidle_device *dev,
 			       struct cpuidle_driver *drv, int index)
 {
+	unsigned long ret;
 	u64 time_start;
 
 	time_start = local_clock_noinstr();
@@ -26,12 +27,16 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
 
 		limit = cpuidle_poll_time(drv, dev);
 
-		while (!need_resched()) {
-			cpu_relax();
-			if (loop_count++ < POLL_IDLE_RELAX_COUNT)
-				continue;
-
+		for (;;) {
 			loop_count = 0;
+
+			ret = smp_cond_load_relaxed(&current_thread_info()->flags,
+						    VAL & _TIF_NEED_RESCHED ||
+						    loop_count++ >= POLL_IDLE_RELAX_COUNT);
+
+			if (!(ret & _TIF_NEED_RESCHED))
+				break;
+
 			if (local_clock_noinstr() - time_start > limit) {
 				dev->poll_time_limit = true;
 				break;
-- 
1.8.3.1

RE: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed

[Tomohiro Misono (Fujitsu)]

Hi,
> Subject: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed
> 
> cpu_relax on ARM64 does a simple "yield". Thus we replace it with
> smp_cond_load_relaxed which basically does a "wfe".
> 
> Suggested-by: Peter Zijlstra <peterz@infradead.org>
> Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
> ---
>  drivers/cpuidle/poll_state.c | 15 ++++++++++-----
>  1 file changed, 10 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
> index 9b6d90a72601..1e45be906e72 100644
> --- a/drivers/cpuidle/poll_state.c
> +++ b/drivers/cpuidle/poll_state.c
> @@ -13,6 +13,7 @@
>  static int __cpuidle poll_idle(struct cpuidle_device *dev,
>  			       struct cpuidle_driver *drv, int index)
>  {
> +	unsigned long ret;
>  	u64 time_start;
> 
>  	time_start = local_clock_noinstr();
> @@ -26,12 +27,16 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
> 
>  		limit = cpuidle_poll_time(drv, dev);
> 
> -		while (!need_resched()) {
> -			cpu_relax();
> -			if (loop_count++ < POLL_IDLE_RELAX_COUNT)
> -				continue;
> -
> +		for (;;) {
>  			loop_count = 0;
> +
> +			ret = smp_cond_load_relaxed(&current_thread_info()->flags,
> +						    VAL & _TIF_NEED_RESCHED ||
> +						    loop_count++ >= POLL_IDLE_RELAX_COUNT);
> +
> +			if (!(ret & _TIF_NEED_RESCHED))
> +				break;

Should this be "if (ret & _TIF_NEED_RESCHED) since we want to break here
if the flag is set, or am I misunderstood?

Regards,
Tomohiro

> +
>  			if (local_clock_noinstr() - time_start > limit) {
>  				dev->poll_time_limit = true;
>  				break;
> --
> 1.8.3.1
> 
> 
> _______________________________________________
> linux-arm-kernel mailing list
> linux-arm-kernel@lists.infradead.org
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

RE: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed

[Ankur Arora]

Tomohiro Misono (Fujitsu) <misono.tomohiro@fujitsu.com> writes:

> Hi,
> > Subject: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed
> > 
> > cpu_relax on ARM64 does a simple "yield". Thus we replace it with
> > smp_cond_load_relaxed which basically does a "wfe".
> >
> > Suggested-by: Peter Zijlstra <peterz@infradead.org>
> > Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
> > ---
> >  drivers/cpuidle/poll_state.c | 15 ++++++++++-----
> >  1 file changed, 10 insertions(+), 5 deletions(-)
> >
> > diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
> > index 9b6d90a72601..1e45be906e72 100644
> > --- a/drivers/cpuidle/poll_state.c
> > +++ b/drivers/cpuidle/poll_state.c
> > @@ -13,6 +13,7 @@
> >  static int __cpuidle poll_idle(struct cpuidle_device *dev,
> >                              struct cpuidle_driver *drv, int index)
> >  {
> > +     unsigned long ret;
> >       u64 time_start;
> >
> >       time_start = local_clock_noinstr();
> > @@ -26,12 +27,16 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
> >
> >               limit = cpuidle_poll_time(drv, dev);
> >
> > -             while (!need_resched()) {
> > -                     cpu_relax();
> > -                     if (loop_count++ < POLL_IDLE_RELAX_COUNT)
> > -                             continue;
> > -
> > +             for (;;) {
> >                       loop_count = 0;
> > +
> > +                     ret = smp_cond_load_relaxed(&current_thread_info()->flags,
> > +                                                 VAL & _TIF_NEED_RESCHED ||
> > +                                                 loop_count++ >= POLL_IDLE_RELAX_COUNT);
> > +
> > +                     if (!(ret & _TIF_NEED_RESCHED))
> > +                             break;
> 
> Should this be "if (ret & _TIF_NEED_RESCHED) since we want to break here
> if the flag is set, or am I misunderstood?

Yeah, you are right. The check is inverted.

I'll be re-spinning this series. Will fix. Though, it probably makes sense
to just keep the original "while (!need_resched())" check.

Thanks for the review.

--
ankur

Re: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed

[Okanovic, Haris]

On Thu, 2024-02-15 at 09:41 +0200, Mihai Carabas wrote:
> cpu_relax on ARM64 does a simple "yield". Thus we replace it with
> smp_cond_load_relaxed which basically does a "wfe".
> 
> Suggested-by: Peter Zijlstra <peterz@infradead.org>
> Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
> ---
>  drivers/cpuidle/poll_state.c | 15 ++++++++++-----
>  1 file changed, 10 insertions(+), 5 deletions(-)
> 
> diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
> index 9b6d90a72601..1e45be906e72 100644
> --- a/drivers/cpuidle/poll_state.c
> +++ b/drivers/cpuidle/poll_state.c
> @@ -13,6 +13,7 @@
>  static int __cpuidle poll_idle(struct cpuidle_device *dev,
>  			       struct cpuidle_driver *drv, int index)
>  {
> +	unsigned long ret;
>  	u64 time_start;
>  
>  	time_start = local_clock_noinstr();
> @@ -26,12 +27,16 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
>  
>  		limit = cpuidle_poll_time(drv, dev);
>  
> -		while (!need_resched()) {
> -			cpu_relax();
> -			if (loop_count++ < POLL_IDLE_RELAX_COUNT)
> -				continue;
> -
> +		for (;;) {
>  			loop_count = 0;
> +
> +			ret = smp_cond_load_relaxed(&current_thread_info()->flags,
> +						    VAL & _TIF_NEED_RESCHED ||
> +						    loop_count++ >= POLL_IDLE_RELAX_COUNT);

Is it necessary to repeat this 200 times with a wfe poll? Does kvm not
implement a timeout period?

Could you make it configurable? This patch improves certain workloads
on AWS Graviton instances as well, but blocks up to 6ms in 200 * 30us
increments before going to wfi, which is a bit excessive.

> +
> +			if (!(ret & _TIF_NEED_RESCHED))
> +				break;
> +
>  			if (local_clock_noinstr() - time_start > limit) {
>  				dev->poll_time_limit = true;
>  				break;

Re: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed

[Ankur Arora]

Okanovic, Haris <harisokn@amazon.com> writes:

> On Thu, 2024-02-15 at 09:41 +0200, Mihai Carabas wrote:
>> cpu_relax on ARM64 does a simple "yield". Thus we replace it with
>> smp_cond_load_relaxed which basically does a "wfe".
>>
>> Suggested-by: Peter Zijlstra <peterz@infradead.org>
>> Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
>> ---
>>  drivers/cpuidle/poll_state.c | 15 ++++++++++-----
>>  1 file changed, 10 insertions(+), 5 deletions(-)
>>
>> diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
>> index 9b6d90a72601..1e45be906e72 100644
>> --- a/drivers/cpuidle/poll_state.c
>> +++ b/drivers/cpuidle/poll_state.c
>> @@ -13,6 +13,7 @@
>>  static int __cpuidle poll_idle(struct cpuidle_device *dev,
>>  			       struct cpuidle_driver *drv, int index)
>>  {
>> +	unsigned long ret;
>>  	u64 time_start;
>>
>>  	time_start = local_clock_noinstr();
>> @@ -26,12 +27,16 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
>>
>>  		limit = cpuidle_poll_time(drv, dev);
>>
>> -		while (!need_resched()) {
>> -			cpu_relax();
>> -			if (loop_count++ < POLL_IDLE_RELAX_COUNT)
>> -				continue;
>> -
>> +		for (;;) {
>>  			loop_count = 0;
>> +
>> +			ret = smp_cond_load_relaxed(&current_thread_info()->flags,
>> +						    VAL & _TIF_NEED_RESCHED ||
>> +						    loop_count++ >= POLL_IDLE_RELAX_COUNT);
>
> Is it necessary to repeat this 200 times with a wfe poll?

The POLL_IDLE_RELAX_COUNT is there because on x86 each cpu_relax()
iteration is much shorter.

With WFE, it makes less sense.

> Does kvm not implement a timeout period?

Not yet, but it does become more useful after a WFE haltpoll is
available on ARM64.

Haltpoll does have a timeout, which you should be able to tune via
/sys/module/haltpoll/parameters/ but that, of course, won't help here.

> Could you make it configurable? This patch improves certain workloads
> on AWS Graviton instances as well, but blocks up to 6ms in 200 * 30us
> increments before going to wfi, which is a bit excessive.

Yeah, this looks like a problem. We could solve it by making it an
architectural parameter. Though I worry about ARM platforms with
much smaller default timeouts.
The other possibility is using WFET in the primitive, but then we
have that dependency and that's a bigger change.

Will address this in the next version.

Thanks for pointing this out.

--
ankur

Re: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed

[Okanovic, Haris]

On Fri, 2024-04-05 at 16:14 -0700, Ankur Arora wrote:
> CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
> 
> 
> 
> Okanovic, Haris <harisokn@amazon.com> writes:
> 
> > On Thu, 2024-02-15 at 09:41 +0200, Mihai Carabas wrote:
> > > cpu_relax on ARM64 does a simple "yield". Thus we replace it with
> > > smp_cond_load_relaxed which basically does a "wfe".
> > > 
> > > Suggested-by: Peter Zijlstra <peterz@infradead.org>
> > > Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
> > > ---
> > >  drivers/cpuidle/poll_state.c | 15 ++++++++++-----
> > >  1 file changed, 10 insertions(+), 5 deletions(-)
> > > 
> > > diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
> > > index 9b6d90a72601..1e45be906e72 100644
> > > --- a/drivers/cpuidle/poll_state.c
> > > +++ b/drivers/cpuidle/poll_state.c
> > > @@ -13,6 +13,7 @@
> > >  static int __cpuidle poll_idle(struct cpuidle_device *dev,
> > >                             struct cpuidle_driver *drv, int index)
> > >  {
> > > +    unsigned long ret;
> > >      u64 time_start;
> > > 
> > >      time_start = local_clock_noinstr();
> > > @@ -26,12 +27,16 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
> > > 
> > >              limit = cpuidle_poll_time(drv, dev);
> > > 
> > > -            while (!need_resched()) {
> > > -                    cpu_relax();
> > > -                    if (loop_count++ < POLL_IDLE_RELAX_COUNT)
> > > -                            continue;
> > > -
> > > +            for (;;) {
> > >                      loop_count = 0;
> > > +
> > > +                    ret = smp_cond_load_relaxed(&current_thread_info()->flags,
> > > +                                                VAL & _TIF_NEED_RESCHED ||
> > > +                                                loop_count++ >= POLL_IDLE_RELAX_COUNT);
> > 
> > Is it necessary to repeat this 200 times with a wfe poll?
> 
> The POLL_IDLE_RELAX_COUNT is there because on x86 each cpu_relax()
> iteration is much shorter.
> 
> With WFE, it makes less sense.
> 
> > Does kvm not implement a timeout period?
> 
> Not yet, but it does become more useful after a WFE haltpoll is
> available on ARM64.

Note that kvm conditionally traps WFE and WFI based on number of host
CPU tasks. VMs will sometimes see hardware behavior - potentially
polling for a long time before entering WFI.

https://elixir.bootlin.com/linux/latest/source/arch/arm64/kvm/arm.c#L459

> 
> Haltpoll does have a timeout, which you should be able to tune via
> /sys/module/haltpoll/parameters/ but that, of course, won't help here.
> 
> > Could you make it configurable? This patch improves certain workloads
> > on AWS Graviton instances as well, but blocks up to 6ms in 200 * 30us
> > increments before going to wfi, which is a bit excessive.
> 
> Yeah, this looks like a problem. We could solve it by making it an
> architectural parameter. Though I worry about ARM platforms with
> much smaller default timeouts.
> The other possibility is using WFET in the primitive, but then we
> have that dependency and that's a bigger change.

See arm64's delay() for inspiration:

https://elixir.bootlin.com/linux/v6.9-rc2/source/arch/arm64/lib/delay.c#L26

> 
> Will address this in the next version.
> 
> Thanks for pointing this out.
> 
> --
> ankur

Re: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed

[Ankur Arora]

Okanovic, Haris <harisokn@amazon.com> writes:

> On Fri, 2024-04-05 at 16:14 -0700, Ankur Arora wrote:
>> CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
>>
>>
>>
>> Okanovic, Haris <harisokn@amazon.com> writes:
>>
>> > On Thu, 2024-02-15 at 09:41 +0200, Mihai Carabas wrote:
>> > > cpu_relax on ARM64 does a simple "yield". Thus we replace it with
>> > > smp_cond_load_relaxed which basically does a "wfe".
>> > >
>> > > Suggested-by: Peter Zijlstra <peterz@infradead.org>
>> > > Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
>> > > ---
>> > >  drivers/cpuidle/poll_state.c | 15 ++++++++++-----
>> > >  1 file changed, 10 insertions(+), 5 deletions(-)
>> > >
>> > > diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
>> > > index 9b6d90a72601..1e45be906e72 100644
>> > > --- a/drivers/cpuidle/poll_state.c
>> > > +++ b/drivers/cpuidle/poll_state.c
>> > > @@ -13,6 +13,7 @@
>> > >  static int __cpuidle poll_idle(struct cpuidle_device *dev,
>> > >                             struct cpuidle_driver *drv, int index)
>> > >  {
>> > > +    unsigned long ret;
>> > >      u64 time_start;
>> > >
>> > >      time_start = local_clock_noinstr();
>> > > @@ -26,12 +27,16 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
>> > >
>> > >              limit = cpuidle_poll_time(drv, dev);
>> > >
>> > > -            while (!need_resched()) {
>> > > -                    cpu_relax();
>> > > -                    if (loop_count++ < POLL_IDLE_RELAX_COUNT)
>> > > -                            continue;
>> > > -
>> > > +            for (;;) {
>> > >                      loop_count = 0;
>> > > +
>> > > +                    ret = smp_cond_load_relaxed(&current_thread_info()->flags,
>> > > +                                                VAL & _TIF_NEED_RESCHED ||
>> > > +                                                loop_count++ >= POLL_IDLE_RELAX_COUNT);
>> >
>> > Is it necessary to repeat this 200 times with a wfe poll?
>>
>> The POLL_IDLE_RELAX_COUNT is there because on x86 each cpu_relax()
>> iteration is much shorter.
>>
>> With WFE, it makes less sense.
>>
>> > Does kvm not implement a timeout period?
>>
>> Not yet, but it does become more useful after a WFE haltpoll is
>> available on ARM64.
>
> Note that kvm conditionally traps WFE and WFI based on number of host
> CPU tasks. VMs will sometimes see hardware behavior - potentially
> polling for a long time before entering WFI.
>
> https://elixir.bootlin.com/linux/latest/source/arch/arm64/kvm/arm.c#L459

Yeah. There was a discussion on this
https://lore.kernel.org/lkml/871qc6qufy.fsf@oracle.com/.

>> Haltpoll does have a timeout, which you should be able to tune via
>> /sys/module/haltpoll/parameters/ but that, of course, won't help here.
>>
>> > Could you make it configurable? This patch improves certain workloads
>> > on AWS Graviton instances as well, but blocks up to 6ms in 200 * 30us
>> > increments before going to wfi, which is a bit excessive.
>>
>> Yeah, this looks like a problem. We could solve it by making it an
>> architectural parameter. Though I worry about ARM platforms with
>> much smaller default timeouts.
>> The other possibility is using WFET in the primitive, but then we
>> have that dependency and that's a bigger change.
>
> See arm64's delay() for inspiration:
>
> https://elixir.bootlin.com/linux/v6.9-rc2/source/arch/arm64/lib/delay.c#L26

Sure, that part is straight-forward enough. However, this will need a fallback
the case when WFET is not available. And, because this path is used on x86,
so we need a cross platform smp_cond*timeout(). Though given that the x86
version is based on cpu_relax() then that could just fold the sched_clock()
check in.

Maybe another place to do this would be by KVM forcing a WFE timeout. Arguably
that is needed regardless of whether we use a smp_cond*timeout() or not.

--
ankur

Re: [PATCH v4 7/8] cpuidle/poll_state: replace cpu_relax with smp_cond_load_relaxed

[Okanovic, Haris]

On Mon, 2024-04-08 at 11:46 -0700, Ankur Arora wrote:
> CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
> 
> 
> 
> Okanovic, Haris <harisokn@amazon.com> writes:
> 
> > On Fri, 2024-04-05 at 16:14 -0700, Ankur Arora wrote:
> > > CAUTION: This email originated from outside of the organization. Do not click links or open attachments unless you can confirm the sender and know the content is safe.
> > > 
> > > 
> > > 
> > > Okanovic, Haris <harisokn@amazon.com> writes:
> > > 
> > > > On Thu, 2024-02-15 at 09:41 +0200, Mihai Carabas wrote:
> > > > > cpu_relax on ARM64 does a simple "yield". Thus we replace it with
> > > > > smp_cond_load_relaxed which basically does a "wfe".
> > > > > 
> > > > > Suggested-by: Peter Zijlstra <peterz@infradead.org>
> > > > > Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
> > > > > ---
> > > > >  drivers/cpuidle/poll_state.c | 15 ++++++++++-----
> > > > >  1 file changed, 10 insertions(+), 5 deletions(-)
> > > > > 
> > > > > diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c
> > > > > index 9b6d90a72601..1e45be906e72 100644
> > > > > --- a/drivers/cpuidle/poll_state.c
> > > > > +++ b/drivers/cpuidle/poll_state.c
> > > > > @@ -13,6 +13,7 @@
> > > > >  static int __cpuidle poll_idle(struct cpuidle_device *dev,
> > > > >                             struct cpuidle_driver *drv, int index)
> > > > >  {
> > > > > +    unsigned long ret;
> > > > >      u64 time_start;
> > > > > 
> > > > >      time_start = local_clock_noinstr();
> > > > > @@ -26,12 +27,16 @@ static int __cpuidle poll_idle(struct cpuidle_device *dev,
> > > > > 
> > > > >              limit = cpuidle_poll_time(drv, dev);
> > > > > 
> > > > > -            while (!need_resched()) {
> > > > > -                    cpu_relax();
> > > > > -                    if (loop_count++ < POLL_IDLE_RELAX_COUNT)
> > > > > -                            continue;
> > > > > -
> > > > > +            for (;;) {
> > > > >                      loop_count = 0;
> > > > > +
> > > > > +                    ret = smp_cond_load_relaxed(&current_thread_info()->flags,
> > > > > +                                                VAL & _TIF_NEED_RESCHED ||
> > > > > +                                                loop_count++ >= POLL_IDLE_RELAX_COUNT);
> > > > 
> > > > Is it necessary to repeat this 200 times with a wfe poll?
> > > 
> > > The POLL_IDLE_RELAX_COUNT is there because on x86 each cpu_relax()
> > > iteration is much shorter.
> > > 
> > > With WFE, it makes less sense.
> > > 
> > > > Does kvm not implement a timeout period?
> > > 
> > > Not yet, but it does become more useful after a WFE haltpoll is
> > > available on ARM64.
> > 
> > Note that kvm conditionally traps WFE and WFI based on number of host
> > CPU tasks. VMs will sometimes see hardware behavior - potentially
> > polling for a long time before entering WFI.
> > 
> > https://elixir.bootlin.com/linux/latest/source/arch/arm64/kvm/arm.c#L459
> 
> Yeah. There was a discussion on this
> https://lore.kernel.org/lkml/871qc6qufy.fsf@oracle.com/.
> 
> > > Haltpoll does have a timeout, which you should be able to tune via
> > > /sys/module/haltpoll/parameters/ but that, of course, won't help here.
> > > 
> > > > Could you make it configurable? This patch improves certain workloads
> > > > on AWS Graviton instances as well, but blocks up to 6ms in 200 * 30us
> > > > increments before going to wfi, which is a bit excessive.
> > > 
> > > Yeah, this looks like a problem. We could solve it by making it an
> > > architectural parameter. Though I worry about ARM platforms with
> > > much smaller default timeouts.
> > > The other possibility is using WFET in the primitive, but then we
> > > have that dependency and that's a bigger change.
> > 
> > See arm64's delay() for inspiration:
> > 
> > https://elixir.bootlin.com/linux/v6.9-rc2/source/arch/arm64/lib/delay.c#L26
> 
> Sure, that part is straight-forward enough. However, this will need a fallback
> the case when WFET is not available. And, because this path is used on x86,
> so we need a cross platform smp_cond*timeout(). Though given that the x86
> version is based on cpu_relax() then that could just fold the sched_clock()
> check in.

I was trying to point out how delay() handles different configurations:
It prefers WFET when available, falls back to WFE when event stream is
available, and finally falls back to cpu_relax() as last resort. Same
logic can apply here. The x86 case can always use cpu_relax() fallback,
for same behavior as smp_cond_load_relaxed().

Re your concern about "ARM platforms with much smaller default
timeouts": You could do something different when arch_timer_get_rate()
is too small. Although I'm not sure this is a huge concern, given that
delay() doesn't seem to care in the WFE case.

-- Haris Okanovic

> 
> Maybe another place to do this would be by KVM forcing a WFE timeout. Arguably
> that is needed regardless of whether we use a smp_cond*timeout() or not.
> 
> --
> ankur

[PATCH v4 8/8] cpuidle: replace with HAS_CPU_RELAX with HAS_WANTS_IDLE_POLL

[Mihai Carabas]

Replace ARCH_HAS_CPU_RELAX with ARCH_WANTS_IDLE_POLL for clarity as it controls
the building of poll_state.

Suggested-by: Will Deacon <will@kernel.org>
Signed-off-by: Ankur arora <ankur.a.arora@oracle.com>
Signed-off-by: Mihai Carabas <mihai.carabas@oracle.com>
---
 arch/Kconfig                  | 2 +-
 arch/arm64/Kconfig            | 2 +-
 arch/x86/Kconfig              | 2 +-
 drivers/acpi/processor_idle.c | 4 ++--
 drivers/cpuidle/Makefile      | 2 +-
 include/linux/cpuidle.h       | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/arch/Kconfig b/arch/Kconfig
index 5b2e8a88853c..e7659a3a7d58 100644
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -1363,7 +1363,7 @@ config RELR
 config ARCH_HAS_MEM_ENCRYPT
 	bool
 
-config ARCH_HAS_CPU_RELAX
+config ARCH_WANTS_IDLE_POLL
 	bool
 
 config ARCH_HAS_CC_PLATFORM
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index bc628a3165eb..7c963f7c10e4 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -107,7 +107,7 @@ config ARM64
 	select ARCH_WANT_LD_ORPHAN_WARN
 	select ARCH_WANTS_NO_INSTR
 	select ARCH_WANTS_THP_SWAP if ARM64_4K_PAGES
-	select ARCH_HAS_CPU_RELAX
+	select ARCH_WANTS_IDLE_POLL
 	select ARCH_HAS_UBSAN_SANITIZE_ALL
 	select ARM_AMBA
 	select ARM_ARCH_TIMER
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 8c4312133832..90f5d16be8c0 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -73,7 +73,7 @@ config X86
 	select ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
 	select ARCH_HAS_CPU_FINALIZE_INIT
 	select ARCH_HAS_CPU_PASID		if IOMMU_SVA
-	select ARCH_HAS_CPU_RELAX
+	select ARCH_WANTS_IDLE_POLL
 	select ARCH_HAS_CURRENT_STACK_POINTER
 	select ARCH_HAS_DEBUG_VIRTUAL
 	select ARCH_HAS_DEBUG_VM_PGTABLE	if !X86_PAE
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
index 55437f5e0c3a..6a0a1f16a5c3 100644
--- a/drivers/acpi/processor_idle.c
+++ b/drivers/acpi/processor_idle.c
@@ -36,7 +36,7 @@
 #include <asm/cpu.h>
 #endif
 
-#define ACPI_IDLE_STATE_START	(IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX) ? 1 : 0)
+#define ACPI_IDLE_STATE_START	(IS_ENABLED(CONFIG_ARCH_WANTS_IDLE_POLL) ? 1 : 0)
 
 static unsigned int max_cstate __read_mostly = ACPI_PROCESSOR_MAX_POWER;
 module_param(max_cstate, uint, 0400);
@@ -787,7 +787,7 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr)
 	if (max_cstate == 0)
 		max_cstate = 1;
 
-	if (IS_ENABLED(CONFIG_ARCH_HAS_CPU_RELAX)) {
+	if (IS_ENABLED(CONFIG_ARCH_WANTS_IDLE_POLL)) {
 		cpuidle_poll_state_init(drv);
 		count = 1;
 	} else {
diff --git a/drivers/cpuidle/Makefile b/drivers/cpuidle/Makefile
index d103342b7cfc..23f48d99f0f2 100644
--- a/drivers/cpuidle/Makefile
+++ b/drivers/cpuidle/Makefile
@@ -7,7 +7,7 @@ obj-y += cpuidle.o driver.o governor.o sysfs.o governors/
 obj-$(CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED) += coupled.o
 obj-$(CONFIG_DT_IDLE_STATES)		  += dt_idle_states.o
 obj-$(CONFIG_DT_IDLE_GENPD)		  += dt_idle_genpd.o
-obj-$(CONFIG_ARCH_HAS_CPU_RELAX)	  += poll_state.o
+obj-$(CONFIG_ARCH_WANTS_IDLE_POLL)	  += poll_state.o
 obj-$(CONFIG_HALTPOLL_CPUIDLE)		  += cpuidle-haltpoll.o
 
 ##################################################################################
diff --git a/include/linux/cpuidle.h b/include/linux/cpuidle.h
index 3183aeb7f5b4..53e55a91d55d 100644
--- a/include/linux/cpuidle.h
+++ b/include/linux/cpuidle.h
@@ -275,7 +275,7 @@ static inline void cpuidle_coupled_parallel_barrier(struct cpuidle_device *dev,
 }
 #endif
 
-#if defined(CONFIG_CPU_IDLE) && defined(CONFIG_ARCH_HAS_CPU_RELAX)
+#if defined(CONFIG_CPU_IDLE) && defined(CONFIG_ARCH_WANTS_IDLE_POLL)
 void cpuidle_poll_state_init(struct cpuidle_driver *drv);
 #else
 static inline void cpuidle_poll_state_init(struct cpuidle_driver *drv) {}
-- 
1.8.3.1