[PATCH RFC 3/3] x86: use mwait for trigger API

[PATCH RFC 3/3] x86: use mwait for trigger API

[Jeremy Fitzhardinge]

If the cpu implements monitor/mwait, use it for the trigger API.

TODO: work out if any mwait hints are going to be useful here.

Signed-off-by: Jeremy Fitzhardinge <jeremy.fitzhardinge@citrix.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
Cc: Christian Borntraeger <borntraeger@de.ibm.com>
Cc: Rusty Russell <rusty@rustcorp.com.au>
---
 arch/x86/kernel/paravirt-spinlocks.c |   22 ++++++++++++
 arch/x86/kernel/trigger.c            |   59 ++++++++++++++++++++++++++++++++++
 include/asm-x86/trigger.h            |   27 +++++++++++++--
 3 files changed, 104 insertions(+), 4 deletions(-)

===================================================================
--- a/arch/x86/kernel/paravirt-spinlocks.c
+++ b/arch/x86/kernel/paravirt-spinlocks.c
@@ -7,6 +7,7 @@
 #include <linux/module.h>
 
 #include <asm/paravirt.h>
+#include <asm/cpufeature.h>
 
 static void default_spin_lock_flags(struct raw_spinlock *lock, unsigned long flags)
 {
@@ -41,3 +42,24 @@
 	pv_lock_ops.spin_unlock = __byte_spin_unlock;
 #endif
 }
+
+static int __init use_mwait_trigger(void)
+{
+	/*
+	 * If we're using normal native trigger operations and the cpu
+	 * supports mwait, then switch to using it.
+	 */
+	if (pv_lock_ops.trigger_wait == native_trigger_wait &&
+	    pv_lock_ops.trigger_reset == native_trigger_reset &&
+	    pv_lock_ops.trigger_kick == native_trigger_kick &&
+	    pv_lock_ops.trigger_finish == paravirt_nop &&
+	    boot_cpu_has(X86_FEATURE_MWAIT)) {
+		pv_lock_ops.trigger_reset = mwait_trigger_reset;
+		pv_lock_ops.trigger_wait = mwait_trigger_wait;
+		pv_lock_ops.trigger_kick = mwait_trigger_kick;
+		pv_lock_ops.trigger_finish = mwait_trigger_finish;
+	}
+
+	return 0;
+}
+early_initcall(use_mwait_trigger);
===================================================================
--- a/arch/x86/kernel/trigger.c
+++ b/arch/x86/kernel/trigger.c
@@ -1,5 +1,9 @@
 #include <linux/trigger.h>
 #include <linux/smp.h>
+#include <linux/percpu.h>
+#include <linux/irqflags.h>
+
+#include <asm/processor.h>
 
 void native_trigger_reset(trigger_t *t)
 {
@@ -20,4 +24,59 @@
 {
 	smp_wmb();
 	cpus_setall(t->cpus);
+	smp_wmb();
 }
+
+/*
+ * mwait requires interrupts to be disabled between monitor and mwait.
+ * On monitor, we disable interrupts and store the previous state
+ * here.  When mwaiting, we enable interrupts if they were originally
+ * enabled.
+ */
+static DEFINE_PER_CPU(unsigned long, mwait_saved_flags);
+
+void mwait_trigger_reset(trigger_t *t)
+{
+	unsigned long flags;
+
+	t->trigger = 0;
+
+	local_save_flags(flags);
+	__get_cpu_var(mwait_saved_flags) = flags;
+
+	__monitor(&t->trigger, 0, 0);
+}
+
+void mwait_trigger_wait(trigger_t *t)
+{
+	unsigned long flags = __get_cpu_var(mwait_saved_flags);
+	int cpu = smp_processor_id();
+
+	if (irqs_disabled_flags(flags)) {
+		while(!cpu_isset(cpu, t->cpus)) {
+			__mwait(0, 0);
+			barrier();
+			__monitor(&t->trigger, 0, 0);
+		}
+	} else {
+		while(!cpu_isset(cpu, t->cpus)) {
+			__sti_mwait(0, 0);
+			barrier();
+			local_irq_disable();
+			__monitor(&t->trigger, 0, 0);
+		}
+	}
+}
+
+void mwait_trigger_finish(trigger_t *t)
+{
+	local_irq_restore(__get_cpu_var(mwait_saved_flags));
+}
+
+void mwait_trigger_kick(trigger_t *t)
+{
+	cpus_setall(t->cpus);
+	smp_wmb();
+	t->trigger = 1;
+	smp_wmb();
+}
===================================================================
--- a/include/asm-x86/trigger.h
+++ b/include/asm-x86/trigger.h
@@ -1,5 +1,7 @@
 #ifndef _ASM_X86_TRIGGER_H
 #define _ASM_X86_TRIGGER_H
+
+#include <asm/cpufeature.h>
 
 #include <linux/cpumask.h>
 
@@ -30,27 +32,44 @@
 extern void native_trigger_wait(trigger_t *t);
 extern void native_trigger_kick(trigger_t *t);
 
+extern void mwait_trigger_reset(trigger_t *);
+extern void mwait_trigger_wait(trigger_t *);
+extern void mwait_trigger_finish(trigger_t *);
+extern void mwait_trigger_kick(trigger_t *);
+
 #ifdef CONFIG_PARAVIRT
 #include <asm/paravirt.h>
 #else  /* CONFIG_PARAVIRT */
 static inline void __raw_trigger_reset(trigger_t *t)
 {
-	native_trigger_reset(t);
+	if (boot_cpu_has(X86_FEATURE_MWAIT))
+		mwait_trigger_reset(t);
+	else
+		native_trigger_reset(t);
 }
 
 static inline void __raw_trigger_wait(trigger_t *t)
 {
-	native_trigger_wait(t);
+	if (boot_cpu_has(X86_FEATURE_MWAIT))
+		mwait_trigger_wait(t);
+	else
+		native_trigger_wait(t);
 }
 
 static inline void __raw_trigger_finish(trigger_t *t)
 {
-	native_trigger_finish(t);
+	if (boot_cpu_has(X86_FEATURE_MWAIT))
+		mwait_trigger_finish(t);
+	else
+		native_trigger_finish(t);
 }
 
 static inline void __raw_trigger_kick(trigger_t *t)
 {
-	native_trigger_kick(t);
+	if (boot_cpu_has(X86_FEATURE_MWAIT))
+		mwait_trigger_finish(t);
+	else
+		native_trigger_finish(t);
 }
 #endif	/* CONFIG_PARAVIRT */
 

Re: [PATCH RFC 3/3] x86: use mwait for trigger API

[Arjan van de Ven]

On Sat, 16 Aug 2008 09:34:26 -0700
Jeremy Fitzhardinge <jeremy@goop.org> wrote:

> If the cpu implements monitor/mwait, use it for the trigger API.
> 
> TODO: work out if any mwait hints are going to be useful here.


monitor/mwait is rather really expensive.. are we really sure we want
to use this?
(from an Intel cpu perspective the answer is very likely no; but I don't
know what AMD does here)


-- 
If you want to reach me at my work email, use arjan@linux.intel.com
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org

Re: [PATCH RFC 3/3] x86: use mwait for trigger API

[Jeremy Fitzhardinge]

Arjan van de Ven wrote:
> monitor/mwait is rather really expensive.. are we really sure we want
> to use this?
> (from an Intel cpu perspective the answer is very likely no; but I don't
> know what AMD does here)
>   

The intended use is when you're going to be waiting for a while (on the
order of microseconds or more).  In the Xen case, I use this to block
the vcpu if we pass a few iterations without the condition being true. 
While the mwait patch doesn't do this at present, it could.

    J

Re: [PATCH RFC 3/3] x86: use mwait for trigger API

[Arjan van de Ven]

On Sat, 16 Aug 2008 14:50:34 -0700
Jeremy Fitzhardinge <jeremy@goop.org> wrote:

> Arjan van de Ven wrote:
> > monitor/mwait is rather really expensive.. are we really sure we
> > want to use this?
> > (from an Intel cpu perspective the answer is very likely no; but I
> > don't know what AMD does here)
> >   
> 
> The intended use is when you're going to be waiting for a while (on
> the order of microseconds or more). 

well mwait really is not cheap, I'd not be surprised if it's in that
same order.

> In the Xen case, I use this to
> block the vcpu if we pass a few iterations without the condition
> being true. While the mwait patch doesn't do this at present, it
> could.

that's another hard one.. passing C-state hints into mwait needs ACPI
help; the BIOS tells us which mwait values are legal/valid at any point
in time.. but this gets tricky to put into these spinpletions.


-- 
If you want to reach me at my work email, use arjan@linux.intel.com
For development, discussion and tips for power savings, 
visit http://www.lesswatts.org

Re: [PATCH RFC 3/3] x86: use mwait for trigger API

[Christian Borntraeger]

Am Samstag, 16. August 2008 schrieb Jeremy Fitzhardinge:

Seems that this cant work. We never reset the t->cpus bits. That means we 
never mwait after a kick.

See:

> +void mwait_trigger_reset(trigger_t *t)
> +{
> +	unsigned long flags;
> +
> +	t->trigger = 0;
> +
> +	local_save_flags(flags);
> +	__get_cpu_var(mwait_saved_flags) = flags;
> +
> +	__monitor(&t->trigger, 0, 0);
> +}
> +
> +void mwait_trigger_wait(trigger_t *t)
> +{
> +	unsigned long flags = __get_cpu_var(mwait_saved_flags);
> +	int cpu = smp_processor_id();
> +
> +	if (irqs_disabled_flags(flags)) {
> +		while(!cpu_isset(cpu, t->cpus)) {

We check the bits here

> +			__mwait(0, 0);
> +			barrier();
> +			__monitor(&t->trigger, 0, 0);
> +		}
> +	} else {
> +		while(!cpu_isset(cpu, t->cpus)) {

and here

> +			__sti_mwait(0, 0);
> +			barrier();
> +			local_irq_disable();
> +			__monitor(&t->trigger, 0, 0);
> +		}
> +	}
> +}
> +
> +void mwait_trigger_finish(trigger_t *t)
> +{
> +	local_irq_restore(__get_cpu_var(mwait_saved_flags));
> +}
> +
> +void mwait_trigger_kick(trigger_t *t)
> +{
> +	cpus_setall(t->cpus);

We set the bits here.

> +	smp_wmb();
> +	t->trigger = 1;
> +	smp_wmb();
> +}

Nothing else.

Re: [PATCH RFC 3/3] x86: use mwait for trigger API

[Jeremy Fitzhardinge]

Christian Borntraeger wrote:
> Am Samstag, 16. August 2008 schrieb Jeremy Fitzhardinge:
>
> Seems that this cant work. We never reset the t->cpus bits. That means we 
> never mwait after a kick.
>   

Yeah, I have to admit I never tested this code (it was an RFC, after
all).  And after Arjan said that mwait is unusable, I didn't spend any
more effort on it.

> See:
>
>   
>> +void mwait_trigger_reset(trigger_t *t)
>> +{
>> +	unsigned long flags;
>> +
>> +	t->trigger = 0;
>> +
>> +	local_save_flags(flags);
>> +	__get_cpu_var(mwait_saved_flags) = flags;
>> +
>> +	__monitor(&t->trigger, 0, 0);
>> +}
>> +
>> +void mwait_trigger_wait(trigger_t *t)
>> +{
>> +	unsigned long flags = __get_cpu_var(mwait_saved_flags);
>> +	int cpu = smp_processor_id();
>> +
>> +	if (irqs_disabled_flags(flags)) {
>> +		while(!cpu_isset(cpu, t->cpus)) {
>>     
>
> We check the bits here
>
>   
>> +			__mwait(0, 0);
>> +			barrier();
>> +			__monitor(&t->trigger, 0, 0);
>> +		}
>> +	} else {
>> +		while(!cpu_isset(cpu, t->cpus)) {
>>     
>
> and here
>
>   
>> +			__sti_mwait(0, 0);
>> +			barrier();
>> +			local_irq_disable();
>> +			__monitor(&t->trigger, 0, 0);
>> +		}
>> +	}
>>     

Must have lost a line somewhere.  It's supposed to clear the bit here,
before wait returns.

    J