[PATCH 0/3] arm/arm64: rework mmu_enabled, for spinlock speedup

[PATCH 0/3] arm/arm64: rework mmu_enabled, for spinlock speedup

[Andrew Jones]

Andrew Jones (3):
  arm/arm64: Introduce mmu_disable
  arm/arm64: drop mmu_set_enabled
  arm/arm64: speed up spinlocks and atomic ops

 arm/cstart.S          |  9 +++++++++
 arm/cstart64.S        |  8 ++++++++
 lib/arm/asm/mmu-api.h |  9 +++++++--
 lib/arm/mmu.c         | 32 ++++++++++++++++++++++----------
 lib/arm/smp.c         |  1 -
 5 files changed, 46 insertions(+), 13 deletions(-)

-- 
2.4.3

[PATCH 1/3] arm/arm64: Introduce mmu_disable

[Andrew Jones]

Allow unit test cpus to disable the MMU. Why not? We want the
test framework to be as flexible as possible. Callers will have
to deal with the cache coherency fallout... Cache flush support
is still forthcoming to the framework though.

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 arm/cstart.S          | 9 +++++++++
 arm/cstart64.S        | 8 ++++++++
 lib/arm/asm/mmu-api.h | 1 +
 lib/arm/mmu.c         | 8 ++++++++
 4 files changed, 26 insertions(+)

diff --git a/arm/cstart.S b/arm/cstart.S
index 2b7f8f3d200ef..3943867d2f219 100644
--- a/arm/cstart.S
+++ b/arm/cstart.S
@@ -159,6 +159,15 @@ asm_mmu_enable:
 
 	mov     pc, lr
 
+.globl asm_mmu_disable
+asm_mmu_disable:
+	/* SCTLR */
+	mrc	p15, 0, r0, c1, c0, 0
+	bic	r0, #CR_M
+	mcr	p15, 0, r0, c1, c0, 0
+	isb
+	mov     pc, lr
+
 /*
  * Vector stubs
  * Simplified version of the Linux kernel implementation
diff --git a/arm/cstart64.S b/arm/cstart64.S
index cdda13c17af9e..44cff32d0f18e 100644
--- a/arm/cstart64.S
+++ b/arm/cstart64.S
@@ -146,6 +146,14 @@ asm_mmu_enable:
 
 	ret
 
+.globl asm_mmu_disable
+asm_mmu_disable:
+	mrs	x0, sctlr_el1
+	bic	x0, x0, SCTLR_EL1_M
+	msr	sctlr_el1, x0
+	isb
+	ret
+
 /*
  * Vectors
  * Adapted from arch/arm64/kernel/entry.S
diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
index 68dc707d67241..c46c4b08b14cc 100644
--- a/lib/arm/asm/mmu-api.h
+++ b/lib/arm/asm/mmu-api.h
@@ -4,6 +4,7 @@ extern pgd_t *mmu_idmap;
 extern bool mmu_enabled(void);
 extern void mmu_set_enabled(void);
 extern void mmu_enable(pgd_t *pgtable);
+extern void mmu_disable(void);
 extern void mmu_enable_idmap(void);
 extern void mmu_init_io_sect(pgd_t *pgtable, unsigned long virt_offset);
 extern void mmu_set_range_sect(pgd_t *pgtable, unsigned long virt_offset,
diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
index 732000a8eb088..5966b408cb455 100644
--- a/lib/arm/mmu.c
+++ b/lib/arm/mmu.c
@@ -35,6 +35,14 @@ void mmu_enable(pgd_t *pgtable)
 	mmu_set_enabled();
 }
 
+extern void asm_mmu_disable(void);
+void mmu_disable(void)
+{
+	struct thread_info *ti = current_thread_info();
+	cpumask_clear_cpu(ti->cpu, &mmu_enabled_cpumask);
+	asm_mmu_disable();
+}
+
 void mmu_set_range_ptes(pgd_t *pgtable, unsigned long virt_offset,
 			unsigned long phys_start, unsigned long phys_end,
 			pgprot_t prot)
-- 
2.4.3

Re: [PATCH 1/3] arm/arm64: Introduce mmu_disable

[Paolo Bonzini]

On 25/06/2015 20:45, Andrew Jones wrote:
> Allow unit test cpus to disable the MMU. Why not? We want the
> test framework to be as flexible as possible. Callers will have
> to deal with the cache coherency fallout... Cache flush support
> is still forthcoming to the framework though.
> 
> Signed-off-by: Andrew Jones <drjones@redhat.com>
> ---
>  arm/cstart.S          | 9 +++++++++
>  arm/cstart64.S        | 8 ++++++++
>  lib/arm/asm/mmu-api.h | 1 +
>  lib/arm/mmu.c         | 8 ++++++++
>  4 files changed, 26 insertions(+)
> 
> diff --git a/arm/cstart.S b/arm/cstart.S
> index 2b7f8f3d200ef..3943867d2f219 100644
> --- a/arm/cstart.S
> +++ b/arm/cstart.S
> @@ -159,6 +159,15 @@ asm_mmu_enable:
>  
>  	mov     pc, lr
>  
> +.globl asm_mmu_disable
> +asm_mmu_disable:
> +	/* SCTLR */
> +	mrc	p15, 0, r0, c1, c0, 0
> +	bic	r0, #CR_M
> +	mcr	p15, 0, r0, c1, c0, 0
> +	isb
> +	mov     pc, lr
> +
>  /*
>   * Vector stubs
>   * Simplified version of the Linux kernel implementation
> diff --git a/arm/cstart64.S b/arm/cstart64.S
> index cdda13c17af9e..44cff32d0f18e 100644
> --- a/arm/cstart64.S
> +++ b/arm/cstart64.S
> @@ -146,6 +146,14 @@ asm_mmu_enable:
>  
>  	ret
>  
> +.globl asm_mmu_disable
> +asm_mmu_disable:
> +	mrs	x0, sctlr_el1
> +	bic	x0, x0, SCTLR_EL1_M
> +	msr	sctlr_el1, x0
> +	isb
> +	ret
> +
>  /*
>   * Vectors
>   * Adapted from arch/arm64/kernel/entry.S
> diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
> index 68dc707d67241..c46c4b08b14cc 100644
> --- a/lib/arm/asm/mmu-api.h
> +++ b/lib/arm/asm/mmu-api.h
> @@ -4,6 +4,7 @@ extern pgd_t *mmu_idmap;
>  extern bool mmu_enabled(void);
>  extern void mmu_set_enabled(void);
>  extern void mmu_enable(pgd_t *pgtable);
> +extern void mmu_disable(void);
>  extern void mmu_enable_idmap(void);
>  extern void mmu_init_io_sect(pgd_t *pgtable, unsigned long virt_offset);
>  extern void mmu_set_range_sect(pgd_t *pgtable, unsigned long virt_offset,
> diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
> index 732000a8eb088..5966b408cb455 100644
> --- a/lib/arm/mmu.c
> +++ b/lib/arm/mmu.c
> @@ -35,6 +35,14 @@ void mmu_enable(pgd_t *pgtable)
>  	mmu_set_enabled();
>  }
>  
> +extern void asm_mmu_disable(void);
> +void mmu_disable(void)
> +{
> +	struct thread_info *ti = current_thread_info();
> +	cpumask_clear_cpu(ti->cpu, &mmu_enabled_cpumask);
> +	asm_mmu_disable();
> +}
> +
>  void mmu_set_range_ptes(pgd_t *pgtable, unsigned long virt_offset,
>  			unsigned long phys_start, unsigned long phys_end,
>  			pgprot_t prot)
> 

Applied, thanks.

Paolo

[PATCH 2/3] arm/arm64: drop mmu_set_enabled

[Andrew Jones]

The mmu is enabled automatically for all cpus, they must disable it
themselves if they don't want it on. Switch from managing a cpumask
of enabled cpus to one of disabled cpus. This allows us to remove
the mmu_set_enabled call from secondary_cinit, and the function all
together.

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 lib/arm/asm/mmu-api.h |  1 -
 lib/arm/mmu.c         | 21 ++++++++++-----------
 lib/arm/smp.c         |  1 -
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
index c46c4b08b14cc..12fdc57918c27 100644
--- a/lib/arm/asm/mmu-api.h
+++ b/lib/arm/asm/mmu-api.h
@@ -2,7 +2,6 @@
 #define __ASMARM_MMU_API_H_
 extern pgd_t *mmu_idmap;
 extern bool mmu_enabled(void);
-extern void mmu_set_enabled(void);
 extern void mmu_enable(pgd_t *pgtable);
 extern void mmu_disable(void);
 extern void mmu_enable_idmap(void);
diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
index 5966b408cb455..ad558e177dd1c 100644
--- a/lib/arm/mmu.c
+++ b/lib/arm/mmu.c
@@ -14,32 +14,31 @@ extern unsigned long etext;
 
 pgd_t *mmu_idmap;
 
-static cpumask_t mmu_enabled_cpumask;
+static cpumask_t mmu_disabled_cpumask;
+
 bool mmu_enabled(void)
 {
-	struct thread_info *ti = current_thread_info();
-	return cpumask_test_cpu(ti->cpu, &mmu_enabled_cpumask);
-}
+	int cpu = current_thread_info()->cpu;
 
-void mmu_set_enabled(void)
-{
-	struct thread_info *ti = current_thread_info();
-	cpumask_set_cpu(ti->cpu, &mmu_enabled_cpumask);
+	return !cpumask_test_cpu(cpu, &mmu_disabled_cpumask);
 }
 
 extern void asm_mmu_enable(phys_addr_t pgtable);
 void mmu_enable(pgd_t *pgtable)
 {
+	int cpu = current_thread_info()->cpu;
+
 	asm_mmu_enable(__pa(pgtable));
 	flush_tlb_all();
-	mmu_set_enabled();
+	cpumask_clear_cpu(cpu, &mmu_disabled_cpumask);
 }
 
 extern void asm_mmu_disable(void);
 void mmu_disable(void)
 {
-	struct thread_info *ti = current_thread_info();
-	cpumask_clear_cpu(ti->cpu, &mmu_enabled_cpumask);
+	int cpu = current_thread_info()->cpu;
+
+	cpumask_set_cpu(cpu, &mmu_disabled_cpumask);
 	asm_mmu_disable();
 }
 
diff --git a/lib/arm/smp.c b/lib/arm/smp.c
index f389ba6598faa..ca435dcd5f4a2 100644
--- a/lib/arm/smp.c
+++ b/lib/arm/smp.c
@@ -23,7 +23,6 @@ secondary_entry_fn secondary_cinit(void)
 	secondary_entry_fn entry;
 
 	thread_info_init(ti, 0);
-	mmu_set_enabled();
 
 	/*
 	 * Save secondary_data.entry locally to avoid opening a race
-- 
2.4.3

Re: [PATCH 2/3] arm/arm64: drop mmu_set_enabled

[Paolo Bonzini]

On 25/06/2015 20:45, Andrew Jones wrote:
> The mmu is enabled automatically for all cpus, they must disable it
> themselves if they don't want it on. Switch from managing a cpumask
> of enabled cpus to one of disabled cpus. This allows us to remove
> the mmu_set_enabled call from secondary_cinit, and the function all
> together.
> 
> Signed-off-by: Andrew Jones <drjones@redhat.com>
> ---
>  lib/arm/asm/mmu-api.h |  1 -
>  lib/arm/mmu.c         | 21 ++++++++++-----------
>  lib/arm/smp.c         |  1 -
>  3 files changed, 10 insertions(+), 13 deletions(-)
> 
> diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
> index c46c4b08b14cc..12fdc57918c27 100644
> --- a/lib/arm/asm/mmu-api.h
> +++ b/lib/arm/asm/mmu-api.h
> @@ -2,7 +2,6 @@
>  #define __ASMARM_MMU_API_H_
>  extern pgd_t *mmu_idmap;
>  extern bool mmu_enabled(void);
> -extern void mmu_set_enabled(void);
>  extern void mmu_enable(pgd_t *pgtable);
>  extern void mmu_disable(void);
>  extern void mmu_enable_idmap(void);
> diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
> index 5966b408cb455..ad558e177dd1c 100644
> --- a/lib/arm/mmu.c
> +++ b/lib/arm/mmu.c
> @@ -14,32 +14,31 @@ extern unsigned long etext;
>  
>  pgd_t *mmu_idmap;
>  
> -static cpumask_t mmu_enabled_cpumask;
> +static cpumask_t mmu_disabled_cpumask;
> +
>  bool mmu_enabled(void)
>  {
> -	struct thread_info *ti = current_thread_info();
> -	return cpumask_test_cpu(ti->cpu, &mmu_enabled_cpumask);
> -}
> +	int cpu = current_thread_info()->cpu;
>  
> -void mmu_set_enabled(void)
> -{
> -	struct thread_info *ti = current_thread_info();
> -	cpumask_set_cpu(ti->cpu, &mmu_enabled_cpumask);
> +	return !cpumask_test_cpu(cpu, &mmu_disabled_cpumask);
>  }
>  
>  extern void asm_mmu_enable(phys_addr_t pgtable);
>  void mmu_enable(pgd_t *pgtable)
>  {
> +	int cpu = current_thread_info()->cpu;
> +
>  	asm_mmu_enable(__pa(pgtable));
>  	flush_tlb_all();
> -	mmu_set_enabled();
> +	cpumask_clear_cpu(cpu, &mmu_disabled_cpumask);
>  }
>  
>  extern void asm_mmu_disable(void);
>  void mmu_disable(void)
>  {
> -	struct thread_info *ti = current_thread_info();
> -	cpumask_clear_cpu(ti->cpu, &mmu_enabled_cpumask);
> +	int cpu = current_thread_info()->cpu;
> +
> +	cpumask_set_cpu(cpu, &mmu_disabled_cpumask);
>  	asm_mmu_disable();
>  }
>  
> diff --git a/lib/arm/smp.c b/lib/arm/smp.c
> index f389ba6598faa..ca435dcd5f4a2 100644
> --- a/lib/arm/smp.c
> +++ b/lib/arm/smp.c
> @@ -23,7 +23,6 @@ secondary_entry_fn secondary_cinit(void)
>  	secondary_entry_fn entry;
>  
>  	thread_info_init(ti, 0);
> -	mmu_set_enabled();
>  
>  	/*
>  	 * Save secondary_data.entry locally to avoid opening a race
> 

This is needed too on a Cubietruck:

diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
index 12fdc57..73bac70 100644
--- a/lib/arm/asm/mmu-api.h
+++ b/lib/arm/asm/mmu-api.h
@@ -3,6 +3,7 @@
 extern pgd_t *mmu_idmap;
 extern bool mmu_enabled(void);
 extern void mmu_enable(pgd_t *pgtable);
+extern void mmu_mark_disabled(int cpu);
 extern void mmu_disable(void);
 extern void mmu_enable_idmap(void);
 extern void mmu_init_io_sect(pgd_t *pgtable, unsigned long virt_offset);
diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
index ad558e1..8ff659b 100644
--- a/lib/arm/mmu.c
+++ b/lib/arm/mmu.c
@@ -14,7 +14,8 @@ extern unsigned long etext;

 pgd_t *mmu_idmap;

-static cpumask_t mmu_disabled_cpumask;
+/* CPU 0 starts with disabled MMU */
+static cpumask_t mmu_disabled_cpumask = { {1} };

 bool mmu_enabled(void)
 {
@@ -33,6 +34,11 @@ void mmu_enable(pgd_t *pgtable)
 	cpumask_clear_cpu(cpu, &mmu_disabled_cpumask);
 }

+void mmu_mark_disabled(int cpu)
+{
+	cpumask_set_cpu(cpu, &mmu_disabled_cpumask);
+}
+
 extern void asm_mmu_disable(void);
 void mmu_disable(void)
 {
diff --git a/lib/arm/psci.c b/lib/arm/psci.c
index aca8885..c4469c9 100644
--- a/lib/arm/psci.c
+++ b/lib/arm/psci.c
@@ -9,6 +9,7 @@
 #include <asm/psci.h>
 #include <asm/setup.h>
 #include <asm/page.h>
+#include <asm/mmu-api.h>

 #define T PSCI_INVOKE_ARG_TYPE
 __attribute__((noinline))
@@ -29,6 +30,7 @@ int psci_cpu_on(unsigned long cpuid, unsigned long
entry_point)
 extern void secondary_entry(void);
 int cpu_psci_cpu_boot(unsigned int cpu)
 {
+	mmu_mark_disabled(cpu);
 	int err = psci_cpu_on(cpus[cpu], __pa(secondary_entry));
 	if (err)
 		printf("failed to boot CPU%d (%d)\n", cpu, err);

Re: [PATCH 2/3] arm/arm64: drop mmu_set_enabled

[Andrew Jones]

On Fri, Jul 03, 2015 at 07:39:13PM +0200, Paolo Bonzini wrote:
> 
> 
> On 25/06/2015 20:45, Andrew Jones wrote:
> > The mmu is enabled automatically for all cpus, they must disable it
> > themselves if they don't want it on. Switch from managing a cpumask
> > of enabled cpus to one of disabled cpus. This allows us to remove
> > the mmu_set_enabled call from secondary_cinit, and the function all
> > together.
> > 
> > Signed-off-by: Andrew Jones <drjones@redhat.com>
> > ---
> >  lib/arm/asm/mmu-api.h |  1 -
> >  lib/arm/mmu.c         | 21 ++++++++++-----------
> >  lib/arm/smp.c         |  1 -
> >  3 files changed, 10 insertions(+), 13 deletions(-)
> > 
> > diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
> > index c46c4b08b14cc..12fdc57918c27 100644
> > --- a/lib/arm/asm/mmu-api.h
> > +++ b/lib/arm/asm/mmu-api.h
> > @@ -2,7 +2,6 @@
> >  #define __ASMARM_MMU_API_H_
> >  extern pgd_t *mmu_idmap;
> >  extern bool mmu_enabled(void);
> > -extern void mmu_set_enabled(void);
> >  extern void mmu_enable(pgd_t *pgtable);
> >  extern void mmu_disable(void);
> >  extern void mmu_enable_idmap(void);
> > diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
> > index 5966b408cb455..ad558e177dd1c 100644
> > --- a/lib/arm/mmu.c
> > +++ b/lib/arm/mmu.c
> > @@ -14,32 +14,31 @@ extern unsigned long etext;
> >  
> >  pgd_t *mmu_idmap;
> >  
> > -static cpumask_t mmu_enabled_cpumask;
> > +static cpumask_t mmu_disabled_cpumask;
> > +
> >  bool mmu_enabled(void)
> >  {
> > -	struct thread_info *ti = current_thread_info();
> > -	return cpumask_test_cpu(ti->cpu, &mmu_enabled_cpumask);
> > -}
> > +	int cpu = current_thread_info()->cpu;
> >  
> > -void mmu_set_enabled(void)
> > -{
> > -	struct thread_info *ti = current_thread_info();
> > -	cpumask_set_cpu(ti->cpu, &mmu_enabled_cpumask);
> > +	return !cpumask_test_cpu(cpu, &mmu_disabled_cpumask);
> >  }
> >  
> >  extern void asm_mmu_enable(phys_addr_t pgtable);
> >  void mmu_enable(pgd_t *pgtable)
> >  {
> > +	int cpu = current_thread_info()->cpu;
> > +
> >  	asm_mmu_enable(__pa(pgtable));
> >  	flush_tlb_all();
> > -	mmu_set_enabled();
> > +	cpumask_clear_cpu(cpu, &mmu_disabled_cpumask);
> >  }
> >  
> >  extern void asm_mmu_disable(void);
> >  void mmu_disable(void)
> >  {
> > -	struct thread_info *ti = current_thread_info();
> > -	cpumask_clear_cpu(ti->cpu, &mmu_enabled_cpumask);
> > +	int cpu = current_thread_info()->cpu;
> > +
> > +	cpumask_set_cpu(cpu, &mmu_disabled_cpumask);
> >  	asm_mmu_disable();
> >  }
> >  
> > diff --git a/lib/arm/smp.c b/lib/arm/smp.c
> > index f389ba6598faa..ca435dcd5f4a2 100644
> > --- a/lib/arm/smp.c
> > +++ b/lib/arm/smp.c
> > @@ -23,7 +23,6 @@ secondary_entry_fn secondary_cinit(void)
> >  	secondary_entry_fn entry;
> >  
> >  	thread_info_init(ti, 0);
> > -	mmu_set_enabled();
> >  
> >  	/*
> >  	 * Save secondary_data.entry locally to avoid opening a race
> > 
> 
> This is needed too on a Cubietruck:

Oh right! We definitely need to return false from mmu_enabled
for cubietruck on early boot (that was the whole reason to
introduce mmu_enabled()). Thanks for fixing it up!

> 
> diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
> index 12fdc57..73bac70 100644
> --- a/lib/arm/asm/mmu-api.h
> +++ b/lib/arm/asm/mmu-api.h
> @@ -3,6 +3,7 @@
>  extern pgd_t *mmu_idmap;
>  extern bool mmu_enabled(void);
>  extern void mmu_enable(pgd_t *pgtable);
> +extern void mmu_mark_disabled(int cpu);
>  extern void mmu_disable(void);
>  extern void mmu_enable_idmap(void);
>  extern void mmu_init_io_sect(pgd_t *pgtable, unsigned long virt_offset);
> diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
> index ad558e1..8ff659b 100644
> --- a/lib/arm/mmu.c
> +++ b/lib/arm/mmu.c
> @@ -14,7 +14,8 @@ extern unsigned long etext;
> 
>  pgd_t *mmu_idmap;
> 
> -static cpumask_t mmu_disabled_cpumask;
> +/* CPU 0 starts with disabled MMU */
> +static cpumask_t mmu_disabled_cpumask = { {1} };
> 
>  bool mmu_enabled(void)
>  {
> @@ -33,6 +34,11 @@ void mmu_enable(pgd_t *pgtable)
>  	cpumask_clear_cpu(cpu, &mmu_disabled_cpumask);
>  }
> 
> +void mmu_mark_disabled(int cpu)
> +{
> +	cpumask_set_cpu(cpu, &mmu_disabled_cpumask);
> +}
> +
>  extern void asm_mmu_disable(void);
>  void mmu_disable(void)
>  {
> diff --git a/lib/arm/psci.c b/lib/arm/psci.c
> index aca8885..c4469c9 100644
> --- a/lib/arm/psci.c
> +++ b/lib/arm/psci.c
> @@ -9,6 +9,7 @@
>  #include <asm/psci.h>
>  #include <asm/setup.h>
>  #include <asm/page.h>
> +#include <asm/mmu-api.h>
> 
>  #define T PSCI_INVOKE_ARG_TYPE
>  __attribute__((noinline))
> @@ -29,6 +30,7 @@ int psci_cpu_on(unsigned long cpuid, unsigned long
> entry_point)
>  extern void secondary_entry(void);
>  int cpu_psci_cpu_boot(unsigned int cpu)
>  {
> +	mmu_mark_disabled(cpu);
>  	int err = psci_cpu_on(cpus[cpu], __pa(secondary_entry));
>  	if (err)
>  		printf("failed to boot CPU%d (%d)\n", cpu, err);
> --
> To unsubscribe from this list: send the line "unsubscribe kvm" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH 3/3] arm/arm64: speed up spinlocks and atomic ops

[Andrew Jones]

spinlock torture tests made it clear that checking mmu_enabled()
every time we call spin_lock is a bad idea. As most tests will
want the MMU enabled the entire time, then we can inline a light
weight "nobody disabled the mmu" check, and bail out early.

Adding a light weight inlined check vs. a compile-time flag
suggested by Paolo.

Signed-off-by: Andrew Jones <drjones@redhat.com>
---
 lib/arm/asm/mmu-api.h | 7 ++++++-
 lib/arm/mmu.c         | 9 +++++++--
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
index 12fdc57918c27..b2fc900a30add 100644
--- a/lib/arm/asm/mmu-api.h
+++ b/lib/arm/asm/mmu-api.h
@@ -1,7 +1,12 @@
 #ifndef __ASMARM_MMU_API_H_
 #define __ASMARM_MMU_API_H_
 extern pgd_t *mmu_idmap;
-extern bool mmu_enabled(void);
+extern unsigned int mmu_disabled_cpu_count;
+extern bool __mmu_enabled(void);
+static inline bool mmu_enabled(void)
+{
+	return mmu_disabled_cpu_count == 0 || __mmu_enabled();
+}
 extern void mmu_enable(pgd_t *pgtable);
 extern void mmu_disable(void);
 extern void mmu_enable_idmap(void);
diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
index ad558e177dd1c..e661d4f26e4b7 100644
--- a/lib/arm/mmu.c
+++ b/lib/arm/mmu.c
@@ -15,8 +15,9 @@ extern unsigned long etext;
 pgd_t *mmu_idmap;
 
 static cpumask_t mmu_disabled_cpumask;
+unsigned int mmu_disabled_cpu_count;
 
-bool mmu_enabled(void)
+bool __mmu_enabled(void)
 {
 	int cpu = current_thread_info()->cpu;
 
@@ -30,7 +31,9 @@ void mmu_enable(pgd_t *pgtable)
 
 	asm_mmu_enable(__pa(pgtable));
 	flush_tlb_all();
-	cpumask_clear_cpu(cpu, &mmu_disabled_cpumask);
+
+	if (cpumask_test_and_clear_cpu(cpu, &mmu_disabled_cpumask))
+		--mmu_disabled_cpu_count;
 }
 
 extern void asm_mmu_disable(void);
@@ -39,6 +42,8 @@ void mmu_disable(void)
 	int cpu = current_thread_info()->cpu;
 
 	cpumask_set_cpu(cpu, &mmu_disabled_cpumask);
+	++mmu_disabled_cpu_count;
+
 	asm_mmu_disable();
 }
 
-- 
2.4.3

Re: [PATCH 3/3] arm/arm64: speed up spinlocks and atomic ops

[Paolo Bonzini]

On 25/06/2015 20:45, Andrew Jones wrote:
> spinlock torture tests made it clear that checking mmu_enabled()
> every time we call spin_lock is a bad idea. As most tests will
> want the MMU enabled the entire time, then we can inline a light
> weight "nobody disabled the mmu" check, and bail out early.
> 
> Adding a light weight inlined check vs. a compile-time flag
> suggested by Paolo.
> 
> Signed-off-by: Andrew Jones <drjones@redhat.com>
> ---
>  lib/arm/asm/mmu-api.h | 7 ++++++-
>  lib/arm/mmu.c         | 9 +++++++--
>  2 files changed, 13 insertions(+), 3 deletions(-)
> 
> diff --git a/lib/arm/asm/mmu-api.h b/lib/arm/asm/mmu-api.h
> index 12fdc57918c27..b2fc900a30add 100644
> --- a/lib/arm/asm/mmu-api.h
> +++ b/lib/arm/asm/mmu-api.h
> @@ -1,7 +1,12 @@
>  #ifndef __ASMARM_MMU_API_H_
>  #define __ASMARM_MMU_API_H_
>  extern pgd_t *mmu_idmap;
> -extern bool mmu_enabled(void);
> +extern unsigned int mmu_disabled_cpu_count;
> +extern bool __mmu_enabled(void);
> +static inline bool mmu_enabled(void)
> +{
> +	return mmu_disabled_cpu_count == 0 || __mmu_enabled();
> +}
>  extern void mmu_enable(pgd_t *pgtable);
>  extern void mmu_disable(void);
>  extern void mmu_enable_idmap(void);
> diff --git a/lib/arm/mmu.c b/lib/arm/mmu.c
> index ad558e177dd1c..e661d4f26e4b7 100644
> --- a/lib/arm/mmu.c
> +++ b/lib/arm/mmu.c
> @@ -15,8 +15,9 @@ extern unsigned long etext;
>  pgd_t *mmu_idmap;
>  
>  static cpumask_t mmu_disabled_cpumask;
> +unsigned int mmu_disabled_cpu_count;
>  
> -bool mmu_enabled(void)
> +bool __mmu_enabled(void)
>  {
>  	int cpu = current_thread_info()->cpu;
>  
> @@ -30,7 +31,9 @@ void mmu_enable(pgd_t *pgtable)
>  
>  	asm_mmu_enable(__pa(pgtable));
>  	flush_tlb_all();
> -	cpumask_clear_cpu(cpu, &mmu_disabled_cpumask);
> +
> +	if (cpumask_test_and_clear_cpu(cpu, &mmu_disabled_cpumask))
> +		--mmu_disabled_cpu_count;
>  }
>  
>  extern void asm_mmu_disable(void);
> @@ -39,6 +42,8 @@ void mmu_disable(void)
>  	int cpu = current_thread_info()->cpu;
>  
>  	cpumask_set_cpu(cpu, &mmu_disabled_cpumask);
> +	++mmu_disabled_cpu_count;
> +
>  	asm_mmu_disable();
>  }
>  
> 

Applied, thanks.

Paolo