[PATCH 1/4] powerpc/mm/radix: Don't iterate all sets when flushing the PWC

[PATCH 1/4] powerpc/mm/radix: Don't iterate all sets when flushing the PWC

[Benjamin Herrenschmidt]

The PWC flush only needs a single set call, just like the
full (RIC=2) flush.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
---
 arch/powerpc/mm/tlb-radix.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 02e7140..5403419 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -52,12 +52,15 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
 	 */
 	__tlbiel_pid(pid, 0, ric);
 
-	if (ric == RIC_FLUSH_ALL)
-		/* For the remaining sets, just flush the TLB */
-		ric = RIC_FLUSH_TLB;
+	/* For PWC, only one flush is needed */
+	if (ric == RIC_FLUSH_PWC) {
+		asm volatile("ptesync": : :"memory");
+		return;
+	}
 
+	/* For the remaining sets, just flush the TLB */
 	for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
-		__tlbiel_pid(pid, set, ric);
+		__tlbiel_pid(pid, set, RIC_FLUSH_TLB);
 
 	asm volatile("ptesync": : :"memory");
 	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
-- 
2.9.4

[PATCH 2/4] powerpc/mm/radix: Improve TLB/PWC flushes

[Benjamin Herrenschmidt]

At the moment we have to rather sub-optimal flushing behaviours:

 - flush_tlb_mm() will flush the PWC which is unnecessary (for example
   when doing a fork)

 - A large unmap will call flush_tlb_pwc() multiple times causing us
   to perform that fairly expensive operation repeatedly. This happens
   often in batches of 3 on every new process.

So we change flush_tlb_mm() to only flush the TLB, and we use the
existing "need_flush_all" flag in struct mmu_gather to indicate
that the PWC needs flushing.

Unfortunately, flush_tlb_range() still needs to do a full flush
for now as it's used by the THP collapsing. We will fix that later.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
---
 .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  4 +-
 arch/powerpc/mm/tlb-radix.c                        | 66 +++++++++-------------
 2 files changed, 28 insertions(+), 42 deletions(-)

diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index cc7fbde..7196999 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -22,22 +22,20 @@ extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end
 
 extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
 extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
-extern void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
 extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 					      int psize);
 extern void radix__tlb_flush(struct mmu_gather *tlb);
 #ifdef CONFIG_SMP
 extern void radix__flush_tlb_mm(struct mm_struct *mm);
 extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
-extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
 extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 					int psize);
 #else
 #define radix__flush_tlb_mm(mm)		radix__local_flush_tlb_mm(mm)
 #define radix__flush_tlb_page(vma,addr)	radix__local_flush_tlb_page(vma,addr)
 #define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
-#define radix__flush_tlb_pwc(tlb, addr)	radix__local_flush_tlb_pwc(tlb, addr)
 #endif
+extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
 extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
 				     unsigned long page_size);
 extern void radix__flush_tlb_lpid(unsigned long lpid);
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 5403419..f65c928 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -66,17 +66,6 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
 	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
 }
 
-static inline void tlbiel_pwc(unsigned long pid)
-{
-	asm volatile("ptesync": : :"memory");
-
-	/* For PWC flush, we don't look at set number */
-	__tlbiel_pid(pid, 0, RIC_FLUSH_PWC);
-
-	asm volatile("ptesync": : :"memory");
-	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
-}
-
 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
 {
 	unsigned long rb,rs,prs,r;
@@ -144,31 +133,23 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
 	preempt_disable();
 	pid = mm->context.id;
 	if (pid != MMU_NO_CONTEXT)
-		_tlbiel_pid(pid, RIC_FLUSH_ALL);
+		_tlbiel_pid(pid, RIC_FLUSH_TLB);
 	preempt_enable();
 }
 EXPORT_SYMBOL(radix__local_flush_tlb_mm);
 
-void radix__local_flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+#ifndef CONFIG_SMP
+static void radix__local_flush_all_mm(struct mm_struct *mm)
 {
 	unsigned long pid;
-	struct mm_struct *mm = tlb->mm;
-	/*
-	 * If we are doing a full mm flush, we will do a tlb flush
-	 * with RIC_FLUSH_ALL later.
-	 */
-	if (tlb->fullmm)
-		return;
 
 	preempt_disable();
-
 	pid = mm->context.id;
 	if (pid != MMU_NO_CONTEXT)
-		tlbiel_pwc(pid);
-
+		_tlbiel_pid(pid, RIC_FLUSH_ALL);
 	preempt_enable();
 }
-EXPORT_SYMBOL(radix__local_flush_tlb_pwc);
+#endif /* CONFIG_SMP */
 
 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 				       int psize)
@@ -206,38 +187,35 @@ void radix__flush_tlb_mm(struct mm_struct *mm)
 		goto no_context;
 
 	if (!mm_is_thread_local(mm))
-		_tlbie_pid(pid, RIC_FLUSH_ALL);
+		_tlbie_pid(pid, RIC_FLUSH_TLB);
 	else
-		_tlbiel_pid(pid, RIC_FLUSH_ALL);
+		_tlbiel_pid(pid, RIC_FLUSH_TLB);
 no_context:
 	preempt_enable();
 }
 EXPORT_SYMBOL(radix__flush_tlb_mm);
 
-void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+static void radix__flush_all_mm(struct mm_struct *mm)
 {
 	unsigned long pid;
-	struct mm_struct *mm = tlb->mm;
 
-	/*
-	 * If we are doing a full mm flush, we will do a tlb flush
-	 * with RIC_FLUSH_ALL later.
-	 */
-	if (tlb->fullmm)
-		return;
 	preempt_disable();
-
 	pid = mm->context.id;
 	if (unlikely(pid == MMU_NO_CONTEXT))
 		goto no_context;
 
 	if (!mm_is_thread_local(mm))
-		_tlbie_pid(pid, RIC_FLUSH_PWC);
+		_tlbie_pid(pid, RIC_FLUSH_ALL);
 	else
-		tlbiel_pwc(pid);
+		_tlbiel_pid(pid, RIC_FLUSH_ALL);
 no_context:
 	preempt_enable();
 }
+
+void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr)
+{
+	tlb->need_flush_all = 1;
+}
 EXPORT_SYMBOL(radix__flush_tlb_pwc);
 
 void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
@@ -269,6 +247,8 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
 }
 EXPORT_SYMBOL(radix__flush_tlb_page);
 
+#else /* CONFIG_SMP */
+#define radix__flush_all_mm radix__local_flush_all_mm
 #endif /* CONFIG_SMP */
 
 void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -286,7 +266,12 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 
 {
 	struct mm_struct *mm = vma->vm_mm;
-	radix__flush_tlb_mm(mm);
+
+	/*
+	 * This is currently used when collapsing THPs so we need to
+	 * flush the PWC. We should fix this.
+	 */
+	radix__flush_all_mm(mm);
 }
 EXPORT_SYMBOL(radix__flush_tlb_range);
 
@@ -317,7 +302,10 @@ void radix__tlb_flush(struct mmu_gather *tlb)
 	 */
 	if (psize != -1 && !tlb->fullmm && !tlb->need_flush_all)
 		radix__flush_tlb_range_psize(mm, tlb->start, tlb->end, psize);
-	else
+	else if (tlb->need_flush_all) {
+		tlb->need_flush_all = 0;
+		radix__flush_all_mm(mm);
+	} else
 		radix__flush_tlb_mm(mm);
 }
 
-- 
2.9.4

[PATCH 3/4] powerpc/mm/radix: Avoid flushing the PWC on every flush_tlb_range

[Benjamin Herrenschmidt]

We do that because it's used by THP pmd collapsing, so use
instead a dedicated flush function.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
---
 .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  1 +
 arch/powerpc/mm/pgtable-radix.c                    |  5 ++-
 arch/powerpc/mm/tlb-radix.c                        | 41 +++++++++++++++++++---
 3 files changed, 41 insertions(+), 6 deletions(-)

diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 7196999..9b433a6 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -36,6 +36,7 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad
 #define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
 #endif
 extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
+extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
 extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
 				     unsigned long page_size);
 extern void radix__flush_tlb_lpid(unsigned long lpid);
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index c28165d..83d70f7 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -706,9 +706,12 @@ pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addre
 	 */
 	pmd = *pmdp;
 	pmd_clear(pmdp);
+
 	/*FIXME!!  Verify whether we need this kick below */
 	kick_all_cpus_sync();
-	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+
+	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
+
 	return pmd;
 }
 
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index f65c928..73d3fbf 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -267,11 +267,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
 {
 	struct mm_struct *mm = vma->vm_mm;
 
-	/*
-	 * This is currently used when collapsing THPs so we need to
-	 * flush the PWC. We should fix this.
-	 */
-	radix__flush_all_mm(mm);
+	radix__flush_tlb_mm(mm);
 }
 EXPORT_SYMBOL(radix__flush_tlb_range);
 
@@ -350,6 +346,41 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
 	preempt_enable();
 }
 
+void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
+{
+	int local = mm_is_thread_local(mm);
+	unsigned long ap = mmu_get_ap(mmu_virtual_psize);
+	unsigned long pid, end;
+
+
+	pid = mm ? mm->context.id : 0;
+	if (unlikely(pid == MMU_NO_CONTEXT))
+		goto no_context;
+
+	/* 4k page size, just blow the world */
+	if (PAGE_SIZE == 0x1000) {
+		radix__flush_all_mm(mm);
+		return;
+	}
+
+	/* Otherwise first do the PWC */
+	if (local)
+		_tlbiel_pid(pid, RIC_FLUSH_PWC);
+	else
+		_tlbie_pid(pid, RIC_FLUSH_PWC);
+
+	/* Then iterate the pages */
+	end = addr + HPAGE_PMD_SIZE;
+	for (; addr < end; addr += PAGE_SIZE) {
+		if (local)
+			_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
+		else
+			_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
+	}
+no_context:
+	preempt_enable();
+}
+
 void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
 			      unsigned long page_size)
 {
-- 
2.9.4

Re: [PATCH 3/4] powerpc/mm/radix: Avoid flushing the PWC on every flush_tlb_range

[Aneesh Kumar K.V]

Benjamin Herrenschmidt <benh@kernel.crashing.org> writes:

> We do that because it's used by THP pmd collapsing, so use
> instead a dedicated flush function.
>
> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> ---
>  .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  1 +
>  arch/powerpc/mm/pgtable-radix.c                    |  5 ++-
>  arch/powerpc/mm/tlb-radix.c                        | 41 +++++++++++++++++++---
>  3 files changed, 41 insertions(+), 6 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> index 7196999..9b433a6 100644
> --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> @@ -36,6 +36,7 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad
>  #define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
>  #endif
>  extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr);
> +extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr);
>  extern void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
>  				     unsigned long page_size);
>  extern void radix__flush_tlb_lpid(unsigned long lpid);
> diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
> index c28165d..83d70f7 100644
> --- a/arch/powerpc/mm/pgtable-radix.c
> +++ b/arch/powerpc/mm/pgtable-radix.c
> @@ -706,9 +706,12 @@ pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long addre
>  	 */
>  	pmd = *pmdp;
>  	pmd_clear(pmdp);
> +
>  	/*FIXME!!  Verify whether we need this kick below */
>  	kick_all_cpus_sync();
> -	flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
> +
> +	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
> +
>  	return pmd;
>  }
>
> diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
> index f65c928..73d3fbf 100644
> --- a/arch/powerpc/mm/tlb-radix.c
> +++ b/arch/powerpc/mm/tlb-radix.c
> @@ -267,11 +267,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
>  {
>  	struct mm_struct *mm = vma->vm_mm;
>
> -	/*
> -	 * This is currently used when collapsing THPs so we need to
> -	 * flush the PWC. We should fix this.
> -	 */
> -	radix__flush_all_mm(mm);
> +	radix__flush_tlb_mm(mm);
>  }
>  EXPORT_SYMBOL(radix__flush_tlb_range);
>
> @@ -350,6 +346,41 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
>  	preempt_enable();
>  }
>
> +void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
> +{
> +	int local = mm_is_thread_local(mm);
> +	unsigned long ap = mmu_get_ap(mmu_virtual_psize);
> +	unsigned long pid, end;
> +
> +
> +	pid = mm ? mm->context.id : 0;
> +	if (unlikely(pid == MMU_NO_CONTEXT))
> +		goto no_context;
> +
> +	/* 4k page size, just blow the world */
> +	if (PAGE_SIZE == 0x1000) {
> +		radix__flush_all_mm(mm);
> +		return;
> +	}

What is special here ? because it is 4096 tlb flushes ?


> +
> +	/* Otherwise first do the PWC */
> +	if (local)
> +		_tlbiel_pid(pid, RIC_FLUSH_PWC);
> +	else
> +		_tlbie_pid(pid, RIC_FLUSH_PWC);
> +
> +	/* Then iterate the pages */
> +	end = addr + HPAGE_PMD_SIZE;
> +	for (; addr < end; addr += PAGE_SIZE) {
> +		if (local)
> +			_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
> +		else
> +			_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
> +	}
> +no_context:
> +	preempt_enable();
> +}
> +
>  void radix__flush_tlb_lpid_va(unsigned long lpid, unsigned long gpa,
>  			      unsigned long page_size)
>  {
> -- 
> 2.9.4

Re: [PATCH 3/4] powerpc/mm/radix: Avoid flushing the PWC on every flush_tlb_range

[Benjamin Herrenschmidt]

On Fri, 2017-07-14 at 11:14 +0530, Aneesh Kumar K.V wrote:
> > +     pid = mm ? mm->context.id : 0;
> > +     if (unlikely(pid == MMU_NO_CONTEXT))
> > +             goto no_context;
> > +
> > +     /* 4k page size, just blow the world */
> > +     if (PAGE_SIZE == 0x1000) {
> > +             radix__flush_all_mm(mm);
> > +             return;
> > +     }
> 
> What is special here ? because it is 4096 tlb flushes ?

Yes, keeping in line with the existing logic in flush_tlb_pages(), more
than 32, do a full flush.

Cheers,
Ben.

Re: [PATCH 3/4] powerpc/mm/radix: Avoid flushing the PWC on every flush_tlb_range

[kbuild test robot]

[-- Attachment #1: Type: text/plain, Size: 3744 bytes --]

Hi Benjamin,

[auto build test ERROR on powerpc/next]
[also build test ERROR on v4.13-rc1 next-20170717]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]

url:    https://github.com/0day-ci/linux/commits/Benjamin-Herrenschmidt/powerpc-mm-radix-Don-t-iterate-all-sets-when-flushing-the-PWC/20170715-044140
base:   https://git.kernel.org/pub/scm/linux/kernel/git/powerpc/linux.git next
config: powerpc-pasemi_defconfig (attached as .config)
compiler: powerpc64-linux-gnu-gcc (Debian 6.1.1-9) 6.1.1 20160705
reproduce:
        wget https://raw.githubusercontent.com/01org/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
        chmod +x ~/bin/make.cross
        # save the attached .config to linux build tree
        make.cross ARCH=powerpc 

All error/warnings (new ones prefixed by >>):

   In file included from include/asm-generic/bug.h:4:0,
                    from arch/powerpc/include/asm/bug.h:127,
                    from include/linux/bug.h:4,
                    from include/linux/mmdebug.h:4,
                    from include/linux/mm.h:8,
                    from arch/powerpc/mm/tlb-radix.c:12:
   arch/powerpc/mm/tlb-radix.c: In function 'radix__flush_tlb_collapsed_pmd':
>> include/linux/compiler.h:529:38: error: call to '__compiletime_assert_378' declared with attribute error: BUILD_BUG failed
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
                                         ^
   include/linux/compiler.h:512:4: note: in definition of macro '__compiletime_assert'
       prefix ## suffix();    \
       ^~~~~~
   include/linux/compiler.h:529:2: note: in expansion of macro '_compiletime_assert'
     _compiletime_assert(condition, msg, __compiletime_assert_, __LINE__)
     ^~~~~~~~~~~~~~~~~~~
   include/linux/bug.h:54:37: note: in expansion of macro 'compiletime_assert'
    #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
                                        ^~~~~~~~~~~~~~~~~~
   include/linux/bug.h:88:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
    #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
                        ^~~~~~~~~~~~~~~~
>> include/linux/huge_mm.h:218:27: note: in expansion of macro 'BUILD_BUG'
    #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; })
                              ^~~~~~~~~
>> arch/powerpc/mm/tlb-radix.c:378:15: note: in expansion of macro 'HPAGE_PMD_SIZE'
     end = addr + HPAGE_PMD_SIZE;
                  ^~~~~~~~~~~~~~

vim +/HPAGE_PMD_SIZE +378 arch/powerpc/mm/tlb-radix.c

   353	
   354	void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
   355	{
   356		int local = mm_is_thread_local(mm);
   357		unsigned long ap = mmu_get_ap(mmu_virtual_psize);
   358		unsigned long pid, end;
   359	
   360	
   361		pid = mm ? mm->context.id : 0;
   362		if (unlikely(pid == MMU_NO_CONTEXT))
   363			goto no_context;
   364	
   365		/* 4k page size, just blow the world */
   366		if (PAGE_SIZE == 0x1000) {
   367			radix__flush_all_mm(mm);
   368			return;
   369		}
   370	
   371		/* Otherwise first do the PWC */
   372		if (local)
   373			_tlbiel_pid(pid, RIC_FLUSH_PWC);
   374		else
   375			_tlbie_pid(pid, RIC_FLUSH_PWC);
   376	
   377		/* Then iterate the pages */
 > 378		end = addr + HPAGE_PMD_SIZE;
   379		for (; addr < end; addr += PAGE_SIZE) {
   380			if (local)
   381				_tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
   382			else
   383				_tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
   384		}
   385	no_context:
   386		preempt_enable();
   387	}
   388	

---
0-DAY kernel test infrastructure                Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all                   Intel Corporation

[-- Attachment #2: .config.gz --]
[-- Type: application/gzip, Size: 20321 bytes --]

[PATCH 4/4] powerpc/mm/radix: Workaround prefetch issue with KVM

[Benjamin Herrenschmidt]

There's a somewhat architectural issue with Radix MMU and KVM.

When coming out of a guest with AIL (ie, MMU enabled), we start
executing hypervisor code with the PID register still containing
whatever the guest has been using.

The problem is that the CPU can (and will) then start prefetching
or speculatively load from whatever host context has that same
PID (if any), thus bringing translations for that context into
the TLB, which Linux doesn't know about.

This can cause stale translations and subsequent crashes.

Fixing this in a way that is neither racy nor a huge performance
impact is difficult. We could just make the host invalidations
always use broadcast forms but that would hurt single threaded
programs for example.

We chose to fix it instead by partitioning the PID space between
guest and host. This is possible because today Linux only use 19
out of the 20 bits of PID space, so existing guests will work
if we make the host use the top half of the 20 bits space.

We additionally add a property to indicate to Linux the size of
the PID register which will be useful if we eventually have
processors with a larger PID space available.

There is still an issue with malicious guests purposefully setting
the PID register to a value in the host range. Hopefully future HW
can prevent that, but in the meantime, we handle it with a pair of
kludges:

 - On the way out of a guest, before we clear the current VCPU
in the PACA, we check the PID and if it's outside of the permitted
range we flush the TLB for that PID.

 - When context switching, if the mm is "new" on that CPU (the
corresponding bit was set for the first time in the mm cpumask), we
check if any sibling thread is in KVM (has a non-NULL VCPU pointer
in the PACA). If that is the case, we also flush the PID for that
CPU (core).

This second part is needed to handle the case where a process is
migrated (or starts a new pthread) on a sibling thread of the CPU
coming out of KVM, as there's a window where stale translations
can exist before we detect it and flush them out.

A future optimization could be added by keeping track of whether
the PID has ever been used and avoid doing that for completely
fresh PIDs. We could similarily mark PIDs that have been the subject of
a global invalidation as "fresh". But for now this will do.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
---
---
 arch/powerpc/include/asm/book3s/64/mmu.h           | 15 ++++----
 .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  3 ++
 arch/powerpc/include/asm/mmu_context.h             | 25 +++++++++----
 arch/powerpc/kernel/head_32.S                      |  6 +--
 arch/powerpc/kernel/swsusp.c                       |  2 +-
 arch/powerpc/kvm/book3s_32_sr.S                    |  2 +-
 arch/powerpc/kvm/book3s_hv_rmhandlers.S            | 12 ++++++
 arch/powerpc/mm/mmu_context_book3s64.c             | 43 ++++++++++++++++++++--
 arch/powerpc/mm/mmu_context_nohash.c               |  4 +-
 arch/powerpc/mm/pgtable-radix.c                    | 31 +++++++++++++++-
 arch/powerpc/mm/pgtable_64.c                       |  3 ++
 arch/powerpc/mm/tlb-radix.c                        | 10 +++--
 drivers/cpufreq/pmac32-cpufreq.c                   |  2 +-
 drivers/macintosh/via-pmu.c                        |  4 +-
 14 files changed, 131 insertions(+), 31 deletions(-)

diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
index 77529a3..5b4023c 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu.h
@@ -59,13 +59,14 @@ extern struct patb_entry *partition_tb;
 #define PRTS_MASK	0x1f		/* process table size field */
 #define PRTB_MASK	0x0ffffffffffff000UL
 
-/*
- * Limit process table to PAGE_SIZE table. This
- * also limit the max pid we can support.
- * MAX_USER_CONTEXT * 16 bytes of space.
- */
-#define PRTB_SIZE_SHIFT	(CONTEXT_BITS + 4)
-#define PRTB_ENTRIES	(1ul << CONTEXT_BITS)
+/* Number of supported PID bits */
+extern unsigned int mmu_pid_bits;
+
+/* Base PID to allocate from */
+extern unsigned int mmu_base_pid;
+
+#define PRTB_SIZE_SHIFT	(mmu_pid_bits + 4)
+#define PRTB_ENTRIES	(1ul << mmu_pid_bits)
 
 /*
  * Power9 currently only support 64K partition table size.
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 9b433a6..2e0a6b4 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -21,10 +21,13 @@ extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long sta
 extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
 
 extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
+extern void radix__local_flush_all_mm(struct mm_struct *mm);
 extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
 extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 					      int psize);
 extern void radix__tlb_flush(struct mmu_gather *tlb);
+extern void radix_flush_pid(unsigned long pid);
+
 #ifdef CONFIG_SMP
 extern void radix__flush_tlb_mm(struct mm_struct *mm);
 extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index da7e943..26a587a 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -45,13 +45,15 @@ extern void set_context(unsigned long id, pgd_t *pgd);
 
 #ifdef CONFIG_PPC_BOOK3S_64
 extern void radix__switch_mmu_context(struct mm_struct *prev,
-				     struct mm_struct *next);
+				      struct mm_struct *next,
+				      bool new_on_cpu);
 static inline void switch_mmu_context(struct mm_struct *prev,
 				      struct mm_struct *next,
-				      struct task_struct *tsk)
+				      struct task_struct *tsk,
+				      bool new_on_cpu)
 {
 	if (radix_enabled())
-		return radix__switch_mmu_context(prev, next);
+		return radix__switch_mmu_context(prev, next, new_on_cpu);
 	return switch_slb(tsk, next);
 }
 
@@ -60,8 +62,13 @@ extern void hash__reserve_context_id(int id);
 extern void __destroy_context(int context_id);
 static inline void mmu_context_init(void) { }
 #else
-extern void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
-			       struct task_struct *tsk);
+extern void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
+				 struct task_struct *tsk);
+static inline void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
+				      struct task_struct *tsk, bool new_on_cpu)
+{
+	__switch_mmu_context(prev, next, tsk);
+}
 extern unsigned long __init_new_context(void);
 extern void __destroy_context(unsigned long context_id);
 extern void mmu_context_init(void);
@@ -79,9 +86,13 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
 				      struct mm_struct *next,
 				      struct task_struct *tsk)
 {
+	bool new_on_cpu = false;
+
 	/* Mark this context has been used on the new CPU */
-	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next)))
+	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next))) {
 		cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
+		new_on_cpu = true;
+	}
 
 	/* 32-bit keeps track of the current PGDIR in the thread struct */
 #ifdef CONFIG_PPC32
@@ -113,7 +124,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
 	 * The actual HW switching method differs between the various
 	 * sub architectures. Out of line for now
 	 */
-	switch_mmu_context(prev, next, tsk);
+	switch_mmu_context(prev, next, tsk, new_on_cpu);
 }
 
 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S
index e227342..a530124 100644
--- a/arch/powerpc/kernel/head_32.S
+++ b/arch/powerpc/kernel/head_32.S
@@ -1003,11 +1003,11 @@ start_here:
 	RFI
 
 /*
- * void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
+ * void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
  *
  * Set up the segment registers for a new context.
  */
-_ENTRY(switch_mmu_context)
+_ENTRY(__switch_mmu_context)
 	lwz	r3,MMCONTEXTID(r4)
 	cmpwi	cr0,r3,0
 	blt-	4f
@@ -1040,7 +1040,7 @@ _ENTRY(switch_mmu_context)
 4:	trap
 	EMIT_BUG_ENTRY 4b,__FILE__,__LINE__,0
 	blr
-EXPORT_SYMBOL(switch_mmu_context)
+EXPORT_SYMBOL(__switch_mmu_context)
 
 /*
  * An undocumented "feature" of 604e requires that the v bit
diff --git a/arch/powerpc/kernel/swsusp.c b/arch/powerpc/kernel/swsusp.c
index 0050b2d..9c32cfd 100644
--- a/arch/powerpc/kernel/swsusp.c
+++ b/arch/powerpc/kernel/swsusp.c
@@ -32,6 +32,6 @@ void save_processor_state(void)
 void restore_processor_state(void)
 {
 #ifdef CONFIG_PPC32
-	switch_mmu_context(current->active_mm, current->active_mm, NULL);
+	__switch_mmu_context(current->active_mm, current->active_mm, NULL);
 #endif
 }
diff --git a/arch/powerpc/kvm/book3s_32_sr.S b/arch/powerpc/kvm/book3s_32_sr.S
index 7e06a6f..a1705d4 100644
--- a/arch/powerpc/kvm/book3s_32_sr.S
+++ b/arch/powerpc/kvm/book3s_32_sr.S
@@ -138,6 +138,6 @@
 	lwz	r4, MM(r4)
 	tophys(r4, r4)
 	/* This only clobbers r0, r3, r4 and r5 */
-	bl	switch_mmu_context
+	bl	__switch_mmu_context
 
 .endm
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index 6ea4b53..4fb3581b 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -1522,6 +1522,18 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
 	std	r6, VCPU_BESCR(r9)
 	stw	r7, VCPU_GUEST_PID(r9)
 	std	r8, VCPU_WORT(r9)
+
+	/* Handle the case where the guest used an illegal PID */
+	LOAD_REG_ADDR(r4, mmu_base_pid)
+	lwz	r3, 0(r4)
+	cmpw	cr0,r7,r3
+	blt	1f
+
+	/* Illegal PID, flush the TLB */
+	bl	radix_flush_pid
+	ld	r9, HSTATE_KVM_VCPU(r13)
+1:
+
 BEGIN_FTR_SECTION
 	mfspr	r5, SPRN_TCSCR
 	mfspr	r6, SPRN_ACOP
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index abed1fe..183a67b 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -25,6 +25,10 @@
 #include <asm/mmu_context.h>
 #include <asm/pgalloc.h>
 
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+#include <asm/kvm_book3s_asm.h>
+#endif
+
 #include "icswx.h"
 
 static DEFINE_SPINLOCK(mmu_context_lock);
@@ -126,9 +130,10 @@ static int hash__init_new_context(struct mm_struct *mm)
 static int radix__init_new_context(struct mm_struct *mm)
 {
 	unsigned long rts_field;
-	int index;
+	int index, max_id;
 
-	index = alloc_context_id(1, PRTB_ENTRIES - 1);
+	max_id = (1 << mmu_pid_bits) - 1;
+	index = alloc_context_id(mmu_base_pid, max_id);
 	if (index < 0)
 		return index;
 
@@ -247,8 +252,40 @@ void destroy_context(struct mm_struct *mm)
 }
 
 #ifdef CONFIG_PPC_RADIX_MMU
-void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
+void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
+			       bool new_on_cpu)
 {
+	/*
+	 * If this context hasn't run on that CPU before and KVM is
+	 * around, there's a slim chance that the guest on another
+	 * CPU just brought in obsolete translation into the TLB of
+	 * this CPU due to a bad prefetch using the guest PID on
+	 * the way into the hypervisor.
+	 *
+	 * We work around this here. If KVM is possible, we check if
+	 * any sibling thread is in KVM. If it is, the window may exist
+	 * and thus we flush that PID from the core.
+	 *
+	 * A potential future improvement would be to mark which PIDs
+	 * have never been used on the system and avoid it if the PID
+	 * is new and the process has no other cpumask bit set.
+	 */
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+	if (cpu_has_feature(CPU_FTR_HVMODE) && new_on_cpu) {
+		int cpu = smp_processor_id();
+		int sib = cpu_first_thread_sibling(cpu);
+		bool flush = false;
+
+		for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) {
+			if (sib == cpu)
+				continue;
+			if (paca[sib].kvm_hstate.kvm_vcpu)
+				flush = true;
+		}
+		if (flush)
+			radix__local_flush_all_mm(next);
+	}
+#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
 
 	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
 		isync();
diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c
index 4554d65..47d84d0 100644
--- a/arch/powerpc/mm/mmu_context_nohash.c
+++ b/arch/powerpc/mm/mmu_context_nohash.c
@@ -226,8 +226,8 @@ static void context_check_map(void)
 static void context_check_map(void) { }
 #endif
 
-void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
-			struct task_struct *tsk)
+void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
+			  struct task_struct *tsk)
 {
 	unsigned int i, id, cpu = smp_processor_id();
 	unsigned long *map;
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index 83d70f7..3888aea 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -168,11 +168,34 @@ static void __init radix_init_pgtable(void)
 	for_each_memblock(memory, reg)
 		WARN_ON(create_physical_mapping(reg->base,
 						reg->base + reg->size));
+
+	/* Find out how many PID bits are supported */
+	if (cpu_has_feature(CPU_FTR_HVMODE)) {
+		if (!mmu_pid_bits)
+			mmu_pid_bits = 20;
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+		/*
+		 * When KVM is possible, we only use the top half of the
+		 * PID space to avoid collisions between host and guest PIDs
+		 * which can cause problems due to prefetch when exiting the
+		 * guest with AIL=3
+		 */
+		mmu_base_pid = 1 << (mmu_pid_bits - 1);
+#else
+		mmu_base_pid = 1;
+#endif
+	} else {
+		/* The guest uses the bottom half of the PID space */
+		if (!mmu_pid_bits)
+			mmu_pid_bits = 19;
+		mmu_base_pid = 1;
+	}
+
 	/*
 	 * Allocate Partition table and process table for the
 	 * host.
 	 */
-	BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 36), "Process table size too large.");
+	BUG_ON(PRTB_SIZE_SHIFT > 36);
 	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT);
 	/*
 	 * Fill in the process table.
@@ -245,6 +268,12 @@ static int __init radix_dt_scan_page_sizes(unsigned long node,
 	if (type == NULL || strcmp(type, "cpu") != 0)
 		return 0;
 
+	/* Find MMU PID size */
+	prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size);
+	if (prop && size == 4)
+		mmu_pid_bits = be32_to_cpup(prop);
+
+	/* Grab page size encodings */
 	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
 	if (!prop)
 		return 0;
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index b165bab..5cc73f1 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -68,6 +68,9 @@
  */
 struct prtb_entry *process_tb;
 struct patb_entry *partition_tb;
+unsigned int mmu_pid_bits;
+unsigned int mmu_base_pid;
+
 /*
  * page table size
  */
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 73d3fbf..54d5c75 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -66,6 +66,12 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
 	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
 }
 
+/* Used by KVM */
+void radix_flush_pid(unsigned long pid)
+{
+	_tlbiel_pid(pid, RIC_FLUSH_ALL);
+}
+
 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
 {
 	unsigned long rb,rs,prs,r;
@@ -138,8 +144,7 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
 }
 EXPORT_SYMBOL(radix__local_flush_tlb_mm);
 
-#ifndef CONFIG_SMP
-static void radix__local_flush_all_mm(struct mm_struct *mm)
+void radix__local_flush_all_mm(struct mm_struct *mm)
 {
 	unsigned long pid;
 
@@ -149,7 +154,6 @@ static void radix__local_flush_all_mm(struct mm_struct *mm)
 		_tlbiel_pid(pid, RIC_FLUSH_ALL);
 	preempt_enable();
 }
-#endif /* CONFIG_SMP */
 
 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 				       int psize)
diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c
index ff44016..51026f2 100644
--- a/drivers/cpufreq/pmac32-cpufreq.c
+++ b/drivers/cpufreq/pmac32-cpufreq.c
@@ -300,7 +300,7 @@ static int pmu_set_cpu_speed(int low_speed)
  		_set_L3CR(save_l3cr);
 
 	/* Restore userland MMU context */
-	switch_mmu_context(NULL, current->active_mm, NULL);
+	__switch_mmu_context(NULL, current->active_mm, NULL);
 
 #ifdef DEBUG_FREQ
 	printk(KERN_DEBUG "HID1, after: %x\n", mfspr(SPRN_HID1));
diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
index cce99f7..c06f08d 100644
--- a/drivers/macintosh/via-pmu.c
+++ b/drivers/macintosh/via-pmu.c
@@ -1851,7 +1851,7 @@ static int powerbook_sleep_grackle(void)
  		_set_L2CR(save_l2cr);
 	
 	/* Restore userland MMU context */
-	switch_mmu_context(NULL, current->active_mm, NULL);
+	__switch_mmu_context(NULL, current->active_mm, NULL);
 
 	/* Power things up */
 	pmu_unlock();
@@ -1940,7 +1940,7 @@ powerbook_sleep_Core99(void)
  		_set_L3CR(save_l3cr);
 	
 	/* Restore userland MMU context */
-	switch_mmu_context(NULL, current->active_mm, NULL);
+	__switch_mmu_context(NULL, current->active_mm, NULL);
 
 	/* Tell PMU we are ready */
 	pmu_unlock();
-- 
2.9.4

Re: [PATCH 4/4] powerpc/mm/radix: Workaround prefetch issue with KVM

[Aneesh Kumar K.V]

Benjamin Herrenschmidt <benh@kernel.crashing.org> writes:

> There's a somewhat architectural issue with Radix MMU and KVM.
>
> When coming out of a guest with AIL (ie, MMU enabled), we start
> executing hypervisor code with the PID register still containing
> whatever the guest has been using.
>
> The problem is that the CPU can (and will) then start prefetching
> or speculatively load from whatever host context has that same
> PID (if any), thus bringing translations for that context into
> the TLB, which Linux doesn't know about.
>
> This can cause stale translations and subsequent crashes.
>
> Fixing this in a way that is neither racy nor a huge performance
> impact is difficult. We could just make the host invalidations
> always use broadcast forms but that would hurt single threaded
> programs for example.
>
> We chose to fix it instead by partitioning the PID space between
> guest and host. This is possible because today Linux only use 19
> out of the 20 bits of PID space, so existing guests will work
> if we make the host use the top half of the 20 bits space.
>
> We additionally add a property to indicate to Linux the size of
> the PID register which will be useful if we eventually have
> processors with a larger PID space available.
>
> There is still an issue with malicious guests purposefully setting
> the PID register to a value in the host range. Hopefully future HW
> can prevent that, but in the meantime, we handle it with a pair of
> kludges:
>
>  - On the way out of a guest, before we clear the current VCPU
> in the PACA, we check the PID and if it's outside of the permitted
> range we flush the TLB for that PID.

Can you explain more about this ? We still run with guest PIDR at this
point right ? How does flushing TLB here prevent a further speculative
access of Quadrant 0 ? Or is it that beyound this point such access is
not possible ?

>
>  - When context switching, if the mm is "new" on that CPU (the
> corresponding bit was set for the first time in the mm cpumask), we
> check if any sibling thread is in KVM (has a non-NULL VCPU pointer
> in the PACA). If that is the case, we also flush the PID for that
> CPU (core).

This is needed so that we don't run with old stale translation details
which was left in this core right ?


>
> This second part is needed to handle the case where a process is
> migrated (or starts a new pthread) on a sibling thread of the CPU
> coming out of KVM, as there's a window where stale translations
> can exist before we detect it and flush them out.
>
> A future optimization could be added by keeping track of whether
> the PID has ever been used and avoid doing that for completely
> fresh PIDs. We could similarily mark PIDs that have been the subject of
> a global invalidation as "fresh". But for now this will do.
>
> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> ---
> ---
>  arch/powerpc/include/asm/book3s/64/mmu.h           | 15 ++++----
>  .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  3 ++
>  arch/powerpc/include/asm/mmu_context.h             | 25 +++++++++----
>  arch/powerpc/kernel/head_32.S                      |  6 +--
>  arch/powerpc/kernel/swsusp.c                       |  2 +-
>  arch/powerpc/kvm/book3s_32_sr.S                    |  2 +-
>  arch/powerpc/kvm/book3s_hv_rmhandlers.S            | 12 ++++++
>  arch/powerpc/mm/mmu_context_book3s64.c             | 43 ++++++++++++++++++++--
>  arch/powerpc/mm/mmu_context_nohash.c               |  4 +-
>  arch/powerpc/mm/pgtable-radix.c                    | 31 +++++++++++++++-
>  arch/powerpc/mm/pgtable_64.c                       |  3 ++
>  arch/powerpc/mm/tlb-radix.c                        | 10 +++--
>  drivers/cpufreq/pmac32-cpufreq.c                   |  2 +-
>  drivers/macintosh/via-pmu.c                        |  4 +-
>  14 files changed, 131 insertions(+), 31 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
> index 77529a3..5b4023c 100644
> --- a/arch/powerpc/include/asm/book3s/64/mmu.h
> +++ b/arch/powerpc/include/asm/book3s/64/mmu.h
> @@ -59,13 +59,14 @@ extern struct patb_entry *partition_tb;
>  #define PRTS_MASK	0x1f		/* process table size field */
>  #define PRTB_MASK	0x0ffffffffffff000UL
>
> -/*
> - * Limit process table to PAGE_SIZE table. This
> - * also limit the max pid we can support.
> - * MAX_USER_CONTEXT * 16 bytes of space.
> - */
> -#define PRTB_SIZE_SHIFT	(CONTEXT_BITS + 4)
> -#define PRTB_ENTRIES	(1ul << CONTEXT_BITS)
> +/* Number of supported PID bits */
> +extern unsigned int mmu_pid_bits;
> +
> +/* Base PID to allocate from */
> +extern unsigned int mmu_base_pid;
> +
> +#define PRTB_SIZE_SHIFT	(mmu_pid_bits + 4)
> +#define PRTB_ENTRIES	(1ul << mmu_pid_bits)
>
>  /*
>   * Power9 currently only support 64K partition table size.
> diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> index 9b433a6..2e0a6b4 100644
> --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> @@ -21,10 +21,13 @@ extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long sta
>  extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
>
>  extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
> +extern void radix__local_flush_all_mm(struct mm_struct *mm);
>  extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
>  extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
>  					      int psize);
>  extern void radix__tlb_flush(struct mmu_gather *tlb);
> +extern void radix_flush_pid(unsigned long pid);
> +
>  #ifdef CONFIG_SMP
>  extern void radix__flush_tlb_mm(struct mm_struct *mm);
>  extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
> diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
> index da7e943..26a587a 100644
> --- a/arch/powerpc/include/asm/mmu_context.h
> +++ b/arch/powerpc/include/asm/mmu_context.h
> @@ -45,13 +45,15 @@ extern void set_context(unsigned long id, pgd_t *pgd);
>
>  #ifdef CONFIG_PPC_BOOK3S_64
>  extern void radix__switch_mmu_context(struct mm_struct *prev,
> -				     struct mm_struct *next);
> +				      struct mm_struct *next,
> +				      bool new_on_cpu);
>  static inline void switch_mmu_context(struct mm_struct *prev,
>  				      struct mm_struct *next,
> -				      struct task_struct *tsk)
> +				      struct task_struct *tsk,
> +				      bool new_on_cpu)
>  {
>  	if (radix_enabled())
> -		return radix__switch_mmu_context(prev, next);
> +		return radix__switch_mmu_context(prev, next, new_on_cpu);
>  	return switch_slb(tsk, next);
>  }
>
> @@ -60,8 +62,13 @@ extern void hash__reserve_context_id(int id);
>  extern void __destroy_context(int context_id);
>  static inline void mmu_context_init(void) { }
>  #else
> -extern void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> -			       struct task_struct *tsk);
> +extern void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> +				 struct task_struct *tsk);
> +static inline void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> +				      struct task_struct *tsk, bool new_on_cpu)
> +{
> +	__switch_mmu_context(prev, next, tsk);
> +}
>  extern unsigned long __init_new_context(void);
>  extern void __destroy_context(unsigned long context_id);
>  extern void mmu_context_init(void);
> @@ -79,9 +86,13 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
>  				      struct mm_struct *next,
>  				      struct task_struct *tsk)
>  {
> +	bool new_on_cpu = false;
> +
>  	/* Mark this context has been used on the new CPU */
> -	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next)))
> +	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next))) {
>  		cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
> +		new_on_cpu = true;
> +	}
>
>  	/* 32-bit keeps track of the current PGDIR in the thread struct */
>  #ifdef CONFIG_PPC32
> @@ -113,7 +124,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
>  	 * The actual HW switching method differs between the various
>  	 * sub architectures. Out of line for now
>  	 */
> -	switch_mmu_context(prev, next, tsk);
> +	switch_mmu_context(prev, next, tsk, new_on_cpu);
>  }
>
>  static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
> diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S
> index e227342..a530124 100644
> --- a/arch/powerpc/kernel/head_32.S
> +++ b/arch/powerpc/kernel/head_32.S
> @@ -1003,11 +1003,11 @@ start_here:
>  	RFI
>
>  /*
> - * void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
> + * void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
>   *
>   * Set up the segment registers for a new context.
>   */
> -_ENTRY(switch_mmu_context)
> +_ENTRY(__switch_mmu_context)
>  	lwz	r3,MMCONTEXTID(r4)
>  	cmpwi	cr0,r3,0
>  	blt-	4f
> @@ -1040,7 +1040,7 @@ _ENTRY(switch_mmu_context)
>  4:	trap
>  	EMIT_BUG_ENTRY 4b,__FILE__,__LINE__,0
>  	blr
> -EXPORT_SYMBOL(switch_mmu_context)
> +EXPORT_SYMBOL(__switch_mmu_context)
>
>  /*
>   * An undocumented "feature" of 604e requires that the v bit
> diff --git a/arch/powerpc/kernel/swsusp.c b/arch/powerpc/kernel/swsusp.c
> index 0050b2d..9c32cfd 100644
> --- a/arch/powerpc/kernel/swsusp.c
> +++ b/arch/powerpc/kernel/swsusp.c
> @@ -32,6 +32,6 @@ void save_processor_state(void)
>  void restore_processor_state(void)
>  {
>  #ifdef CONFIG_PPC32
> -	switch_mmu_context(current->active_mm, current->active_mm, NULL);
> +	__switch_mmu_context(current->active_mm, current->active_mm, NULL);
>  #endif
>  }
> diff --git a/arch/powerpc/kvm/book3s_32_sr.S b/arch/powerpc/kvm/book3s_32_sr.S
> index 7e06a6f..a1705d4 100644
> --- a/arch/powerpc/kvm/book3s_32_sr.S
> +++ b/arch/powerpc/kvm/book3s_32_sr.S
> @@ -138,6 +138,6 @@
>  	lwz	r4, MM(r4)
>  	tophys(r4, r4)
>  	/* This only clobbers r0, r3, r4 and r5 */
> -	bl	switch_mmu_context
> +	bl	__switch_mmu_context
>
>  .endm
> diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> index 6ea4b53..4fb3581b 100644
> --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> @@ -1522,6 +1522,18 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
>  	std	r6, VCPU_BESCR(r9)
>  	stw	r7, VCPU_GUEST_PID(r9)
>  	std	r8, VCPU_WORT(r9)
> +
> +	/* Handle the case where the guest used an illegal PID */
> +	LOAD_REG_ADDR(r4, mmu_base_pid)
> +	lwz	r3, 0(r4)
> +	cmpw	cr0,r7,r3
> +	blt	1f
> +
> +	/* Illegal PID, flush the TLB */
> +	bl	radix_flush_pid
> +	ld	r9, HSTATE_KVM_VCPU(r13)
> +1:
> +
>  BEGIN_FTR_SECTION
>  	mfspr	r5, SPRN_TCSCR
>  	mfspr	r6, SPRN_ACOP
> diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
> index abed1fe..183a67b 100644
> --- a/arch/powerpc/mm/mmu_context_book3s64.c
> +++ b/arch/powerpc/mm/mmu_context_book3s64.c
> @@ -25,6 +25,10 @@
>  #include <asm/mmu_context.h>
>  #include <asm/pgalloc.h>
>
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +#include <asm/kvm_book3s_asm.h>
> +#endif
> +
>  #include "icswx.h"
>
>  static DEFINE_SPINLOCK(mmu_context_lock);
> @@ -126,9 +130,10 @@ static int hash__init_new_context(struct mm_struct *mm)
>  static int radix__init_new_context(struct mm_struct *mm)
>  {
>  	unsigned long rts_field;
> -	int index;
> +	int index, max_id;
>
> -	index = alloc_context_id(1, PRTB_ENTRIES - 1);
> +	max_id = (1 << mmu_pid_bits) - 1;
> +	index = alloc_context_id(mmu_base_pid, max_id);
>  	if (index < 0)
>  		return index;
>
> @@ -247,8 +252,40 @@ void destroy_context(struct mm_struct *mm)
>  }
>
>  #ifdef CONFIG_PPC_RADIX_MMU
> -void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
> +void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> +			       bool new_on_cpu)
>  {
> +	/*
> +	 * If this context hasn't run on that CPU before and KVM is
> +	 * around, there's a slim chance that the guest on another
> +	 * CPU just brought in obsolete translation into the TLB of
> +	 * this CPU due to a bad prefetch using the guest PID on
> +	 * the way into the hypervisor.
> +	 *
> +	 * We work around this here. If KVM is possible, we check if
> +	 * any sibling thread is in KVM. If it is, the window may exist
> +	 * and thus we flush that PID from the core.
> +	 *
> +	 * A potential future improvement would be to mark which PIDs
> +	 * have never been used on the system and avoid it if the PID
> +	 * is new and the process has no other cpumask bit set.
> +	 */
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +	if (cpu_has_feature(CPU_FTR_HVMODE) && new_on_cpu) {
> +		int cpu = smp_processor_id();
> +		int sib = cpu_first_thread_sibling(cpu);
> +		bool flush = false;
> +
> +		for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) {
> +			if (sib == cpu)
> +				continue;
> +			if (paca[sib].kvm_hstate.kvm_vcpu)
> +				flush = true;
> +		}
> +		if (flush)
> +			radix__local_flush_all_mm(next);
> +	}
> +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */

Is it possible to avoid that #ifdef and add kvm helper for this ? That will also
avoid that kvm header inclusion arc/powerpc/mm/*.c files. 


>
>  	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
>  		isync();
> diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c
> index 4554d65..47d84d0 100644
> --- a/arch/powerpc/mm/mmu_context_nohash.c
> +++ b/arch/powerpc/mm/mmu_context_nohash.c
> @@ -226,8 +226,8 @@ static void context_check_map(void)
>  static void context_check_map(void) { }
>  #endif
>
> -void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> -			struct task_struct *tsk)
> +void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> +			  struct task_struct *tsk)
>  {
>  	unsigned int i, id, cpu = smp_processor_id();
>  	unsigned long *map;
> diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
> index 83d70f7..3888aea 100644
> --- a/arch/powerpc/mm/pgtable-radix.c
> +++ b/arch/powerpc/mm/pgtable-radix.c
> @@ -168,11 +168,34 @@ static void __init radix_init_pgtable(void)
>  	for_each_memblock(memory, reg)
>  		WARN_ON(create_physical_mapping(reg->base,
>  						reg->base + reg->size));
> +
> +	/* Find out how many PID bits are supported */
> +	if (cpu_has_feature(CPU_FTR_HVMODE)) {
> +		if (!mmu_pid_bits)
> +			mmu_pid_bits = 20;
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +		/*
> +		 * When KVM is possible, we only use the top half of the
> +		 * PID space to avoid collisions between host and guest PIDs
> +		 * which can cause problems due to prefetch when exiting the
> +		 * guest with AIL=3
> +		 */
> +		mmu_base_pid = 1 << (mmu_pid_bits - 1);
> +#else
> +		mmu_base_pid = 1;
> +#endif
> +	} else {
> +		/* The guest uses the bottom half of the PID space */
> +		if (!mmu_pid_bits)
> +			mmu_pid_bits = 19;
> +		mmu_base_pid = 1;
> +	}
> +
>  	/*
>  	 * Allocate Partition table and process table for the
>  	 * host.
>  	 */
> -	BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 36), "Process table size too large.");
> +	BUG_ON(PRTB_SIZE_SHIFT > 36);
>  	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT);
>  	/*
>  	 * Fill in the process table.
> @@ -245,6 +268,12 @@ static int __init radix_dt_scan_page_sizes(unsigned long node,
>  	if (type == NULL || strcmp(type, "cpu") != 0)
>  		return 0;
>
> +	/* Find MMU PID size */
> +	prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size);
> +	if (prop && size == 4)
> +		mmu_pid_bits = be32_to_cpup(prop);
> +
> +	/* Grab page size encodings */
>  	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
>  	if (!prop)
>  		return 0;
> diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
> index b165bab..5cc73f1 100644
> --- a/arch/powerpc/mm/pgtable_64.c
> +++ b/arch/powerpc/mm/pgtable_64.c
> @@ -68,6 +68,9 @@
>   */
>  struct prtb_entry *process_tb;
>  struct patb_entry *partition_tb;
> +unsigned int mmu_pid_bits;
> +unsigned int mmu_base_pid;
> +
>  /*
>   * page table size
>   */
> diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
> index 73d3fbf..54d5c75 100644
> --- a/arch/powerpc/mm/tlb-radix.c
> +++ b/arch/powerpc/mm/tlb-radix.c
> @@ -66,6 +66,12 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
>  	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
>  }
>
> +/* Used by KVM */
> +void radix_flush_pid(unsigned long pid)
> +{
> +	_tlbiel_pid(pid, RIC_FLUSH_ALL);
> +}
> +
>  static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
>  {
>  	unsigned long rb,rs,prs,r;
> @@ -138,8 +144,7 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
>  }
>  EXPORT_SYMBOL(radix__local_flush_tlb_mm);
>
> -#ifndef CONFIG_SMP
> -static void radix__local_flush_all_mm(struct mm_struct *mm)
> +void radix__local_flush_all_mm(struct mm_struct *mm)
>  {
>  	unsigned long pid;
>
> @@ -149,7 +154,6 @@ static void radix__local_flush_all_mm(struct mm_struct *mm)
>  		_tlbiel_pid(pid, RIC_FLUSH_ALL);
>  	preempt_enable();
>  }
> -#endif /* CONFIG_SMP */
>
>  void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
>  				       int psize)
> diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c
> index ff44016..51026f2 100644
> --- a/drivers/cpufreq/pmac32-cpufreq.c
> +++ b/drivers/cpufreq/pmac32-cpufreq.c
> @@ -300,7 +300,7 @@ static int pmu_set_cpu_speed(int low_speed)
>   		_set_L3CR(save_l3cr);
>
>  	/* Restore userland MMU context */
> -	switch_mmu_context(NULL, current->active_mm, NULL);
> +	__switch_mmu_context(NULL, current->active_mm, NULL);
>
>  #ifdef DEBUG_FREQ
>  	printk(KERN_DEBUG "HID1, after: %x\n", mfspr(SPRN_HID1));
> diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
> index cce99f7..c06f08d 100644
> --- a/drivers/macintosh/via-pmu.c
> +++ b/drivers/macintosh/via-pmu.c
> @@ -1851,7 +1851,7 @@ static int powerbook_sleep_grackle(void)
>   		_set_L2CR(save_l2cr);
>  	
>  	/* Restore userland MMU context */
> -	switch_mmu_context(NULL, current->active_mm, NULL);
> +	__switch_mmu_context(NULL, current->active_mm, NULL);
>
>  	/* Power things up */
>  	pmu_unlock();
> @@ -1940,7 +1940,7 @@ powerbook_sleep_Core99(void)
>   		_set_L3CR(save_l3cr);
>  	
>  	/* Restore userland MMU context */
> -	switch_mmu_context(NULL, current->active_mm, NULL);
> +	__switch_mmu_context(NULL, current->active_mm, NULL);
>
>  	/* Tell PMU we are ready */
>  	pmu_unlock();
> -- 
> 2.9.4

Re: [PATCH 4/4] powerpc/mm/radix: Workaround prefetch issue with KVM

[Benjamin Herrenschmidt]

On Fri, 2017-07-14 at 11:21 +0530, Aneesh Kumar K.V wrote:
> 
> > There is still an issue with malicious guests purposefully setting
> > the PID register to a value in the host range. Hopefully future HW
> > can prevent that, but in the meantime, we handle it with a pair of
> > kludges:
> > 
> >  - On the way out of a guest, before we clear the current VCPU
> > in the PACA, we check the PID and if it's outside of the permitted
> > range we flush the TLB for that PID.
> 
> Can you explain more about this ? We still run with guest PIDR at this
> point right ? How does flushing TLB here prevent a further speculative
> access of Quadrant 0 ? Or is it that beyound this point such access is
> not possible ?

No, the code is buggy, it's supposed to be doing the test after we
restored the PIDR to the host PID but I somewhat fumbled that up.

I'll respin.
> > 
> >  - When context switching, if the mm is "new" on that CPU (the
> > corresponding bit was set for the first time in the mm cpumask), we
> > check if any sibling thread is in KVM (has a non-NULL VCPU pointer
> > in the PACA). If that is the case, we also flush the PID for that
> > CPU (core).
> 
> This is needed so that we don't run with old stale translation details
> which was left in this core right ?

Sort of. What can happen is that before the above flushes the TLB, some
translations can come in, then be invalidated on the remote CPU but not
on this one*, then another thread of this core picks up the PID and
hits the now stale translations.

Very unlikely but theorically possible. 

> 
> > 
> > This second part is needed to handle the case where a process is
> > migrated (or starts a new pthread) on a sibling thread of the CPU
> > coming out of KVM, as there's a window where stale translations
> > can exist before we detect it and flush them out.
> > 
> > A future optimization could be added by keeping track of whether
> > the PID has ever been used and avoid doing that for completely
> > fresh PIDs. We could similarily mark PIDs that have been the subject of
> > a global invalidation as "fresh". But for now this will do.
> > 
> > Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> > ---
> > ---
> >  arch/powerpc/include/asm/book3s/64/mmu.h           | 15 ++++----
> >  .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  3 ++
> >  arch/powerpc/include/asm/mmu_context.h             | 25 +++++++++----
> >  arch/powerpc/kernel/head_32.S                      |  6 +--
> >  arch/powerpc/kernel/swsusp.c                       |  2 +-
> >  arch/powerpc/kvm/book3s_32_sr.S                    |  2 +-
> >  arch/powerpc/kvm/book3s_hv_rmhandlers.S            | 12 ++++++
> >  arch/powerpc/mm/mmu_context_book3s64.c             | 43 ++++++++++++++++++++--
> >  arch/powerpc/mm/mmu_context_nohash.c               |  4 +-
> >  arch/powerpc/mm/pgtable-radix.c                    | 31 +++++++++++++++-
> >  arch/powerpc/mm/pgtable_64.c                       |  3 ++
> >  arch/powerpc/mm/tlb-radix.c                        | 10 +++--
> >  drivers/cpufreq/pmac32-cpufreq.c                   |  2 +-
> >  drivers/macintosh/via-pmu.c                        |  4 +-
> >  14 files changed, 131 insertions(+), 31 deletions(-)
> > 
> > diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
> > index 77529a3..5b4023c 100644
> > --- a/arch/powerpc/include/asm/book3s/64/mmu.h
> > +++ b/arch/powerpc/include/asm/book3s/64/mmu.h
> > @@ -59,13 +59,14 @@ extern struct patb_entry *partition_tb;
> >  #define PRTS_MASK	0x1f		/* process table size field */
> >  #define PRTB_MASK	0x0ffffffffffff000UL
> > 
> > -/*
> > - * Limit process table to PAGE_SIZE table. This
> > - * also limit the max pid we can support.
> > - * MAX_USER_CONTEXT * 16 bytes of space.
> > - */
> > -#define PRTB_SIZE_SHIFT	(CONTEXT_BITS + 4)
> > -#define PRTB_ENTRIES	(1ul << CONTEXT_BITS)
> > +/* Number of supported PID bits */
> > +extern unsigned int mmu_pid_bits;
> > +
> > +/* Base PID to allocate from */
> > +extern unsigned int mmu_base_pid;
> > +
> > +#define PRTB_SIZE_SHIFT	(mmu_pid_bits + 4)
> > +#define PRTB_ENTRIES	(1ul << mmu_pid_bits)
> > 
> >  /*
> >   * Power9 currently only support 64K partition table size.
> > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> > index 9b433a6..2e0a6b4 100644
> > --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> > @@ -21,10 +21,13 @@ extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long sta
> >  extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
> > 
> >  extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
> > +extern void radix__local_flush_all_mm(struct mm_struct *mm);
> >  extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
> >  extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
> >  					      int psize);
> >  extern void radix__tlb_flush(struct mmu_gather *tlb);
> > +extern void radix_flush_pid(unsigned long pid);
> > +
> >  #ifdef CONFIG_SMP
> >  extern void radix__flush_tlb_mm(struct mm_struct *mm);
> >  extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
> > diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
> > index da7e943..26a587a 100644
> > --- a/arch/powerpc/include/asm/mmu_context.h
> > +++ b/arch/powerpc/include/asm/mmu_context.h
> > @@ -45,13 +45,15 @@ extern void set_context(unsigned long id, pgd_t *pgd);
> > 
> >  #ifdef CONFIG_PPC_BOOK3S_64
> >  extern void radix__switch_mmu_context(struct mm_struct *prev,
> > -				     struct mm_struct *next);
> > +				      struct mm_struct *next,
> > +				      bool new_on_cpu);
> >  static inline void switch_mmu_context(struct mm_struct *prev,
> >  				      struct mm_struct *next,
> > -				      struct task_struct *tsk)
> > +				      struct task_struct *tsk,
> > +				      bool new_on_cpu)
> >  {
> >  	if (radix_enabled())
> > -		return radix__switch_mmu_context(prev, next);
> > +		return radix__switch_mmu_context(prev, next, new_on_cpu);
> >  	return switch_slb(tsk, next);
> >  }
> > 
> > @@ -60,8 +62,13 @@ extern void hash__reserve_context_id(int id);
> >  extern void __destroy_context(int context_id);
> >  static inline void mmu_context_init(void) { }
> >  #else
> > -extern void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> > -			       struct task_struct *tsk);
> > +extern void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> > +				 struct task_struct *tsk);
> > +static inline void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> > +				      struct task_struct *tsk, bool new_on_cpu)
> > +{
> > +	__switch_mmu_context(prev, next, tsk);
> > +}
> >  extern unsigned long __init_new_context(void);
> >  extern void __destroy_context(unsigned long context_id);
> >  extern void mmu_context_init(void);
> > @@ -79,9 +86,13 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
> >  				      struct mm_struct *next,
> >  				      struct task_struct *tsk)
> >  {
> > +	bool new_on_cpu = false;
> > +
> >  	/* Mark this context has been used on the new CPU */
> > -	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next)))
> > +	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next))) {
> >  		cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
> > +		new_on_cpu = true;
> > +	}
> > 
> >  	/* 32-bit keeps track of the current PGDIR in the thread struct */
> >  #ifdef CONFIG_PPC32
> > @@ -113,7 +124,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
> >  	 * The actual HW switching method differs between the various
> >  	 * sub architectures. Out of line for now
> >  	 */
> > -	switch_mmu_context(prev, next, tsk);
> > +	switch_mmu_context(prev, next, tsk, new_on_cpu);
> >  }
> > 
> >  static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
> > diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S
> > index e227342..a530124 100644
> > --- a/arch/powerpc/kernel/head_32.S
> > +++ b/arch/powerpc/kernel/head_32.S
> > @@ -1003,11 +1003,11 @@ start_here:
> >  	RFI
> > 
> >  /*
> > - * void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
> > + * void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
> >   *
> >   * Set up the segment registers for a new context.
> >   */
> > -_ENTRY(switch_mmu_context)
> > +_ENTRY(__switch_mmu_context)
> >  	lwz	r3,MMCONTEXTID(r4)
> >  	cmpwi	cr0,r3,0
> >  	blt-	4f
> > @@ -1040,7 +1040,7 @@ _ENTRY(switch_mmu_context)
> >  4:	trap
> >  	EMIT_BUG_ENTRY 4b,__FILE__,__LINE__,0
> >  	blr
> > -EXPORT_SYMBOL(switch_mmu_context)
> > +EXPORT_SYMBOL(__switch_mmu_context)
> > 
> >  /*
> >   * An undocumented "feature" of 604e requires that the v bit
> > diff --git a/arch/powerpc/kernel/swsusp.c b/arch/powerpc/kernel/swsusp.c
> > index 0050b2d..9c32cfd 100644
> > --- a/arch/powerpc/kernel/swsusp.c
> > +++ b/arch/powerpc/kernel/swsusp.c
> > @@ -32,6 +32,6 @@ void save_processor_state(void)
> >  void restore_processor_state(void)
> >  {
> >  #ifdef CONFIG_PPC32
> > -	switch_mmu_context(current->active_mm, current->active_mm, NULL);
> > +	__switch_mmu_context(current->active_mm, current->active_mm, NULL);
> >  #endif
> >  }
> > diff --git a/arch/powerpc/kvm/book3s_32_sr.S b/arch/powerpc/kvm/book3s_32_sr.S
> > index 7e06a6f..a1705d4 100644
> > --- a/arch/powerpc/kvm/book3s_32_sr.S
> > +++ b/arch/powerpc/kvm/book3s_32_sr.S
> > @@ -138,6 +138,6 @@
> >  	lwz	r4, MM(r4)
> >  	tophys(r4, r4)
> >  	/* This only clobbers r0, r3, r4 and r5 */
> > -	bl	switch_mmu_context
> > +	bl	__switch_mmu_context
> > 
> >  .endm
> > diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> > index 6ea4b53..4fb3581b 100644
> > --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> > +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> > @@ -1522,6 +1522,18 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
> >  	std	r6, VCPU_BESCR(r9)
> >  	stw	r7, VCPU_GUEST_PID(r9)
> >  	std	r8, VCPU_WORT(r9)
> > +
> > +	/* Handle the case where the guest used an illegal PID */
> > +	LOAD_REG_ADDR(r4, mmu_base_pid)
> > +	lwz	r3, 0(r4)
> > +	cmpw	cr0,r7,r3
> > +	blt	1f
> > +
> > +	/* Illegal PID, flush the TLB */
> > +	bl	radix_flush_pid
> > +	ld	r9, HSTATE_KVM_VCPU(r13)
> > +1:
> > +
> >  BEGIN_FTR_SECTION
> >  	mfspr	r5, SPRN_TCSCR
> >  	mfspr	r6, SPRN_ACOP
> > diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
> > index abed1fe..183a67b 100644
> > --- a/arch/powerpc/mm/mmu_context_book3s64.c
> > +++ b/arch/powerpc/mm/mmu_context_book3s64.c
> > @@ -25,6 +25,10 @@
> >  #include <asm/mmu_context.h>
> >  #include <asm/pgalloc.h>
> > 
> > +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> > +#include <asm/kvm_book3s_asm.h>
> > +#endif
> > +
> >  #include "icswx.h"
> > 
> >  static DEFINE_SPINLOCK(mmu_context_lock);
> > @@ -126,9 +130,10 @@ static int hash__init_new_context(struct mm_struct *mm)
> >  static int radix__init_new_context(struct mm_struct *mm)
> >  {
> >  	unsigned long rts_field;
> > -	int index;
> > +	int index, max_id;
> > 
> > -	index = alloc_context_id(1, PRTB_ENTRIES - 1);
> > +	max_id = (1 << mmu_pid_bits) - 1;
> > +	index = alloc_context_id(mmu_base_pid, max_id);
> >  	if (index < 0)
> >  		return index;
> > 
> > @@ -247,8 +252,40 @@ void destroy_context(struct mm_struct *mm)
> >  }
> > 
> >  #ifdef CONFIG_PPC_RADIX_MMU
> > -void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
> > +void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> > +			       bool new_on_cpu)
> >  {
> > +	/*
> > +	 * If this context hasn't run on that CPU before and KVM is
> > +	 * around, there's a slim chance that the guest on another
> > +	 * CPU just brought in obsolete translation into the TLB of
> > +	 * this CPU due to a bad prefetch using the guest PID on
> > +	 * the way into the hypervisor.
> > +	 *
> > +	 * We work around this here. If KVM is possible, we check if
> > +	 * any sibling thread is in KVM. If it is, the window may exist
> > +	 * and thus we flush that PID from the core.
> > +	 *
> > +	 * A potential future improvement would be to mark which PIDs
> > +	 * have never been used on the system and avoid it if the PID
> > +	 * is new and the process has no other cpumask bit set.
> > +	 */
> > +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> > +	if (cpu_has_feature(CPU_FTR_HVMODE) && new_on_cpu) {
> > +		int cpu = smp_processor_id();
> > +		int sib = cpu_first_thread_sibling(cpu);
> > +		bool flush = false;
> > +
> > +		for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) {
> > +			if (sib == cpu)
> > +				continue;
> > +			if (paca[sib].kvm_hstate.kvm_vcpu)
> > +				flush = true;
> > +		}
> > +		if (flush)
> > +			radix__local_flush_all_mm(next);
> > +	}
> > +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
> 
> Is it possible to avoid that #ifdef and add kvm helper for this ? That will also
> avoid that kvm header inclusion arc/powerpc/mm/*.c files. 
> 
> 
> > 
> >  	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
> >  		isync();
> > diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c
> > index 4554d65..47d84d0 100644
> > --- a/arch/powerpc/mm/mmu_context_nohash.c
> > +++ b/arch/powerpc/mm/mmu_context_nohash.c
> > @@ -226,8 +226,8 @@ static void context_check_map(void)
> >  static void context_check_map(void) { }
> >  #endif
> > 
> > -void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> > -			struct task_struct *tsk)
> > +void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> > +			  struct task_struct *tsk)
> >  {
> >  	unsigned int i, id, cpu = smp_processor_id();
> >  	unsigned long *map;
> > diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
> > index 83d70f7..3888aea 100644
> > --- a/arch/powerpc/mm/pgtable-radix.c
> > +++ b/arch/powerpc/mm/pgtable-radix.c
> > @@ -168,11 +168,34 @@ static void __init radix_init_pgtable(void)
> >  	for_each_memblock(memory, reg)
> >  		WARN_ON(create_physical_mapping(reg->base,
> >  						reg->base + reg->size));
> > +
> > +	/* Find out how many PID bits are supported */
> > +	if (cpu_has_feature(CPU_FTR_HVMODE)) {
> > +		if (!mmu_pid_bits)
> > +			mmu_pid_bits = 20;
> > +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> > +		/*
> > +		 * When KVM is possible, we only use the top half of the
> > +		 * PID space to avoid collisions between host and guest PIDs
> > +		 * which can cause problems due to prefetch when exiting the
> > +		 * guest with AIL=3
> > +		 */
> > +		mmu_base_pid = 1 << (mmu_pid_bits - 1);
> > +#else
> > +		mmu_base_pid = 1;
> > +#endif
> > +	} else {
> > +		/* The guest uses the bottom half of the PID space */
> > +		if (!mmu_pid_bits)
> > +			mmu_pid_bits = 19;
> > +		mmu_base_pid = 1;
> > +	}
> > +
> >  	/*
> >  	 * Allocate Partition table and process table for the
> >  	 * host.
> >  	 */
> > -	BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 36), "Process table size too large.");
> > +	BUG_ON(PRTB_SIZE_SHIFT > 36);
> >  	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT);
> >  	/*
> >  	 * Fill in the process table.
> > @@ -245,6 +268,12 @@ static int __init radix_dt_scan_page_sizes(unsigned long node,
> >  	if (type == NULL || strcmp(type, "cpu") != 0)
> >  		return 0;
> > 
> > +	/* Find MMU PID size */
> > +	prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size);
> > +	if (prop && size == 4)
> > +		mmu_pid_bits = be32_to_cpup(prop);
> > +
> > +	/* Grab page size encodings */
> >  	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
> >  	if (!prop)
> >  		return 0;
> > diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
> > index b165bab..5cc73f1 100644
> > --- a/arch/powerpc/mm/pgtable_64.c
> > +++ b/arch/powerpc/mm/pgtable_64.c
> > @@ -68,6 +68,9 @@
> >   */
> >  struct prtb_entry *process_tb;
> >  struct patb_entry *partition_tb;
> > +unsigned int mmu_pid_bits;
> > +unsigned int mmu_base_pid;
> > +
> >  /*
> >   * page table size
> >   */
> > diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
> > index 73d3fbf..54d5c75 100644
> > --- a/arch/powerpc/mm/tlb-radix.c
> > +++ b/arch/powerpc/mm/tlb-radix.c
> > @@ -66,6 +66,12 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
> >  	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
> >  }
> > 
> > +/* Used by KVM */
> > +void radix_flush_pid(unsigned long pid)
> > +{
> > +	_tlbiel_pid(pid, RIC_FLUSH_ALL);
> > +}
> > +
> >  static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
> >  {
> >  	unsigned long rb,rs,prs,r;
> > @@ -138,8 +144,7 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
> >  }
> >  EXPORT_SYMBOL(radix__local_flush_tlb_mm);
> > 
> > -#ifndef CONFIG_SMP
> > -static void radix__local_flush_all_mm(struct mm_struct *mm)
> > +void radix__local_flush_all_mm(struct mm_struct *mm)
> >  {
> >  	unsigned long pid;
> > 
> > @@ -149,7 +154,6 @@ static void radix__local_flush_all_mm(struct mm_struct *mm)
> >  		_tlbiel_pid(pid, RIC_FLUSH_ALL);
> >  	preempt_enable();
> >  }
> > -#endif /* CONFIG_SMP */
> > 
> >  void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
> >  				       int psize)
> > diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c
> > index ff44016..51026f2 100644
> > --- a/drivers/cpufreq/pmac32-cpufreq.c
> > +++ b/drivers/cpufreq/pmac32-cpufreq.c
> > @@ -300,7 +300,7 @@ static int pmu_set_cpu_speed(int low_speed)
> >   		_set_L3CR(save_l3cr);
> > 
> >  	/* Restore userland MMU context */
> > -	switch_mmu_context(NULL, current->active_mm, NULL);
> > +	__switch_mmu_context(NULL, current->active_mm, NULL);
> > 
> >  #ifdef DEBUG_FREQ
> >  	printk(KERN_DEBUG "HID1, after: %x\n", mfspr(SPRN_HID1));
> > diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
> > index cce99f7..c06f08d 100644
> > --- a/drivers/macintosh/via-pmu.c
> > +++ b/drivers/macintosh/via-pmu.c
> > @@ -1851,7 +1851,7 @@ static int powerbook_sleep_grackle(void)
> >   		_set_L2CR(save_l2cr);
> >  	
> >  	/* Restore userland MMU context */
> > -	switch_mmu_context(NULL, current->active_mm, NULL);
> > +	__switch_mmu_context(NULL, current->active_mm, NULL);
> > 
> >  	/* Power things up */
> >  	pmu_unlock();
> > @@ -1940,7 +1940,7 @@ powerbook_sleep_Core99(void)
> >   		_set_L3CR(save_l3cr);
> >  	
> >  	/* Restore userland MMU context */
> > -	switch_mmu_context(NULL, current->active_mm, NULL);
> > +	__switch_mmu_context(NULL, current->active_mm, NULL);
> > 
> >  	/* Tell PMU we are ready */
> >  	pmu_unlock();
> > -- 
> > 2.9.4

[PATCH v2] powerpc/mm/radix: Workaround prefetch issue with KVM

[Benjamin Herrenschmidt]

There's a somewhat architectural issue with Radix MMU and KVM.

When coming out of a guest with AIL (ie, MMU enabled), we start
executing hypervisor code with the PID register still containing
whatever the guest has been using.

The problem is that the CPU can (and will) then start prefetching
or speculatively load from whatever host context has that same
PID (if any), thus bringing translations for that context into
the TLB, which Linux doesn't know about.

This can cause stale translations and subsequent crashes.

Fixing this in a way that is neither racy nor a huge performance
impact is difficult. We could just make the host invalidations
always use broadcast forms but that would hurt single threaded
programs for example.

We chose to fix it instead by partitioning the PID space between
guest and host. This is possible because today Linux only use 19
out of the 20 bits of PID space, so existing guests will work
if we make the host use the top half of the 20 bits space.

We additionally add a property to indicate to Linux the size of
the PID register which will be useful if we eventually have
processors with a larger PID space available.

There is still an issue with malicious guests purposefully setting
the PID register to a value in the host range. Hopefully future HW
can prevent that, but in the meantime, we handle it with a pair of
kludges:

 - On the way out of a guest, before we clear the current VCPU
in the PACA, we check the PID and if it's outside of the permitted
range we flush the TLB for that PID.

 - When context switching, if the mm is "new" on that CPU (the
corresponding bit was set for the first time in the mm cpumask), we
check if any sibling thread is in KVM (has a non-NULL VCPU pointer
in the PACA). If that is the case, we also flush the PID for that
CPU (core).

This second part is needed to handle the case where a process is
migrated (or starts a new pthread) on a sibling thread of the CPU
coming out of KVM, as there's a window where stale translations
can exist before we detect it and flush them out.

A future optimization could be added by keeping track of whether
the PID has ever been used and avoid doing that for completely
fresh PIDs. We could similarily mark PIDs that have been the subject of
a global invalidation as "fresh". But for now this will do.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
---

v2. Do the check on KVM exit *after* we've restored the host PID

---
 arch/powerpc/include/asm/book3s/64/mmu.h           | 15 ++++----
 .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  3 ++
 arch/powerpc/include/asm/mmu_context.h             | 25 +++++++++----
 arch/powerpc/kernel/head_32.S                      |  6 +--
 arch/powerpc/kernel/swsusp.c                       |  2 +-
 arch/powerpc/kvm/book3s_32_sr.S                    |  2 +-
 arch/powerpc/kvm/book3s_hv_rmhandlers.S            | 14 +++++++
 arch/powerpc/mm/mmu_context_book3s64.c             | 43 ++++++++++++++++++++--
 arch/powerpc/mm/mmu_context_nohash.c               |  4 +-
 arch/powerpc/mm/pgtable-radix.c                    | 31 +++++++++++++++-
 arch/powerpc/mm/pgtable_64.c                       |  3 ++
 arch/powerpc/mm/tlb-radix.c                        | 10 +++--
 drivers/cpufreq/pmac32-cpufreq.c                   |  2 +-
 drivers/macintosh/via-pmu.c                        |  4 +-
 14 files changed, 133 insertions(+), 31 deletions(-)

diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
index 77529a3..5b4023c 100644
--- a/arch/powerpc/include/asm/book3s/64/mmu.h
+++ b/arch/powerpc/include/asm/book3s/64/mmu.h
@@ -59,13 +59,14 @@ extern struct patb_entry *partition_tb;
 #define PRTS_MASK	0x1f		/* process table size field */
 #define PRTB_MASK	0x0ffffffffffff000UL
 
-/*
- * Limit process table to PAGE_SIZE table. This
- * also limit the max pid we can support.
- * MAX_USER_CONTEXT * 16 bytes of space.
- */
-#define PRTB_SIZE_SHIFT	(CONTEXT_BITS + 4)
-#define PRTB_ENTRIES	(1ul << CONTEXT_BITS)
+/* Number of supported PID bits */
+extern unsigned int mmu_pid_bits;
+
+/* Base PID to allocate from */
+extern unsigned int mmu_base_pid;
+
+#define PRTB_SIZE_SHIFT	(mmu_pid_bits + 4)
+#define PRTB_ENTRIES	(1ul << mmu_pid_bits)
 
 /*
  * Power9 currently only support 64K partition table size.
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 9b433a6..2e0a6b4 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -21,10 +21,13 @@ extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long sta
 extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
 
 extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
+extern void radix__local_flush_all_mm(struct mm_struct *mm);
 extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
 extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 					      int psize);
 extern void radix__tlb_flush(struct mmu_gather *tlb);
+extern void radix_flush_pid(unsigned long pid);
+
 #ifdef CONFIG_SMP
 extern void radix__flush_tlb_mm(struct mm_struct *mm);
 extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
index da7e943..26a587a 100644
--- a/arch/powerpc/include/asm/mmu_context.h
+++ b/arch/powerpc/include/asm/mmu_context.h
@@ -45,13 +45,15 @@ extern void set_context(unsigned long id, pgd_t *pgd);
 
 #ifdef CONFIG_PPC_BOOK3S_64
 extern void radix__switch_mmu_context(struct mm_struct *prev,
-				     struct mm_struct *next);
+				      struct mm_struct *next,
+				      bool new_on_cpu);
 static inline void switch_mmu_context(struct mm_struct *prev,
 				      struct mm_struct *next,
-				      struct task_struct *tsk)
+				      struct task_struct *tsk,
+				      bool new_on_cpu)
 {
 	if (radix_enabled())
-		return radix__switch_mmu_context(prev, next);
+		return radix__switch_mmu_context(prev, next, new_on_cpu);
 	return switch_slb(tsk, next);
 }
 
@@ -60,8 +62,13 @@ extern void hash__reserve_context_id(int id);
 extern void __destroy_context(int context_id);
 static inline void mmu_context_init(void) { }
 #else
-extern void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
-			       struct task_struct *tsk);
+extern void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
+				 struct task_struct *tsk);
+static inline void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
+				      struct task_struct *tsk, bool new_on_cpu)
+{
+	__switch_mmu_context(prev, next, tsk);
+}
 extern unsigned long __init_new_context(void);
 extern void __destroy_context(unsigned long context_id);
 extern void mmu_context_init(void);
@@ -79,9 +86,13 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
 				      struct mm_struct *next,
 				      struct task_struct *tsk)
 {
+	bool new_on_cpu = false;
+
 	/* Mark this context has been used on the new CPU */
-	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next)))
+	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next))) {
 		cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
+		new_on_cpu = true;
+	}
 
 	/* 32-bit keeps track of the current PGDIR in the thread struct */
 #ifdef CONFIG_PPC32
@@ -113,7 +124,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
 	 * The actual HW switching method differs between the various
 	 * sub architectures. Out of line for now
 	 */
-	switch_mmu_context(prev, next, tsk);
+	switch_mmu_context(prev, next, tsk, new_on_cpu);
 }
 
 static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S
index e227342..a530124 100644
--- a/arch/powerpc/kernel/head_32.S
+++ b/arch/powerpc/kernel/head_32.S
@@ -1003,11 +1003,11 @@ start_here:
 	RFI
 
 /*
- * void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
+ * void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
  *
  * Set up the segment registers for a new context.
  */
-_ENTRY(switch_mmu_context)
+_ENTRY(__switch_mmu_context)
 	lwz	r3,MMCONTEXTID(r4)
 	cmpwi	cr0,r3,0
 	blt-	4f
@@ -1040,7 +1040,7 @@ _ENTRY(switch_mmu_context)
 4:	trap
 	EMIT_BUG_ENTRY 4b,__FILE__,__LINE__,0
 	blr
-EXPORT_SYMBOL(switch_mmu_context)
+EXPORT_SYMBOL(__switch_mmu_context)
 
 /*
  * An undocumented "feature" of 604e requires that the v bit
diff --git a/arch/powerpc/kernel/swsusp.c b/arch/powerpc/kernel/swsusp.c
index 0050b2d..9c32cfd 100644
--- a/arch/powerpc/kernel/swsusp.c
+++ b/arch/powerpc/kernel/swsusp.c
@@ -32,6 +32,6 @@ void save_processor_state(void)
 void restore_processor_state(void)
 {
 #ifdef CONFIG_PPC32
-	switch_mmu_context(current->active_mm, current->active_mm, NULL);
+	__switch_mmu_context(current->active_mm, current->active_mm, NULL);
 #endif
 }
diff --git a/arch/powerpc/kvm/book3s_32_sr.S b/arch/powerpc/kvm/book3s_32_sr.S
index 7e06a6f..a1705d4 100644
--- a/arch/powerpc/kvm/book3s_32_sr.S
+++ b/arch/powerpc/kvm/book3s_32_sr.S
@@ -138,6 +138,6 @@
 	lwz	r4, MM(r4)
 	tophys(r4, r4)
 	/* This only clobbers r0, r3, r4 and r5 */
-	bl	switch_mmu_context
+	bl	__switch_mmu_context
 
 .endm
diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index 6ea4b53..e744d11 100644
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -1522,6 +1522,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
 	std	r6, VCPU_BESCR(r9)
 	stw	r7, VCPU_GUEST_PID(r9)
 	std	r8, VCPU_WORT(r9)
+
 BEGIN_FTR_SECTION
 	mfspr	r5, SPRN_TCSCR
 	mfspr	r6, SPRN_ACOP
@@ -1728,6 +1729,19 @@ BEGIN_FTR_SECTION
 	mtspr	SPRN_PSSCR, r6
 	mtspr	SPRN_PID, r7
 	mtspr	SPRN_IAMR, r8
+
+	/* Handle the case where the guest used an illegal PID */
+	LOAD_REG_ADDR(r4, mmu_base_pid)
+	lwz	r3, VCPU_GUEST_PID(r9)
+	lwz	r5, 0(r4)
+	cmpw	cr0,r3,r5
+	blt	1f
+
+	/* Illegal PID, flush the TLB */
+	isync
+	bl	radix_flush_pid
+1:
+
 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
 BEGIN_FTR_SECTION
 	PPC_INVALIDATE_ERAT
diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
index abed1fe..183a67b 100644
--- a/arch/powerpc/mm/mmu_context_book3s64.c
+++ b/arch/powerpc/mm/mmu_context_book3s64.c
@@ -25,6 +25,10 @@
 #include <asm/mmu_context.h>
 #include <asm/pgalloc.h>
 
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+#include <asm/kvm_book3s_asm.h>
+#endif
+
 #include "icswx.h"
 
 static DEFINE_SPINLOCK(mmu_context_lock);
@@ -126,9 +130,10 @@ static int hash__init_new_context(struct mm_struct *mm)
 static int radix__init_new_context(struct mm_struct *mm)
 {
 	unsigned long rts_field;
-	int index;
+	int index, max_id;
 
-	index = alloc_context_id(1, PRTB_ENTRIES - 1);
+	max_id = (1 << mmu_pid_bits) - 1;
+	index = alloc_context_id(mmu_base_pid, max_id);
 	if (index < 0)
 		return index;
 
@@ -247,8 +252,40 @@ void destroy_context(struct mm_struct *mm)
 }
 
 #ifdef CONFIG_PPC_RADIX_MMU
-void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
+void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
+			       bool new_on_cpu)
 {
+	/*
+	 * If this context hasn't run on that CPU before and KVM is
+	 * around, there's a slim chance that the guest on another
+	 * CPU just brought in obsolete translation into the TLB of
+	 * this CPU due to a bad prefetch using the guest PID on
+	 * the way into the hypervisor.
+	 *
+	 * We work around this here. If KVM is possible, we check if
+	 * any sibling thread is in KVM. If it is, the window may exist
+	 * and thus we flush that PID from the core.
+	 *
+	 * A potential future improvement would be to mark which PIDs
+	 * have never been used on the system and avoid it if the PID
+	 * is new and the process has no other cpumask bit set.
+	 */
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+	if (cpu_has_feature(CPU_FTR_HVMODE) && new_on_cpu) {
+		int cpu = smp_processor_id();
+		int sib = cpu_first_thread_sibling(cpu);
+		bool flush = false;
+
+		for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) {
+			if (sib == cpu)
+				continue;
+			if (paca[sib].kvm_hstate.kvm_vcpu)
+				flush = true;
+		}
+		if (flush)
+			radix__local_flush_all_mm(next);
+	}
+#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
 
 	if (cpu_has_feature(CPU_FTR_POWER9_DD1)) {
 		isync();
diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c
index 4554d65..47d84d0 100644
--- a/arch/powerpc/mm/mmu_context_nohash.c
+++ b/arch/powerpc/mm/mmu_context_nohash.c
@@ -226,8 +226,8 @@ static void context_check_map(void)
 static void context_check_map(void) { }
 #endif
 
-void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
-			struct task_struct *tsk)
+void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
+			  struct task_struct *tsk)
 {
 	unsigned int i, id, cpu = smp_processor_id();
 	unsigned long *map;
diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c
index 83d70f7..3888aea 100644
--- a/arch/powerpc/mm/pgtable-radix.c
+++ b/arch/powerpc/mm/pgtable-radix.c
@@ -168,11 +168,34 @@ static void __init radix_init_pgtable(void)
 	for_each_memblock(memory, reg)
 		WARN_ON(create_physical_mapping(reg->base,
 						reg->base + reg->size));
+
+	/* Find out how many PID bits are supported */
+	if (cpu_has_feature(CPU_FTR_HVMODE)) {
+		if (!mmu_pid_bits)
+			mmu_pid_bits = 20;
+#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
+		/*
+		 * When KVM is possible, we only use the top half of the
+		 * PID space to avoid collisions between host and guest PIDs
+		 * which can cause problems due to prefetch when exiting the
+		 * guest with AIL=3
+		 */
+		mmu_base_pid = 1 << (mmu_pid_bits - 1);
+#else
+		mmu_base_pid = 1;
+#endif
+	} else {
+		/* The guest uses the bottom half of the PID space */
+		if (!mmu_pid_bits)
+			mmu_pid_bits = 19;
+		mmu_base_pid = 1;
+	}
+
 	/*
 	 * Allocate Partition table and process table for the
 	 * host.
 	 */
-	BUILD_BUG_ON_MSG((PRTB_SIZE_SHIFT > 36), "Process table size too large.");
+	BUG_ON(PRTB_SIZE_SHIFT > 36);
 	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT);
 	/*
 	 * Fill in the process table.
@@ -245,6 +268,12 @@ static int __init radix_dt_scan_page_sizes(unsigned long node,
 	if (type == NULL || strcmp(type, "cpu") != 0)
 		return 0;
 
+	/* Find MMU PID size */
+	prop = of_get_flat_dt_prop(node, "ibm,mmu-pid-bits", &size);
+	if (prop && size == 4)
+		mmu_pid_bits = be32_to_cpup(prop);
+
+	/* Grab page size encodings */
 	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
 	if (!prop)
 		return 0;
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index b165bab..5cc73f1 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -68,6 +68,9 @@
  */
 struct prtb_entry *process_tb;
 struct patb_entry *partition_tb;
+unsigned int mmu_pid_bits;
+unsigned int mmu_base_pid;
+
 /*
  * page table size
  */
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 73d3fbf..54d5c75 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -66,6 +66,12 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
 	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");
 }
 
+/* Used by KVM */
+void radix_flush_pid(unsigned long pid)
+{
+	_tlbiel_pid(pid, RIC_FLUSH_ALL);
+}
+
 static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
 {
 	unsigned long rb,rs,prs,r;
@@ -138,8 +144,7 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
 }
 EXPORT_SYMBOL(radix__local_flush_tlb_mm);
 
-#ifndef CONFIG_SMP
-static void radix__local_flush_all_mm(struct mm_struct *mm)
+void radix__local_flush_all_mm(struct mm_struct *mm)
 {
 	unsigned long pid;
 
@@ -149,7 +154,6 @@ static void radix__local_flush_all_mm(struct mm_struct *mm)
 		_tlbiel_pid(pid, RIC_FLUSH_ALL);
 	preempt_enable();
 }
-#endif /* CONFIG_SMP */
 
 void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
 				       int psize)
diff --git a/drivers/cpufreq/pmac32-cpufreq.c b/drivers/cpufreq/pmac32-cpufreq.c
index ff44016..51026f2 100644
--- a/drivers/cpufreq/pmac32-cpufreq.c
+++ b/drivers/cpufreq/pmac32-cpufreq.c
@@ -300,7 +300,7 @@ static int pmu_set_cpu_speed(int low_speed)
  		_set_L3CR(save_l3cr);
 
 	/* Restore userland MMU context */
-	switch_mmu_context(NULL, current->active_mm, NULL);
+	__switch_mmu_context(NULL, current->active_mm, NULL);
 
 #ifdef DEBUG_FREQ
 	printk(KERN_DEBUG "HID1, after: %x\n", mfspr(SPRN_HID1));
diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
index cce99f7..c06f08d 100644
--- a/drivers/macintosh/via-pmu.c
+++ b/drivers/macintosh/via-pmu.c
@@ -1851,7 +1851,7 @@ static int powerbook_sleep_grackle(void)
  		_set_L2CR(save_l2cr);
 	
 	/* Restore userland MMU context */
-	switch_mmu_context(NULL, current->active_mm, NULL);
+	__switch_mmu_context(NULL, current->active_mm, NULL);
 
 	/* Power things up */
 	pmu_unlock();
@@ -1940,7 +1940,7 @@ powerbook_sleep_Core99(void)
  		_set_L3CR(save_l3cr);
 	
 	/* Restore userland MMU context */
-	switch_mmu_context(NULL, current->active_mm, NULL);
+	__switch_mmu_context(NULL, current->active_mm, NULL);
 
 	/* Tell PMU we are ready */
 	pmu_unlock();

Re: [PATCH v2] powerpc/mm/radix: Workaround prefetch issue with KVM

[Aneesh Kumar K.V]

Benjamin Herrenschmidt <benh@kernel.crashing.org> writes:

> There's a somewhat architectural issue with Radix MMU and KVM.
>
> When coming out of a guest with AIL (ie, MMU enabled), we start
> executing hypervisor code with the PID register still containing
> whatever the guest has been using.
>
> The problem is that the CPU can (and will) then start prefetching
> or speculatively load from whatever host context has that same
> PID (if any), thus bringing translations for that context into
> the TLB, which Linux doesn't know about.
>
> This can cause stale translations and subsequent crashes.
>
> Fixing this in a way that is neither racy nor a huge performance
> impact is difficult. We could just make the host invalidations
> always use broadcast forms but that would hurt single threaded
> programs for example.
>
> We chose to fix it instead by partitioning the PID space between
> guest and host. This is possible because today Linux only use 19
> out of the 20 bits of PID space, so existing guests will work
> if we make the host use the top half of the 20 bits space.
>
> We additionally add a property to indicate to Linux the size of
> the PID register which will be useful if we eventually have
> processors with a larger PID space available.
>
> There is still an issue with malicious guests purposefully setting
> the PID register to a value in the host range. Hopefully future HW
> can prevent that, but in the meantime, we handle it with a pair of
> kludges:
>
>  - On the way out of a guest, before we clear the current VCPU
> in the PACA, we check the PID and if it's outside of the permitted
> range we flush the TLB for that PID.
>
>  - When context switching, if the mm is "new" on that CPU (the
> corresponding bit was set for the first time in the mm cpumask), we
> check if any sibling thread is in KVM (has a non-NULL VCPU pointer
> in the PACA). If that is the case, we also flush the PID for that
> CPU (core).
>
> This second part is needed to handle the case where a process is
> migrated (or starts a new pthread) on a sibling thread of the CPU
> coming out of KVM, as there's a window where stale translations
> can exist before we detect it and flush them out.
>
> A future optimization could be added by keeping track of whether
> the PID has ever been used and avoid doing that for completely
> fresh PIDs. We could similarily mark PIDs that have been the subject of
> a global invalidation as "fresh". But for now this will do.
>
> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> ---
>
> v2. Do the check on KVM exit *after* we've restored the host PID
>
....

 diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> index 6ea4b53..e744d11 100644
> --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> @@ -1522,6 +1522,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
>  	std	r6, VCPU_BESCR(r9)
>  	stw	r7, VCPU_GUEST_PID(r9)
>  	std	r8, VCPU_WORT(r9)
> +
>  BEGIN_FTR_SECTION
>  	mfspr	r5, SPRN_TCSCR
>  	mfspr	r6, SPRN_ACOP
> @@ -1728,6 +1729,19 @@ BEGIN_FTR_SECTION
>  	mtspr	SPRN_PSSCR, r6
>  	mtspr	SPRN_PID, r7
>  	mtspr	SPRN_IAMR, r8
> +
> +	/* Handle the case where the guest used an illegal PID */
> +	LOAD_REG_ADDR(r4, mmu_base_pid)
> +	lwz	r3, VCPU_GUEST_PID(r9)
> +	lwz	r5, 0(r4)
> +	cmpw	cr0,r3,r5
> +	blt	1f
> +
> +	/* Illegal PID, flush the TLB */
> +	isync
> +	bl	radix_flush_pid
> +1:

this need to be done only for radix right ? Do we need radix feature
check here ?


> +
>  END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
>  BEGIN_FTR_SECTION
>  	PPC_INVALIDATE_ERAT


-aneesh

Re: [PATCH 4/4] powerpc/mm/radix: Workaround prefetch issue with KVM

[Balbir Singh]

On Fri, 2017-07-14 at 11:52 +1000, Benjamin Herrenschmidt wrote:
> There's a somewhat architectural issue with Radix MMU and KVM.
> 
> When coming out of a guest with AIL (ie, MMU enabled), we start
> executing hypervisor code with the PID register still containing
> whatever the guest has been using.
> 
> The problem is that the CPU can (and will) then start prefetching
> or speculatively load from whatever host context has that same
> PID (if any), thus bringing translations for that context into
> the TLB, which Linux doesn't know about.
> 
> This can cause stale translations and subsequent crashes.
> 
> Fixing this in a way that is neither racy nor a huge performance
> impact is difficult. We could just make the host invalidations
> always use broadcast forms but that would hurt single threaded
> programs for example.
> 
> We chose to fix it instead by partitioning the PID space between
> guest and host. This is possible because today Linux only use 19
> out of the 20 bits of PID space, so existing guests will work
> if we make the host use the top half of the 20 bits space.
> 
> We additionally add a property to indicate to Linux the size of
> the PID register which will be useful if we eventually have
> processors with a larger PID space available.
> 
> There is still an issue with malicious guests purposefully setting
> the PID register to a value in the host range. Hopefully future HW
> can prevent that, but in the meantime, we handle it with a pair of
> kludges:
> 
>  - On the way out of a guest, before we clear the current VCPU
> in the PACA, we check the PID and if it's outside of the permitted
> range we flush the TLB for that PID.
> 
>  - When context switching, if the mm is "new" on that CPU (the
> corresponding bit was set for the first time in the mm cpumask), we
> check if any sibling thread is in KVM (has a non-NULL VCPU pointer
> in the PACA). If that is the case, we also flush the PID for that
> CPU (core).
> 
> This second part is needed to handle the case where a process is
> migrated (or starts a new pthread) on a sibling thread of the CPU
> coming out of KVM, as there's a window where stale translations
> can exist before we detect it and flush them out.
> 
> A future optimization could be added by keeping track of whether
> the PID has ever been used and avoid doing that for completely
> fresh PIDs. We could similarily mark PIDs that have been the subject of
> a global invalidation as "fresh". But for now this will do.
> 
> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> ---
> ---
>  arch/powerpc/include/asm/book3s/64/mmu.h           | 15 ++++----
>  .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  3 ++
>  arch/powerpc/include/asm/mmu_context.h             | 25 +++++++++----
>  arch/powerpc/kernel/head_32.S                      |  6 +--
>  arch/powerpc/kernel/swsusp.c                       |  2 +-
>  arch/powerpc/kvm/book3s_32_sr.S                    |  2 +-
>  arch/powerpc/kvm/book3s_hv_rmhandlers.S            | 12 ++++++
>  arch/powerpc/mm/mmu_context_book3s64.c             | 43 ++++++++++++++++++++--
>  arch/powerpc/mm/mmu_context_nohash.c               |  4 +-
>  arch/powerpc/mm/pgtable-radix.c                    | 31 +++++++++++++++-
>  arch/powerpc/mm/pgtable_64.c                       |  3 ++
>  arch/powerpc/mm/tlb-radix.c                        | 10 +++--
>  drivers/cpufreq/pmac32-cpufreq.c                   |  2 +-
>  drivers/macintosh/via-pmu.c                        |  4 +-
>  14 files changed, 131 insertions(+), 31 deletions(-)
> 
> diff --git a/arch/powerpc/include/asm/book3s/64/mmu.h b/arch/powerpc/include/asm/book3s/64/mmu.h
> index 77529a3..5b4023c 100644
> --- a/arch/powerpc/include/asm/book3s/64/mmu.h
> +++ b/arch/powerpc/include/asm/book3s/64/mmu.h
> @@ -59,13 +59,14 @@ extern struct patb_entry *partition_tb;
>  #define PRTS_MASK	0x1f		/* process table size field */
>  #define PRTB_MASK	0x0ffffffffffff000UL
>  
> -/*
> - * Limit process table to PAGE_SIZE table. This
> - * also limit the max pid we can support.
> - * MAX_USER_CONTEXT * 16 bytes of space.
> - */
> -#define PRTB_SIZE_SHIFT	(CONTEXT_BITS + 4)
> -#define PRTB_ENTRIES	(1ul << CONTEXT_BITS)
> +/* Number of supported PID bits */
> +extern unsigned int mmu_pid_bits;
> +
> +/* Base PID to allocate from */
> +extern unsigned int mmu_base_pid;
> +
> +#define PRTB_SIZE_SHIFT	(mmu_pid_bits + 4)
> +#define PRTB_ENTRIES	(1ul << mmu_pid_bits)
>  
>  /*
>   * Power9 currently only support 64K partition table size.
> diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> index 9b433a6..2e0a6b4 100644
> --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> @@ -21,10 +21,13 @@ extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long sta
>  extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
>  
>  extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
> +extern void radix__local_flush_all_mm(struct mm_struct *mm);
>  extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
>  extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
>  					      int psize);
>  extern void radix__tlb_flush(struct mmu_gather *tlb);
> +extern void radix_flush_pid(unsigned long pid);
> +
>  #ifdef CONFIG_SMP
>  extern void radix__flush_tlb_mm(struct mm_struct *mm);
>  extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
> diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h
> index da7e943..26a587a 100644
> --- a/arch/powerpc/include/asm/mmu_context.h
> +++ b/arch/powerpc/include/asm/mmu_context.h
> @@ -45,13 +45,15 @@ extern void set_context(unsigned long id, pgd_t *pgd);
>  
>  #ifdef CONFIG_PPC_BOOK3S_64
>  extern void radix__switch_mmu_context(struct mm_struct *prev,
> -				     struct mm_struct *next);
> +				      struct mm_struct *next,
> +				      bool new_on_cpu);
>  static inline void switch_mmu_context(struct mm_struct *prev,
>  				      struct mm_struct *next,
> -				      struct task_struct *tsk)
> +				      struct task_struct *tsk,
> +				      bool new_on_cpu)
>  {
>  	if (radix_enabled())
> -		return radix__switch_mmu_context(prev, next);
> +		return radix__switch_mmu_context(prev, next, new_on_cpu);
>  	return switch_slb(tsk, next);
>  }
>  
> @@ -60,8 +62,13 @@ extern void hash__reserve_context_id(int id);
>  extern void __destroy_context(int context_id);
>  static inline void mmu_context_init(void) { }
>  #else
> -extern void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> -			       struct task_struct *tsk);
> +extern void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> +				 struct task_struct *tsk);
> +static inline void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> +				      struct task_struct *tsk, bool new_on_cpu)
> +{
> +	__switch_mmu_context(prev, next, tsk);
> +}
>  extern unsigned long __init_new_context(void);
>  extern void __destroy_context(unsigned long context_id);
>  extern void mmu_context_init(void);
> @@ -79,9 +86,13 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
>  				      struct mm_struct *next,
>  				      struct task_struct *tsk)
>  {
> +	bool new_on_cpu = false;
> +
>  	/* Mark this context has been used on the new CPU */
> -	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next)))
> +	if (!cpumask_test_cpu(smp_processor_id(), mm_cpumask(next))) {
>  		cpumask_set_cpu(smp_processor_id(), mm_cpumask(next));
> +		new_on_cpu = true;
> +	}
>  
>  	/* 32-bit keeps track of the current PGDIR in the thread struct */
>  #ifdef CONFIG_PPC32
> @@ -113,7 +124,7 @@ static inline void switch_mm_irqs_off(struct mm_struct *prev,
>  	 * The actual HW switching method differs between the various
>  	 * sub architectures. Out of line for now
>  	 */
> -	switch_mmu_context(prev, next, tsk);
> +	switch_mmu_context(prev, next, tsk, new_on_cpu);
>  }
>  
>  static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
> diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S
> index e227342..a530124 100644
> --- a/arch/powerpc/kernel/head_32.S
> +++ b/arch/powerpc/kernel/head_32.S
> @@ -1003,11 +1003,11 @@ start_here:
>  	RFI
>  
>  /*
> - * void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
> + * void __switch_mmu_context(struct mm_struct *prev, struct mm_struct *next);
>   *
>   * Set up the segment registers for a new context.
>   */
> -_ENTRY(switch_mmu_context)
> +_ENTRY(__switch_mmu_context)
>  	lwz	r3,MMCONTEXTID(r4)
>  	cmpwi	cr0,r3,0
>  	blt-	4f
> @@ -1040,7 +1040,7 @@ _ENTRY(switch_mmu_context)
>  4:	trap
>  	EMIT_BUG_ENTRY 4b,__FILE__,__LINE__,0
>  	blr
> -EXPORT_SYMBOL(switch_mmu_context)
> +EXPORT_SYMBOL(__switch_mmu_context)
>  
>  /*
>   * An undocumented "feature" of 604e requires that the v bit
> diff --git a/arch/powerpc/kernel/swsusp.c b/arch/powerpc/kernel/swsusp.c
> index 0050b2d..9c32cfd 100644
> --- a/arch/powerpc/kernel/swsusp.c
> +++ b/arch/powerpc/kernel/swsusp.c
> @@ -32,6 +32,6 @@ void save_processor_state(void)
>  void restore_processor_state(void)
>  {
>  #ifdef CONFIG_PPC32
> -	switch_mmu_context(current->active_mm, current->active_mm, NULL);
> +	__switch_mmu_context(current->active_mm, current->active_mm, NULL);
>  #endif
>  }
> diff --git a/arch/powerpc/kvm/book3s_32_sr.S b/arch/powerpc/kvm/book3s_32_sr.S
> index 7e06a6f..a1705d4 100644
> --- a/arch/powerpc/kvm/book3s_32_sr.S
> +++ b/arch/powerpc/kvm/book3s_32_sr.S
> @@ -138,6 +138,6 @@
>  	lwz	r4, MM(r4)
>  	tophys(r4, r4)
>  	/* This only clobbers r0, r3, r4 and r5 */
> -	bl	switch_mmu_context
> +	bl	__switch_mmu_context
>  
>  .endm
> diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> index 6ea4b53..4fb3581b 100644
> --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> @@ -1522,6 +1522,18 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
>  	std	r6, VCPU_BESCR(r9)
>  	stw	r7, VCPU_GUEST_PID(r9)
>  	std	r8, VCPU_WORT(r9)
> +
> +	/* Handle the case where the guest used an illegal PID */
> +	LOAD_REG_ADDR(r4, mmu_base_pid)
> +	lwz	r3, 0(r4)
> +	cmpw	cr0,r7,r3
> +	blt	1f

So the boundary is [1..(1<<mmu_pid_bits-1)]? Do we flush the tlb
for pid 0 always?

> +
> +	/* Illegal PID, flush the TLB */
> +	bl	radix_flush_pid
> +	ld	r9, HSTATE_KVM_VCPU(r13)
> +1:
> +
>  BEGIN_FTR_SECTION
>  	mfspr	r5, SPRN_TCSCR
>  	mfspr	r6, SPRN_ACOP
> diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
> index abed1fe..183a67b 100644
> --- a/arch/powerpc/mm/mmu_context_book3s64.c
> +++ b/arch/powerpc/mm/mmu_context_book3s64.c
> @@ -25,6 +25,10 @@
>  #include <asm/mmu_context.h>
>  #include <asm/pgalloc.h>
>  
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +#include <asm/kvm_book3s_asm.h>
> +#endif
> +
>  #include "icswx.h"
>  
>  static DEFINE_SPINLOCK(mmu_context_lock);
> @@ -126,9 +130,10 @@ static int hash__init_new_context(struct mm_struct *mm)
>  static int radix__init_new_context(struct mm_struct *mm)
>  {
>  	unsigned long rts_field;
> -	int index;
> +	int index, max_id;
>  
> -	index = alloc_context_id(1, PRTB_ENTRIES - 1);
> +	max_id = (1 << mmu_pid_bits) - 1;
> +	index = alloc_context_id(mmu_base_pid, max_id);
>  	if (index < 0)
>  		return index;
>  
> @@ -247,8 +252,40 @@ void destroy_context(struct mm_struct *mm)
>  }
>  
>  #ifdef CONFIG_PPC_RADIX_MMU
> -void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
> +void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> +			       bool new_on_cpu)
>  {
> +	/*
> +	 * If this context hasn't run on that CPU before and KVM is
> +	 * around, there's a slim chance that the guest on another
> +	 * CPU just brought in obsolete translation into the TLB of
> +	 * this CPU due to a bad prefetch using the guest PID on
> +	 * the way into the hypervisor.
> +	 *
> +	 * We work around this here. If KVM is possible, we check if
> +	 * any sibling thread is in KVM. If it is, the window may exist
> +	 * and thus we flush that PID from the core.
> +	 *
> +	 * A potential future improvement would be to mark which PIDs
> +	 * have never been used on the system and avoid it if the PID
> +	 * is new and the process has no other cpumask bit set.

Also due to the pid split, the chances of the context bringing in
TLB entries is low, but a bad guest could bring in stale entries

> +	 */
> +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> +	if (cpu_has_feature(CPU_FTR_HVMODE) && new_on_cpu) {
> +		int cpu = smp_processor_id();
> +		int sib = cpu_first_thread_sibling(cpu);
> +		bool flush = false;
> +
> +		for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) {
> +			if (sib == cpu)
> +				continue;
> +			if (paca[sib].kvm_hstate.kvm_vcpu)
> +				flush = true;
> +		}
> +		if (flush)
> +			radix__local_flush_all_mm(next);
> +	}
> +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
>  

Re: [PATCH 4/4] powerpc/mm/radix: Workaround prefetch issue with KVM

[Benjamin Herrenschmidt]

On Wed, 2017-07-19 at 12:29 +1000, Balbir Singh wrote:
> 
> > diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> > index 6ea4b53..4fb3581b 100644
> > --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> > +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
> > @@ -1522,6 +1522,18 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
> >  	std	r6, VCPU_BESCR(r9)
> >  	stw	r7, VCPU_GUEST_PID(r9)
> >  	std	r8, VCPU_WORT(r9)
> > +
> > +	/* Handle the case where the guest used an illegal PID */
> > +	LOAD_REG_ADDR(r4, mmu_base_pid)
> > +	lwz	r3, 0(r4)
> > +	cmpw	cr0,r7,r3
> > +	blt	1f
> 
> So the boundary is [1..(1<<mmu_pid_bits-1)]? Do we flush the tlb
> for pid 0 always?

What do you mean ? mmu_base_pid is going to be 1 << (mmu_pid_bits-1),
we flush the TLB if the guest PID r7 is not lower than the boundary.

Ntoe: this is patch v1, look at v2.

> > +
> > +	/* Illegal PID, flush the TLB */
> > +	bl	radix_flush_pid
> > +	ld	r9, HSTATE_KVM_VCPU(r13)
> > +1:
> > +
> >  BEGIN_FTR_SECTION
> >  	mfspr	r5, SPRN_TCSCR
> >  	mfspr	r6, SPRN_ACOP
> > diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c
> > index abed1fe..183a67b 100644
> > --- a/arch/powerpc/mm/mmu_context_book3s64.c
> > +++ b/arch/powerpc/mm/mmu_context_book3s64.c
> > @@ -25,6 +25,10 @@
> >  #include <asm/mmu_context.h>
> >  #include <asm/pgalloc.h>
> >  
> > +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> > +#include <asm/kvm_book3s_asm.h>
> > +#endif
> > +
> >  #include "icswx.h"
> >  
> >  static DEFINE_SPINLOCK(mmu_context_lock);
> > @@ -126,9 +130,10 @@ static int hash__init_new_context(struct mm_struct *mm)
> >  static int radix__init_new_context(struct mm_struct *mm)
> >  {
> >  	unsigned long rts_field;
> > -	int index;
> > +	int index, max_id;
> >  
> > -	index = alloc_context_id(1, PRTB_ENTRIES - 1);
> > +	max_id = (1 << mmu_pid_bits) - 1;
> > +	index = alloc_context_id(mmu_base_pid, max_id);
> >  	if (index < 0)
> >  		return index;
> >  
> > @@ -247,8 +252,40 @@ void destroy_context(struct mm_struct *mm)
> >  }
> >  
> >  #ifdef CONFIG_PPC_RADIX_MMU
> > -void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next)
> > +void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next,
> > +			       bool new_on_cpu)
> >  {
> > +	/*
> > +	 * If this context hasn't run on that CPU before and KVM is
> > +	 * around, there's a slim chance that the guest on another
> > +	 * CPU just brought in obsolete translation into the TLB of
> > +	 * this CPU due to a bad prefetch using the guest PID on
> > +	 * the way into the hypervisor.
> > +	 *
> > +	 * We work around this here. If KVM is possible, we check if
> > +	 * any sibling thread is in KVM. If it is, the window may exist
> > +	 * and thus we flush that PID from the core.
> > +	 *
> > +	 * A potential future improvement would be to mark which PIDs
> > +	 * have never been used on the system and avoid it if the PID
> > +	 * is new and the process has no other cpumask bit set.
> 
> Also due to the pid split, the chances of the context bringing in
> TLB entries is low, but a bad guest could bring in stale entries
> 
> > +	 */
> > +#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
> > +	if (cpu_has_feature(CPU_FTR_HVMODE) && new_on_cpu) {
> > +		int cpu = smp_processor_id();
> > +		int sib = cpu_first_thread_sibling(cpu);
> > +		bool flush = false;
> > +
> > +		for (; sib <= cpu_last_thread_sibling(cpu) && !flush; sib++) {
> > +			if (sib == cpu)
> > +				continue;
> > +			if (paca[sib].kvm_hstate.kvm_vcpu)
> > +				flush = true;
> > +		}
> > +		if (flush)
> > +			radix__local_flush_all_mm(next);
> > +	}
> > +#endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
> >  

Re: [PATCH 1/4] powerpc/mm/radix: Don't iterate all sets when flushing the PWC

[Aneesh Kumar K.V]

Benjamin Herrenschmidt <benh@kernel.crashing.org> writes:

> The PWC flush only needs a single set call, just like the
> full (RIC=2) flush.
>
> Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
> ---
>  arch/powerpc/mm/tlb-radix.c | 11 +++++++----
>  1 file changed, 7 insertions(+), 4 deletions(-)
>
> diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
> index 02e7140..5403419 100644
> --- a/arch/powerpc/mm/tlb-radix.c
> +++ b/arch/powerpc/mm/tlb-radix.c
> @@ -52,12 +52,15 @@ static inline void _tlbiel_pid(unsigned long pid, unsigned long ric)
>  	 */
>  	__tlbiel_pid(pid, 0, ric);
>
> -	if (ric == RIC_FLUSH_ALL)
> -		/* For the remaining sets, just flush the TLB */
> -		ric = RIC_FLUSH_TLB;
> +	/* For PWC, only one flush is needed */
> +	if (ric == RIC_FLUSH_PWC) {
> +		asm volatile("ptesync": : :"memory");
> +		return;
> +	}
>
> +	/* For the remaining sets, just flush the TLB */
>  	for (set = 1; set < POWER9_TLB_SETS_RADIX ; set++)
> -		__tlbiel_pid(pid, set, ric);
> +		__tlbiel_pid(pid, set, RIC_FLUSH_TLB);
>
>  	asm volatile("ptesync": : :"memory");
>  	asm volatile(PPC_INVALIDATE_ERAT "; isync" : : :"memory");


With the current code, we use tlbiel_pwc() for doing a pwc flush.
and that does what is done this patch. May be we can update this patch
such that we drop tlbiel_pwc and switch all those instance to
tlbiel_pid(pid, RIC_FLUSH_PWC) ?

You already do this in a later patch.

-aneesh

Re: [PATCH 1/4] powerpc/mm/radix: Don't iterate all sets when flushing the PWC

[Benjamin Herrenschmidt]

On Fri, 2017-07-14 at 11:11 +0530, Aneesh Kumar K.V wrote:
> With the current code, we use tlbiel_pwc() for doing a pwc flush.
> and that does what is done this patch. May be we can update this patch
> such that we drop tlbiel_pwc and switch all those instance to
> tlbiel_pid(pid, RIC_FLUSH_PWC) ?
> 
> You already do this in a later patch.

Right, I do it in a later latch and I prefer to keep it this way, to
avoid doing too many changes in a single patch.

Ben.

Re: [PATCH 1/4] powerpc/mm/radix: Don't iterate all sets when flushing the PWC

[Aneesh Kumar K.V]

Benjamin Herrenschmidt <benh@kernel.crashing.org> writes:

> On Fri, 2017-07-14 at 11:11 +0530, Aneesh Kumar K.V wrote:
>> With the current code, we use tlbiel_pwc() for doing a pwc flush.
>> and that does what is done this patch. May be we can update this patch
>> such that we drop tlbiel_pwc and switch all those instance to
>> tlbiel_pid(pid, RIC_FLUSH_PWC) ?
>> 
>> You already do this in a later patch.
>
> Right, I do it in a later latch and I prefer to keep it this way, to
> avoid doing too many changes in a single patch.

The commit message do indicate that we don't do flushing correctly for
PWC flush with the current code. That is not correct. May be update the
commit message then ?

-aneesh

Re: [PATCH 1/4] powerpc/mm/radix: Don't iterate all sets when flushing the PWC

[Benjamin Herrenschmidt]

On Fri, 2017-07-14 at 12:33 +0530, Aneesh Kumar K.V wrote:
> Benjamin Herrenschmidt <benh@kernel.crashing.org> writes:
> 
> > On Fri, 2017-07-14 at 11:11 +0530, Aneesh Kumar K.V wrote:
> > > With the current code, we use tlbiel_pwc() for doing a pwc flush.
> > > and that does what is done this patch. May be we can update this patch
> > > such that we drop tlbiel_pwc and switch all those instance to
> > > tlbiel_pid(pid, RIC_FLUSH_PWC) ?
> > > 
> > > You already do this in a later patch.
> > 
> > Right, I do it in a later latch and I prefer to keep it this way, to
> > avoid doing too many changes in a single patch.
> 
> The commit message do indicate that we don't do flushing correctly for
> PWC flush with the current code. That is not correct. May be update the
> commit message then ?

Yeah ok.

Ben.