* [PATCH 0/7] TLB flush update for radix
@ 2016-05-19 7:58 Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 1/7] powerpc/mm: Use hugetlb flush functions Aneesh Kumar K.V
` (6 more replies)
0 siblings, 7 replies; 8+ messages in thread
From: Aneesh Kumar K.V @ 2016-05-19 7:58 UTC (permalink / raw)
To: benh, paulus, mpe; +Cc: linuxppc-dev, Aneesh Kumar K.V
Hi,
This patch series introduce range based tlb flush and use the same for radix
implementation. We still need to handle the mmu_gather related tlb flush.
That will be done in a later patch.
Aneesh Kumar K.V (7):
powerpc/mm: Use hugetlb flush functions
powerpc/mm: Drop multiple definition of mm_is_core_local
powerpc/mm/radix: Add tlb flush of THP ptes
powerpc/mm/radix: Rename function and drop unused arg
powerpc/mm/radix/hugetlb: Add helper for finding page size from hstate
powerpc/mm/hugetlb: Add flush_hugetlb_tlb_range
powerpc/mm: remove flush_tlb_page_nohash
arch/powerpc/include/asm/book3s/64/hugetlb-radix.h | 15 +++++
arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 5 --
.../powerpc/include/asm/book3s/64/tlbflush-radix.h | 16 +++--
arch/powerpc/include/asm/book3s/64/tlbflush.h | 27 +++++---
arch/powerpc/include/asm/hugetlb.h | 2 +-
arch/powerpc/include/asm/tlb.h | 13 ++++
arch/powerpc/include/asm/tlbflush.h | 1 -
arch/powerpc/mm/hugetlbpage-radix.c | 39 +++++------
arch/powerpc/mm/pgtable-book3s64.c | 4 +-
arch/powerpc/mm/pgtable.c | 2 +-
arch/powerpc/mm/tlb-radix.c | 78 ++++++++++++++++++----
arch/powerpc/mm/tlb_hash32.c | 11 ---
arch/powerpc/mm/tlb_nohash.c | 6 --
mm/hugetlb.c | 10 ++-
14 files changed, 152 insertions(+), 77 deletions(-)
--
2.7.4
^ permalink raw reply [flat|nested] 8+ messages in thread
* [PATCH 1/7] powerpc/mm: Use hugetlb flush functions
2016-05-19 7:58 [PATCH 0/7] TLB flush update for radix Aneesh Kumar K.V
@ 2016-05-19 7:58 ` Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 2/7] powerpc/mm: Drop multiple definition of mm_is_core_local Aneesh Kumar K.V
` (5 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Aneesh Kumar K.V @ 2016-05-19 7:58 UTC (permalink / raw)
To: benh, paulus, mpe; +Cc: linuxppc-dev, Aneesh Kumar K.V
Use flush_hugetlb_page instead of flush_tlb_page when we clear flush the
pte.
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/hugetlb.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index e2d9f4996e5c..c5517f463ec7 100644
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -147,7 +147,7 @@ static inline void huge_ptep_clear_flush(struct vm_area_struct *vma,
{
pte_t pte;
pte = huge_ptep_get_and_clear(vma->vm_mm, addr, ptep);
- flush_tlb_page(vma, addr);
+ flush_hugetlb_page(vma, addr);
}
static inline int huge_pte_none(pte_t pte)
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 2/7] powerpc/mm: Drop multiple definition of mm_is_core_local
2016-05-19 7:58 [PATCH 0/7] TLB flush update for radix Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 1/7] powerpc/mm: Use hugetlb flush functions Aneesh Kumar K.V
@ 2016-05-19 7:58 ` Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 3/7] powerpc/mm/radix: Add tlb flush of THP ptes Aneesh Kumar K.V
` (4 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Aneesh Kumar K.V @ 2016-05-19 7:58 UTC (permalink / raw)
To: benh, paulus, mpe; +Cc: linuxppc-dev, Aneesh Kumar K.V
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/tlb.h | 13 +++++++++++++
arch/powerpc/mm/tlb-radix.c | 6 ------
arch/powerpc/mm/tlb_nohash.c | 6 ------
3 files changed, 13 insertions(+), 12 deletions(-)
diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h
index 20733fa518ae..f6f68f73e858 100644
--- a/arch/powerpc/include/asm/tlb.h
+++ b/arch/powerpc/include/asm/tlb.h
@@ -46,5 +46,18 @@ static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep,
#endif
}
+#ifdef CONFIG_SMP
+static inline int mm_is_core_local(struct mm_struct *mm)
+{
+ return cpumask_subset(mm_cpumask(mm),
+ topology_sibling_cpumask(smp_processor_id()));
+}
+#else
+static inline int mm_is_core_local(struct mm_struct *mm)
+{
+ return 1;
+}
+#endif
+
#endif /* __KERNEL__ */
#endif /* __ASM_POWERPC_TLB_H */
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 0fdaf93a3e09..5807f5d72e1b 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -152,12 +152,6 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
EXPORT_SYMBOL(radix__local_flush_tlb_page);
#ifdef CONFIG_SMP
-static int mm_is_core_local(struct mm_struct *mm)
-{
- return cpumask_subset(mm_cpumask(mm),
- topology_sibling_cpumask(smp_processor_id()));
-}
-
void radix__flush_tlb_mm(struct mm_struct *mm)
{
unsigned int pid;
diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c
index f4668488512c..050badc0ebd3 100644
--- a/arch/powerpc/mm/tlb_nohash.c
+++ b/arch/powerpc/mm/tlb_nohash.c
@@ -215,12 +215,6 @@ EXPORT_SYMBOL(local_flush_tlb_page);
static DEFINE_RAW_SPINLOCK(tlbivax_lock);
-static int mm_is_core_local(struct mm_struct *mm)
-{
- return cpumask_subset(mm_cpumask(mm),
- topology_sibling_cpumask(smp_processor_id()));
-}
-
struct tlb_flush_param {
unsigned long addr;
unsigned int pid;
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 3/7] powerpc/mm/radix: Add tlb flush of THP ptes
2016-05-19 7:58 [PATCH 0/7] TLB flush update for radix Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 1/7] powerpc/mm: Use hugetlb flush functions Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 2/7] powerpc/mm: Drop multiple definition of mm_is_core_local Aneesh Kumar K.V
@ 2016-05-19 7:58 ` Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 4/7] powerpc/mm/radix: Rename function and drop unused arg Aneesh Kumar K.V
` (3 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Aneesh Kumar K.V @ 2016-05-19 7:58 UTC (permalink / raw)
To: benh, paulus, mpe; +Cc: linuxppc-dev, Aneesh Kumar K.V
Instead of flushing the entire mm, implement a flush_pmd_tlb_range
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
.../powerpc/include/asm/book3s/64/tlbflush-radix.h | 4 ++
arch/powerpc/include/asm/book3s/64/tlbflush.h | 9 ++++
arch/powerpc/mm/pgtable-book3s64.c | 4 +-
arch/powerpc/mm/tlb-radix.c | 54 ++++++++++++++++++++++
4 files changed, 69 insertions(+), 2 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 13ef38828dfe..823528d34688 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -10,6 +10,10 @@ static inline int mmu_get_ap(int psize)
return mmu_psize_defs[psize].ap;
}
+extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
+ unsigned long end, int psize);
+extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end);
extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end);
extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index d98424ae356c..f0d6c9d38916 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
@@ -7,6 +7,15 @@
#include <asm/book3s/64/tlbflush-hash.h>
#include <asm/book3s/64/tlbflush-radix.h>
+#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
+static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+{
+ if (radix_enabled())
+ return radix__flush_pmd_tlb_range(vma, start, end);
+ return hash__flush_tlb_range(vma, start, end);
+}
+
static inline void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c
index 670318766545..7bb8acffe876 100644
--- a/arch/powerpc/mm/pgtable-book3s64.c
+++ b/arch/powerpc/mm/pgtable-book3s64.c
@@ -33,7 +33,7 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address,
changed = !pmd_same(*(pmdp), entry);
if (changed) {
__ptep_set_access_flags(pmdp_ptep(pmdp), pmd_pte(entry));
- flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+ flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
}
return changed;
}
@@ -66,7 +66,7 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address,
pmd_t *pmdp)
{
pmd_hugepage_update(vma->vm_mm, address, pmdp, _PAGE_PRESENT, 0);
- flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
+ flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
/*
* This ensures that generic code that rely on IRQ disabling
* to prevent a parallel THP split work as expected.
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 5807f5d72e1b..fe2fc58d2e00 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -243,3 +243,57 @@ void radix__tlb_flush(struct mmu_gather *tlb)
struct mm_struct *mm = tlb->mm;
radix__flush_tlb_mm(mm);
}
+
+#define TLB_FLUSH_ALL -1UL
+/*
+ * Number of pages above which we will do a bcast tlbie. Just a
+ * number at this point copied from x86
+ */
+static unsigned long tlb_single_page_flush_ceiling __read_mostly = 33;
+
+void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
+ unsigned long end, int psize)
+{
+ unsigned int pid;
+ unsigned long addr;
+ int local = mm_is_core_local(mm);
+ unsigned long ap = mmu_get_ap(psize);
+ int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE);
+ unsigned long page_size = 1UL << mmu_psize_defs[psize].shift;
+
+
+ preempt_disable();
+ pid = mm ? mm->context.id : 0;
+ if (unlikely(pid == MMU_NO_CONTEXT))
+ goto err_out;
+
+ if (end == TLB_FLUSH_ALL ||
+ (end - start) > tlb_single_page_flush_ceiling * page_size) {
+ if (local)
+ _tlbiel_pid(pid);
+ else
+ _tlbie_pid(pid);
+ goto err_out;
+ }
+ for (addr = start; addr < end; addr += page_size) {
+
+ if (local)
+ _tlbiel_va(addr, pid, ap);
+ else {
+ if (lock_tlbie)
+ raw_spin_lock(&native_tlbie_lock);
+ _tlbie_va(addr, pid, ap);
+ if (lock_tlbie)
+ raw_spin_unlock(&native_tlbie_lock);
+ }
+ }
+err_out:
+ preempt_enable();
+}
+
+void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+{
+ radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
+}
+EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 4/7] powerpc/mm/radix: Rename function and drop unused arg
2016-05-19 7:58 [PATCH 0/7] TLB flush update for radix Aneesh Kumar K.V
` (2 preceding siblings ...)
2016-05-19 7:58 ` [PATCH 3/7] powerpc/mm/radix: Add tlb flush of THP ptes Aneesh Kumar K.V
@ 2016-05-19 7:58 ` Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 5/7] powerpc/mm/radix/hugetlb: Add helper for finding page size from hstate Aneesh Kumar K.V
` (2 subsequent siblings)
6 siblings, 0 replies; 8+ messages in thread
From: Aneesh Kumar K.V @ 2016-05-19 7:58 UTC (permalink / raw)
To: benh, paulus, mpe; +Cc: linuxppc-dev, Aneesh Kumar K.V
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h | 10 +++++-----
arch/powerpc/mm/hugetlbpage-radix.c | 4 ++--
arch/powerpc/mm/tlb-radix.c | 16 ++++++++--------
3 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 823528d34688..07b2e0031dad 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -20,18 +20,18 @@ extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
-extern void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
- unsigned long ap, int nid);
+extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
+ unsigned long ap);
extern void radix__tlb_flush(struct mmu_gather *tlb);
#ifdef CONFIG_SMP
extern void radix__flush_tlb_mm(struct mm_struct *mm);
extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
-extern void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
- unsigned long ap, int nid);
+extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
+ unsigned long ap);
#else
#define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm)
#define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr)
-#define radix___flush_tlb_page(mm,addr,p,i) radix___local_flush_tlb_page(mm,addr,p,i)
+#define radix__flush_tlb_page_psize(mm,addr,p) radix__local_flush_tlb_page_psize(mm,addr,p)
#endif
#endif
diff --git a/arch/powerpc/mm/hugetlbpage-radix.c b/arch/powerpc/mm/hugetlbpage-radix.c
index 1e11559e1aac..0dfa1816f0c6 100644
--- a/arch/powerpc/mm/hugetlbpage-radix.c
+++ b/arch/powerpc/mm/hugetlbpage-radix.c
@@ -20,7 +20,7 @@ void radix__flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
WARN(1, "Wrong huge page shift\n");
return ;
}
- radix___flush_tlb_page(vma->vm_mm, vmaddr, ap, 0);
+ radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, ap);
}
void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
@@ -37,7 +37,7 @@ void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long v
WARN(1, "Wrong huge page shift\n");
return ;
}
- radix___local_flush_tlb_page(vma->vm_mm, vmaddr, ap, 0);
+ radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, ap);
}
/*
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index fe2fc58d2e00..b1dc4675925d 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -127,8 +127,8 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
}
EXPORT_SYMBOL(radix__local_flush_tlb_mm);
-void radix___local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
- unsigned long ap, int nid)
+void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
+ unsigned long ap)
{
unsigned int pid;
@@ -146,8 +146,8 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
if (vma && is_vm_hugetlb_page(vma))
return __local_flush_hugetlb_page(vma, vmaddr);
#endif
- radix___local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
- mmu_get_ap(mmu_virtual_psize), 0);
+ radix__local_flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
+ mmu_get_ap(mmu_virtual_psize));
}
EXPORT_SYMBOL(radix__local_flush_tlb_page);
@@ -176,8 +176,8 @@ no_context:
}
EXPORT_SYMBOL(radix__flush_tlb_mm);
-void radix___flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
- unsigned long ap, int nid)
+void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
+ unsigned long ap)
{
unsigned int pid;
@@ -205,8 +205,8 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
if (vma && is_vm_hugetlb_page(vma))
return flush_hugetlb_page(vma, vmaddr);
#endif
- radix___flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
- mmu_get_ap(mmu_virtual_psize), 0);
+ radix__flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
+ mmu_get_ap(mmu_virtual_psize));
}
EXPORT_SYMBOL(radix__flush_tlb_page);
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 5/7] powerpc/mm/radix/hugetlb: Add helper for finding page size from hstate
2016-05-19 7:58 [PATCH 0/7] TLB flush update for radix Aneesh Kumar K.V
` (3 preceding siblings ...)
2016-05-19 7:58 ` [PATCH 4/7] powerpc/mm/radix: Rename function and drop unused arg Aneesh Kumar K.V
@ 2016-05-19 7:58 ` Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 6/7] powerpc/mm/hugetlb: Add flush_hugetlb_tlb_range Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 7/7] powerpc/mm: remove flush_tlb_page_nohash Aneesh Kumar K.V
6 siblings, 0 replies; 8+ messages in thread
From: Aneesh Kumar K.V @ 2016-05-19 7:58 UTC (permalink / raw)
To: benh, paulus, mpe; +Cc: linuxppc-dev, Aneesh Kumar K.V
Use the helper instead of open coding the same at multiple place
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/book3s/64/hugetlb-radix.h | 15 +++++++++++
.../powerpc/include/asm/book3s/64/tlbflush-radix.h | 4 +--
arch/powerpc/mm/hugetlbpage-radix.c | 29 ++++++----------------
arch/powerpc/mm/tlb-radix.c | 10 +++++---
4 files changed, 30 insertions(+), 28 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h b/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h
index 60f47649306f..c45189aa7476 100644
--- a/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/hugetlb-radix.h
@@ -11,4 +11,19 @@ extern unsigned long
radix__hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
unsigned long len, unsigned long pgoff,
unsigned long flags);
+
+static inline int hstate_get_psize(struct hstate *hstate)
+{
+ unsigned long shift;
+
+ shift = huge_page_shift(hstate);
+ if (shift == mmu_psize_defs[MMU_PAGE_2M].shift)
+ return MMU_PAGE_2M;
+ else if (shift == mmu_psize_defs[MMU_PAGE_1G].shift)
+ return MMU_PAGE_1G;
+ else {
+ WARN(1, "Wrong huge page shift\n");
+ return mmu_virtual_psize;
+ }
+}
#endif
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 07b2e0031dad..68839e6adcf1 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -21,13 +21,13 @@ extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
- unsigned long ap);
+ int psize);
extern void radix__tlb_flush(struct mmu_gather *tlb);
#ifdef CONFIG_SMP
extern void radix__flush_tlb_mm(struct mm_struct *mm);
extern void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr);
extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
- unsigned long ap);
+ int psize);
#else
#define radix__flush_tlb_mm(mm) radix__local_flush_tlb_mm(mm)
#define radix__flush_tlb_page(vma,addr) radix__local_flush_tlb_page(vma,addr)
diff --git a/arch/powerpc/mm/hugetlbpage-radix.c b/arch/powerpc/mm/hugetlbpage-radix.c
index 0dfa1816f0c6..1eca0deaf89b 100644
--- a/arch/powerpc/mm/hugetlbpage-radix.c
+++ b/arch/powerpc/mm/hugetlbpage-radix.c
@@ -5,39 +5,24 @@
#include <asm/cacheflush.h>
#include <asm/machdep.h>
#include <asm/mman.h>
+#include <asm/tlb.h>
void radix__flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
- unsigned long ap, shift;
+ int psize;
struct hstate *hstate = hstate_file(vma->vm_file);
- shift = huge_page_shift(hstate);
- if (shift == mmu_psize_defs[MMU_PAGE_2M].shift)
- ap = mmu_get_ap(MMU_PAGE_2M);
- else if (shift == mmu_psize_defs[MMU_PAGE_1G].shift)
- ap = mmu_get_ap(MMU_PAGE_1G);
- else {
- WARN(1, "Wrong huge page shift\n");
- return ;
- }
- radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, ap);
+ psize = hstate_get_psize(hstate);
+ radix__flush_tlb_page_psize(vma->vm_mm, vmaddr, psize);
}
void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
{
- unsigned long ap, shift;
+ int psize;
struct hstate *hstate = hstate_file(vma->vm_file);
- shift = huge_page_shift(hstate);
- if (shift == mmu_psize_defs[MMU_PAGE_2M].shift)
- ap = mmu_get_ap(MMU_PAGE_2M);
- else if (shift == mmu_psize_defs[MMU_PAGE_1G].shift)
- ap = mmu_get_ap(MMU_PAGE_1G);
- else {
- WARN(1, "Wrong huge page shift\n");
- return ;
- }
- radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, ap);
+ psize = hstate_get_psize(hstate);
+ radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, psize);
}
/*
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index b1dc4675925d..7bc3d1402c63 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -128,9 +128,10 @@ void radix__local_flush_tlb_mm(struct mm_struct *mm)
EXPORT_SYMBOL(radix__local_flush_tlb_mm);
void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
- unsigned long ap)
+ int psize)
{
unsigned int pid;
+ unsigned long ap = mmu_get_ap(psize);
preempt_disable();
pid = mm ? mm->context.id : 0;
@@ -147,7 +148,7 @@ void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmadd
return __local_flush_hugetlb_page(vma, vmaddr);
#endif
radix__local_flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
- mmu_get_ap(mmu_virtual_psize));
+ mmu_virtual_psize);
}
EXPORT_SYMBOL(radix__local_flush_tlb_page);
@@ -177,9 +178,10 @@ no_context:
EXPORT_SYMBOL(radix__flush_tlb_mm);
void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmaddr,
- unsigned long ap)
+ int psize)
{
unsigned int pid;
+ unsigned long ap = mmu_get_ap(psize);
preempt_disable();
pid = mm ? mm->context.id : 0;
@@ -206,7 +208,7 @@ void radix__flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
return flush_hugetlb_page(vma, vmaddr);
#endif
radix__flush_tlb_page_psize(vma ? vma->vm_mm : NULL, vmaddr,
- mmu_get_ap(mmu_virtual_psize));
+ mmu_virtual_psize);
}
EXPORT_SYMBOL(radix__flush_tlb_page);
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 6/7] powerpc/mm/hugetlb: Add flush_hugetlb_tlb_range
2016-05-19 7:58 [PATCH 0/7] TLB flush update for radix Aneesh Kumar K.V
` (4 preceding siblings ...)
2016-05-19 7:58 ` [PATCH 5/7] powerpc/mm/radix/hugetlb: Add helper for finding page size from hstate Aneesh Kumar K.V
@ 2016-05-19 7:58 ` Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 7/7] powerpc/mm: remove flush_tlb_page_nohash Aneesh Kumar K.V
6 siblings, 0 replies; 8+ messages in thread
From: Aneesh Kumar K.V @ 2016-05-19 7:58 UTC (permalink / raw)
To: benh, paulus, mpe; +Cc: linuxppc-dev, Aneesh Kumar K.V
Some archs like ppc64 need to do special things when flushing tlb for
hugepage. Add a new helper to flush hugetlb tlb range. This helps us to
avoid flushing the entire tlb mapping for the pid.
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/book3s/64/tlbflush-radix.h | 2 ++
arch/powerpc/include/asm/book3s/64/tlbflush.h | 10 ++++++++++
arch/powerpc/mm/hugetlbpage-radix.c | 10 ++++++++++
mm/hugetlb.c | 10 +++++++++-
4 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 68839e6adcf1..73953a44d4e3 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -10,6 +10,8 @@ static inline int mmu_get_ap(int psize)
return mmu_psize_defs[psize].ap;
}
+extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end);
extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
unsigned long end, int psize);
extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index f0d6c9d38916..3b3e5e944af7 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
@@ -16,6 +16,16 @@ static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
return hash__flush_tlb_range(vma, start, end);
}
+#define __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
+static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end)
+{
+ if (radix_enabled())
+ return radix__flush_hugetlb_tlb_range(vma, start, end);
+ return hash__flush_tlb_range(vma, start, end);
+}
+
static inline void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
diff --git a/arch/powerpc/mm/hugetlbpage-radix.c b/arch/powerpc/mm/hugetlbpage-radix.c
index 1eca0deaf89b..35254a678456 100644
--- a/arch/powerpc/mm/hugetlbpage-radix.c
+++ b/arch/powerpc/mm/hugetlbpage-radix.c
@@ -25,6 +25,16 @@ void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long v
radix__local_flush_tlb_page_psize(vma->vm_mm, vmaddr, psize);
}
+void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start,
+ unsigned long end)
+{
+ int psize;
+ struct hstate *hstate = hstate_file(vma->vm_file);
+
+ psize = hstate_get_psize(hstate);
+ radix__flush_tlb_range_psize(vma->vm_mm, start, end, psize);
+}
+
/*
* A vairant of hugetlb_get_unmapped_area doing topdown search
* FIXME!! should we do as x86 does or non hugetlb area does ?
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 19d0d08b396f..076a57ee8790 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -3893,6 +3893,14 @@ same_page:
return i ? i : -EFAULT;
}
+#ifndef __HAVE_ARCH_FLUSH_HUGETLB_TLB_RANGE
+/*
+ * ARCHes with special requirements for evicting HUGETLB backing TLB entries can
+ * implement this.
+ */
+#define flush_hugetlb_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
+#endif
+
unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
unsigned long address, unsigned long end, pgprot_t newprot)
{
@@ -3953,7 +3961,7 @@ unsigned long hugetlb_change_protection(struct vm_area_struct *vma,
* once we release i_mmap_rwsem, another task can do the final put_page
* and that page table be reused and filled with junk.
*/
- flush_tlb_range(vma, start, end);
+ flush_hugetlb_tlb_range(vma, start, end);
mmu_notifier_invalidate_range(mm, start, end);
i_mmap_unlock_write(vma->vm_file->f_mapping);
mmu_notifier_invalidate_range_end(mm, start, end);
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
* [PATCH 7/7] powerpc/mm: remove flush_tlb_page_nohash
2016-05-19 7:58 [PATCH 0/7] TLB flush update for radix Aneesh Kumar K.V
` (5 preceding siblings ...)
2016-05-19 7:58 ` [PATCH 6/7] powerpc/mm/hugetlb: Add flush_hugetlb_tlb_range Aneesh Kumar K.V
@ 2016-05-19 7:58 ` Aneesh Kumar K.V
6 siblings, 0 replies; 8+ messages in thread
From: Aneesh Kumar K.V @ 2016-05-19 7:58 UTC (permalink / raw)
To: benh, paulus, mpe; +Cc: linuxppc-dev, Aneesh Kumar K.V
This should be same as flush_tlb_page except for hash32. For hash32
I guess the existing code is wrong, because we don't seem to be
flushing tlb for Hash != 0 case at all. Fix this by switching to
calling flush_tlb_page() which does the right thing by flushing
tlb for both hash and nohash case with hash32
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
---
arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 5 -----
arch/powerpc/include/asm/book3s/64/tlbflush.h | 8 --------
arch/powerpc/include/asm/tlbflush.h | 1 -
arch/powerpc/mm/pgtable.c | 2 +-
arch/powerpc/mm/tlb_hash32.c | 11 -----------
5 files changed, 1 insertion(+), 26 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
index f12ddf5e8de5..2f6373144e2c 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h
@@ -75,11 +75,6 @@ static inline void hash__flush_tlb_page(struct vm_area_struct *vma,
{
}
-static inline void hash__flush_tlb_page_nohash(struct vm_area_struct *vma,
- unsigned long vmaddr)
-{
-}
-
static inline void hash__flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index 3b3e5e944af7..ea29cc3318d2 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
@@ -57,14 +57,6 @@ static inline void local_flush_tlb_page(struct vm_area_struct *vma,
return hash__local_flush_tlb_page(vma, vmaddr);
}
-static inline void flush_tlb_page_nohash(struct vm_area_struct *vma,
- unsigned long vmaddr)
-{
- if (radix_enabled())
- return radix__flush_tlb_page(vma, vmaddr);
- return hash__flush_tlb_page_nohash(vma, vmaddr);
-}
-
static inline void tlb_flush(struct mmu_gather *tlb)
{
if (radix_enabled())
diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h
index 1b38eea28e5a..13dbcd41885e 100644
--- a/arch/powerpc/include/asm/tlbflush.h
+++ b/arch/powerpc/include/asm/tlbflush.h
@@ -54,7 +54,6 @@ extern void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
#define flush_tlb_page(vma,addr) local_flush_tlb_page(vma,addr)
#define __flush_tlb_page(mm,addr,p,i) __local_flush_tlb_page(mm,addr,p,i)
#endif
-#define flush_tlb_page_nohash(vma,addr) flush_tlb_page(vma,addr)
#elif defined(CONFIG_PPC_STD_MMU_32)
diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
index 88a307504b5a..0b6fb244d0a1 100644
--- a/arch/powerpc/mm/pgtable.c
+++ b/arch/powerpc/mm/pgtable.c
@@ -225,7 +225,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address,
if (!is_vm_hugetlb_page(vma))
assert_pte_locked(vma->vm_mm, address);
__ptep_set_access_flags(ptep, entry);
- flush_tlb_page_nohash(vma, address);
+ flush_tlb_page(vma, address);
}
return changed;
}
diff --git a/arch/powerpc/mm/tlb_hash32.c b/arch/powerpc/mm/tlb_hash32.c
index 558e30cce33e..702d7689d714 100644
--- a/arch/powerpc/mm/tlb_hash32.c
+++ b/arch/powerpc/mm/tlb_hash32.c
@@ -49,17 +49,6 @@ void flush_hash_entry(struct mm_struct *mm, pte_t *ptep, unsigned long addr)
EXPORT_SYMBOL(flush_hash_entry);
/*
- * Called by ptep_set_access_flags, must flush on CPUs for which the
- * DSI handler can't just "fixup" the TLB on a write fault
- */
-void flush_tlb_page_nohash(struct vm_area_struct *vma, unsigned long addr)
-{
- if (Hash != 0)
- return;
- _tlbie(addr);
-}
-
-/*
* Called at the end of a mmu_gather operation to make sure the
* TLB flush is completely done.
*/
--
2.7.4
^ permalink raw reply related [flat|nested] 8+ messages in thread
end of thread, other threads:[~2016-05-19 7:59 UTC | newest]
Thread overview: 8+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2016-05-19 7:58 [PATCH 0/7] TLB flush update for radix Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 1/7] powerpc/mm: Use hugetlb flush functions Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 2/7] powerpc/mm: Drop multiple definition of mm_is_core_local Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 3/7] powerpc/mm/radix: Add tlb flush of THP ptes Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 4/7] powerpc/mm/radix: Rename function and drop unused arg Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 5/7] powerpc/mm/radix/hugetlb: Add helper for finding page size from hstate Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 6/7] powerpc/mm/hugetlb: Add flush_hugetlb_tlb_range Aneesh Kumar K.V
2016-05-19 7:58 ` [PATCH 7/7] powerpc/mm: remove flush_tlb_page_nohash Aneesh Kumar K.V
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).