From: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
To: linux-mm@kvack.org, akpm@linux-foundation.org
Cc: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>,
Linus Torvalds <torvalds@linux-foundation.org>,
npiggin@gmail.com, kaleshsingh@google.com,
joel@joelfernandes.org, linuxppc-dev@lists.ozlabs.org
Subject: [PATCH v6 06/11] powerpc/mm/book3s64: Update tlb flush routines to take a page walk cache flush argument
Date: Mon, 24 May 2021 14:31:09 +0530 [thread overview]
Message-ID: <20210524090114.63446-7-aneesh.kumar@linux.ibm.com> (raw)
In-Reply-To: <20210524090114.63446-1-aneesh.kumar@linux.ibm.com>
No functional change in this patch
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
---
.../include/asm/book3s/64/tlbflush-radix.h | 19 +++++++-----
arch/powerpc/include/asm/book3s/64/tlbflush.h | 23 ++++++++++++---
arch/powerpc/mm/book3s64/radix_hugetlbpage.c | 4 +--
arch/powerpc/mm/book3s64/radix_tlb.c | 29 +++++++------------
4 files changed, 42 insertions(+), 33 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 8b33601cdb9d..171441a43b35 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -56,15 +56,18 @@ static inline void radix__flush_all_lpid_guest(unsigned int lpid)
}
#endif
-extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end);
-extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize);
-extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end);
-extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ bool flush_pwc);
+void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ bool flush_pwc);
+void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
+ unsigned long end, int psize, bool flush_pwc);
+void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end);
-extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
+void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
+
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
extern void radix__local_flush_all_mm(struct mm_struct *mm);
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index 215973b4cb26..f9f8a3a264f7 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
@@ -45,13 +45,30 @@ static inline void tlbiel_all_lpid(bool radix)
hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
}
+static inline void flush_pmd_tlb_pwc_range(struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end,
+ bool flush_pwc)
+{
+ if (radix_enabled())
+ return radix__flush_pmd_tlb_range(vma, start, end, flush_pwc);
+ return hash__flush_tlb_range(vma, start, end);
+}
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
+{
+ return flush_pmd_tlb_pwc_range(vma, start, end, false);
+}
+
+static inline void flush_hugetlb_tlb_pwc_range(struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end,
+ bool flush_pwc)
{
if (radix_enabled())
- return radix__flush_pmd_tlb_range(vma, start, end);
+ return radix__flush_hugetlb_tlb_range(vma, start, end, flush_pwc);
return hash__flush_tlb_range(vma, start, end);
}
@@ -60,9 +77,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
unsigned long start,
unsigned long end)
{
- if (radix_enabled())
- return radix__flush_hugetlb_tlb_range(vma, start, end);
- return hash__flush_tlb_range(vma, start, end);
+ return flush_hugetlb_tlb_pwc_range(vma, start, end, false);
}
static inline void flush_tlb_range(struct vm_area_struct *vma,
diff --git a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
index cb91071eef52..e62f5679b119 100644
--- a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
+++ b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
@@ -26,13 +26,13 @@ void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long v
}
void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start,
- unsigned long end)
+ unsigned long end, bool flush_pwc)
{
int psize;
struct hstate *hstate = hstate_file(vma->vm_file);
psize = hstate_get_psize(hstate);
- radix__flush_tlb_range_psize(vma->vm_mm, start, end, psize);
+ radix__flush_tlb_pwc_range_psize(vma->vm_mm, start, end, psize, flush_pwc);
}
/*
diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index 817a02ef6032..5a59e19f9e53 100644
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
@@ -1090,7 +1090,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
{
#ifdef CONFIG_HUGETLB_PAGE
if (is_vm_hugetlb_page(vma))
- return radix__flush_hugetlb_tlb_range(vma, start, end);
+ return radix__flush_hugetlb_tlb_range(vma, start, end, false);
#endif
__radix__flush_tlb_range(vma->vm_mm, start, end);
@@ -1151,9 +1151,6 @@ void radix__flush_all_lpid_guest(unsigned int lpid)
_tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
}
-static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize);
-
void radix__tlb_flush(struct mmu_gather *tlb)
{
int psize = 0;
@@ -1177,10 +1174,8 @@ void radix__tlb_flush(struct mmu_gather *tlb)
else
radix__flush_all_mm(mm);
} else {
- if (!tlb->freed_tables)
- radix__flush_tlb_range_psize(mm, start, end, psize);
- else
- radix__flush_tlb_pwc_range_psize(mm, start, end, psize);
+ radix__flush_tlb_pwc_range_psize(mm, start,
+ end, psize, tlb->freed_tables);
}
}
@@ -1254,16 +1249,10 @@ static void __radix__flush_tlb_range_psize(struct mm_struct *mm,
preempt_enable();
}
-void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize)
+void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
+ unsigned long end, int psize, bool flush_pwc)
{
- return __radix__flush_tlb_range_psize(mm, start, end, psize, false);
-}
-
-static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize)
-{
- __radix__flush_tlb_range_psize(mm, start, end, psize, true);
+ __radix__flush_tlb_range_psize(mm, start, end, psize, flush_pwc);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -1315,9 +1304,11 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
+ unsigned long start, unsigned long end,
+ bool flush_pwc)
{
- radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
+ __radix__flush_tlb_range_psize(vma->vm_mm, start,
+ end, MMU_PAGE_2M, flush_pwc);
}
EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
--
2.31.1
WARNING: multiple messages have this Message-ID (diff)
From: "Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
To: linux-mm@kvack.org, akpm@linux-foundation.org
Cc: mpe@ellerman.id.au, linuxppc-dev@lists.ozlabs.org,
kaleshsingh@google.com, npiggin@gmail.com,
joel@joelfernandes.org,
Christophe Leroy <christophe.leroy@csgroup.eu>,
Linus Torvalds <torvalds@linux-foundation.org>,
"Aneesh Kumar K.V" <aneesh.kumar@linux.ibm.com>
Subject: [PATCH v6 06/11] powerpc/mm/book3s64: Update tlb flush routines to take a page walk cache flush argument
Date: Mon, 24 May 2021 14:31:09 +0530 [thread overview]
Message-ID: <20210524090114.63446-7-aneesh.kumar@linux.ibm.com> (raw)
In-Reply-To: <20210524090114.63446-1-aneesh.kumar@linux.ibm.com>
No functional change in this patch
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.ibm.com>
---
.../include/asm/book3s/64/tlbflush-radix.h | 19 +++++++-----
arch/powerpc/include/asm/book3s/64/tlbflush.h | 23 ++++++++++++---
arch/powerpc/mm/book3s64/radix_hugetlbpage.c | 4 +--
arch/powerpc/mm/book3s64/radix_tlb.c | 29 +++++++------------
4 files changed, 42 insertions(+), 33 deletions(-)
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
index 8b33601cdb9d..171441a43b35 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
@@ -56,15 +56,18 @@ static inline void radix__flush_all_lpid_guest(unsigned int lpid)
}
#endif
-extern void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end);
-extern void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize);
-extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end);
-extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ bool flush_pwc);
+void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end,
+ bool flush_pwc);
+void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
+ unsigned long end, int psize, bool flush_pwc);
+void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end);
-extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
+void radix__flush_tlb_kernel_range(unsigned long start, unsigned long end);
+
extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
extern void radix__local_flush_all_mm(struct mm_struct *mm);
diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h b/arch/powerpc/include/asm/book3s/64/tlbflush.h
index 215973b4cb26..f9f8a3a264f7 100644
--- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
+++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
@@ -45,13 +45,30 @@ static inline void tlbiel_all_lpid(bool radix)
hash__tlbiel_all(TLB_INVAL_SCOPE_LPID);
}
+static inline void flush_pmd_tlb_pwc_range(struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end,
+ bool flush_pwc)
+{
+ if (radix_enabled())
+ return radix__flush_pmd_tlb_range(vma, start, end, flush_pwc);
+ return hash__flush_tlb_range(vma, start, end);
+}
#define __HAVE_ARCH_FLUSH_PMD_TLB_RANGE
static inline void flush_pmd_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
+{
+ return flush_pmd_tlb_pwc_range(vma, start, end, false);
+}
+
+static inline void flush_hugetlb_tlb_pwc_range(struct vm_area_struct *vma,
+ unsigned long start,
+ unsigned long end,
+ bool flush_pwc)
{
if (radix_enabled())
- return radix__flush_pmd_tlb_range(vma, start, end);
+ return radix__flush_hugetlb_tlb_range(vma, start, end, flush_pwc);
return hash__flush_tlb_range(vma, start, end);
}
@@ -60,9 +77,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
unsigned long start,
unsigned long end)
{
- if (radix_enabled())
- return radix__flush_hugetlb_tlb_range(vma, start, end);
- return hash__flush_tlb_range(vma, start, end);
+ return flush_hugetlb_tlb_pwc_range(vma, start, end, false);
}
static inline void flush_tlb_range(struct vm_area_struct *vma,
diff --git a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
index cb91071eef52..e62f5679b119 100644
--- a/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
+++ b/arch/powerpc/mm/book3s64/radix_hugetlbpage.c
@@ -26,13 +26,13 @@ void radix__local_flush_hugetlb_page(struct vm_area_struct *vma, unsigned long v
}
void radix__flush_hugetlb_tlb_range(struct vm_area_struct *vma, unsigned long start,
- unsigned long end)
+ unsigned long end, bool flush_pwc)
{
int psize;
struct hstate *hstate = hstate_file(vma->vm_file);
psize = hstate_get_psize(hstate);
- radix__flush_tlb_range_psize(vma->vm_mm, start, end, psize);
+ radix__flush_tlb_pwc_range_psize(vma->vm_mm, start, end, psize, flush_pwc);
}
/*
diff --git a/arch/powerpc/mm/book3s64/radix_tlb.c b/arch/powerpc/mm/book3s64/radix_tlb.c
index 817a02ef6032..5a59e19f9e53 100644
--- a/arch/powerpc/mm/book3s64/radix_tlb.c
+++ b/arch/powerpc/mm/book3s64/radix_tlb.c
@@ -1090,7 +1090,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
{
#ifdef CONFIG_HUGETLB_PAGE
if (is_vm_hugetlb_page(vma))
- return radix__flush_hugetlb_tlb_range(vma, start, end);
+ return radix__flush_hugetlb_tlb_range(vma, start, end, false);
#endif
__radix__flush_tlb_range(vma->vm_mm, start, end);
@@ -1151,9 +1151,6 @@ void radix__flush_all_lpid_guest(unsigned int lpid)
_tlbie_lpid_guest(lpid, RIC_FLUSH_ALL);
}
-static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize);
-
void radix__tlb_flush(struct mmu_gather *tlb)
{
int psize = 0;
@@ -1177,10 +1174,8 @@ void radix__tlb_flush(struct mmu_gather *tlb)
else
radix__flush_all_mm(mm);
} else {
- if (!tlb->freed_tables)
- radix__flush_tlb_range_psize(mm, start, end, psize);
- else
- radix__flush_tlb_pwc_range_psize(mm, start, end, psize);
+ radix__flush_tlb_pwc_range_psize(mm, start,
+ end, psize, tlb->freed_tables);
}
}
@@ -1254,16 +1249,10 @@ static void __radix__flush_tlb_range_psize(struct mm_struct *mm,
preempt_enable();
}
-void radix__flush_tlb_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize)
+void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
+ unsigned long end, int psize, bool flush_pwc)
{
- return __radix__flush_tlb_range_psize(mm, start, end, psize, false);
-}
-
-static void radix__flush_tlb_pwc_range_psize(struct mm_struct *mm, unsigned long start,
- unsigned long end, int psize)
-{
- __radix__flush_tlb_range_psize(mm, start, end, psize, true);
+ __radix__flush_tlb_range_psize(mm, start, end, psize, flush_pwc);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
@@ -1315,9 +1304,11 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
+ unsigned long start, unsigned long end,
+ bool flush_pwc)
{
- radix__flush_tlb_range_psize(vma->vm_mm, start, end, MMU_PAGE_2M);
+ __radix__flush_tlb_range_psize(vma->vm_mm, start,
+ end, MMU_PAGE_2M, flush_pwc);
}
EXPORT_SYMBOL(radix__flush_pmd_tlb_range);
--
2.31.1
next prev parent reply other threads:[~2021-05-24 9:04 UTC|newest]
Thread overview: 32+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-05-24 9:01 [PATCH v6 00/11] Speedup mremap on ppc64 Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 01/11] selftest/mremap_test: Update the test to handle pagesize other than 4K Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 02/11] selftest/mremap_test: Avoid crash with static build Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 03/11] mm/mremap: Convert huge PUD move to separate helper Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 04/11] mm/mremap: Use pmd/pud_poplulate to update page table entries Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 05/11] powerpc/mm/book3s64: Fix possible build error Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V [this message]
2021-05-24 9:01 ` [PATCH v6 06/11] powerpc/mm/book3s64: Update tlb flush routines to take a page walk cache flush argument Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 07/11] mm/mremap: Use range flush that does TLB and page walk cache flush Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 17:02 ` Linus Torvalds
2021-05-24 17:02 ` Linus Torvalds
2021-05-25 13:27 ` Aneesh Kumar K.V
2021-05-25 13:27 ` Aneesh Kumar K.V
2021-05-25 17:08 ` Linus Torvalds
2021-05-25 17:08 ` Linus Torvalds
2021-05-24 9:01 ` [PATCH v6 08/11] mm/mremap: properly flush the TLB on mremap Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 09/11] mm/mremap: Fix race between mremap and pageout Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 13:38 ` [PATCH v6 updated 9/11] " Aneesh Kumar K.V
2021-05-24 13:38 ` Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 10/11] mm/mremap: Allow arch runtime override Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
2021-05-24 9:01 ` [PATCH v6 11/11] powerpc/mm: Enable HAVE_MOVE_PMD support Aneesh Kumar K.V
2021-05-24 9:01 ` Aneesh Kumar K.V
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210524090114.63446-7-aneesh.kumar@linux.ibm.com \
--to=aneesh.kumar@linux.ibm.com \
--cc=akpm@linux-foundation.org \
--cc=joel@joelfernandes.org \
--cc=kaleshsingh@google.com \
--cc=linux-mm@kvack.org \
--cc=linuxppc-dev@lists.ozlabs.org \
--cc=npiggin@gmail.com \
--cc=torvalds@linux-foundation.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.