public inbox for linux-riscv@lists.infradead.org
 help / color / mirror / Atom feed
From: Xu Lu <luxu.kernel@bytedance.com>
To: paul.walmsley@sifive.com, palmer@dabbelt.com,
	aou@eecs.berkeley.edu, ardb@kernel.org, anup@brainfault.org,
	atishp@atishpatra.org
Cc: xieyongji@bytedance.com, lihangjing@bytedance.com,
	punit.agrawal@bytedance.com, linux-kernel@vger.kernel.org,
	linux-riscv@lists.infradead.org,
	Xu Lu <luxu.kernel@bytedance.com>
Subject: [RFC PATCH v2 04/21] riscv: mm: Reimplement page table entry constructor function
Date: Thu,  5 Dec 2024 18:37:12 +0800	[thread overview]
Message-ID: <20241205103729.14798-5-luxu.kernel@bytedance.com> (raw)
In-Reply-To: <20241205103729.14798-1-luxu.kernel@bytedance.com>

This commit reimplements the page table entry constructor. As each page
can contains several hardware pages now, the pte constructor need to
initialize all mapping entries of these hardware pages. Note that the
step path between mapping entries differs in different page table entry
levels. For example, in PTE level, the step path between hardware
mapping entries is hardware page size (aka 4K). In PMD level, the step
path is (2 ^ 9) * hardware page size (aka 2M), etc.

Signed-off-by: Xu Lu <luxu.kernel@bytedance.com>
---
 arch/riscv/include/asm/pgtable-32.h |  5 +++
 arch/riscv/include/asm/pgtable-64.h | 41 +++++++++++++++++++---
 arch/riscv/include/asm/pgtable.h    | 54 ++++++++++++++++++++++++-----
 arch/riscv/mm/pgtable.c             | 47 +++++++++++++++++++++++++
 4 files changed, 133 insertions(+), 14 deletions(-)

diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h
index 159a668c3dd8..2959ab72f926 100644
--- a/arch/riscv/include/asm/pgtable-32.h
+++ b/arch/riscv/include/asm/pgtable-32.h
@@ -37,4 +37,9 @@
 static const __maybe_unused int pgtable_l4_enabled;
 static const __maybe_unused int pgtable_l5_enabled;
 
+static inline int __pgd_present(unsigned long pgdval)
+{
+	return pgdval & _PAGE_PRESENT;
+}
+
 #endif /* _ASM_RISCV_PGTABLE_32_H */
diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h
index e736873d7768..efcf63667f93 100644
--- a/arch/riscv/include/asm/pgtable-64.h
+++ b/arch/riscv/include/asm/pgtable-64.h
@@ -204,9 +204,14 @@ static inline u64 riscv_page_io(void)
 					  _PAGE_USER | _PAGE_GLOBAL |	\
 					  _PAGE_MTMASK))
 
+static inline int __pud_present(unsigned long pudval)
+{
+	return pudval & _PAGE_PRESENT;
+}
+
 static inline int pud_present(pud_t pud)
 {
-	return (pud_val(pud) & _PAGE_PRESENT);
+	return __pud_present(pud_val(pud));
 }
 
 static inline int pud_none(pud_t pud)
@@ -219,11 +224,16 @@ static inline int pud_bad(pud_t pud)
 	return !pud_present(pud);
 }
 
-#define pud_leaf	pud_leaf
+static inline bool __pud_leaf(unsigned long pudval)
+{
+	return __pud_present(pudval) && (pudval & _PAGE_LEAF);
+}
+
 static inline bool pud_leaf(pud_t pud)
 {
-	return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF);
+	return __pud_leaf(pud_val(pud));
 }
+#define pud_leaf	pud_leaf
 
 static inline int pud_user(pud_t pud)
 {
@@ -321,14 +331,30 @@ static inline int p4d_none(p4d_t p4d)
 	return 0;
 }
 
+static inline int __p4d_present(unsigned long p4dval)
+{
+	return p4dval & _PAGE_PRESENT;
+}
+
 static inline int p4d_present(p4d_t p4d)
 {
 	if (pgtable_l4_enabled)
-		return (p4d_val(p4d) & _PAGE_PRESENT);
+		return __p4d_present(p4d_val(p4d));
 
 	return 1;
 }
 
+static inline int __p4d_leaf(unsigned long p4dval)
+{
+	return 0;
+}
+
+static inline int p4d_leaf(p4d_t p4d)
+{
+	return __p4d_leaf(p4d_val(p4d));
+}
+#define p4d_leaf	p4d_leaf
+
 static inline int p4d_bad(p4d_t p4d)
 {
 	if (pgtable_l4_enabled)
@@ -388,10 +414,15 @@ static inline int pgd_none(pgd_t pgd)
 	return 0;
 }
 
+static inline int __pgd_present(unsigned long pgdval)
+{
+	return pgdval & _PAGE_PRESENT;
+}
+
 static inline int pgd_present(pgd_t pgd)
 {
 	if (pgtable_l5_enabled)
-		return (pgd_val(pgd) & _PAGE_PRESENT);
+		return __pgd_present(pgd_val(pgd));
 
 	return 1;
 }
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index f9aed43809b3..1d5f533edbd5 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -220,8 +220,19 @@ static inline unsigned long satp_pfn(unsigned long satp)
 	return hwpfn_to_pfn(hwpfn);
 }
 
+static inline int __pgd_leaf(unsigned long pgdval)
+{
+	return __pgd_present(pgdval) && (pgdval & _PAGE_LEAF);
+}
+
+static inline int pgd_leaf(pgd_t pgd)
+{
+	return __pgd_leaf(pgd_val(pgd));
+}
+#define pgd_leaf	pgd_leaf
+
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
-static inline int pmd_present(pmd_t pmd)
+static inline int __pmd_present(unsigned long pmdval)
 {
 	/*
 	 * Checking for _PAGE_LEAF is needed too because:
@@ -229,15 +240,20 @@ static inline int pmd_present(pmd_t pmd)
 	 * the present bit, in this situation, pmd_present() and
 	 * pmd_trans_huge() still needs to return true.
 	 */
-	return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE | _PAGE_LEAF));
+	return (pmdval & (_PAGE_PRESENT | _PAGE_PROT_NONE | _PAGE_LEAF));
 }
 #else
-static inline int pmd_present(pmd_t pmd)
+static inline int __pmd_present(unsigned long pmdval)
 {
-	return (pmd_val(pmd) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
+	return (pmdval & (_PAGE_PRESENT | _PAGE_PROT_NONE));
 }
 #endif
 
+static inline int pmd_present(pmd_t pmd)
+{
+	return __pmd_present(pmd_val(pmd));
+}
+
 static inline int pmd_none(pmd_t pmd)
 {
 	return (pmd_val(pmd) == 0);
@@ -248,11 +264,16 @@ static inline int pmd_bad(pmd_t pmd)
 	return !pmd_present(pmd) || (pmd_val(pmd) & _PAGE_LEAF);
 }
 
-#define pmd_leaf	pmd_leaf
+static inline bool __pmd_leaf(unsigned long pmdval)
+{
+	return __pmd_present(pmdval) && (pmdval & _PAGE_LEAF);
+}
+
 static inline bool pmd_leaf(pmd_t pmd)
 {
-	return pmd_present(pmd) && (pmd_val(pmd) & _PAGE_LEAF);
+	return __pmd_leaf(pmd_val(pmd));
 }
+#define pmd_leaf	pmd_leaf
 
 static inline void set_pmd(pmd_t *pmdp, pmd_t pmd)
 {
@@ -306,9 +327,14 @@ static __always_inline bool has_svnapot(void)
 	return riscv_has_extension_likely(RISCV_ISA_EXT_SVNAPOT);
 }
 
+static inline unsigned long __pte_napot(unsigned long val)
+{
+	return val & _PAGE_NAPOT;
+}
+
 static inline unsigned long pte_napot(pte_t pte)
 {
-	return pte_val(pte) & _PAGE_NAPOT;
+	return __pte_napot(pte_val(pte));
 }
 
 static inline pte_t pte_mknapot(pte_t pte, unsigned int order)
@@ -324,11 +350,16 @@ static inline pte_t pte_mknapot(pte_t pte, unsigned int order)
 
 static __always_inline bool has_svnapot(void) { return false; }
 
-static inline unsigned long pte_napot(pte_t pte)
+static inline unsigned long __pte_napot(unsigned long pteval)
 {
 	return 0;
 }
 
+static inline unsigned long pte_napot(pte_t pte)
+{
+	return __pte_napot(pte_val(pte));
+}
+
 #endif /* CONFIG_RISCV_ISA_SVNAPOT */
 
 /* Yields the page frame number (PFN) of a page table entry */
@@ -356,9 +387,14 @@ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot)
 
 #define mk_pte(page, prot)       pfn_pte(page_to_pfn(page), prot)
 
+static inline int __pte_present(unsigned long pteval)
+{
+	return (pteval & (_PAGE_PRESENT | _PAGE_PROT_NONE));
+}
+
 static inline int pte_present(pte_t pte)
 {
-	return (pte_val(pte) & (_PAGE_PRESENT | _PAGE_PROT_NONE));
+	return __pte_present(pte_val(pte));
 }
 
 #define pte_accessible pte_accessible
diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c
index 0c6b2fc6be58..f57ada26a183 100644
--- a/arch/riscv/mm/pgtable.c
+++ b/arch/riscv/mm/pgtable.c
@@ -10,6 +10,13 @@
 pte_t __pte(unsigned long pteval)
 {
 	pte_t pte;
+	unsigned int i;
+
+	for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+		pte.ptes[i] = pteval;
+		if (__pte_present(pteval) && !__pte_napot(pteval))
+			pteval += 1 << _PAGE_HWPFN_SHIFT;
+	}
 
 	return pte;
 }
@@ -18,6 +25,16 @@ EXPORT_SYMBOL(__pte);
 pgd_t __pgd(unsigned long pgdval)
 {
 	pgd_t pgd;
+	unsigned int i;
+
+	for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+		pgd.pgds[i] = pgdval;
+		if (__pgd_leaf(pgdval))
+			pgdval += (1 << (PGDIR_SHIFT - PAGE_SHIFT)) <<
+					_PAGE_HWPFN_SHIFT;
+		else if (__pgd_present(pgdval))
+			pgdval += 1 << _PAGE_HWPFN_SHIFT;
+	}
 
 	return pgd;
 }
@@ -27,6 +44,16 @@ EXPORT_SYMBOL(__pgd);
 p4d_t __p4d(unsigned long p4dval)
 {
 	p4d_t p4d;
+	unsigned int i;
+
+	for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+		p4d.p4ds[i] = p4dval;
+		if (__p4d_leaf(p4dval))
+			p4dval += (1 << (P4D_SHIFT - PAGE_SHIFT)) <<
+					_PAGE_HWPFN_SHIFT;
+		else if (__p4d_present(p4dval))
+			p4dval += 1 << _PAGE_HWPFN_SHIFT;
+	}
 
 	return p4d;
 }
@@ -35,6 +62,16 @@ EXPORT_SYMBOL(__p4d);
 pud_t __pud(unsigned long pudval)
 {
 	pud_t pud;
+	unsigned int i;
+
+	for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+		pud.puds[i] = pudval;
+		if (__pud_leaf(pudval))
+			pudval += (1 << (PUD_SHIFT - PAGE_SHIFT)) <<
+					_PAGE_HWPFN_SHIFT;
+		else if (__pud_present(pudval))
+			pudval += 1 << _PAGE_HWPFN_SHIFT;
+	}
 
 	return pud;
 }
@@ -43,6 +80,16 @@ EXPORT_SYMBOL(__pud);
 pmd_t __pmd(unsigned long pmdval)
 {
 	pmd_t pmd;
+	unsigned int i;
+
+	for (i = 0; i < HW_PAGES_PER_PAGE; i++) {
+		pmd.pmds[i] = pmdval;
+		if (__pmd_leaf(pmdval))
+			pmdval += (1 << (PMD_SHIFT - PAGE_SHIFT)) <<
+					_PAGE_HWPFN_SHIFT;
+		else if (__pmd_present(pmdval))
+			pmdval += 1 << _PAGE_HWPFN_SHIFT;
+	}
 
 	return pmd;
 }
-- 
2.20.1


_______________________________________________
linux-riscv mailing list
linux-riscv@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-riscv

  parent reply	other threads:[~2024-12-05 10:44 UTC|newest]

Thread overview: 30+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2024-12-05 10:37 [RFC PATCH v2 00/21] riscv: Introduce 64K base page Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 01/21] riscv: mm: Distinguish hardware base page and software " Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 02/21] riscv: mm: Configure satp with hw page pfn Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 03/21] riscv: mm: Reimplement page table entry structures Xu Lu
2024-12-05 10:37 ` Xu Lu [this message]
2024-12-05 10:37 ` [RFC PATCH v2 05/21] riscv: mm: Reimplement conversion functions between page table entry Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 06/21] riscv: mm: Avoid pte constructor during pte conversion Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 07/21] riscv: mm: Reimplement page table entry get function Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 08/21] riscv: mm: Reimplement page table entry atomic " Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 09/21] riscv: mm: Replace READ_ONCE with atomic pte " Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 10/21] riscv: mm: Reimplement PTE A/D bit check function Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 11/21] riscv: mm: Reimplement mk_huge_pte function Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 12/21] riscv: mm: Reimplement tlb flush function Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 13/21] riscv: mm: Adjust PGDIR/P4D/PUD/PMD_SHIFT Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 14/21] riscv: mm: Only apply svnapot region bigger than software page Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 15/21] riscv: mm: Adjust FIX_BTMAPS_SLOTS for variable PAGE_SIZE Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 16/21] riscv: mm: Adjust FIX_FDT_SIZE for variable PMD_SIZE Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 17/21] riscv: mm: Apply Svnapot for base page mapping if possible Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 18/21] riscv: Kconfig: Introduce 64K page size Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 19/21] riscv: Kconfig: Adjust mmap rnd bits for 64K Page Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 20/21] riscv: mm: Adjust address space layout and init page table " Xu Lu
2024-12-05 10:37 ` [RFC PATCH v2 21/21] riscv: mm: Update EXEC_PAGESIZE " Xu Lu
2024-12-06  2:00 ` [RFC PATCH v2 00/21] riscv: Introduce 64K base page Zi Yan
2024-12-06  2:41   ` [External] " Xu Lu
2024-12-06 10:13   ` David Hildenbrand
2024-12-06 13:42     ` [External] " Xu Lu
2024-12-06 18:48       ` Pedro Falcato
2024-12-07  8:03         ` Xu Lu
2024-12-07 22:02           ` Yu Zhao
2024-12-09  3:36             ` Xu Lu

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20241205103729.14798-5-luxu.kernel@bytedance.com \
    --to=luxu.kernel@bytedance.com \
    --cc=anup@brainfault.org \
    --cc=aou@eecs.berkeley.edu \
    --cc=ardb@kernel.org \
    --cc=atishp@atishpatra.org \
    --cc=lihangjing@bytedance.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-riscv@lists.infradead.org \
    --cc=palmer@dabbelt.com \
    --cc=paul.walmsley@sifive.com \
    --cc=punit.agrawal@bytedance.com \
    --cc=xieyongji@bytedance.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body. 
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox