[PATCH 03/16] mm: pgalloc: introduce {pud|pmd}_populate_sync()

[Yang Shi <yang@os.amperecomputing.com>]

pud_populate() and pmd_populate_kernel() are called by percpu and
vmemmap to populate PUD and PMD.  They need to sync up kernel page table
when percpu page table support for ARM64 is added in the later patch.
So introduce {pud|pmd}_populate_sync() and protect them with
ARCH_HAS_{PUD|PMD}_POPULATE_SYNC kernel configs.  ARM64 specific
implementation will be defined in the later patch.

We can change pud_populate() and pmd_populate_kernel() to take address
as a new parameter, but this change is needed for all architectures and
not all callsites of them need sync page table.  So introducing new APIs
for this usecase seems much more simple.

This is a preparation patch, no functional change.

Signed-off-by: Yang Shi <yang@os.amperecomputing.com>
---
 include/linux/pgalloc.h | 13 +++++++++++++
 mm/Kconfig              |  6 ++++++
 mm/percpu.c             |  4 ++--
 mm/sparse-vmemmap.c     |  4 ++--
 4 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/include/linux/pgalloc.h b/include/linux/pgalloc.h
index 9174fa59bbc5..062c64a5d4bd 100644
--- a/include/linux/pgalloc.h
+++ b/include/linux/pgalloc.h
@@ -26,4 +26,17 @@
 			arch_sync_kernel_mappings(addr, addr);		\
 	} while (0)
 
+#ifndef CONFIG_ARCH_HAS_PUD_POPULATE_SYNC
+#define pud_populate_sync(addr, pud, pmd)				\
+	do {								\
+		pud_populate(&init_mm, pud, pmd);			\
+	} while (0)
+#endif
+
+#ifndef CONFIG_ARCH_HAS_PMD_POPULATE_SYNC
+#define pmd_populate_sync(addr, pmd, pte)				\
+	do {								\
+		pmd_populate_kernel(&init_mm, pmd, pte);		\
+	} while (0)
+#endif
 #endif /* _LINUX_PGALLOC_H */
diff --git a/mm/Kconfig b/mm/Kconfig
index 9e0ca4824905..13ea4b29ee7b 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -1509,6 +1509,12 @@ config LAZY_MMU_MODE_KUNIT_TEST
 
 	  If unsure, say N.
 
+config ARCH_HAS_PMD_POPULATE_SYNC
+	bool
+
+config ARCH_HAS_PUD_POPULATE_SYNC
+	bool
+
 source "mm/damon/Kconfig"
 
 endmenu
diff --git a/mm/percpu.c b/mm/percpu.c
index b0676b8054ed..45c30c6531c5 100644
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -3163,7 +3163,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr)
 	pud = pud_offset(p4d, addr);
 	if (pud_none(*pud)) {
 		pmd = memblock_alloc_or_panic(PMD_TABLE_SIZE, PMD_TABLE_SIZE);
-		pud_populate(&init_mm, pud, pmd);
+		pud_populate_sync(addr, pud, pmd);
 	}
 
 	pmd = pmd_offset(pud, addr);
@@ -3171,7 +3171,7 @@ void __init __weak pcpu_populate_pte(unsigned long addr)
 		pte_t *new;
 
 		new = memblock_alloc_or_panic(PTE_TABLE_SIZE, PTE_TABLE_SIZE);
-		pmd_populate_kernel(&init_mm, pmd, new);
+		pmd_populate_sync(addr, pmd, new);
 	}
 
 	return;
diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c
index 99e2be39671b..463c2f8b611c 100644
--- a/mm/sparse-vmemmap.c
+++ b/mm/sparse-vmemmap.c
@@ -198,7 +198,7 @@ static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, in
 		if (!p)
 			return NULL;
 		kernel_pte_init(p);
-		pmd_populate_kernel(&init_mm, pmd, p);
+		pmd_populate_sync(addr, pmd, p);
 	}
 	return pmd;
 }
@@ -211,7 +211,7 @@ static pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, in
 		if (!p)
 			return NULL;
 		pmd_init(p);
-		pud_populate(&init_mm, pud, p);
+		pud_populate_sync(addr, pud, p);
 	}
 	return pud;
 }
-- 
2.47.0