[PATCH V2 6/6] arm64/mm: Enable ARCH_ENABLE_SPLIT_PMD_PTLOCK

From: Anshuman Khandual <anshuman.khandual@arm.com>
To: linux-arm-kernel@lists.infradead.org
Cc: mark.rutland@arm.com, yuzhao@google.com, Steve.Capper@arm.com,
	marc.zyngier@arm.com, Catalin.Marinas@arm.com,
	suzuki.poulose@arm.com, will.deacon@arm.com, james.morse@arm.com
Subject: [PATCH V2 6/6] arm64/mm: Enable ARCH_ENABLE_SPLIT_PMD_PTLOCK
Date: Mon, 25 Feb 2019 10:33:59 +0530	[thread overview]
Message-ID: <1551071039-20192-7-git-send-email-anshuman.khandual@arm.com> (raw)
In-Reply-To: <1551071039-20192-1-git-send-email-anshuman.khandual@arm.com>

Enabling ARCH_ENABLE_SPLIT_PMD_PTLOCK should help reduce lock contention on
larger systems when the page tables are being modified concurrently. This
moves locking granularity from mm_struct (mm->page_table_lock) to per pmd
page table lock (page->ptl).

Signed-off-by: Anshuman Khandual <anshuman.khandual@arm.com>
---
 arch/arm64/Kconfig               |  4 +++
 arch/arm64/include/asm/pgalloc.h | 43 ++++++++++++++++++++++++++++++--
 2 files changed, 45 insertions(+), 2 deletions(-)

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index a4168d366127..b909e5d3b951 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -285,6 +285,10 @@ config PGTABLE_LEVELS
 	default 3 if ARM64_16K_PAGES && ARM64_VA_BITS_47
 	default 4 if !ARM64_64K_PAGES && ARM64_VA_BITS_48
 
+config ARCH_ENABLE_SPLIT_PMD_PTLOCK
+	def_bool y
+	depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && (PGTABLE_LEVELS > 2)
+
 config ARCH_SUPPORTS_UPROBES
 	def_bool y
 
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index a02a4d1d967d..9e06238be9b6 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -35,15 +35,54 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pte);
 
 #if CONFIG_PGTABLE_LEVELS > 2
 
+/*
+ * FIXME: _workaround might not be the right suffix here but it does
+ * indicate the intent behind it. Generic pgtable_pmd_page_ctor/dtor
+ * constructs neither do account the PMD page towards NR_PAGETABLE
+ * nor do they update page state with the new page type PageTable.
+ * Ideally pgtable_pmd_page_ctor/dtor should have been just taking care
+ * of page->pmd_huge_pte when applicable along with what is already
+ * achieved with pgtable_page_ctor/dtor constructs. Unfortunately that
+ * is not the case currently and changing those generic mm constructs
+ * might impact other archs. For now lets do the right thing here and
+ * drop this when generic PMD constructs accommodate required changes.
+ */
+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS
+static inline void pgtable_pmd_page_ctor_workaround(struct page *page)
+{
+	page->pmd_huge_pte = NULL;
+}
+
+static inline void pgtable_pmd_page_dtor_workaround(struct page *page)
+{
+	VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
+}
+#else
+static inline void pgtable_pmd_page_ctor_workaround(struct page *page) { }
+static inline void pgtable_pmd_page_dtor_workaround(struct page *page) { }
+#endif
+
 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
 {
-	return (pmd_t *)pte_alloc_one_virt(mm);
+	pgtable_t ptr;
+
+	ptr = pte_alloc_one(mm);
+	if (!ptr)
+		return 0;
+
+	pgtable_pmd_page_ctor_workaround(ptr);
+	return (pmd_t *)page_to_virt(ptr);
 }
 
 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmdp)
 {
+	struct page *page;
+
 	BUG_ON((unsigned long)pmdp & (PAGE_SIZE-1));
-	pte_free(mm, virt_to_page(pmdp));
+	page = virt_to_page(pmdp);
+
+	pgtable_pmd_page_dtor_workaround(page);
+	pte_free(mm, page);
 }
 
 static inline void __pud_populate(pud_t *pudp, phys_addr_t pmdp, pudval_t prot)
-- 
2.20.1


_______________________________________________
linux-arm-kernel mailing list
linux-arm-kernel@lists.infradead.org
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
