[PATCH 7/7] abstract pagetable locking and pte updates

Nick Piggin
2004-10-29T07:23:43Z

[-- Attachment #1: Type: text/plain, Size: 4 bytes --]

7/7

[-- Attachment #2: vm-x86_64-lockless-page-table.patch --]
[-- Type: text/x-patch, Size: 3103 bytes --]

x86_64: implement lockless page tables using cmpxchg

---

linux-2.6-npiggin/arch/x86_64/mm/ioremap.c | 19 ++++++++++++++-----
linux-2.6-npiggin/include/asm-x86_64/pgtable.h | 24 ++++++++++++++++++++++++
2 files changed, 38 insertions(+), 5 deletions(-)

diff -puN include/asm-x86_64/pgtable.h~vm-x86_64-lockless-page-table include/asm-x86_64/pgtable.h
--- linux-2.6/include/asm-x86_64/pgtable.h~vm-x86_64-lockless-page-table 2004-10-29 16:48:39.000000000 +1000
+++ linux-2.6-npiggin/include/asm-x86_64/pgtable.h 2004-10-29 16:48:39.000000000 +1000
@@ -417,6 +417,30 @@ extern inline pte_t pte_modify(pte_t pte
#define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) })
#define __swp_entry_to_pte(x) ((pte_t) { (x).val })

+#define __HAVE_ARCH_PTEP_CMPXCHG
+
+#define ptep_cmpxchg(xp, oldval, newval) \
+ (cmpxchg(&(xp)->pte, pte_val(oldval), pte_val(newval)) != pte_val(oldval))
+
+#define PGD_NONE 0UL
+
+#define pgd_test_and_populate(__mm, ___pgd, ___pmd) \
+({ \
+ unlikely(cmpxchg((int *)___pgd, PGD_NONE, _PAGE_TABLE | __pa(___pmd)) != PGD_NONE); \
+})
+
+#define PMD_NONE 0UL
+
+#define pmd_test_and_populate(__mm, ___pmd, ___pte) \
+({ \
+ unlikely(cmpxchg((int *)___pmd, PMD_NONE, _PAGE_TABLE | (page_to_pfn(___pte) << PAGE_SHIFT)) != PMD_NONE); \
+})
+
+#define pmd_test_and_populate_kernel(__mm, ___pmd, ___pte) \
+({ \
+ unlikely(cmpxchg((int *)___pmd, PMD_NONE, _PAGE_TABLE | __pa(___pte))); \
+})
+
#endif /* !__ASSEMBLY__ */

extern int kern_addr_valid(unsigned long addr);
diff -puN arch/x86_64/mm/ioremap.c~vm-x86_64-lockless-page-table arch/x86_64/mm/ioremap.c
--- linux-2.6/arch/x86_64/mm/ioremap.c~vm-x86_64-lockless-page-table 2004-10-29 16:48:39.000000000 +1000
+++ linux-2.6-npiggin/arch/x86_64/mm/ioremap.c 2004-10-29 16:48:39.000000000 +1000
@@ -32,12 +32,21 @@ static inline void remap_area_pte(pte_t
BUG();
pfn = phys_addr >> PAGE_SHIFT;
do {
- if (!pte_none(*pte)) {
+ struct pte_modify pmod;
+ pte_t new;
+again:
+ new = ptep_begin_modify(&pmod, &init_mm, pte);
+ if (!pte_none(new)) {
printk("remap_area_pte: page already exists\n");
BUG();
}
- set_pte(pte, pfn_pte(pfn, __pgprot(_PAGE_PRESENT | _PAGE_RW |
- _PAGE_GLOBAL | _PAGE_DIRTY | _PAGE_ACCESSED | flags)));
+ new = pfn_pte(pfn, __pgprot(_PAGE_PRESENT | _PAGE_RW |
+ _PAGE_GLOBAL | _PAGE_DIRTY |
+ _PAGE_ACCESSED | flags));
+ if (ptep_commit(&pmod, &init_mm, pte, new)) {
+ printk("remap_area_pte: ptep_commit raced\n");
+ goto again;
+ }
address += PAGE_SIZE;
pfn++;
pte++;
@@ -79,7 +88,7 @@ static int remap_area_pages(unsigned lon
flush_cache_all();
if (address >= end)
BUG();
- spin_lock(&init_mm.page_table_lock);
+ mm_lock_page_table(&init_mm);
do {
pmd_t *pmd;
pmd = pmd_alloc(&init_mm, dir, address);
@@ -93,7 +102,7 @@ static int remap_area_pages(unsigned lon
address = (address + PGDIR_SIZE) & PGDIR_MASK;
dir++;
} while (address && (address < end));
- spin_unlock(&init_mm.page_table_lock);
+ mm_unlock_page_table(&init_mm);
flush_tlb_all();
return error;
}

_