[PATCH 6/7] abstract pagetable locking and pte updates

[Nick Piggin <nickpiggin@yahoo.com.au>]

[-- Attachment #1: Type: text/plain, Size: 4 bytes --]

6/7

[-- Attachment #2: vm-i386-lockless-page-table.patch --]
[-- Type: text/x-patch, Size: 4471 bytes --]



i386: Implement lockless pagetables using cmpxchg


---

 linux-2.6-npiggin/include/asm-i386/pgtable-2level.h |    3 +
 linux-2.6-npiggin/include/asm-i386/pgtable-3level.h |   23 +++-------
 linux-2.6-npiggin/include/asm-i386/pgtable.h        |   46 ++++++++++++++++++++
 3 files changed, 58 insertions(+), 14 deletions(-)

diff -puN include/asm-i386/pgtable.h~vm-i386-lockless-page-table include/asm-i386/pgtable.h
--- linux-2.6/include/asm-i386/pgtable.h~vm-i386-lockless-page-table	2004-10-29 16:28:16.000000000 +1000
+++ linux-2.6-npiggin/include/asm-i386/pgtable.h	2004-10-29 16:41:46.000000000 +1000
@@ -398,6 +398,52 @@ extern pte_t *lookup_address(unsigned lo
 		}							  \
 	} while (0)
 
+#define __HAVE_ARCH_PTEP_CMPXCHG
+
+#ifdef CONFIG_X86_PAE
+#define __HAVE_ARCH_PTEP_ATOMIC_READ
+#define ptep_atomic_read(__ptep)					\
+({									\
+	unsigned long long ret = get_64bit((unsigned long long *)__ptep); \
+ 	*((pte_t *)&ret);						\
+})
+#endif
+
+#define pgd_test_and_populate(__mm, ___pgd, ___page)			\
+({									\
+	BUG();								\
+	0;								\
+})
+
+#define PMD_NONE 0
+
+#ifndef CONFIG_X86_PAE
+#define pmd_test_and_populate(__mm, ___pmd, ___page)			\
+({									\
+	unlikely(cmpxchg((unsigned long *)___pmd, PMD_NONE,		\
+	_PAGE_TABLE + (page_to_pfn(___page) << PAGE_SHIFT)) != PMD_NONE); \
+})
+
+#define pmd_test_and_populate_kernel(__mm, ___pmd, ___page)		\
+({									\
+	unlikely(cmpxchg((unsigned long *)___pmd, PMD_NONE,		\
+			_PAGE_TABLE + __pa(___page)) != PMD_NONE);	\
+})
+#else
+#define pmd_test_and_populate(__mm, ___pmd, ___page)			\
+({									\
+	unlikely(cmpxchg8b((unsigned long long *)___pmd, PMD_NONE,	\
+	_PAGE_TABLE + ((unsigned long long)page_to_pfn(___page) << PAGE_SHIFT)) != PMD_NONE); \
+})
+
+#define pmd_test_and_populate_kernel(__mm, ___pmd, ___page)		\
+({									\
+	unlikely(cmpxchg8b((unsigned long long *)___pmd, PMD_NONE,	\
+		_PAGE_TABLE + (unsigned long long)__pa(___page)) != PMD_NONE); \
+})
+#endif
+
+
 #endif /* !__ASSEMBLY__ */
 
 #ifndef CONFIG_DISCONTIGMEM
diff -puN include/asm-i386/pgtable-2level.h~vm-i386-lockless-page-table include/asm-i386/pgtable-2level.h
--- linux-2.6/include/asm-i386/pgtable-2level.h~vm-i386-lockless-page-table	2004-10-29 16:28:16.000000000 +1000
+++ linux-2.6-npiggin/include/asm-i386/pgtable-2level.h	2004-10-29 16:28:16.000000000 +1000
@@ -82,4 +82,7 @@ static inline int pte_exec_kernel(pte_t 
 #define __pte_to_swp_entry(pte)		((swp_entry_t) { (pte).pte_low })
 #define __swp_entry_to_pte(x)		((pte_t) { (x).val })
 
+#define ptep_cmpxchg(ptep, old, new)					\
+({ cmpxchg(&(ptep)->pte_low, (old).pte_low, (new).pte_low) != (old).pte_low; })
+
 #endif /* _I386_PGTABLE_2LEVEL_H */
diff -puN include/asm-i386/pgtable-3level.h~vm-i386-lockless-page-table include/asm-i386/pgtable-3level.h
--- linux-2.6/include/asm-i386/pgtable-3level.h~vm-i386-lockless-page-table	2004-10-29 16:28:16.000000000 +1000
+++ linux-2.6-npiggin/include/asm-i386/pgtable-3level.h	2004-10-29 16:28:16.000000000 +1000
@@ -42,26 +42,15 @@ static inline int pte_exec_kernel(pte_t 
 	return pte_x(pte);
 }
 
-/* Rules for using set_pte: the pte being assigned *must* be
- * either not present or in a state where the hardware will
- * not attempt to update the pte.  In places where this is
- * not possible, use pte_get_and_clear to obtain the old pte
- * value and then use set_pte to update it.  -ben
- */
-static inline void set_pte(pte_t *ptep, pte_t pte)
-{
-	ptep->pte_high = pte.pte_high;
-	smp_wmb();
-	ptep->pte_low = pte.pte_low;
-}
-#define __HAVE_ARCH_SET_PTE_ATOMIC
-#define set_pte_atomic(pteptr,pteval) \
+#define set_pte(pteptr,pteval) \
 		set_64bit((unsigned long long *)(pteptr),pte_val(pteval))
 #define set_pmd(pmdptr,pmdval) \
 		set_64bit((unsigned long long *)(pmdptr),pmd_val(pmdval))
 #define set_pgd(pgdptr,pgdval) \
 		set_64bit((unsigned long long *)(pgdptr),pgd_val(pgdval))
 
+#define set_pte_atomic(pteptr,pteval) set_pte(pteptr,pteval)
+
 /*
  * Pentium-II erratum A13: in PAE mode we explicitly have to flush
  * the TLB via cr3 if the top-level pgd is changed...
@@ -142,4 +131,10 @@ static inline pmd_t pfn_pmd(unsigned lon
 #define __pte_to_swp_entry(pte)		((swp_entry_t){ (pte).pte_high })
 #define __swp_entry_to_pte(x)		((pte_t){ 0, (x).val })
 
+#define ptep_cmpxchg(ptep, old, new)					\
+({									\
+	cmpxchg8b(((unsigned long long *)ptep), pte_val(old), pte_val(new)) \
+ 			!= pte_val(old);				\
+})
+
 #endif /* _I386_PGTABLE_3LEVEL_H */

_