[PATCH 1/7] abstract pagetable locking and pte updates

Nick Piggin
2004-10-29T07:20:52Z

[-- Attachment #1: Type: text/plain, Size: 4 bytes --]

1/7

[-- Attachment #2: vm-free-pgtables-late.patch --]
[-- Type: text/x-patch, Size: 3566 bytes --]

Moves page table destruction to after vma destruction. This
makes pinning a vma pin the page tables, which is needed to make
rmap.c safe without the page table lock.

---

linux-2.6-npiggin/mm/mmap.c | 49 ++++++++++++++++++++++++++++++++++----------
1 files changed, 38 insertions(+), 11 deletions(-)

diff -puN mm/mmap.c~vm-free-pgtables-late mm/mmap.c
--- linux-2.6/mm/mmap.c~vm-free-pgtables-late 2004-10-23 19:40:06.000000000 +1000
+++ linux-2.6-npiggin/mm/mmap.c 2004-10-23 19:40:16.000000000 +1000
@@ -1559,7 +1559,6 @@ static void unmap_vma_list(struct mm_str
*/
static void unmap_region(struct mm_struct *mm,
struct vm_area_struct *vma,
- struct vm_area_struct *prev,
unsigned long start,
unsigned long end)
{
@@ -1567,15 +1566,31 @@ static void unmap_region(struct mm_struc
unsigned long nr_accounted = 0;

lru_add_drain();
+
+ spin_lock(&mm->page_table_lock);
tlb = tlb_gather_mmu(mm, 0);
unmap_vmas(&tlb, mm, vma, start, end, &nr_accounted, NULL);
+ tlb_finish_mmu(tlb, start, end);
+ spin_unlock(&mm->page_table_lock);
+
vm_unacct_memory(nr_accounted);
+}

+static void free_dangling_pgtables_region(struct mm_struct *mm,
+ struct vm_area_struct *prev,
+ unsigned long start,
+ unsigned long end)
+{
+ struct mmu_gather *tlb;
+
+ spin_lock(&mm->page_table_lock);
+ tlb = tlb_gather_mmu(mm, 0);
if (is_hugepage_only_range(start, end - start))
hugetlb_free_pgtables(tlb, prev, start, end);
else
free_pgtables(tlb, prev, start, end);
tlb_finish_mmu(tlb, start, end);
+ spin_unlock(&mm->page_table_lock);
}

/*
@@ -1709,13 +1724,18 @@ int do_munmap(struct mm_struct *mm, unsi
* Remove the vma's, and unmap the actual pages
*/
detach_vmas_to_be_unmapped(mm, mpnt, prev, end);
- spin_lock(&mm->page_table_lock);
- unmap_region(mm, mpnt, prev, start, end);
- spin_unlock(&mm->page_table_lock);
+
+ unmap_region(mm, mpnt, start, end);

/* Fix up all other VM information */
unmap_vma_list(mm, mpnt);

+ /*
+ * Free the page tables. Nothing will reference them at this
+ * point.
+ */
+ free_dangling_pgtables_region(mm, prev, start, end);
+
return 0;
}

@@ -1833,16 +1853,16 @@ void exit_mmap(struct mm_struct *mm)
lru_add_drain();

spin_lock(&mm->page_table_lock);
-
tlb = tlb_gather_mmu(mm, 1);
flush_cache_mm(mm);
/* Use ~0UL here to ensure all VMAs in the mm are unmapped */
mm->map_count -= unmap_vmas(&tlb, mm, mm->mmap, 0,
~0UL, &nr_accounted, NULL);
+ tlb_finish_mmu(tlb, 0, MM_VM_SIZE(mm));
+ spin_unlock(&mm->page_table_lock);
+
vm_unacct_memory(nr_accounted);
BUG_ON(mm->map_count); /* This is just debugging */
- clear_page_tables(tlb, FIRST_USER_PGD_NR, USER_PTRS_PER_PGD);
- tlb_finish_mmu(tlb, 0, MM_VM_SIZE(mm));

vma = mm->mmap;
mm->mmap = mm->mmap_cache = NULL;
@@ -1851,17 +1871,24 @@ void exit_mmap(struct mm_struct *mm)
mm->total_vm = 0;
mm->locked_vm = 0;

- spin_unlock(&mm->page_table_lock);
-
/*
- * Walk the list again, actually closing and freeing it
- * without holding any MM locks.
+ * Walk the list again, actually closing and freeing it.
*/
while (vma) {
struct vm_area_struct *next = vma->vm_next;
remove_vm_struct(vma);
vma = next;
}
+
+ /*
+ * Finally, free the pagetables. By this point, nothing should
+ * refer to them.
+ */
+ spin_lock(&mm->page_table_lock);
+ tlb = tlb_gather_mmu(mm, 1);
+ clear_page_tables(tlb, FIRST_USER_PGD_NR, USER_PTRS_PER_PGD);
+ tlb_finish_mmu(tlb, 0, MM_VM_SIZE(mm));
+ spin_unlock(&mm->page_table_lock);
}

/* Insert vm structure into process list sorted by address

_