* [PATCH 1/7] mm: Move anon_vma ref out from under CONFIG_KSM
2010-04-02 14:16 [PATCH 0/7] mm: preemptibility Peter Zijlstra
@ 2010-04-02 14:16 ` Peter Zijlstra
2010-04-02 14:16 ` [PATCH 2/7] mm: Make use of the anon_vma ref count Peter Zijlstra
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Peter Zijlstra @ 2010-04-02 14:16 UTC (permalink / raw)
To: Andrea Arcangeli, Avi Kivity, Thomas Gleixner, Rik van Riel,
Ingo Molnar, akpm
Cc: linux-kernel, Benjamin Herrenschmidt, Hugh Dickins, Mel Gorman,
Nick Piggin, Peter Zijlstra
[-- Attachment #1: mm-anon_vma-ref.patch --]
[-- Type: text/plain, Size: 2632 bytes --]
We need an anon_vma refcount for preemptible anon_vma->lock as well
as memory compaction, so move it out into generic code.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
Index: linux-2.6/include/linux/rmap.h
===================================================================
--- linux-2.6.orig/include/linux/rmap.h
+++ linux-2.6/include/linux/rmap.h
@@ -26,9 +26,7 @@
*/
struct anon_vma {
spinlock_t lock; /* Serialize access to vma list */
-#ifdef CONFIG_KSM
- atomic_t ksm_refcount;
-#endif
+ atomic_t ref;
/*
* NOTE: the LSB of the head.next is set by
* mm_take_all_locks() _after_ taking the above lock. So the
@@ -61,26 +59,6 @@ struct anon_vma_chain {
};
#ifdef CONFIG_MMU
-#ifdef CONFIG_KSM
-static inline void ksm_refcount_init(struct anon_vma *anon_vma)
-{
- atomic_set(&anon_vma->ksm_refcount, 0);
-}
-
-static inline int ksm_refcount(struct anon_vma *anon_vma)
-{
- return atomic_read(&anon_vma->ksm_refcount);
-}
-#else
-static inline void ksm_refcount_init(struct anon_vma *anon_vma)
-{
-}
-
-static inline int ksm_refcount(struct anon_vma *anon_vma)
-{
- return 0;
-}
-#endif /* CONFIG_KSM */
static inline struct anon_vma *page_anon_vma(struct page *page)
{
Index: linux-2.6/mm/ksm.c
===================================================================
--- linux-2.6.orig/mm/ksm.c
+++ linux-2.6/mm/ksm.c
@@ -318,14 +318,14 @@ static void hold_anon_vma(struct rmap_it
struct anon_vma *anon_vma)
{
rmap_item->anon_vma = anon_vma;
- atomic_inc(&anon_vma->ksm_refcount);
+ atomic_inc(&anon_vma->ref);
}
static void drop_anon_vma(struct rmap_item *rmap_item)
{
struct anon_vma *anon_vma = rmap_item->anon_vma;
- if (atomic_dec_and_lock(&anon_vma->ksm_refcount, &anon_vma->lock)) {
+ if (atomic_dec_and_lock(&anon_vma->ref, &anon_vma->lock)) {
int empty = list_empty(&anon_vma->head);
spin_unlock(&anon_vma->lock);
if (empty)
Index: linux-2.6/mm/rmap.c
===================================================================
--- linux-2.6.orig/mm/rmap.c
+++ linux-2.6/mm/rmap.c
@@ -248,7 +248,7 @@ static void anon_vma_unlink(struct anon_
list_del(&anon_vma_chain->same_anon_vma);
/* We must garbage collect the anon_vma if it's empty */
- empty = list_empty(&anon_vma->head) && !ksm_refcount(anon_vma);
+ empty = list_empty(&anon_vma->head) && !atomic_read(&anon_vma->ref);
spin_unlock(&anon_vma->lock);
if (empty)
@@ -272,7 +272,7 @@ static void anon_vma_ctor(void *data)
struct anon_vma *anon_vma = data;
spin_lock_init(&anon_vma->lock);
- ksm_refcount_init(anon_vma);
+ atomic_set(&anon_vma->ref, 0);
INIT_LIST_HEAD(&anon_vma->head);
}
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH 2/7] mm: Make use of the anon_vma ref count
2010-04-02 14:16 [PATCH 0/7] mm: preemptibility Peter Zijlstra
2010-04-02 14:16 ` [PATCH 1/7] mm: Move anon_vma ref out from under CONFIG_KSM Peter Zijlstra
@ 2010-04-02 14:16 ` Peter Zijlstra
2010-04-02 14:16 ` [PATCH 3/7] mm: Preemptible mmu_gather Peter Zijlstra
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Peter Zijlstra @ 2010-04-02 14:16 UTC (permalink / raw)
To: Andrea Arcangeli, Avi Kivity, Thomas Gleixner, Rik van Riel,
Ingo Molnar, akpm
Cc: linux-kernel, Benjamin Herrenschmidt, Hugh Dickins, Mel Gorman,
Nick Piggin, Peter Zijlstra
[-- Attachment #1: mm-use-anon_vma-ref.patch --]
[-- Type: text/plain, Size: 6153 bytes --]
This patch changes the anon_vma refcount to be 0 when the object is
free. It does this by adding 1 ref to being in use in the anon_vma
structure (iow. the anon_vma->head list is not empty).
This allows a simpler release scheme without having to check both the
refcount and the list as well as avoids taking a ref for each entry
on the list.
We then use this new refcount in the migration code to avoid a long
RCU read side section and convert page_lock_anon_vma() over to use
refcounts.
This later is done for each of convertion of anon_vma from spinlock
to mutex.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/rmap.h | 7 +++++++
mm/ksm.c | 9 +--------
mm/migrate.c | 17 ++++++-----------
mm/rmap.c | 43 +++++++++++++++++++++++++++++--------------
4 files changed, 43 insertions(+), 33 deletions(-)
Index: linux-2.6/include/linux/rmap.h
===================================================================
--- linux-2.6.orig/include/linux/rmap.h
+++ linux-2.6/include/linux/rmap.h
@@ -100,6 +100,13 @@ static inline void anon_vma_merge(struct
unlink_anon_vmas(next);
}
+struct anon_vma *anon_vma_get(struct page *page);
+static inline void anon_vma_put(struct anon_vma *anon_vma)
+{
+ if (atomic_dec_and_test(&anon_vma->ref))
+ anon_vma_free(anon_vma);
+}
+
/*
* rmap interfaces called when adding or removing pte of page
*/
Index: linux-2.6/mm/ksm.c
===================================================================
--- linux-2.6.orig/mm/ksm.c
+++ linux-2.6/mm/ksm.c
@@ -323,14 +323,7 @@ static void hold_anon_vma(struct rmap_it
static void drop_anon_vma(struct rmap_item *rmap_item)
{
- struct anon_vma *anon_vma = rmap_item->anon_vma;
-
- if (atomic_dec_and_lock(&anon_vma->ref, &anon_vma->lock)) {
- int empty = list_empty(&anon_vma->head);
- spin_unlock(&anon_vma->lock);
- if (empty)
- anon_vma_free(anon_vma);
- }
+ anon_vma_put(rmap_item->anon_vma);
}
/*
Index: linux-2.6/mm/migrate.c
===================================================================
--- linux-2.6.orig/mm/migrate.c
+++ linux-2.6/mm/migrate.c
@@ -544,7 +544,7 @@ static int unmap_and_move(new_page_t get
int rc = 0;
int *result = NULL;
struct page *newpage = get_new_page(page, private, &result);
- int rcu_locked = 0;
+ struct anon_vma *anon_vma = NULL;
int charge = 0;
struct mem_cgroup *mem = NULL;
@@ -600,10 +600,8 @@ static int unmap_and_move(new_page_t get
* File Caches may use write_page() or lock_page() in migration, then,
* just care Anon page here.
*/
- if (PageAnon(page)) {
- rcu_read_lock();
- rcu_locked = 1;
- }
+ if (PageAnon(page))
+ anon_vma = anon_vma_get(page);
/*
* Corner case handling:
@@ -621,10 +619,7 @@ static int unmap_and_move(new_page_t get
if (!PageAnon(page) && page_has_private(page)) {
/*
* Go direct to try_to_free_buffers() here because
- * a) that's what try_to_release_page() would do anyway
- * b) we may be under rcu_read_lock() here, so we can't
- * use GFP_KERNEL which is what try_to_release_page()
- * needs to be effective.
+ * that's what try_to_release_page() would do anyway
*/
try_to_free_buffers(page);
goto rcu_unlock;
@@ -642,8 +637,8 @@ skip_unmap:
if (rc)
remove_migration_ptes(page, page);
rcu_unlock:
- if (rcu_locked)
- rcu_read_unlock();
+ if (anon_vma)
+ anon_vma_put(anon_vma);
uncharge:
if (!charge)
mem_cgroup_end_migration(mem, page, newpage);
Index: linux-2.6/mm/rmap.c
===================================================================
--- linux-2.6.orig/mm/rmap.c
+++ linux-2.6/mm/rmap.c
@@ -66,11 +66,18 @@ static struct kmem_cache *anon_vma_chain
static inline struct anon_vma *anon_vma_alloc(void)
{
- return kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
+ struct anon_vma *anon_vma;
+
+ anon_vma = kmem_cache_alloc(anon_vma_cachep, GFP_KERNEL);
+ if (anon_vma)
+ atomic_set(&anon_vma->ref, 1);
+
+ return anon_vma;
}
void anon_vma_free(struct anon_vma *anon_vma)
{
+ VM_BUG_ON(atomic_read(&anon_vma->ref));
kmem_cache_free(anon_vma_cachep, anon_vma);
}
@@ -149,7 +156,7 @@ int anon_vma_prepare(struct vm_area_stru
spin_unlock(&anon_vma->lock);
if (unlikely(allocated)) {
- anon_vma_free(allocated);
+ anon_vma_put(allocated);
anon_vma_chain_free(avc);
}
}
@@ -230,7 +237,7 @@ int anon_vma_fork(struct vm_area_struct
return 0;
out_error_free_anon_vma:
- anon_vma_free(anon_vma);
+ anon_vma_put(anon_vma);
out_error:
return -ENOMEM;
}
@@ -246,13 +253,11 @@ static void anon_vma_unlink(struct anon_
spin_lock(&anon_vma->lock);
list_del(&anon_vma_chain->same_anon_vma);
-
- /* We must garbage collect the anon_vma if it's empty */
- empty = list_empty(&anon_vma->head) && !atomic_read(&anon_vma->ref);
+ empty = list_empty(&anon_vma->head);
spin_unlock(&anon_vma->lock);
if (empty)
- anon_vma_free(anon_vma);
+ anon_vma_put(anon_vma);
}
void unlink_anon_vmas(struct vm_area_struct *vma)
@@ -285,11 +290,11 @@ void __init anon_vma_init(void)
/*
* Getting a lock on a stable anon_vma from a page off the LRU is
- * tricky: page_lock_anon_vma rely on RCU to guard against the races.
+ * tricky: page_lock_anon_vma relies on RCU to guard against the races.
*/
-struct anon_vma *page_lock_anon_vma(struct page *page)
+struct anon_vma *anon_vma_get(struct page *page)
{
- struct anon_vma *anon_vma;
+ struct anon_vma *anon_vma = NULL;
unsigned long anon_mapping;
rcu_read_lock();
@@ -300,17 +305,27 @@ struct anon_vma *page_lock_anon_vma(stru
goto out;
anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
- spin_lock(&anon_vma->lock);
- return anon_vma;
+ if (!atomic_inc_not_zero(&anon_vma->ref))
+ anon_vma = NULL;
out:
rcu_read_unlock();
- return NULL;
+ return anon_vma;
+}
+
+struct anon_vma *page_lock_anon_vma(struct page *page)
+{
+ struct anon_vma *anon_vma = anon_vma_get(page);
+
+ if (anon_vma)
+ spin_lock(&anon_vma->lock);
+
+ return anon_vma;
}
void page_unlock_anon_vma(struct anon_vma *anon_vma)
{
spin_unlock(&anon_vma->lock);
- rcu_read_unlock();
+ anon_vma_put(anon_vma);
}
/*
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH 3/7] mm: Preemptible mmu_gather
2010-04-02 14:16 [PATCH 0/7] mm: preemptibility Peter Zijlstra
2010-04-02 14:16 ` [PATCH 1/7] mm: Move anon_vma ref out from under CONFIG_KSM Peter Zijlstra
2010-04-02 14:16 ` [PATCH 2/7] mm: Make use of the anon_vma ref count Peter Zijlstra
@ 2010-04-02 14:16 ` Peter Zijlstra
2010-04-02 14:16 ` [PATCH 4/7] lockdep, mutex: Provide mutex_lock_nest_lock Peter Zijlstra
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Peter Zijlstra @ 2010-04-02 14:16 UTC (permalink / raw)
To: Andrea Arcangeli, Avi Kivity, Thomas Gleixner, Rik van Riel,
Ingo Molnar, akpm
Cc: linux-kernel, Benjamin Herrenschmidt, Hugh Dickins, Mel Gorman,
Nick Piggin, Peter Zijlstra
[-- Attachment #1: mm-preempt-tlb-gather.patch --]
[-- Type: text/plain, Size: 9828 bytes --]
Make mmu_gather preemptible by using a small on stack list and use
an option allocation to speed things up.
This breaks at least PPC for which there is a patch in -rt which
still needs porting.
Preemptible mmu_gather is desired in general and usable once
i_mmap_lock becomes a mutex. Doing it before the mutex conversion
saves us from having to rework the code by moving the mmu_gather
bits inside the i_mmap_lock.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
fs/exec.c | 10 ++++-----
include/asm-generic/tlb.h | 51 +++++++++++++++++++++++++++++-----------------
include/linux/mm.h | 2 -
mm/memory.c | 27 +++++-------------------
mm/mmap.c | 16 +++++++-------
5 files changed, 53 insertions(+), 53 deletions(-)
Index: linux-2.6/fs/exec.c
===================================================================
--- linux-2.6.orig/fs/exec.c
+++ linux-2.6/fs/exec.c
@@ -503,7 +503,7 @@ static int shift_arg_pages(struct vm_are
unsigned long length = old_end - old_start;
unsigned long new_start = old_start - shift;
unsigned long new_end = old_end - shift;
- struct mmu_gather *tlb;
+ struct mmu_gather tlb;
BUG_ON(new_start > new_end);
@@ -529,12 +529,12 @@ static int shift_arg_pages(struct vm_are
return -ENOMEM;
lru_add_drain();
- tlb = tlb_gather_mmu(mm, 0);
+ tlb_gather_mmu(&tlb, mm, 0);
if (new_end > old_start) {
/*
* when the old and new regions overlap clear from new_end.
*/
- free_pgd_range(tlb, new_end, old_end, new_end,
+ free_pgd_range(&tlb, new_end, old_end, new_end,
vma->vm_next ? vma->vm_next->vm_start : 0);
} else {
/*
@@ -543,10 +543,10 @@ static int shift_arg_pages(struct vm_are
* have constraints on va-space that make this illegal (IA64) -
* for the others its just a little faster.
*/
- free_pgd_range(tlb, old_start, old_end, new_end,
+ free_pgd_range(&tlb, old_start, old_end, new_end,
vma->vm_next ? vma->vm_next->vm_start : 0);
}
- tlb_finish_mmu(tlb, new_end, old_end);
+ tlb_finish_mmu(&tlb, new_end, old_end);
/*
* Shrink the vma to just the new range. Always succeeds.
Index: linux-2.6/include/asm-generic/tlb.h
===================================================================
--- linux-2.6.orig/include/asm-generic/tlb.h
+++ linux-2.6/include/asm-generic/tlb.h
@@ -22,14 +22,8 @@
* and page free order so much..
*/
#ifdef CONFIG_SMP
- #ifdef ARCH_FREE_PTR_NR
- #define FREE_PTR_NR ARCH_FREE_PTR_NR
- #else
- #define FREE_PTE_NR 506
- #endif
#define tlb_fast_mode(tlb) ((tlb)->nr == ~0U)
#else
- #define FREE_PTE_NR 1
#define tlb_fast_mode(tlb) 1
#endif
@@ -39,30 +33,48 @@
struct mmu_gather {
struct mm_struct *mm;
unsigned int nr; /* set to ~0U means fast mode */
+ unsigned int max; /* nr < max */
unsigned int need_flush;/* Really unmapped some ptes? */
unsigned int fullmm; /* non-zero means full mm flush */
- struct page * pages[FREE_PTE_NR];
+#ifdef HAVE_ARCH_MMU_GATHER
+ struct arch_mmu_gather arch;
+#endif
+ struct page **pages;
+ struct page *local[8];
};
-/* Users of the generic TLB shootdown code must declare this storage space. */
-DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
+static inline void __tlb_alloc_pages(struct mmu_gather *tlb)
+{
+ unsigned long addr = __get_free_pages(GFP_ATOMIC, 0);
+
+ if (addr) {
+ tlb->pages = (void *)addr;
+ tlb->max = PAGE_SIZE / sizeof(struct page *);
+ }
+}
/* tlb_gather_mmu
* Return a pointer to an initialized struct mmu_gather.
*/
-static inline struct mmu_gather *
-tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
+static inline void
+tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int full_mm_flush)
{
- struct mmu_gather *tlb = &get_cpu_var(mmu_gathers);
-
tlb->mm = mm;
- /* Use fast mode if only one CPU is online */
- tlb->nr = num_online_cpus() > 1 ? 0U : ~0U;
+ tlb->max = ARRAY_SIZE(tlb->local);
+ tlb->pages = tlb->local;
+
+ if (num_online_cpus() > 1) {
+ tlb->nr = 0;
+ __tlb_alloc_pages(tlb);
+ } else /* Use fast mode if only one CPU is online */
+ tlb->nr = ~0U;
tlb->fullmm = full_mm_flush;
- return tlb;
+#ifdef HAVE_ARCH_MMU_GATHER
+ tlb->arch = ARCH_MMU_GATHER_INIT;
+#endif
}
static inline void
@@ -75,6 +87,8 @@ tlb_flush_mmu(struct mmu_gather *tlb, un
if (!tlb_fast_mode(tlb)) {
free_pages_and_swap_cache(tlb->pages, tlb->nr);
tlb->nr = 0;
+ if (tlb->pages == tlb->local)
+ __tlb_alloc_pages(tlb);
}
}
@@ -90,7 +104,8 @@ tlb_finish_mmu(struct mmu_gather *tlb, u
/* keep the page table cache within bounds */
check_pgt_cache();
- put_cpu_var(mmu_gathers);
+ if (tlb->pages != tlb->local)
+ free_pages((unsigned long)tlb->pages, 0);
}
/* tlb_remove_page
@@ -106,7 +121,7 @@ static inline void tlb_remove_page(struc
return;
}
tlb->pages[tlb->nr++] = page;
- if (tlb->nr >= FREE_PTE_NR)
+ if (tlb->nr >= tlb->max)
tlb_flush_mmu(tlb, 0, 0);
}
Index: linux-2.6/include/linux/mm.h
===================================================================
--- linux-2.6.orig/include/linux/mm.h
+++ linux-2.6/include/linux/mm.h
@@ -760,7 +760,7 @@ int zap_vma_ptes(struct vm_area_struct *
unsigned long size);
unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address,
unsigned long size, struct zap_details *);
-unsigned long unmap_vmas(struct mmu_gather **tlb,
+unsigned long unmap_vmas(struct mmu_gather *tlb,
struct vm_area_struct *start_vma, unsigned long start_addr,
unsigned long end_addr, unsigned long *nr_accounted,
struct zap_details *);
Index: linux-2.6/mm/memory.c
===================================================================
--- linux-2.6.orig/mm/memory.c
+++ linux-2.6/mm/memory.c
@@ -1095,17 +1095,14 @@ static unsigned long unmap_page_range(st
* ensure that any thus-far unmapped pages are flushed before unmap_vmas()
* drops the lock and schedules.
*/
-unsigned long unmap_vmas(struct mmu_gather **tlbp,
+unsigned long unmap_vmas(struct mmu_gather *tlb,
struct vm_area_struct *vma, unsigned long start_addr,
unsigned long end_addr, unsigned long *nr_accounted,
struct zap_details *details)
{
long zap_work = ZAP_BLOCK_SIZE;
- unsigned long tlb_start = 0; /* For tlb_finish_mmu */
- int tlb_start_valid = 0;
unsigned long start = start_addr;
spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
- int fullmm = (*tlbp)->fullmm;
struct mm_struct *mm = vma->vm_mm;
mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
@@ -1126,11 +1123,6 @@ unsigned long unmap_vmas(struct mmu_gath
untrack_pfn_vma(vma, 0, 0);
while (start != end) {
- if (!tlb_start_valid) {
- tlb_start = start;
- tlb_start_valid = 1;
- }
-
if (unlikely(is_vm_hugetlb_page(vma))) {
/*
* It is undesirable to test vma->vm_file as it
@@ -1151,7 +1143,7 @@ unsigned long unmap_vmas(struct mmu_gath
start = end;
} else
- start = unmap_page_range(*tlbp, vma,
+ start = unmap_page_range(tlb, vma,
start, end, &zap_work, details);
if (zap_work > 0) {
@@ -1159,19 +1151,13 @@ unsigned long unmap_vmas(struct mmu_gath
break;
}
- tlb_finish_mmu(*tlbp, tlb_start, start);
-
if (need_resched() ||
(i_mmap_lock && spin_needbreak(i_mmap_lock))) {
- if (i_mmap_lock) {
- *tlbp = NULL;
+ if (i_mmap_lock)
goto out;
- }
cond_resched();
}
- *tlbp = tlb_gather_mmu(vma->vm_mm, fullmm);
- tlb_start_valid = 0;
zap_work = ZAP_BLOCK_SIZE;
}
}
@@ -1191,16 +1177,15 @@ unsigned long zap_page_range(struct vm_a
unsigned long size, struct zap_details *details)
{
struct mm_struct *mm = vma->vm_mm;
- struct mmu_gather *tlb;
+ struct mmu_gather tlb;
unsigned long end = address + size;
unsigned long nr_accounted = 0;
lru_add_drain();
- tlb = tlb_gather_mmu(mm, 0);
+ tlb_gather_mmu(&tlb, mm, 0);
update_hiwater_rss(mm);
end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details);
- if (tlb)
- tlb_finish_mmu(tlb, address, end);
+ tlb_finish_mmu(&tlb, address, end);
return end;
}
Index: linux-2.6/mm/mmap.c
===================================================================
--- linux-2.6.orig/mm/mmap.c
+++ linux-2.6/mm/mmap.c
@@ -1844,17 +1844,17 @@ static void unmap_region(struct mm_struc
unsigned long start, unsigned long end)
{
struct vm_area_struct *next = prev? prev->vm_next: mm->mmap;
- struct mmu_gather *tlb;
+ struct mmu_gather tlb;
unsigned long nr_accounted = 0;
lru_add_drain();
- tlb = tlb_gather_mmu(mm, 0);
+ tlb_gather_mmu(&tlb, mm, 0);
update_hiwater_rss(mm);
unmap_vmas(&tlb, vma, start, end, &nr_accounted, NULL);
vm_unacct_memory(nr_accounted);
- free_pgtables(tlb, vma, prev? prev->vm_end: FIRST_USER_ADDRESS,
+ free_pgtables(&tlb, vma, prev? prev->vm_end: FIRST_USER_ADDRESS,
next? next->vm_start: 0);
- tlb_finish_mmu(tlb, start, end);
+ tlb_finish_mmu(&tlb, start, end);
}
/*
@@ -2190,7 +2190,7 @@ EXPORT_SYMBOL(do_brk);
/* Release all mmaps. */
void exit_mmap(struct mm_struct *mm)
{
- struct mmu_gather *tlb;
+ struct mmu_gather tlb;
struct vm_area_struct *vma;
unsigned long nr_accounted = 0;
unsigned long end;
@@ -2215,14 +2215,14 @@ void exit_mmap(struct mm_struct *mm)
lru_add_drain();
flush_cache_mm(mm);
- tlb = tlb_gather_mmu(mm, 1);
+ tlb_gather_mmu(&tlb, mm, 1);
/* update_hiwater_rss(mm) here? but nobody should be looking */
/* Use -1 here to ensure all VMAs in the mm are unmapped */
end = unmap_vmas(&tlb, vma, 0, -1, &nr_accounted, NULL);
vm_unacct_memory(nr_accounted);
- free_pgtables(tlb, vma, FIRST_USER_ADDRESS, 0);
- tlb_finish_mmu(tlb, 0, end);
+ free_pgtables(&tlb, vma, FIRST_USER_ADDRESS, 0);
+ tlb_finish_mmu(&tlb, 0, end);
/*
* Walk the list again, actually closing and freeing it,
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH 4/7] lockdep, mutex: Provide mutex_lock_nest_lock
2010-04-02 14:16 [PATCH 0/7] mm: preemptibility Peter Zijlstra
` (2 preceding siblings ...)
2010-04-02 14:16 ` [PATCH 3/7] mm: Preemptible mmu_gather Peter Zijlstra
@ 2010-04-02 14:16 ` Peter Zijlstra
2010-04-02 14:16 ` [PATCH 5/7] mutex: Provide mutex_is_contended Peter Zijlstra
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Peter Zijlstra @ 2010-04-02 14:16 UTC (permalink / raw)
To: Andrea Arcangeli, Avi Kivity, Thomas Gleixner, Rik van Riel,
Ingo Molnar, akpm
Cc: linux-kernel, Benjamin Herrenschmidt, Hugh Dickins, Mel Gorman,
Nick Piggin, Peter Zijlstra
[-- Attachment #1: mutex_lock_nest_lock.patch --]
[-- Type: text/plain, Size: 5442 bytes --]
Provide the mutex_lock_nest_lock() annotation.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/lockdep.h | 3 +++
include/linux/mutex.h | 9 +++++++++
kernel/mutex.c | 25 +++++++++++++++++--------
3 files changed, 29 insertions(+), 8 deletions(-)
Index: linux-2.6/include/linux/lockdep.h
===================================================================
--- linux-2.6.orig/include/linux/lockdep.h
+++ linux-2.6/include/linux/lockdep.h
@@ -484,12 +484,15 @@ static inline void print_irqtrace_events
#ifdef CONFIG_DEBUG_LOCK_ALLOC
# ifdef CONFIG_PROVE_LOCKING
# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 2, NULL, i)
+# define mutex_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 2, n, i)
# else
# define mutex_acquire(l, s, t, i) lock_acquire(l, s, t, 0, 1, NULL, i)
+# define mutex_acquire_nest(l, s, t, n, i) lock_acquire(l, s, t, 0, 1, n, i)
# endif
# define mutex_release(l, n, i) lock_release(l, n, i)
#else
# define mutex_acquire(l, s, t, i) do { } while (0)
+# define mutex_acquire_nest(l, s, t, n, i) do { } while (0)
# define mutex_release(l, n, i) do { } while (0)
#endif
Index: linux-2.6/include/linux/mutex.h
===================================================================
--- linux-2.6.orig/include/linux/mutex.h
+++ linux-2.6/include/linux/mutex.h
@@ -124,6 +124,7 @@ static inline int mutex_is_locked(struct
*/
#ifdef CONFIG_DEBUG_LOCK_ALLOC
extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass);
+extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock);
extern int __must_check mutex_lock_interruptible_nested(struct mutex *lock,
unsigned int subclass);
extern int __must_check mutex_lock_killable_nested(struct mutex *lock,
@@ -132,6 +133,13 @@ extern int __must_check mutex_lock_killa
#define mutex_lock(lock) mutex_lock_nested(lock, 0)
#define mutex_lock_interruptible(lock) mutex_lock_interruptible_nested(lock, 0)
#define mutex_lock_killable(lock) mutex_lock_killable_nested(lock, 0)
+
+#define mutex_lock_nest_lock(lock, nest_lock) \
+do { \
+ typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \
+ _mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \
+} while (0)
+
#else
extern void mutex_lock(struct mutex *lock);
extern int __must_check mutex_lock_interruptible(struct mutex *lock);
@@ -140,6 +148,7 @@ extern int __must_check mutex_lock_killa
# define mutex_lock_nested(lock, subclass) mutex_lock(lock)
# define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock)
# define mutex_lock_killable_nested(lock, subclass) mutex_lock_killable(lock)
+# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock)
#endif
/*
Index: linux-2.6/kernel/mutex.c
===================================================================
--- linux-2.6.orig/kernel/mutex.c
+++ linux-2.6/kernel/mutex.c
@@ -140,14 +140,14 @@ EXPORT_SYMBOL(mutex_unlock);
*/
static inline int __sched
__mutex_lock_common(struct mutex *lock, long state, unsigned int subclass,
- unsigned long ip)
+ struct lockdep_map *nest_lock, unsigned long ip)
{
struct task_struct *task = current;
struct mutex_waiter waiter;
unsigned long flags;
preempt_disable();
- mutex_acquire(&lock->dep_map, subclass, 0, ip);
+ mutex_acquire_nest(&lock->dep_map, subclass, 0, nest_lock, ip);
#ifdef CONFIG_MUTEX_SPIN_ON_OWNER
/*
@@ -278,16 +278,25 @@ void __sched
mutex_lock_nested(struct mutex *lock, unsigned int subclass)
{
might_sleep();
- __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, _RET_IP_);
+ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, subclass, NULL, _RET_IP_);
}
EXPORT_SYMBOL_GPL(mutex_lock_nested);
+void __sched
+_mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest)
+{
+ might_sleep();
+ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, nest, _RET_IP_);
+}
+
+EXPORT_SYMBOL_GPL(_mutex_lock_nest_lock);
+
int __sched
mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass)
{
might_sleep();
- return __mutex_lock_common(lock, TASK_KILLABLE, subclass, _RET_IP_);
+ return __mutex_lock_common(lock, TASK_KILLABLE, subclass, NULL, _RET_IP_);
}
EXPORT_SYMBOL_GPL(mutex_lock_killable_nested);
@@ -296,7 +305,7 @@ mutex_lock_interruptible_nested(struct m
{
might_sleep();
return __mutex_lock_common(lock, TASK_INTERRUPTIBLE,
- subclass, _RET_IP_);
+ subclass, NULL, _RET_IP_);
}
EXPORT_SYMBOL_GPL(mutex_lock_interruptible_nested);
@@ -402,7 +411,7 @@ __mutex_lock_slowpath(atomic_t *lock_cou
{
struct mutex *lock = container_of(lock_count, struct mutex, count);
- __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, _RET_IP_);
+ __mutex_lock_common(lock, TASK_UNINTERRUPTIBLE, 0, NULL, _RET_IP_);
}
static noinline int __sched
@@ -410,7 +419,7 @@ __mutex_lock_killable_slowpath(atomic_t
{
struct mutex *lock = container_of(lock_count, struct mutex, count);
- return __mutex_lock_common(lock, TASK_KILLABLE, 0, _RET_IP_);
+ return __mutex_lock_common(lock, TASK_KILLABLE, 0, NULL, _RET_IP_);
}
static noinline int __sched
@@ -418,7 +427,7 @@ __mutex_lock_interruptible_slowpath(atom
{
struct mutex *lock = container_of(lock_count, struct mutex, count);
- return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, _RET_IP_);
+ return __mutex_lock_common(lock, TASK_INTERRUPTIBLE, 0, NULL, _RET_IP_);
}
#endif
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH 5/7] mutex: Provide mutex_is_contended
2010-04-02 14:16 [PATCH 0/7] mm: preemptibility Peter Zijlstra
` (3 preceding siblings ...)
2010-04-02 14:16 ` [PATCH 4/7] lockdep, mutex: Provide mutex_lock_nest_lock Peter Zijlstra
@ 2010-04-02 14:16 ` Peter Zijlstra
2010-04-02 14:16 ` [PATCH 6/7] mm: Convert i_mmap_lock and anon_vma->lock to mutexes Peter Zijlstra
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Peter Zijlstra @ 2010-04-02 14:16 UTC (permalink / raw)
To: Andrea Arcangeli, Avi Kivity, Thomas Gleixner, Rik van Riel,
Ingo Molnar, akpm
Cc: linux-kernel, Benjamin Herrenschmidt, Hugh Dickins, Mel Gorman,
Nick Piggin, Peter Zijlstra
[-- Attachment #1: mutex-is-contended.patch --]
[-- Type: text/plain, Size: 677 bytes --]
Usable for lock-breaks and such.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
include/linux/mutex.h | 5 +++++
1 file changed, 5 insertions(+)
Index: linux-2.6/include/linux/mutex.h
===================================================================
--- linux-2.6.orig/include/linux/mutex.h
+++ linux-2.6/include/linux/mutex.h
@@ -118,6 +118,11 @@ static inline int mutex_is_locked(struct
return atomic_read(&lock->count) != 1;
}
+static inline int mutex_is_contended(struct mutex *lock)
+{
+ return atomic_read(&lock->count) < 0;
+}
+
/*
* See kernel/mutex.c for detailed documentation of these APIs.
* Also see Documentation/mutex-design.txt.
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH 6/7] mm: Convert i_mmap_lock and anon_vma->lock to mutexes
2010-04-02 14:16 [PATCH 0/7] mm: preemptibility Peter Zijlstra
` (4 preceding siblings ...)
2010-04-02 14:16 ` [PATCH 5/7] mutex: Provide mutex_is_contended Peter Zijlstra
@ 2010-04-02 14:16 ` Peter Zijlstra
2010-04-02 14:16 ` [PATCH 7/7] mm: Optimize page_lock_anon_vma Peter Zijlstra
2010-04-02 15:14 ` [PATCH 0/7] mm: preemptibility Andrea Arcangeli
7 siblings, 0 replies; 9+ messages in thread
From: Peter Zijlstra @ 2010-04-02 14:16 UTC (permalink / raw)
To: Andrea Arcangeli, Avi Kivity, Thomas Gleixner, Rik van Riel,
Ingo Molnar, akpm
Cc: linux-kernel, Benjamin Herrenschmidt, Hugh Dickins, Mel Gorman,
Nick Piggin, Peter Zijlstra
[-- Attachment #1: mm-mutex.patch --]
[-- Type: text/plain, Size: 22758 bytes --]
Straight fwd conversion of i_mmap_lock and anon_vma->lock to mutexes.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
arch/x86/mm/hugetlbpage.c | 4 ++--
fs/hugetlbfs/inode.c | 4 ++--
fs/inode.c | 2 +-
include/linux/fs.h | 2 +-
include/linux/mm.h | 2 +-
include/linux/rmap.h | 10 +++++-----
kernel/fork.c | 4 ++--
mm/filemap_xip.c | 4 ++--
mm/fremap.c | 4 ++--
mm/hugetlb.c | 12 ++++++------
mm/ksm.c | 16 ++++++++--------
mm/memory-failure.c | 4 ++--
mm/memory.c | 14 +++++++-------
mm/mmap.c | 20 ++++++++++----------
mm/mremap.c | 4 ++--
mm/rmap.c | 42 +++++++++++++++++++++---------------------
16 files changed, 74 insertions(+), 74 deletions(-)
Index: linux-2.6/arch/x86/mm/hugetlbpage.c
===================================================================
--- linux-2.6.orig/arch/x86/mm/hugetlbpage.c
+++ linux-2.6/arch/x86/mm/hugetlbpage.c
@@ -73,7 +73,7 @@ static void huge_pmd_share(struct mm_str
if (!vma_shareable(vma, addr))
return;
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
vma_prio_tree_foreach(svma, &iter, &mapping->i_mmap, idx, idx) {
if (svma == vma)
continue;
@@ -98,7 +98,7 @@ static void huge_pmd_share(struct mm_str
put_page(virt_to_page(spte));
spin_unlock(&mm->page_table_lock);
out:
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
}
/*
Index: linux-2.6/fs/hugetlbfs/inode.c
===================================================================
--- linux-2.6.orig/fs/hugetlbfs/inode.c
+++ linux-2.6/fs/hugetlbfs/inode.c
@@ -429,10 +429,10 @@ static int hugetlb_vmtruncate(struct ino
pgoff = offset >> PAGE_SHIFT;
i_size_write(inode, offset);
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
if (!prio_tree_empty(&mapping->i_mmap))
hugetlb_vmtruncate_list(&mapping->i_mmap, pgoff);
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
truncate_hugepages(inode, offset);
return 0;
}
Index: linux-2.6/fs/inode.c
===================================================================
--- linux-2.6.orig/fs/inode.c
+++ linux-2.6/fs/inode.c
@@ -258,7 +258,7 @@ void inode_init_once(struct inode *inode
INIT_LIST_HEAD(&inode->i_devices);
INIT_RADIX_TREE(&inode->i_data.page_tree, GFP_ATOMIC);
spin_lock_init(&inode->i_data.tree_lock);
- spin_lock_init(&inode->i_data.i_mmap_lock);
+ mutex_init(&inode->i_data.i_mmap_lock);
INIT_LIST_HEAD(&inode->i_data.private_list);
spin_lock_init(&inode->i_data.private_lock);
INIT_RAW_PRIO_TREE_ROOT(&inode->i_data.i_mmap);
Index: linux-2.6/include/linux/fs.h
===================================================================
--- linux-2.6.orig/include/linux/fs.h
+++ linux-2.6/include/linux/fs.h
@@ -627,7 +627,7 @@ struct address_space {
unsigned int i_mmap_writable;/* count VM_SHARED mappings */
struct prio_tree_root i_mmap; /* tree of private and shared mappings */
struct list_head i_mmap_nonlinear;/*list VM_NONLINEAR mappings */
- spinlock_t i_mmap_lock; /* protect tree, count, list */
+ struct mutex i_mmap_lock; /* protect tree, count, list */
unsigned int truncate_count; /* Cover race condition with truncate */
unsigned long nrpages; /* number of total pages */
pgoff_t writeback_index;/* writeback starts here */
Index: linux-2.6/include/linux/mm.h
===================================================================
--- linux-2.6.orig/include/linux/mm.h
+++ linux-2.6/include/linux/mm.h
@@ -749,7 +749,7 @@ struct zap_details {
struct address_space *check_mapping; /* Check page->mapping if set */
pgoff_t first_index; /* Lowest page->index to unmap */
pgoff_t last_index; /* Highest page->index to unmap */
- spinlock_t *i_mmap_lock; /* For unmap_mapping_range: */
+ struct mutex *i_mmap_lock; /* For unmap_mapping_range: */
unsigned long truncate_count; /* Compare vm_truncate_count */
};
Index: linux-2.6/include/linux/rmap.h
===================================================================
--- linux-2.6.orig/include/linux/rmap.h
+++ linux-2.6/include/linux/rmap.h
@@ -7,7 +7,7 @@
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/mm.h>
-#include <linux/spinlock.h>
+#include <linux/mutex.h>
#include <linux/memcontrol.h>
/*
@@ -25,8 +25,7 @@
* pointing to this anon_vma once its vma list is empty.
*/
struct anon_vma {
- spinlock_t lock; /* Serialize access to vma list */
- atomic_t ref;
+ struct mutex lock; /* Serialize access to vma list */
/*
* NOTE: the LSB of the head.next is set by
* mm_take_all_locks() _after_ taking the above lock. So the
@@ -36,6 +35,7 @@ struct anon_vma {
* mm_take_all_locks() (mm_all_locks_mutex).
*/
struct list_head head; /* Chain of private "related" vmas */
+ atomic_t ref;
};
/*
@@ -72,14 +72,14 @@ static inline void anon_vma_lock(struct
{
struct anon_vma *anon_vma = vma->anon_vma;
if (anon_vma)
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
}
static inline void anon_vma_unlock(struct vm_area_struct *vma)
{
struct anon_vma *anon_vma = vma->anon_vma;
if (anon_vma)
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
}
/*
Index: linux-2.6/kernel/fork.c
===================================================================
--- linux-2.6.orig/kernel/fork.c
+++ linux-2.6/kernel/fork.c
@@ -355,7 +355,7 @@ static int dup_mmap(struct mm_struct *mm
get_file(file);
if (tmp->vm_flags & VM_DENYWRITE)
atomic_dec(&inode->i_writecount);
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
if (tmp->vm_flags & VM_SHARED)
mapping->i_mmap_writable++;
tmp->vm_truncate_count = mpnt->vm_truncate_count;
@@ -363,7 +363,7 @@ static int dup_mmap(struct mm_struct *mm
/* insert tmp into the share list, just after mpnt */
vma_prio_tree_add(tmp, mpnt);
flush_dcache_mmap_unlock(mapping);
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
}
/*
Index: linux-2.6/mm/filemap_xip.c
===================================================================
--- linux-2.6.orig/mm/filemap_xip.c
+++ linux-2.6/mm/filemap_xip.c
@@ -182,7 +182,7 @@ __xip_unmap (struct address_space * mapp
return;
retry:
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
mm = vma->vm_mm;
address = vma->vm_start +
@@ -200,7 +200,7 @@ retry:
page_cache_release(page);
}
}
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
if (locked) {
mutex_unlock(&xip_sparse_mutex);
Index: linux-2.6/mm/fremap.c
===================================================================
--- linux-2.6.orig/mm/fremap.c
+++ linux-2.6/mm/fremap.c
@@ -208,13 +208,13 @@ SYSCALL_DEFINE5(remap_file_pages, unsign
}
goto out;
}
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
flush_dcache_mmap_lock(mapping);
vma->vm_flags |= VM_NONLINEAR;
vma_prio_tree_remove(vma, &mapping->i_mmap);
vma_nonlinear_insert(vma, &mapping->i_mmap_nonlinear);
flush_dcache_mmap_unlock(mapping);
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
}
if (vma->vm_flags & VM_LOCKED) {
Index: linux-2.6/mm/hugetlb.c
===================================================================
--- linux-2.6.orig/mm/hugetlb.c
+++ linux-2.6/mm/hugetlb.c
@@ -2210,9 +2210,9 @@ void __unmap_hugepage_range(struct vm_ar
void unmap_hugepage_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end, struct page *ref_page)
{
- spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
+ mutex_lock(&vma->vm_file->f_mapping->i_mmap_lock);
__unmap_hugepage_range(vma, start, end, ref_page);
- spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
+ mutex_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
}
/*
@@ -2244,7 +2244,7 @@ static int unmap_ref_private(struct mm_s
* this mapping should be shared between all the VMAs,
* __unmap_hugepage_range() is called as the lock is already held
*/
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
/* Do not unmap the current VMA */
if (iter_vma == vma)
@@ -2262,7 +2262,7 @@ static int unmap_ref_private(struct mm_s
address, address + huge_page_size(h),
page);
}
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
return 1;
}
@@ -2678,7 +2678,7 @@ void hugetlb_change_protection(struct vm
BUG_ON(address >= end);
flush_cache_range(vma, address, end);
- spin_lock(&vma->vm_file->f_mapping->i_mmap_lock);
+ mutex_lock(&vma->vm_file->f_mapping->i_mmap_lock);
spin_lock(&mm->page_table_lock);
for (; address < end; address += huge_page_size(h)) {
ptep = huge_pte_offset(mm, address);
@@ -2693,7 +2693,7 @@ void hugetlb_change_protection(struct vm
}
}
spin_unlock(&mm->page_table_lock);
- spin_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
+ mutex_unlock(&vma->vm_file->f_mapping->i_mmap_lock);
flush_tlb_range(vma, start, end);
}
Index: linux-2.6/mm/ksm.c
===================================================================
--- linux-2.6.orig/mm/ksm.c
+++ linux-2.6/mm/ksm.c
@@ -1559,7 +1559,7 @@ again:
struct anon_vma_chain *vmac;
struct vm_area_struct *vma;
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
list_for_each_entry(vmac, &anon_vma->head, same_anon_vma) {
vma = vmac->vma;
if (rmap_item->address < vma->vm_start ||
@@ -1582,7 +1582,7 @@ again:
if (!search_new_forks || !mapcount)
break;
}
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
if (!mapcount)
goto out;
}
@@ -1612,7 +1612,7 @@ again:
struct anon_vma_chain *vmac;
struct vm_area_struct *vma;
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
list_for_each_entry(vmac, &anon_vma->head, same_anon_vma) {
vma = vmac->vma;
if (rmap_item->address < vma->vm_start ||
@@ -1630,11 +1630,11 @@ again:
ret = try_to_unmap_one(page, vma,
rmap_item->address, flags);
if (ret != SWAP_AGAIN || !page_mapped(page)) {
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
goto out;
}
}
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
}
if (!search_new_forks++)
goto again;
@@ -1664,7 +1664,7 @@ again:
struct anon_vma_chain *vmac;
struct vm_area_struct *vma;
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
list_for_each_entry(vmac, &anon_vma->head, same_anon_vma) {
vma = vmac->vma;
if (rmap_item->address < vma->vm_start ||
@@ -1681,11 +1681,11 @@ again:
ret = rmap_one(page, vma, rmap_item->address, arg);
if (ret != SWAP_AGAIN) {
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
goto out;
}
}
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
}
if (!search_new_forks++)
goto again;
Index: linux-2.6/mm/memory-failure.c
===================================================================
--- linux-2.6.orig/mm/memory-failure.c
+++ linux-2.6/mm/memory-failure.c
@@ -421,7 +421,7 @@ static void collect_procs_file(struct pa
*/
read_lock(&tasklist_lock);
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
for_each_process(tsk) {
pgoff_t pgoff = page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
@@ -441,7 +441,7 @@ static void collect_procs_file(struct pa
add_to_kill(tsk, page, vma, to_kill, tkc);
}
}
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
read_unlock(&tasklist_lock);
}
Index: linux-2.6/mm/memory.c
===================================================================
--- linux-2.6.orig/mm/memory.c
+++ linux-2.6/mm/memory.c
@@ -1102,7 +1102,7 @@ unsigned long unmap_vmas(struct mmu_gath
{
long zap_work = ZAP_BLOCK_SIZE;
unsigned long start = start_addr;
- spinlock_t *i_mmap_lock = details? details->i_mmap_lock: NULL;
+ struct mutex *i_mmap_lock = details ? details->i_mmap_lock : NULL;
struct mm_struct *mm = vma->vm_mm;
mmu_notifier_invalidate_range_start(mm, start_addr, end_addr);
@@ -1152,7 +1152,7 @@ unsigned long unmap_vmas(struct mmu_gath
}
if (need_resched() ||
- (i_mmap_lock && spin_needbreak(i_mmap_lock))) {
+ (i_mmap_lock && mutex_is_contended(i_mmap_lock))) {
if (i_mmap_lock)
goto out;
cond_resched();
@@ -2427,7 +2427,7 @@ again:
restart_addr = zap_page_range(vma, start_addr,
end_addr - start_addr, details);
- need_break = need_resched() || spin_needbreak(details->i_mmap_lock);
+ need_break = need_resched() || mutex_is_contended(details->i_mmap_lock);
if (restart_addr >= end_addr) {
/* We have now completed this vma: mark it so */
@@ -2441,9 +2441,9 @@ again:
goto again;
}
- spin_unlock(details->i_mmap_lock);
+ mutex_unlock(details->i_mmap_lock);
cond_resched();
- spin_lock(details->i_mmap_lock);
+ mutex_lock(details->i_mmap_lock);
return -EINTR;
}
@@ -2539,7 +2539,7 @@ void unmap_mapping_range(struct address_
details.last_index = ULONG_MAX;
details.i_mmap_lock = &mapping->i_mmap_lock;
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
/* Protect against endless unmapping loops */
mapping->truncate_count++;
@@ -2554,7 +2554,7 @@ void unmap_mapping_range(struct address_
unmap_mapping_range_tree(&mapping->i_mmap, &details);
if (unlikely(!list_empty(&mapping->i_mmap_nonlinear)))
unmap_mapping_range_list(&mapping->i_mmap_nonlinear, &details);
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
}
EXPORT_SYMBOL(unmap_mapping_range);
Index: linux-2.6/mm/mmap.c
===================================================================
--- linux-2.6.orig/mm/mmap.c
+++ linux-2.6/mm/mmap.c
@@ -216,9 +216,9 @@ void unlink_file_vma(struct vm_area_stru
if (file) {
struct address_space *mapping = file->f_mapping;
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
__remove_shared_vm_struct(vma, file, mapping);
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
}
}
@@ -449,7 +449,7 @@ static void vma_link(struct mm_struct *m
mapping = vma->vm_file->f_mapping;
if (mapping) {
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
vma->vm_truncate_count = mapping->truncate_count;
}
anon_vma_lock(vma);
@@ -459,7 +459,7 @@ static void vma_link(struct mm_struct *m
anon_vma_unlock(vma);
if (mapping)
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
mm->map_count++;
validate_mm(mm);
@@ -565,7 +565,7 @@ again: remove_next = 1 + (end > next->
mapping = file->f_mapping;
if (!(vma->vm_flags & VM_NONLINEAR))
root = &mapping->i_mmap;
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
if (importer &&
vma->vm_truncate_count != next->vm_truncate_count) {
/*
@@ -626,7 +626,7 @@ again: remove_next = 1 + (end > next->
}
if (mapping)
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
if (remove_next) {
if (file) {
@@ -2440,7 +2440,7 @@ static void vm_lock_anon_vma(struct mm_s
* The LSB of head.next can't change from under us
* because we hold the mm_all_locks_mutex.
*/
- spin_lock_nest_lock(&anon_vma->lock, &mm->mmap_sem);
+ mutex_lock_nest_lock(&anon_vma->lock, &mm->mmap_sem);
/*
* We can safely modify head.next after taking the
* anon_vma->lock. If some other vma in this mm shares
@@ -2470,7 +2470,7 @@ static void vm_lock_mapping(struct mm_st
*/
if (test_and_set_bit(AS_MM_ALL_LOCKS, &mapping->flags))
BUG();
- spin_lock_nest_lock(&mapping->i_mmap_lock, &mm->mmap_sem);
+ mutex_lock_nest_lock(&mapping->i_mmap_lock, &mm->mmap_sem);
}
}
@@ -2558,7 +2558,7 @@ static void vm_unlock_anon_vma(struct an
if (!__test_and_clear_bit(0, (unsigned long *)
&anon_vma->head.next))
BUG();
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
}
}
@@ -2569,7 +2569,7 @@ static void vm_unlock_mapping(struct add
* AS_MM_ALL_LOCKS can't change to 0 from under us
* because we hold the mm_all_locks_mutex.
*/
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
if (!test_and_clear_bit(AS_MM_ALL_LOCKS,
&mapping->flags))
BUG();
Index: linux-2.6/mm/mremap.c
===================================================================
--- linux-2.6.orig/mm/mremap.c
+++ linux-2.6/mm/mremap.c
@@ -91,7 +91,7 @@ static void move_ptes(struct vm_area_str
* and we propagate stale pages into the dst afterward.
*/
mapping = vma->vm_file->f_mapping;
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
if (new_vma->vm_truncate_count &&
new_vma->vm_truncate_count != vma->vm_truncate_count)
new_vma->vm_truncate_count = 0;
@@ -123,7 +123,7 @@ static void move_ptes(struct vm_area_str
pte_unmap_nested(new_pte - 1);
pte_unmap_unlock(old_pte - 1, old_ptl);
if (mapping)
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
mmu_notifier_invalidate_range_end(vma->vm_mm, old_start, old_end);
}
Index: linux-2.6/mm/rmap.c
===================================================================
--- linux-2.6.orig/mm/rmap.c
+++ linux-2.6/mm/rmap.c
@@ -140,7 +140,7 @@ int anon_vma_prepare(struct vm_area_stru
goto out_enomem_free_avc;
allocated = anon_vma;
}
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
/* page_table_lock to protect against threads */
spin_lock(&mm->page_table_lock);
@@ -154,7 +154,7 @@ int anon_vma_prepare(struct vm_area_stru
}
spin_unlock(&mm->page_table_lock);
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
if (unlikely(allocated)) {
anon_vma_put(allocated);
anon_vma_chain_free(avc);
@@ -176,9 +176,9 @@ static void anon_vma_chain_link(struct v
avc->anon_vma = anon_vma;
list_add(&avc->same_vma, &vma->anon_vma_chain);
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
list_add_tail(&avc->same_anon_vma, &anon_vma->head);
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
}
/*
@@ -251,10 +251,10 @@ static void anon_vma_unlink(struct anon_
if (!anon_vma)
return;
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
list_del(&anon_vma_chain->same_anon_vma);
empty = list_empty(&anon_vma->head);
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
if (empty)
anon_vma_put(anon_vma);
@@ -276,7 +276,7 @@ static void anon_vma_ctor(void *data)
{
struct anon_vma *anon_vma = data;
- spin_lock_init(&anon_vma->lock);
+ mutex_init(&anon_vma->lock);
atomic_set(&anon_vma->ref, 0);
INIT_LIST_HEAD(&anon_vma->head);
}
@@ -317,14 +317,14 @@ struct anon_vma *page_lock_anon_vma(stru
struct anon_vma *anon_vma = anon_vma_get(page);
if (anon_vma)
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
return anon_vma;
}
void page_unlock_anon_vma(struct anon_vma *anon_vma)
{
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
anon_vma_put(anon_vma);
}
@@ -569,7 +569,7 @@ static int page_referenced_file(struct p
*/
BUG_ON(!PageLocked(page));
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
/*
* i_mmap_lock does not stabilize mapcount at all, but mapcount
@@ -594,7 +594,7 @@ static int page_referenced_file(struct p
break;
}
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
return referenced;
}
@@ -681,7 +681,7 @@ static int page_mkclean_file(struct addr
BUG_ON(PageAnon(page));
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
if (vma->vm_flags & VM_SHARED) {
unsigned long address = vma_address(page, vma);
@@ -690,7 +690,7 @@ static int page_mkclean_file(struct addr
ret += page_mkclean_one(page, vma, address);
}
}
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
return ret;
}
@@ -1196,7 +1196,7 @@ static int try_to_unmap_file(struct page
unsigned long max_nl_size = 0;
unsigned int mapcount;
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
unsigned long address = vma_address(page, vma);
if (address == -EFAULT)
@@ -1242,7 +1242,7 @@ static int try_to_unmap_file(struct page
mapcount = page_mapcount(page);
if (!mapcount)
goto out;
- cond_resched_lock(&mapping->i_mmap_lock);
+ cond_resched();
max_nl_size = (max_nl_size + CLUSTER_SIZE - 1) & CLUSTER_MASK;
if (max_nl_cursor == 0)
@@ -1264,7 +1264,7 @@ static int try_to_unmap_file(struct page
}
vma->vm_private_data = (void *) max_nl_cursor;
}
- cond_resched_lock(&mapping->i_mmap_lock);
+ cond_resched();
max_nl_cursor += CLUSTER_SIZE;
} while (max_nl_cursor <= max_nl_size);
@@ -1276,7 +1276,7 @@ static int try_to_unmap_file(struct page
list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list)
vma->vm_private_data = NULL;
out:
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
return ret;
}
@@ -1361,7 +1361,7 @@ static int rmap_walk_anon(struct page *p
anon_vma = page_anon_vma(page);
if (!anon_vma)
return ret;
- spin_lock(&anon_vma->lock);
+ mutex_lock(&anon_vma->lock);
list_for_each_entry(avc, &anon_vma->head, same_anon_vma) {
struct vm_area_struct *vma = avc->vma;
unsigned long address = vma_address(page, vma);
@@ -1371,7 +1371,7 @@ static int rmap_walk_anon(struct page *p
if (ret != SWAP_AGAIN)
break;
}
- spin_unlock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
return ret;
}
@@ -1386,7 +1386,7 @@ static int rmap_walk_file(struct page *p
if (!mapping)
return ret;
- spin_lock(&mapping->i_mmap_lock);
+ mutex_lock(&mapping->i_mmap_lock);
vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
unsigned long address = vma_address(page, vma);
if (address == -EFAULT)
@@ -1400,7 +1400,7 @@ static int rmap_walk_file(struct page *p
* never contain migration ptes. Decide what to do about this
* limitation to linear when we need rmap_walk() on nonlinear.
*/
- spin_unlock(&mapping->i_mmap_lock);
+ mutex_unlock(&mapping->i_mmap_lock);
return ret;
}
^ permalink raw reply [flat|nested] 9+ messages in thread* [PATCH 7/7] mm: Optimize page_lock_anon_vma
2010-04-02 14:16 [PATCH 0/7] mm: preemptibility Peter Zijlstra
` (5 preceding siblings ...)
2010-04-02 14:16 ` [PATCH 6/7] mm: Convert i_mmap_lock and anon_vma->lock to mutexes Peter Zijlstra
@ 2010-04-02 14:16 ` Peter Zijlstra
2010-04-02 15:14 ` [PATCH 0/7] mm: preemptibility Andrea Arcangeli
7 siblings, 0 replies; 9+ messages in thread
From: Peter Zijlstra @ 2010-04-02 14:16 UTC (permalink / raw)
To: Andrea Arcangeli, Avi Kivity, Thomas Gleixner, Rik van Riel,
Ingo Molnar, akpm
Cc: linux-kernel, Benjamin Herrenschmidt, Hugh Dickins, Mel Gorman,
Nick Piggin, Peter Zijlstra
[-- Attachment #1: mm-opt-rmap.patch --]
[-- Type: text/plain, Size: 3130 bytes --]
Optimize page_lock_anon_vma() by removing the atomic ref count
ops from the fast path.
Rather complicates the code a lot, but might be worth it.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@chello.nl>
---
mm/rmap.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
1 file changed, 61 insertions(+), 5 deletions(-)
Index: linux-2.6/mm/rmap.c
===================================================================
--- linux-2.6.orig/mm/rmap.c
+++ linux-2.6/mm/rmap.c
@@ -78,6 +78,12 @@ static inline struct anon_vma *anon_vma_
void anon_vma_free(struct anon_vma *anon_vma)
{
VM_BUG_ON(atomic_read(&anon_vma->ref));
+ /*
+ * Sync against the anon_vma->lock, so that we can hold the
+ * lock without requiring a reference. See page_lock_anon_vma().
+ */
+ mutex_lock(&anon_vma->lock);
+ mutex_unlock(&anon_vma->lock);
kmem_cache_free(anon_vma_cachep, anon_vma);
}
@@ -290,7 +296,7 @@ void __init anon_vma_init(void)
/*
* Getting a lock on a stable anon_vma from a page off the LRU is
- * tricky: page_lock_anon_vma relies on RCU to guard against the races.
+ * tricky: anon_vma_get relies on RCU to guard against the races.
*/
struct anon_vma *anon_vma_get(struct page *page)
{
@@ -312,20 +318,70 @@ out:
return anon_vma;
}
+/*
+ * Similar to anon_vma_get(), however it relies on the anon_vma->lock
+ * to pin the object. However since we cannot wait for the mutex
+ * acquisition inside the RCU read lock, we use the ref count
+ * in the slow path.
+ */
struct anon_vma *page_lock_anon_vma(struct page *page)
{
- struct anon_vma *anon_vma = anon_vma_get(page);
+ struct anon_vma *anon_vma = NULL;
+ unsigned long anon_mapping;
- if (anon_vma)
- mutex_lock(&anon_vma->lock);
+ rcu_read_lock();
+ anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping);
+ if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
+ goto unlock;
+ if (!page_mapped(page))
+ goto unlock;
+
+ anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
+ if (!mutex_trylock(&anon_vma->lock)) {
+ /*
+ * We failed to acquire the lock, take a ref so we can
+ * drop the RCU read lock and sleep on it.
+ */
+ if (!atomic_inc_not_zero(&anon_vma->ref)) {
+ /*
+ * Failed to get a ref, we're dead, bail.
+ */
+ anon_vma = NULL;
+ goto unlock;
+ }
+ rcu_read_unlock();
+ mutex_lock(&anon_vma->lock);
+ /*
+ * We got the lock, drop the temp. ref, if it was the last
+ * one free it and bail.
+ */
+ if (atomic_dec_and_test(&anon_vma->ref)) {
+ mutex_unlock(&anon_vma->lock);
+ anon_vma_free(anon_vma);
+ anon_vma = NULL;
+ }
+ goto out;
+ }
+ /*
+ * Got the lock, check we're still alive. Seeing a ref
+ * here guarantees the object will stay alive due to
+ * anon_vma_free() syncing against the lock we now hold.
+ */
+ smp_rmb(); /* Order against anon_vma_put() */
+ if (!atomic_read(&anon_vma->ref)) {
+ mutex_unlock(&anon_vma->lock);
+ anon_vma = NULL;
+ }
+unlock:
+ rcu_read_unlock();
+out:
return anon_vma;
}
void page_unlock_anon_vma(struct anon_vma *anon_vma)
{
mutex_unlock(&anon_vma->lock);
- anon_vma_put(anon_vma);
}
/*
^ permalink raw reply [flat|nested] 9+ messages in thread* Re: [PATCH 0/7] mm: preemptibility
2010-04-02 14:16 [PATCH 0/7] mm: preemptibility Peter Zijlstra
` (6 preceding siblings ...)
2010-04-02 14:16 ` [PATCH 7/7] mm: Optimize page_lock_anon_vma Peter Zijlstra
@ 2010-04-02 15:14 ` Andrea Arcangeli
7 siblings, 0 replies; 9+ messages in thread
From: Andrea Arcangeli @ 2010-04-02 15:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Avi Kivity, Thomas Gleixner, Rik van Riel, Ingo Molnar, akpm,
linux-kernel, Benjamin Herrenschmidt, Hugh Dickins, Mel Gorman,
Nick Piggin
On Fri, Apr 02, 2010 at 04:16:01PM +0200, Peter Zijlstra wrote:
> Hi,
>
> This (incomplete) patch-set makes part of the mm a lot more preemptible.
> It converts i_mmap_lock and anon_vma->lock to mutexes.
> On the way there it also makes mmu_gather preemptible.
>
> The main motivation was making mm_take_all_locks() preemptible, since
> it appears people are nesting hundreds of spinlocks there.
>
> The side-effects are that we can finally make mmu_gather preemptible,
> something which lots of people have wanted to do for a long time.
>
> It also gets us anon_vma refcounting which seems to be wanted by
> KSM as well as Mel's compaction work.
>
> This patch set seems to build and boot on my x86_64 machine and even
> builds a kernel. I'll work on getting PPC working again and audit other
> architectures their mmu_gather implementation.
This is also the needed bit to allow to schedule in the mmu notifier
methods by switching from rcu to srcu there. That will make XPMEM
happy by allowing all mmu notifier methods to schedule. Considering
this may not slowdown the more important fast paths, it seems overall
beneficial effort to me.
Thanks,
Andrea
^ permalink raw reply [flat|nested] 9+ messages in thread