From mboxrd@z Thu Jan 1 00:00:00 1970 From: Peter Zijlstra Subject: [PATCH 20/20] mm: Optimize page_lock_anon_vma() fast-path Date: Sat, 28 Aug 2010 16:16:57 +0200 Message-ID: <20100828142456.804579956@chello.nl> References: <20100828141637.421594670@chello.nl> Return-path: Content-Disposition: inline; filename=mm-opt-page_lock_anon_vma.patch Sender: linux-kernel-owner@vger.kernel.org To: Andrea Arcangeli , Avi Kivity , Thomas Gleixner , Rik van Riel , Ingo Molnar , akpm@linux-fou Cc: linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org, Benjamin Herrenschmidt , David Miller , Hugh Dickins , Mel Gorman , Nick Piggin , Peter Zijlstra , Paul McKenney , Yanmin Zhang , Stephen Rothwell List-Id: linux-arch.vger.kernel.org Optimize the page_lock_anon_vma() fast path to be one LOCKed op, instead of two. Signed-off-by: Peter Zijlstra --- mm/rmap.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 4 deletions(-) Index: linux-2.6/mm/rmap.c =================================================================== --- linux-2.6.orig/mm/rmap.c +++ linux-2.6/mm/rmap.c @@ -353,20 +353,69 @@ out: return anon_vma; } +/* + * Similar to page_get_anon_vma() except it locks the anon_vma. + * + * Its a little more complex as it tries to keep the fast path to a single + * atomic op -- the trylock. If we fail the trylock, we fall back to getting a + * reference like with page_get_anon_vma() and then block on the mutex. + */ struct anon_vma *page_lock_anon_vma(struct page *page) { - struct anon_vma *anon_vma = page_get_anon_vma(page); + struct anon_vma *anon_vma = NULL; + unsigned long anon_mapping; + + rcu_read_lock(); + anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping); + if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) + goto out; + if (!page_mapped(page)) + goto out; + + anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); + if (mutex_trylock(&anon_vma->root->lock)) { + /* + * If we observe a !0 refcount, then holding the lock ensures + * the anon_vma will not go away, see __put_anon_vma(). + */ + if (!atomic_read(&anon_vma->refcount)) { + anon_vma_unlock(anon_vma); + anon_vma = NULL; + } + goto out; + } + + /* trylock failed, we got to sleep */ + if (!atomic_inc_not_zero(&anon_vma->refcount)) { + anon_vma = NULL; + goto out; + } + + /* we pinned the anon_vma, its safe to sleep */ + rcu_read_unlock(); + anon_vma_lock(anon_vma); + + if (atomic_dec_and_test(&anon_vma->refcount)) { + /* + * Oops, we held the last refcount, release the lock + * and bail -- can't simply use put_anon_vma() because + * we'll deadlock on the anon_vma_lock() recursion. + */ + anon_vma_unlock(anon_vma); + __put_anon_vma(anon_vma); + anon_vma = NULL; + } - if (anon_vma) - anon_vma_lock(anon_vma); + return anon_vma; +out: + rcu_read_unlock(); return anon_vma; } void page_unlock_anon_vma(struct anon_vma *anon_vma) { anon_vma_unlock(anon_vma); - put_anon_vma(anon_vma); } /* @@ -1458,6 +1507,14 @@ int try_to_munlock(struct page *page) void __put_anon_vma(struct anon_vma *anon_vma) { + /* + * Synchronize against page_lock_anon_vma() such that + * we can safely hold the lock without the anon_vma getting + * freed. + */ + anon_vma_lock(anon_vma); + anon_vma_unlock(anon_vma); + if (anon_vma->root != anon_vma) put_anon_vma(anon_vma->root); anon_vma_free(anon_vma); From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from casper.infradead.org ([85.118.1.10]:42773 "EHLO casper.infradead.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753526Ab0H1O1r (ORCPT ); Sat, 28 Aug 2010 10:27:47 -0400 Message-ID: <20100828142456.804579956@chello.nl> Date: Sat, 28 Aug 2010 16:16:57 +0200 From: Peter Zijlstra Subject: [PATCH 20/20] mm: Optimize page_lock_anon_vma() fast-path References: <20100828141637.421594670@chello.nl> Content-Disposition: inline; filename=mm-opt-page_lock_anon_vma.patch Sender: linux-arch-owner@vger.kernel.org List-ID: To: Andrea Arcangeli , Avi Kivity , Thomas Gleixner , Rik van Riel , Ingo Molnar , akpm@linux-foundation.org, Linus Torvalds Cc: linux-kernel@vger.kernel.org, linux-arch@vger.kernel.org, Benjamin Herrenschmidt , David Miller , Hugh Dickins , Mel Gorman , Nick Piggin , Peter Zijlstra , Paul McKenney , Yanmin Zhang , Stephen Rothwell Message-ID: <20100828141657.s_ttov8_1Y0D6NY4NwnCn17g6Sr61gUC_Y4pOmYxgZs@z> Optimize the page_lock_anon_vma() fast path to be one LOCKed op, instead of two. Signed-off-by: Peter Zijlstra --- mm/rmap.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 4 deletions(-) Index: linux-2.6/mm/rmap.c =================================================================== --- linux-2.6.orig/mm/rmap.c +++ linux-2.6/mm/rmap.c @@ -353,20 +353,69 @@ out: return anon_vma; } +/* + * Similar to page_get_anon_vma() except it locks the anon_vma. + * + * Its a little more complex as it tries to keep the fast path to a single + * atomic op -- the trylock. If we fail the trylock, we fall back to getting a + * reference like with page_get_anon_vma() and then block on the mutex. + */ struct anon_vma *page_lock_anon_vma(struct page *page) { - struct anon_vma *anon_vma = page_get_anon_vma(page); + struct anon_vma *anon_vma = NULL; + unsigned long anon_mapping; + + rcu_read_lock(); + anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping); + if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON) + goto out; + if (!page_mapped(page)) + goto out; + + anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON); + if (mutex_trylock(&anon_vma->root->lock)) { + /* + * If we observe a !0 refcount, then holding the lock ensures + * the anon_vma will not go away, see __put_anon_vma(). + */ + if (!atomic_read(&anon_vma->refcount)) { + anon_vma_unlock(anon_vma); + anon_vma = NULL; + } + goto out; + } + + /* trylock failed, we got to sleep */ + if (!atomic_inc_not_zero(&anon_vma->refcount)) { + anon_vma = NULL; + goto out; + } + + /* we pinned the anon_vma, its safe to sleep */ + rcu_read_unlock(); + anon_vma_lock(anon_vma); + + if (atomic_dec_and_test(&anon_vma->refcount)) { + /* + * Oops, we held the last refcount, release the lock + * and bail -- can't simply use put_anon_vma() because + * we'll deadlock on the anon_vma_lock() recursion. + */ + anon_vma_unlock(anon_vma); + __put_anon_vma(anon_vma); + anon_vma = NULL; + } - if (anon_vma) - anon_vma_lock(anon_vma); + return anon_vma; +out: + rcu_read_unlock(); return anon_vma; } void page_unlock_anon_vma(struct anon_vma *anon_vma) { anon_vma_unlock(anon_vma); - put_anon_vma(anon_vma); } /* @@ -1458,6 +1507,14 @@ int try_to_munlock(struct page *page) void __put_anon_vma(struct anon_vma *anon_vma) { + /* + * Synchronize against page_lock_anon_vma() such that + * we can safely hold the lock without the anon_vma getting + * freed. + */ + anon_vma_lock(anon_vma); + anon_vma_unlock(anon_vma); + if (anon_vma->root != anon_vma) put_anon_vma(anon_vma->root); anon_vma_free(anon_vma);