From: Konstantin Khlebnikov <khlebnikov@openvz.org>
To: linux-mm@kvack.org
Cc: linux-kernel@vger.kernel.org
Subject: [PATCH RFC 15/15] mm: split zone->lru_lock
Date: Thu, 16 Feb 2012 02:58:02 +0400 [thread overview]
Message-ID: <20120215225802.22050.78935.stgit@zurg> (raw)
In-Reply-To: <20120215224221.22050.80605.stgit@zurg>
Looks like all ready for splitting zone->lru_lock into small per-book pieces.
Protect lock loop with rcu, memory controller alread release its mem_cgroup_per_node via rcu,
books embedded into zones never released.
Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
---
include/linux/mm_inline.h | 67 ++++++++++++++++++++++++++++++++++-----------
include/linux/mmzone.h | 2 +
mm/compaction.c | 3 +-
mm/memcontrol.c | 1 +
mm/page_alloc.c | 2 +
mm/swap.c | 2 +
6 files changed, 56 insertions(+), 21 deletions(-)
diff --git a/include/linux/mm_inline.h b/include/linux/mm_inline.h
index 9cb3a7e..5d6df15 100644
--- a/include/linux/mm_inline.h
+++ b/include/linux/mm_inline.h
@@ -38,22 +38,22 @@ static inline struct pglist_data *book_node(struct book *book)
static inline void lock_book(struct book *book, unsigned long *flags)
{
- spin_lock_irqsave(&book_zone(book)->lru_lock, *flags);
+ spin_lock_irqsave(&book->lru_lock, *flags);
}
static inline void lock_book_irq(struct book *book)
{
- spin_lock_irq(&book_zone(book)->lru_lock);
+ spin_lock_irq(&book->lru_lock);
}
static inline void unlock_book(struct book *book, unsigned long *flags)
{
- spin_unlock_irqrestore(&book_zone(book)->lru_lock, *flags);
+ spin_unlock_irqrestore(&book->lru_lock, *flags);
}
static inline void unlock_book_irq(struct book *book)
{
- spin_unlock_irq(&book_zone(book)->lru_lock);
+ spin_unlock_irq(&book->lru_lock);
}
#ifdef CONFIG_MEMORY_BOOKKEEPING
@@ -61,27 +61,47 @@ static inline void unlock_book_irq(struct book *book)
static inline struct book *lock_page_book(struct page *page,
unsigned long *flags)
{
- struct zone *zone = page_zone(page);
+ struct book *locked_book, *book;
- spin_lock_irqsave(&zone->lru_lock, *flags);
- return page_book(page);
+ rcu_read_lock();
+ local_irq_save(*flags);
+ book = page_book(page);
+ do {
+ spin_lock(&book->lru_lock);
+ locked_book = book;
+ book = page_book(page);
+ if (likely(book == locked_book)) {
+ rcu_read_unlock();
+ return book;
+ }
+ spin_unlock(&locked_book->lru_lock);
+ } while (1);
}
static inline struct book *lock_page_book_irq(struct page *page)
{
- struct zone *zone = page_zone(page);
+ struct book *locked_book, *book;
- spin_lock_irq(&zone->lru_lock);
- return page_book(page);
+ rcu_read_lock();
+ local_irq_disable();
+ book = page_book(page);
+ do {
+ spin_lock(&book->lru_lock);
+ locked_book = book;
+ book = page_book(page);
+ if (likely(book == locked_book)) {
+ rcu_read_unlock();
+ return book;
+ }
+ spin_unlock(&locked_book->lru_lock);
+ } while (1);
}
static inline struct book *relock_page_book(struct book *locked_book,
struct page *page,
unsigned long *flags)
{
- struct zone *zone = page_zone(page);
-
- if (!locked_book || zone != book_zone(locked_book)) {
+ if (unlikely(locked_book != page_book(page))) {
if (locked_book)
unlock_book(locked_book, flags);
locked_book = lock_page_book(page, flags);
@@ -93,9 +113,7 @@ static inline struct book *relock_page_book(struct book *locked_book,
static inline struct book *relock_page_book_irq(struct book *locked_book,
struct page *page)
{
- struct zone *zone = page_zone(page);
-
- if (!locked_book || zone != book_zone(locked_book)) {
+ if (unlikely(locked_book != page_book(page))) {
if (locked_book)
unlock_book_irq(locked_book);
locked_book = lock_page_book_irq(page);
@@ -111,7 +129,22 @@ static inline struct book *relock_page_book_irq(struct book *locked_book,
static inline struct book *__relock_page_book(struct book *locked_book,
struct page *page)
{
- return page_book(page);
+ struct book *book;
+
+ if (likely(locked_book == page_book(page)))
+ return locked_book;
+
+ rcu_read_lock();
+ do {
+ book = page_book(page);
+ if (book == locked_book) {
+ rcu_read_unlock();
+ return book;
+ }
+ spin_unlock(&locked_book->lru_lock);
+ spin_lock(&book->lru_lock);
+ locked_book = book;
+ } while (1);
}
#else /* CONFIG_MEMORY_BOOKKEEPING */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 5bcd5b1..629c6bd 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -301,6 +301,7 @@ struct book {
struct pglist_data *node;
struct zone *zone;
#endif
+ spinlock_t lru_lock;
struct list_head pages_lru[NR_LRU_LISTS];
unsigned long pages_count[NR_LRU_LISTS];
@@ -382,7 +383,6 @@ struct zone {
ZONE_PADDING(_pad1_)
/* Fields commonly accessed by the page reclaim scanner */
- spinlock_t lru_lock;
struct book book;
unsigned long pages_scanned; /* since last reclaim */
diff --git a/mm/compaction.c b/mm/compaction.c
index f521edf..cb9266a 100644
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -310,7 +310,8 @@ static isolate_migrate_t isolate_migratepages(struct zone *zone,
unlock_book_irq(book);
book = NULL;
}
- if (need_resched() || spin_is_contended(&zone->lru_lock)) {
+ if (need_resched() ||
+ (book && spin_is_contended(&book->lru_lock))) {
if (book)
unlock_book_irq(book);
book = NULL;
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 90e21d2..eabc2ef 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -4640,6 +4640,7 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
INIT_LIST_HEAD(&mz->book.pages_lru[l]);
mz->book.pages_count[l] = 0;
}
+ spin_lock_init(&mz->book.lru_lock);
mz->book.node = NODE_DATA(node);
mz->book.zone = &NODE_DATA(node)->node_zones[zone];
spin_lock(&mz->book.zone->lock);
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a2df69e..144cef8 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -4305,7 +4305,7 @@ static void __paginginit free_area_init_core(struct pglist_data *pgdat,
#endif
zone->name = zone_names[j];
spin_lock_init(&zone->lock);
- spin_lock_init(&zone->lru_lock);
+ spin_lock_init(&zone->book.lru_lock);
zone_seqlock_init(zone);
zone->zone_pgdat = pgdat;
diff --git a/mm/swap.c b/mm/swap.c
index 652e691..58ea4f3 100644
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -629,7 +629,7 @@ void lru_add_page_tail(struct book *book,
VM_BUG_ON(!PageHead(page));
VM_BUG_ON(PageCompound(page_tail));
VM_BUG_ON(PageLRU(page_tail));
- VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&book_zone(book)->lru_lock));
+ VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&book->lru_lock));
SetPageLRU(page_tail);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom internet charges in Canada: sign http://stopthemeter.ca/
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
next prev parent reply other threads:[~2012-02-15 22:58 UTC|newest]
Thread overview: 31+ messages / expand[flat|nested] mbox.gz Atom feed top
2012-02-15 22:57 [PATCH RFC 00/15] mm: memory book keeping and lru_lock splitting Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 01/15] mm: rename struct lruvec into struct book Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 02/15] mm: memory bookkeeping core Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 03/15] mm: add book->pages_count Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 04/15] mm: unify inactive_list_is_low() Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 05/15] mm: add book->reclaim_stat Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 06/15] mm: kill struct mem_cgroup_zone Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 07/15] mm: move page-to-book translation upper Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 08/15] mm: introduce book locking primitives Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 09/15] mm: handle book relocks on lumpy reclaim Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 10/15] mm: handle book relocks in compaction Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 11/15] mm: handle book relock in memory controller Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 12/15] mm: optimize books in update_page_reclaim_stat() Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 13/15] mm: optimize books in pagevec_lru_move_fn() Konstantin Khlebnikov
2012-02-15 22:57 ` [PATCH RFC 14/15] mm: optimize putback for 0-order reclaim Konstantin Khlebnikov
2012-02-15 22:58 ` Konstantin Khlebnikov [this message]
2012-02-16 2:04 ` [PATCH RFC 00/15] mm: memory book keeping and lru_lock splitting KAMEZAWA Hiroyuki
2012-02-16 5:43 ` Konstantin Khlebnikov
2012-02-16 8:24 ` KAMEZAWA Hiroyuki
2012-02-16 11:02 ` Konstantin Khlebnikov
2012-02-16 15:54 ` Konstantin Khlebnikov
2012-02-16 23:54 ` KAMEZAWA Hiroyuki
2012-02-18 9:09 ` Konstantin Khlebnikov
2012-02-16 2:37 ` Hugh Dickins
2012-02-16 4:51 ` Konstantin Khlebnikov
2012-02-16 21:37 ` Hugh Dickins
2012-02-17 19:56 ` Konstantin Khlebnikov
2012-02-18 2:13 ` Hugh Dickins
2012-02-18 6:35 ` Konstantin Khlebnikov
2012-02-18 7:14 ` Hugh Dickins
2012-02-20 0:32 ` KAMEZAWA Hiroyuki
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20120215225802.22050.78935.stgit@zurg \
--to=khlebnikov@openvz.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-mm@kvack.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).