* [RFC 5/6] Swapless V1: Rip out swap migration code
From: Christoph Lameter @ 2006-04-04 6:58 UTC (permalink / raw)
To: linux-mm
Cc: Lee Schermerhorn, Christoph Lameter, lhms-devel,
Hirokazu Takahashi, Marcelo Tosatti, KAMEZAWA Hiroyuki
In-Reply-To: <20060404065739.24532.95451.sendpatchset@schroedinger.engr.sgi.com>
Rip the page migration logic out
Remove all code that has to do with swapping during page migration.
This also guts the ability to migrate pages to swap. No one used that
so lets let it go for good.
Page migration should be a bit broken after this patch.
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Index: linux-2.6.17-rc1/mm/migrate.c
===================================================================
--- linux-2.6.17-rc1.orig/mm/migrate.c 2006-04-03 22:07:40.000000000 -0700
+++ linux-2.6.17-rc1/mm/migrate.c 2006-04-03 22:07:56.000000000 -0700
@@ -70,10 +70,6 @@ int isolate_lru_page(struct page *page,
*/
int migrate_prep(void)
{
- /* Must have swap device for migration */
- if (nr_swap_pages <= 0)
- return -ENODEV;
-
/*
* Clear the LRU lists so pages can be isolated.
* Note that pages may be moved off the LRU after we have
@@ -129,53 +125,6 @@ int fail_migrate_page(struct page *newpa
EXPORT_SYMBOL(fail_migrate_page);
/*
- * swapout a single page
- * page is locked upon entry, unlocked on exit
- */
-static int swap_page(struct page *page)
-{
- struct address_space *mapping = page_mapping(page);
-
- if (page_mapped(page) && mapping)
- if (try_to_unmap(page, 1) != SWAP_SUCCESS)
- goto unlock_retry;
-
- if (PageDirty(page)) {
- /* Page is dirty, try to write it out here */
- switch(pageout(page, mapping)) {
- case PAGE_KEEP:
- case PAGE_ACTIVATE:
- goto unlock_retry;
-
- case PAGE_SUCCESS:
- goto retry;
-
- case PAGE_CLEAN:
- ; /* try to free the page below */
- }
- }
-
- if (PagePrivate(page)) {
- if (!try_to_release_page(page, GFP_KERNEL) ||
- (!mapping && page_count(page) == 1))
- goto unlock_retry;
- }
-
- if (remove_mapping(mapping, page)) {
- /* Success */
- unlock_page(page);
- return 0;
- }
-
-unlock_retry:
- unlock_page(page);
-
-retry:
- return -EAGAIN;
-}
-EXPORT_SYMBOL(swap_page);
-
-/*
* Remove references for a page and establish the new page with the correct
* basic settings to be able to stop accesses to the page.
*/
@@ -336,8 +285,7 @@ EXPORT_SYMBOL(migrate_page);
* Two lists are passed to this function. The first list
* contains the pages isolated from the LRU to be migrated.
* The second list contains new pages that the pages isolated
- * can be moved to. If the second list is NULL then all
- * pages are swapped out.
+ * can be moved to.
*
* The function returns after 10 attempts or if no pages
* are movable anymore because to has become empty
@@ -393,30 +341,13 @@ redo:
* Only wait on writeback if we have already done a pass where
* we we may have triggered writeouts for lots of pages.
*/
- if (pass > 0) {
+ if (pass > 0)
wait_on_page_writeback(page);
- } else {
+ else {
if (PageWriteback(page))
goto unlock_page;
}
- /*
- * Anonymous pages must have swap cache references otherwise
- * the information contained in the page maps cannot be
- * preserved.
- */
- if (PageAnon(page) && !PageSwapCache(page)) {
- if (!add_to_swap(page, GFP_KERNEL)) {
- rc = -ENOMEM;
- goto unlock_page;
- }
- }
-
- if (!to) {
- rc = swap_page(page);
- goto next;
- }
-
newpage = lru_to_page(to);
lock_page(newpage);
@@ -470,24 +401,6 @@ redo:
goto unlock_both;
}
- /*
- * On early passes with mapped pages simply
- * retry. There may be a lock held for some
- * buffers that may go away. Later
- * swap them out.
- */
- if (pass > 4) {
- /*
- * Persistently unable to drop buffers..... As a
- * measure of last resort we fall back to
- * swap_page().
- */
- unlock_page(newpage);
- newpage = NULL;
- rc = swap_page(page);
- goto next;
- }
-
unlock_both:
unlock_page(newpage);
Index: linux-2.6.17-rc1/mm/swapfile.c
===================================================================
--- linux-2.6.17-rc1.orig/mm/swapfile.c 2006-04-03 22:07:46.000000000 -0700
+++ linux-2.6.17-rc1/mm/swapfile.c 2006-04-03 22:07:56.000000000 -0700
@@ -618,15 +618,6 @@ static int unuse_mm(struct mm_struct *mm
return 0;
}
-#ifdef CONFIG_MIGRATION
-int remove_vma_swap(struct vm_area_struct *vma, struct page *page)
-{
- swp_entry_t entry = { .val = page_private(page) };
-
- return unuse_vma(vma, entry, page);
-}
-#endif
-
/*
* Scan swap_map from current position to next entry still in use.
* Recycle to start on reaching the end, returning 0 when empty.
Index: linux-2.6.17-rc1/mm/rmap.c
===================================================================
--- linux-2.6.17-rc1.orig/mm/rmap.c 2006-04-03 22:07:55.000000000 -0700
+++ linux-2.6.17-rc1/mm/rmap.c 2006-04-03 22:07:56.000000000 -0700
@@ -205,44 +205,6 @@ out:
return anon_vma;
}
-#ifdef CONFIG_MIGRATION
-/*
- * Remove an anonymous page from swap replacing the swap pte's
- * through real pte's pointing to valid pages and then releasing
- * the page from the swap cache.
- *
- * Must hold page lock on page and mmap_sem of one vma that contains
- * the page.
- */
-void remove_from_swap(struct page *page)
-{
- struct anon_vma *anon_vma;
- struct vm_area_struct *vma;
- unsigned long mapping;
-
- if (!PageSwapCache(page))
- return;
-
- mapping = (unsigned long)page->mapping;
-
- if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0)
- return;
-
- /*
- * We hold the mmap_sem lock. So no need to call page_lock_anon_vma.
- */
- anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON);
- spin_lock(&anon_vma->lock);
-
- list_for_each_entry(vma, &anon_vma->head, anon_vma_node)
- remove_vma_swap(vma, page);
-
- spin_unlock(&anon_vma->lock);
- delete_from_swap_cache(page);
-}
-EXPORT_SYMBOL(remove_from_swap);
-#endif
-
/*
* At what user virtual address is page expected in vma?
*/
Index: linux-2.6.17-rc1/include/linux/rmap.h
===================================================================
--- linux-2.6.17-rc1.orig/include/linux/rmap.h 2006-04-03 22:07:55.000000000 -0700
+++ linux-2.6.17-rc1/include/linux/rmap.h 2006-04-03 22:07:56.000000000 -0700
@@ -92,7 +92,6 @@ static inline void page_dup_rmap(struct
*/
int page_referenced(struct page *, int is_locked);
int try_to_unmap(struct page *, int ignore_refs);
-void remove_from_swap(struct page *page);
/*
* Called from mm/filemap_xip.c to unmap empty zero page
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [RFC 6/6] Swapless V1: Revise main migration logic
From: Christoph Lameter @ 2006-04-04 6:58 UTC (permalink / raw)
To: linux-mm
Cc: Lee Schermerhorn, Christoph Lameter, lhms-devel,
Hirokazu Takahashi, Marcelo Tosatti, KAMEZAWA Hiroyuki
In-Reply-To: <20060404065739.24532.95451.sendpatchset@schroedinger.engr.sgi.com>
New migration scheme
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Index: linux-2.6.17-rc1/mm/migrate.c
===================================================================
--- linux-2.6.17-rc1.orig/mm/migrate.c 2006-04-03 23:44:31.000000000 -0700
+++ linux-2.6.17-rc1/mm/migrate.c 2006-04-03 23:48:02.000000000 -0700
@@ -151,27 +151,21 @@ int migrate_page_remove_references(struc
* indicates that the page is in use or truncate has removed
* the page.
*/
- if (!mapping || page_mapcount(page) + nr_refs != page_count(page))
- return -EAGAIN;
+ if (!page->mapping ||
+ page_mapcount(page) + nr_refs + !!mapping != page_count(page))
+ return -EAGAIN;
/*
- * Establish swap ptes for anonymous pages or destroy pte
+ * Establish migration ptes for anonymous pages or destroy pte
* maps for files.
*
* In order to reestablish file backed mappings the fault handlers
* will take the radix tree_lock which may then be used to stop
* processses from accessing this page until the new page is ready.
*
- * A process accessing via a swap pte (an anonymous page) will take a
- * page_lock on the old page which will block the process until the
- * migration attempt is complete. At that time the PageSwapCache bit
- * will be examined. If the page was migrated then the PageSwapCache
- * bit will be clear and the operation to retrieve the page will be
- * retried which will find the new page in the radix tree. Then a new
- * direct mapping may be generated based on the radix tree contents.
- *
- * If the page was not migrated then the PageSwapCache bit
- * is still set and the operation may continue.
+ * A process accessing via a migration pte (an anonymous page) will
+ * take a page_lock on the old page which will block the process
+ * until the migration attempt is complete.
*/
if (try_to_unmap(page, 1) == SWAP_FAIL)
/* A vma has VM_LOCKED set -> permanent failure */
@@ -183,13 +177,19 @@ int migrate_page_remove_references(struc
if (page_mapcount(page))
return -EAGAIN;
+ if (!mapping)
+ return 0; /* Anonymous page without swap */
+
+ /*
+ * Page has a mapping that we need to change
+ */
write_lock_irq(&mapping->tree_lock);
radix_pointer = (struct page **)radix_tree_lookup_slot(
&mapping->page_tree,
page_index(page));
- if (!page_mapping(page) || page_count(page) != nr_refs ||
+ if (!page_mapping(page) || page_count(page) != nr_refs + 1 ||
*radix_pointer != page) {
write_unlock_irq(&mapping->tree_lock);
return -EAGAIN;
@@ -206,11 +206,12 @@ int migrate_page_remove_references(struc
get_page(newpage);
newpage->index = page->index;
newpage->mapping = page->mapping;
+#ifdef CONFIG_SWAP
if (PageSwapCache(page)) {
SetPageSwapCache(newpage);
set_page_private(newpage, page_private(page));
}
-
+#endif
*radix_pointer = newpage;
__put_page(page);
write_unlock_irq(&mapping->tree_lock);
@@ -244,7 +245,9 @@ void migrate_page_copy(struct page *newp
set_page_dirty(newpage);
}
+#ifdef CONFIG_SWAP
ClearPageSwapCache(page);
+#endif
ClearPageActive(page);
ClearPagePrivate(page);
set_page_private(page, 0);
@@ -271,10 +274,12 @@ int migrate_page(struct page *newpage, s
BUG_ON(PageWriteback(page)); /* Writeback must be complete */
- rc = migrate_page_remove_references(newpage, page, 2);
+ rc = migrate_page_remove_references(newpage, page, 1);
- if (rc)
+ if (rc) {
+ remove_migration_ptes(page, page);
return rc;
+ }
migrate_page_copy(newpage, page);
@@ -286,7 +291,7 @@ int migrate_page(struct page *newpage, s
* waiting on the page lock to use the new page via the page tables
* before the new page is unlocked.
*/
- remove_from_swap(newpage);
+ remove_migration_ptes(page, newpage);
return 0;
}
EXPORT_SYMBOL(migrate_page);
@@ -368,9 +373,12 @@ redo:
* Try to migrate the page.
*/
mapping = page_mapping(page);
- if (!mapping)
+ if (!mapping) {
+
+ rc = migrate_page(newpage, page);
goto unlock_both;
+ } else
if (mapping->a_ops->migratepage) {
/*
* Most pages have a mapping and most filesystems
@@ -462,7 +470,7 @@ int buffer_migrate_page(struct page *new
head = page_buffers(page);
- rc = migrate_page_remove_references(newpage, page, 3);
+ rc = migrate_page_remove_references(newpage, page, 2);
if (rc)
return rc;
Index: linux-2.6.17-rc1/mm/Kconfig
===================================================================
--- linux-2.6.17-rc1.orig/mm/Kconfig 2006-04-02 20:22:10.000000000 -0700
+++ linux-2.6.17-rc1/mm/Kconfig 2006-04-03 23:44:31.000000000 -0700
@@ -138,8 +138,8 @@ config SPLIT_PTLOCK_CPUS
#
config MIGRATION
bool "Page migration"
- def_bool y if NUMA
- depends on SWAP && NUMA
+ def_bool y
+ depends on NUMA
help
Allows the migration of the physical location of pages of processes
while the virtual addresses are not changed. This is useful for
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [patch 0/3] lockless pagecache
From: Nick Piggin @ 2006-04-04 9:31 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux Kernel, Nick Piggin, Linux Memory Management
Hi,
I'd like to submit the lockless pagecache for -mm. A scan through -mm
reveals that there shouldn't be any problems, except for reiser4, which
looks like it has a broken ->releasepage (it shouldn't be removing the
page from pagecache itself).
Thanks,
Nick
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [patch 1/3] radix tree: RCU lockless read-side
From: Nick Piggin @ 2006-04-04 9:31 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux Kernel, Nick Piggin, Linux Memory Management
In-Reply-To: <20060219020140.9923.43378.sendpatchset@linux.site>
Make radix tree lookups safe to be performed without locks. Readers
are protected against nodes being deleted by using RCU based freeing.
Readers are protected against new node insertion by using memory
barriers to ensure the node itself will be properly written before it
is visible in the radix tree.
Each radix tree node keeps a record of their height (above leaf
nodes). This height does not change after insertion -- when the radix
tree is extended, higher nodes are only inserted in the top. So a
lookup can take the pointer to what is *now* the root node, and
traverse down it even if the tree is concurrently extended and this
node becomes a subtree of a new root.
When a reader wants to traverse the next branch, they will take a
copy of the pointer. This pointer will be either NULL (and the branch
is empty) or non-NULL (and will point to a valid node).
Also introduce a lockfree gang_lookup_slot which will be used by a
future patch.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Index: linux-2.6/lib/radix-tree.c
===================================================================
--- linux-2.6.orig/lib/radix-tree.c
+++ linux-2.6/lib/radix-tree.c
@@ -30,6 +30,7 @@
#include <linux/gfp.h>
#include <linux/string.h>
#include <linux/bitops.h>
+#include <linux/rcupdate.h>
#ifdef __KERNEL__
@@ -45,7 +46,9 @@
((RADIX_TREE_MAP_SIZE + BITS_PER_LONG - 1) / BITS_PER_LONG)
struct radix_tree_node {
+ unsigned int height; /* Height from the bottom */
unsigned int count;
+ struct rcu_head rcu_head;
void *slots[RADIX_TREE_MAP_SIZE];
unsigned long tags[RADIX_TREE_MAX_TAGS][RADIX_TREE_TAG_LONGS];
};
@@ -97,10 +100,17 @@ radix_tree_node_alloc(struct radix_tree_
return ret;
}
+static void radix_tree_node_rcu_free(struct rcu_head *head)
+{
+ struct radix_tree_node *node =
+ container_of(head, struct radix_tree_node, rcu_head);
+ kmem_cache_free(radix_tree_node_cachep, node);
+}
+
static inline void
radix_tree_node_free(struct radix_tree_node *node)
{
- kmem_cache_free(radix_tree_node_cachep, node);
+ call_rcu(&node->rcu_head, radix_tree_node_rcu_free);
}
/*
@@ -206,6 +216,7 @@ static int radix_tree_extend(struct radi
}
do {
+ unsigned int newheight;
if (!(node = radix_tree_node_alloc(root)))
return -ENOMEM;
@@ -218,9 +229,11 @@ static int radix_tree_extend(struct radi
tag_set(node, tag, 0);
}
+ newheight = root->height+1;
+ node->height = newheight;
node->count = 1;
- root->rnode = node;
- root->height++;
+ rcu_assign_pointer(root->rnode, node);
+ root->height = newheight;
} while (height > root->height);
out:
return 0;
@@ -260,11 +273,12 @@ int radix_tree_insert(struct radix_tree_
/* Have to add a child node. */
if (!(slot = radix_tree_node_alloc(root)))
return -ENOMEM;
+ slot->height = height;
if (node) {
- node->slots[offset] = slot;
+ rcu_assign_pointer(node->slots[offset], slot);
node->count++;
} else
- root->rnode = slot;
+ rcu_assign_pointer(root->rnode, slot);
}
/* Go a level down */
@@ -280,7 +294,7 @@ int radix_tree_insert(struct radix_tree_
BUG_ON(!node);
node->count++;
- node->slots[offset] = item;
+ rcu_assign_pointer(node->slots[offset], item);
BUG_ON(tag_get(node, 0, offset));
BUG_ON(tag_get(node, 1, offset));
@@ -292,25 +306,29 @@ static inline void **__lookup_slot(struc
unsigned long index)
{
unsigned int height, shift;
- struct radix_tree_node **slot;
+ struct radix_tree_node *node, **slot;
- height = root->height;
+ /* Must take a copy now because root->rnode may change */
+ node = rcu_dereference(root->rnode);
+ if (node == NULL)
+ return NULL;
+
+ height = node->height;
if (index > radix_tree_maxindex(height))
return NULL;
shift = (height-1) * RADIX_TREE_MAP_SHIFT;
- slot = &root->rnode;
- while (height > 0) {
- if (*slot == NULL)
+ do {
+ slot = (struct radix_tree_node **)
+ (node->slots + ((index>>shift) & RADIX_TREE_MAP_MASK));
+ node = rcu_dereference(*slot);
+ if (node == NULL)
return NULL;
- slot = (struct radix_tree_node **)
- ((*slot)->slots +
- ((index >> shift) & RADIX_TREE_MAP_MASK));
shift -= RADIX_TREE_MAP_SHIFT;
height--;
- }
+ } while (height > 0);
return (void **)slot;
}
@@ -322,6 +340,12 @@ static inline void **__lookup_slot(struc
*
* Lookup the slot corresponding to the position @index in the radix tree
* @root. This is useful for update-if-exists operations.
+ *
+ * This function can be called under rcu_read_lock, however it is the
+ * duty of the caller to manage the lifetimes of the leaf nodes (ie.
+ * they would usually be RCU protected as well). Also, dereferencing
+ * the slot pointer would require rcu_dereference, and modifying it
+ * would require rcu_assign_pointer.
*/
void **radix_tree_lookup_slot(struct radix_tree_root *root, unsigned long index)
{
@@ -335,13 +359,18 @@ EXPORT_SYMBOL(radix_tree_lookup_slot);
* @index: index key
*
* Lookup the item at the position @index in the radix tree @root.
+ *
+ * Like radix_tree_lookup_slot, this function can be called under
+ * rcu_read_lock, and likewise the caller must manage lifetimes of
+ * leaf nodes. No RCU barriers are required to access or modify the
+ * returned item, however.
*/
void *radix_tree_lookup(struct radix_tree_root *root, unsigned long index)
{
void **slot;
slot = __lookup_slot(root, index);
- return slot != NULL ? *slot : NULL;
+ return slot != NULL ? rcu_dereference(*slot) : NULL;
}
EXPORT_SYMBOL(radix_tree_lookup);
@@ -505,7 +534,7 @@ EXPORT_SYMBOL(radix_tree_tag_get);
#endif
static unsigned int
-__lookup(struct radix_tree_root *root, void **results, unsigned long index,
+__lookup(struct radix_tree_root *root, void ***results, unsigned long index,
unsigned int max_items, unsigned long *next_index)
{
unsigned int nr_found = 0;
@@ -513,18 +542,20 @@ __lookup(struct radix_tree_root *root, v
struct radix_tree_node *slot;
unsigned long i;
- height = root->height;
- if (height == 0)
+ slot = rcu_dereference(root->rnode);
+ if (!slot || slot->height == 0)
goto out;
+ height = slot->height;
shift = (height-1) * RADIX_TREE_MAP_SHIFT;
- slot = root->rnode;
for ( ; height > 1; height--) {
+ struct radix_tree_node *__s;
for (i = (index >> shift) & RADIX_TREE_MAP_MASK ;
i < RADIX_TREE_MAP_SIZE; i++) {
- if (slot->slots[i] != NULL)
+ __s = rcu_dereference(slot->slots[i]);
+ if (__s != NULL)
break;
index &= ~((1UL << shift) - 1);
index += 1UL << shift;
@@ -535,14 +566,14 @@ __lookup(struct radix_tree_root *root, v
goto out;
shift -= RADIX_TREE_MAP_SHIFT;
- slot = slot->slots[i];
+ slot = __s;
}
/* Bottom level: grab some items */
for (i = index & RADIX_TREE_MAP_MASK; i < RADIX_TREE_MAP_SIZE; i++) {
index++;
if (slot->slots[i]) {
- results[nr_found++] = slot->slots[i];
+ results[nr_found++] = &slot->slots[i];
if (nr_found == max_items)
goto out;
}
@@ -572,6 +603,46 @@ radix_tree_gang_lookup(struct radix_tree
const unsigned long max_index = radix_tree_maxindex(root->height);
unsigned long cur_index = first_index;
unsigned int ret = 0;
+ void ***__results = (void ***)results; /* use results as a temporary
+ * store for the pointers to
+ * the actual results */
+
+ while (ret < max_items) {
+ unsigned int nr_found, i;
+ unsigned long next_index; /* Index of next search */
+
+ if (cur_index > max_index)
+ break;
+ nr_found = __lookup(root, __results + ret, cur_index,
+ max_items - ret, &next_index);
+ for (i = 0; i < nr_found; i++)
+ results[ret + i] = *rcu_dereference(__results[ret + i]);
+ ret += nr_found;
+ if (next_index == 0)
+ break;
+ cur_index = next_index;
+ }
+ return ret;
+}
+EXPORT_SYMBOL(radix_tree_gang_lookup);
+
+/**
+ * radix_tree_gang_lookup_slot - perform multiple lookup on a radix tree
+ * @root: radix tree root
+ * @results: where the results of the lookup are placed
+ * @first_index: start the lookup from this key
+ * @max_items: place up to this many items at *results
+ *
+ * Same as radix_tree_gang_lookup, but returns an array of pointers
+ * (slots) to the stored items instead of the items themselves.
+ */
+unsigned int
+radix_tree_gang_lookup_slot(struct radix_tree_root *root, void ***results,
+ unsigned long first_index, unsigned int max_items)
+{
+ const unsigned long max_index = radix_tree_maxindex(root->height);
+ unsigned long cur_index = first_index;
+ unsigned int ret = 0;
while (ret < max_items) {
unsigned int nr_found;
@@ -588,7 +659,8 @@ radix_tree_gang_lookup(struct radix_tree
}
return ret;
}
-EXPORT_SYMBOL(radix_tree_gang_lookup);
+EXPORT_SYMBOL_GPL(radix_tree_gang_lookup_slot);
+
/*
* FIXME: the two tag_get()s here should use find_next_bit() instead of
@@ -694,6 +766,11 @@ static inline void radix_tree_shrink(str
root->rnode->slots[0]) {
struct radix_tree_node *to_free = root->rnode;
+ /*
+ * this doesn't need an rcu_assign_pointer, because
+ * we aren't touching the object that to_free->slots[0]
+ * points to.
+ */
root->rnode = to_free->slots[0];
root->height--;
/* must only free zeroed nodes into the slab */
@@ -809,7 +886,7 @@ EXPORT_SYMBOL(radix_tree_delete);
int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag)
{
struct radix_tree_node *rnode;
- rnode = root->rnode;
+ rnode = rcu_dereference(root->rnode);
if (!rnode)
return 0;
return any_tag_set(rnode, tag);
Index: linux-2.6/include/linux/radix-tree.h
===================================================================
--- linux-2.6.orig/include/linux/radix-tree.h
+++ linux-2.6/include/linux/radix-tree.h
@@ -54,6 +54,9 @@ void *radix_tree_delete(struct radix_tre
unsigned int
radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
unsigned long first_index, unsigned int max_items);
+unsigned int
+radix_tree_gang_lookup_slot(struct radix_tree_root *root, void ***results,
+ unsigned long first_index, unsigned int max_items);
int radix_tree_preload(gfp_t gfp_mask);
void radix_tree_init(void);
void *radix_tree_tag_set(struct radix_tree_root *root,
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [patch 2/3] mm: speculative get_page
From: Nick Piggin @ 2006-04-04 9:32 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux Kernel, Nick Piggin, Linux Memory Management
In-Reply-To: <20060219020140.9923.43378.sendpatchset@linux.site>
If we can be sure that elevating the page_count on a pagecache
page will pin it, we can speculatively run this operation, and
subsequently check to see if we hit the right page rather than
relying on holding a lock or otherwise pinning a reference to the
page.
This can be done if get_page/put_page behaves consistently
throughout the whole tree (ie. if we "get" the page after it has
been used for something else, we must be able to free it with a
put_page).
Actually, there is a period where the count behaves differently:
when the page is free or if it is a constituent page of a compound
page. We need an atomic_inc_not_zero operation to ensure we don't
try to grab the page in either case.
Signed-off-by: Nick Piggin <npiggin@suse.de>
Index: linux-2.6/include/linux/page-flags.h
===================================================================
--- linux-2.6.orig/include/linux/page-flags.h
+++ linux-2.6/include/linux/page-flags.h
@@ -76,6 +76,9 @@
#define PG_nosave_free 18 /* Free, should not be written */
#define PG_uncached 19 /* Page has been mapped as uncached */
+#define PG_nonewrefs 20 /* Block concurrent pagecache lookups
+ * while testing refcount */
+
/*
* Global page accounting. One instance per CPU. Only unsigned longs are
* allowed.
@@ -346,6 +349,11 @@ extern void __mod_page_state_offset(unsi
#define SetPageUncached(page) set_bit(PG_uncached, &(page)->flags)
#define ClearPageUncached(page) clear_bit(PG_uncached, &(page)->flags)
+#define PageNoNewRefs(page) test_bit(PG_nonewrefs, &(page)->flags)
+#define SetPageNoNewRefs(page) set_bit(PG_nonewrefs, &(page)->flags)
+#define ClearPageNoNewRefs(page) clear_bit(PG_nonewrefs, &(page)->flags)
+#define __ClearPageNoNewRefs(page) __clear_bit(PG_nonewrefs, &(page)->flags)
+
struct page; /* forward declaration */
int test_clear_page_dirty(struct page *page);
Index: linux-2.6/include/linux/pagemap.h
===================================================================
--- linux-2.6.orig/include/linux/pagemap.h
+++ linux-2.6/include/linux/pagemap.h
@@ -11,6 +11,8 @@
#include <linux/compiler.h>
#include <asm/uaccess.h>
#include <linux/gfp.h>
+#include <linux/page-flags.h>
+#include <linux/hardirq.h> /* for in_interrupt() */
/*
* Bits in mapping->flags. The lower __GFP_BITS_SHIFT bits are the page
@@ -51,6 +53,91 @@ static inline void mapping_set_gfp_mask(
#define page_cache_release(page) put_page(page)
void release_pages(struct page **pages, int nr, int cold);
+static inline struct page *page_cache_get_speculative(struct page **pagep)
+{
+ struct page *page;
+
+ VM_BUG_ON(in_interrupt());
+
+#ifndef CONFIG_SMP
+ page = *pagep;
+ if (unlikely(!page))
+ return NULL;
+
+ VM_BUG_ON(!in_atomic());
+ /*
+ * Preempt must be disabled here - we rely on rcu_read_lock doing
+ * this for us.
+ *
+ * Pagecache won't be truncated from interrupt context, so if we have
+ * found a page in the radix tree here, we have pinned its refcount by
+ * disabling preempt, and hence no need for the "speculative get" that
+ * SMP requires.
+ */
+ VM_BUG_ON(page_count(page) == 0);
+ atomic_inc(&page->_count);
+ VM_BUG_ON(page != *pagep);
+
+#else
+ again:
+ page = rcu_dereference(*pagep);
+ if (unlikely(!page))
+ return NULL;
+
+ if (unlikely(!get_page_unless_zero(page)))
+ goto again; /* page has been freed */
+
+ /*
+ * Note that get_page_unless_zero provides a memory barrier.
+ * This is needed to ensure PageNoNewRefs is evaluated after the
+ * page refcount has been raised. See below comment.
+ */
+
+ /*
+ * PageNoNewRefs is set in order to prevent new references to the
+ * page (eg. before it gets removed from pagecache). Wait until it
+ * becomes clear (and checks below will ensure we still have the
+ * correct one).
+ */
+ while (unlikely(PageNoNewRefs(page)))
+ cpu_relax();
+
+ /*
+ * smp_rmb is to ensure the load of page->flags (for PageNoNewRefs())
+ * is performed before the load of *pagep in the below comparison.
+ *
+ * Those places that set PageNoNewRefs have the following pattern:
+ * SetPageNoNewRefs(page)
+ * wmb();
+ * if (page_count(page) == X)
+ * remove page from pagecache
+ * wmb();
+ * ClearPageNoNewRefs(page)
+ *
+ * So PageNoNewRefs() becomes clear _after_ we've elevated page
+ * refcount, then either the page will be safely pinned in pagecache,
+ * or it will have been already removed. In the latter case, *pagep
+ * will be changed in the below test - provided it is loaded after
+ * testing PageNoNewRefs() (which is what the smp_rmb is for).
+ *
+ * If the load was out of order, *pagep might be loaded before the
+ * page is removed from pagecache while PageNoNewRefs evaluated after
+ * the ClearPageNoNewRefs().
+ */
+ smp_rmb();
+
+ if (unlikely(page != *pagep)) {
+ /* page no longer at *pagep */
+ put_page(page);
+ goto again;
+ }
+
+#endif
+ VM_BUG_ON(PageCompound(page) && (struct page *)page_private(page) != page);
+
+ return page;
+}
+
#ifdef CONFIG_NUMA
extern struct page *page_cache_alloc(struct address_space *x);
extern struct page *page_cache_alloc_cold(struct address_space *x);
Index: linux-2.6/mm/vmscan.c
===================================================================
--- linux-2.6.orig/mm/vmscan.c
+++ linux-2.6/mm/vmscan.c
@@ -365,6 +365,7 @@ int remove_mapping(struct address_space
if (!mapping)
return 0; /* truncate got there first */
+ SetPageNoNewRefs(page);
write_lock_irq(&mapping->tree_lock);
/*
@@ -383,17 +384,20 @@ int remove_mapping(struct address_space
__delete_from_swap_cache(page);
write_unlock_irq(&mapping->tree_lock);
swap_free(swap);
- __put_page(page); /* The pagecache ref */
- return 1;
+ goto free_it;
}
__remove_from_page_cache(page);
write_unlock_irq(&mapping->tree_lock);
- __put_page(page);
+
+free_it:
+ __ClearPageNoNewRefs(page);
+ __put_page(page); /* The pagecache ref */
return 1;
cannot_free:
write_unlock_irq(&mapping->tree_lock);
+ ClearPageNoNewRefs(page);
return 0;
}
Index: linux-2.6/mm/filemap.c
===================================================================
--- linux-2.6.orig/mm/filemap.c
+++ linux-2.6/mm/filemap.c
@@ -407,6 +407,7 @@ int add_to_page_cache(struct page *page,
int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
if (error == 0) {
+ SetPageNoNewRefs(page);
write_lock_irq(&mapping->tree_lock);
error = radix_tree_insert(&mapping->page_tree, offset, page);
if (!error) {
@@ -418,6 +419,7 @@ int add_to_page_cache(struct page *page,
pagecache_acct(1);
}
write_unlock_irq(&mapping->tree_lock);
+ ClearPageNoNewRefs(page);
radix_tree_preload_end();
}
return error;
Index: linux-2.6/mm/swap_state.c
===================================================================
--- linux-2.6.orig/mm/swap_state.c
+++ linux-2.6/mm/swap_state.c
@@ -78,6 +78,7 @@ static int __add_to_swap_cache(struct pa
BUG_ON(PagePrivate(page));
error = radix_tree_preload(gfp_mask);
if (!error) {
+ SetPageNoNewRefs(page);
write_lock_irq(&swapper_space.tree_lock);
error = radix_tree_insert(&swapper_space.page_tree,
entry.val, page);
@@ -90,6 +91,7 @@ static int __add_to_swap_cache(struct pa
pagecache_acct(1);
}
write_unlock_irq(&swapper_space.tree_lock);
+ ClearPageNoNewRefs(page);
radix_tree_preload_end();
}
return error;
Index: linux-2.6/mm/migrate.c
===================================================================
--- linux-2.6.orig/mm/migrate.c
+++ linux-2.6/mm/migrate.c
@@ -28,8 +28,6 @@
#include "internal.h"
-#include "internal.h"
-
/* The maximum number of pages to take off the LRU for migration */
#define MIGRATE_CHUNK_SIZE 256
@@ -225,6 +223,7 @@ int migrate_page_remove_references(struc
if (page_mapcount(page))
return -EAGAIN;
+ SetPageNoNewRefs(page);
write_lock_irq(&mapping->tree_lock);
radix_pointer = (struct page **)radix_tree_lookup_slot(
@@ -234,6 +233,7 @@ int migrate_page_remove_references(struc
if (!page_mapping(page) || page_count(page) != nr_refs ||
*radix_pointer != page) {
write_unlock_irq(&mapping->tree_lock);
+ ClearPageNoNewRefs(page);
return 1;
}
@@ -253,9 +253,13 @@ int migrate_page_remove_references(struc
set_page_private(newpage, page_private(page));
}
- *radix_pointer = newpage;
+ SetPageNoNewRefs(newpage);
+ rcu_assign_pointer(*radix_pointer, newpage);
+
+ write_unlock_irq(&mapping->tree_lock);
__put_page(page);
- write_unlock_irq(&mapping->tree_lock);
+ ClearPageNoNewRefs(page);
+ ClearPageNoNewRefs(newpage);
return 0;
}
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [patch 3/3] mm: lockless pagecache lookups
From: Nick Piggin @ 2006-04-04 9:32 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux Kernel, Nick Piggin, Linux Memory Management
In-Reply-To: <20060219020140.9923.43378.sendpatchset@linux.site>
Use page_cache_get_speculative and lockless radix tree lookups to
introduce lockless page cache lookups (ie. no mapping->tree_lock).
The only atomicity changes this should introduce is the use of a
non atomic pagevec lookup for truncate, however what atomicity
guarantees that there might have been were not used anyway, because
the size of the pagevec is not guaranteed (eg. it might be 1).
Signed-off-by: Nick Piggin <npiggin@suse.de>
Index: linux-2.6/mm/filemap.c
===================================================================
--- linux-2.6.orig/mm/filemap.c
+++ linux-2.6/mm/filemap.c
@@ -424,7 +424,6 @@ int add_to_page_cache(struct page *page,
}
return error;
}
-
EXPORT_SYMBOL(add_to_page_cache);
int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
@@ -547,21 +546,21 @@ void fastcall __lock_page(struct page *p
EXPORT_SYMBOL(__lock_page);
/*
- * a rather lightweight function, finding and getting a reference to a
- * hashed page atomically.
+ * find_get_page finds and gets a reference to a pagecache page.
*/
-struct page * find_get_page(struct address_space *mapping, unsigned long offset)
+struct page *find_get_page(struct address_space *mapping, unsigned long offset)
{
- struct page *page;
+ struct page **pagep;
+ struct page *page = NULL;
- read_lock_irq(&mapping->tree_lock);
- page = radix_tree_lookup(&mapping->page_tree, offset);
- if (page)
- page_cache_get(page);
- read_unlock_irq(&mapping->tree_lock);
+ rcu_read_lock();
+ pagep = (struct page **)radix_tree_lookup_slot(&mapping->page_tree,
+ offset);
+ if (likely(pagep))
+ page = page_cache_get_speculative(pagep);
+ rcu_read_unlock();
return page;
}
-
EXPORT_SYMBOL(find_get_page);
/*
@@ -597,26 +596,17 @@ struct page *find_lock_page(struct addre
{
struct page *page;
- read_lock_irq(&mapping->tree_lock);
repeat:
- page = radix_tree_lookup(&mapping->page_tree, offset);
+ page = find_get_page(mapping, offset);
if (page) {
- page_cache_get(page);
- if (TestSetPageLocked(page)) {
- read_unlock_irq(&mapping->tree_lock);
- __lock_page(page);
- read_lock_irq(&mapping->tree_lock);
-
- /* Has the page been truncated while we slept? */
- if (unlikely(page->mapping != mapping ||
- page->index != offset)) {
- unlock_page(page);
- page_cache_release(page);
- goto repeat;
- }
+ lock_page(page);
+ /* Has the page been truncated while we slept? */
+ if (unlikely(page->mapping != mapping)) {
+ unlock_page(page);
+ page_cache_release(page);
+ goto repeat;
}
}
- read_unlock_irq(&mapping->tree_lock);
return page;
}
@@ -699,6 +689,32 @@ unsigned find_get_pages(struct address_s
return ret;
}
+unsigned find_get_pages_nonatomic(struct address_space *mapping, pgoff_t start,
+ unsigned int nr_pages, struct page **pages)
+{
+ unsigned int i;
+ unsigned int nr_found;
+ unsigned int ret;
+
+ /*
+ * We do some unsightly casting to use the array first for storing
+ * pointers to the page pointers, and then for the pointers to
+ * the pages themselves that the caller wants.
+ */
+ rcu_read_lock();
+ nr_found = radix_tree_gang_lookup_slot(&mapping->page_tree,
+ (void ***)pages, start, nr_pages);
+ ret = 0;
+ for (i = 0; i < nr_found; i++) {
+ struct page *page;
+ page = page_cache_get_speculative(((struct page ***)pages)[i]);
+ if (likely(page))
+ pages[ret++] = page;
+ }
+ rcu_read_unlock();
+ return ret;
+}
+
/*
* Like find_get_pages, except we only return pages which are tagged with
* `tag'. We update *index to index the next page for the traversal.
Index: linux-2.6/mm/readahead.c
===================================================================
--- linux-2.6.orig/mm/readahead.c
+++ linux-2.6/mm/readahead.c
@@ -286,27 +286,26 @@ __do_page_cache_readahead(struct address
/*
* Preallocate as many pages as we will need.
*/
- read_lock_irq(&mapping->tree_lock);
for (page_idx = 0; page_idx < nr_to_read; page_idx++) {
pgoff_t page_offset = offset + page_idx;
if (page_offset > end_index)
break;
+ /* Don't need mapping->tree_lock - lookup can be racy */
+ rcu_read_lock();
page = radix_tree_lookup(&mapping->page_tree, page_offset);
+ rcu_read_unlock();
if (page)
continue;
- read_unlock_irq(&mapping->tree_lock);
page = page_cache_alloc_cold(mapping);
- read_lock_irq(&mapping->tree_lock);
if (!page)
break;
page->index = page_offset;
list_add(&page->lru, &page_pool);
ret++;
}
- read_unlock_irq(&mapping->tree_lock);
/*
* Now start the IO. We ignore I/O errors - if the page is not
Index: linux-2.6/include/linux/pagemap.h
===================================================================
--- linux-2.6.orig/include/linux/pagemap.h
+++ linux-2.6/include/linux/pagemap.h
@@ -165,6 +165,8 @@ extern struct page * find_or_create_page
unsigned long index, gfp_t gfp_mask);
unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
unsigned int nr_pages, struct page **pages);
+unsigned find_get_pages_nonatomic(struct address_space *mapping, pgoff_t start,
+ unsigned int nr_pages, struct page **pages);
unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
int tag, unsigned int nr_pages, struct page **pages);
Index: linux-2.6/include/linux/pagevec.h
===================================================================
--- linux-2.6.orig/include/linux/pagevec.h
+++ linux-2.6/include/linux/pagevec.h
@@ -28,6 +28,8 @@ void __pagevec_lru_add_active(struct pag
void pagevec_strip(struct pagevec *pvec);
unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
pgoff_t start, unsigned nr_pages);
+unsigned pagevec_lookup_nonatomic(struct pagevec *pvec,
+ struct address_space *mapping, pgoff_t start, unsigned nr_pages);
unsigned pagevec_lookup_tag(struct pagevec *pvec,
struct address_space *mapping, pgoff_t *index, int tag,
unsigned nr_pages);
Index: linux-2.6/mm/swap.c
===================================================================
--- linux-2.6.orig/mm/swap.c
+++ linux-2.6/mm/swap.c
@@ -427,6 +427,20 @@ unsigned pagevec_lookup(struct pagevec *
EXPORT_SYMBOL(pagevec_lookup);
+/**
+ * pagevec_lookup_nonatomic - non atomic pagevec_lookup
+ *
+ * This routine is non-atomic in that it may return blah.
+ */
+unsigned pagevec_lookup_nonatomic(struct pagevec *pvec,
+ struct address_space *mapping, pgoff_t start, unsigned nr_pages)
+{
+ pvec->nr = find_get_pages_nonatomic(mapping, start,
+ nr_pages, pvec->pages);
+ return pagevec_count(pvec);
+}
+EXPORT_SYMBOL(pagevec_lookup_nonatomic);
+
unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
pgoff_t *index, int tag, unsigned nr_pages)
{
Index: linux-2.6/mm/truncate.c
===================================================================
--- linux-2.6.orig/mm/truncate.c
+++ linux-2.6/mm/truncate.c
@@ -124,7 +124,7 @@ void truncate_inode_pages_range(struct a
pagevec_init(&pvec, 0);
next = start;
while (next <= end &&
- pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
+ pagevec_lookup_nonatomic(&pvec, mapping, next, PAGEVEC_SIZE)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
pgoff_t page_index = page->index;
@@ -163,7 +163,7 @@ void truncate_inode_pages_range(struct a
next = start;
for ( ; ; ) {
cond_resched();
- if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
+ if (!pagevec_lookup_nonatomic(&pvec, mapping, next, PAGEVEC_SIZE)) {
if (next == start)
break;
next = start;
@@ -227,7 +227,7 @@ unsigned long invalidate_mapping_pages(s
pagevec_init(&pvec, 0);
while (next <= end &&
- pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
+ pagevec_lookup_nonatomic(&pvec, mapping, next, PAGEVEC_SIZE)) {
for (i = 0; i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
@@ -284,7 +284,7 @@ int invalidate_inode_pages2_range(struct
pagevec_init(&pvec, 0);
next = start;
while (next <= end && !ret && !wrapped &&
- pagevec_lookup(&pvec, mapping, next,
+ pagevec_lookup_nonatomic(&pvec, mapping, next,
min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
for (i = 0; !ret && i < pagevec_count(&pvec); i++) {
struct page *page = pvec.pages[i];
Index: linux-2.6/mm/page-writeback.c
===================================================================
--- linux-2.6.orig/mm/page-writeback.c
+++ linux-2.6/mm/page-writeback.c
@@ -816,17 +816,15 @@ int test_set_page_writeback(struct page
EXPORT_SYMBOL(test_set_page_writeback);
/*
- * Return true if any of the pages in the mapping are marged with the
+ * Return true if any of the pages in the mapping are marked with the
* passed tag.
*/
int mapping_tagged(struct address_space *mapping, int tag)
{
- unsigned long flags;
int ret;
-
- read_lock_irqsave(&mapping->tree_lock, flags);
+ rcu_read_lock();
ret = radix_tree_tagged(&mapping->page_tree, tag);
- read_unlock_irqrestore(&mapping->tree_lock, flags);
+ rcu_read_unlock();
return ret;
}
EXPORT_SYMBOL(mapping_tagged);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [patch 2/3] mm: speculative get_page
From: Andrew Morton @ 2006-04-04 9:47 UTC (permalink / raw)
To: Nick Piggin; +Cc: linux-kernel, linux-mm
In-Reply-To: <20060219020159.9923.94877.sendpatchset@linux.site>
Nick Piggin <npiggin@suse.de> wrote:
>
> +static inline struct page *page_cache_get_speculative(struct page **pagep)
Seems rather large to inline.
> +{
> + struct page *page;
> +
> + VM_BUG_ON(in_interrupt());
> +
> +#ifndef CONFIG_SMP
> + page = *pagep;
> + if (unlikely(!page))
> + return NULL;
> +
> + VM_BUG_ON(!in_atomic());
This will go blam if !CONFIG_PREEMPT.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [patch 2/3] mm: speculative get_page
From: Nick Piggin @ 2006-04-04 10:21 UTC (permalink / raw)
To: Andrew Morton; +Cc: Nick Piggin, linux-kernel, linux-mm
In-Reply-To: <20060404024715.6555d8e2.akpm@osdl.org>
On Tue, Apr 04, 2006 at 02:47:15AM -0700, Andrew Morton wrote:
> Nick Piggin <npiggin@suse.de> wrote:
> >
> > +static inline struct page *page_cache_get_speculative(struct page **pagep)
>
> Seems rather large to inline.
>
Possibly... with all the debugging turned off, it is only atomic_inc
on UP, and atomic_inc_not_zero + several branches and barriers on SMP.
With only two callsites, I figure it is probably OK to be inline. It
probably looks bigger than it is...
> > +{
> > + struct page *page;
> > +
> > + VM_BUG_ON(in_interrupt());
> > +
> > +#ifndef CONFIG_SMP
> > + page = *pagep;
> > + if (unlikely(!page))
> > + return NULL;
> > +
> > + VM_BUG_ON(!in_atomic());
>
> This will go blam if !CONFIG_PREEMPT.
Hmm yes. Is there a safe way to do that? I guess it is pretty trivally
safely under rcu_read_lock , so that can probably just be removed.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 5/6] Swapless V1: Rip out swap migration code
From: KAMEZAWA Hiroyuki @ 2006-04-04 10:37 UTC (permalink / raw)
To: Christoph Lameter
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <20060404065805.24532.65008.sendpatchset@schroedinger.engr.sgi.com>
On Mon, 3 Apr 2006 23:58:05 -0700 (PDT)
Christoph Lameter <clameter@sgi.com> wrote:
> Rip the page migration logic out
>
Thank you. I like this removal, especially removing remove_from_swap() :)
-- Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 6/6] Swapless V1: Revise main migration logic
From: KAMEZAWA Hiroyuki @ 2006-04-04 10:58 UTC (permalink / raw)
To: Christoph Lameter
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <20060404065810.24532.30027.sendpatchset@schroedinger.engr.sgi.com>
On Mon, 3 Apr 2006 23:58:10 -0700 (PDT)
Christoph Lameter <clameter@sgi.com> wrote:
> New migration scheme
>
> Signed-off-by: Christoph Lameter <clameter@sgi.com>
>
> Index: linux-2.6.17-rc1/mm/migrate.c
> ===================================================================
> --- linux-2.6.17-rc1.orig/mm/migrate.c 2006-04-03 23:44:31.000000000 -0700
> +++ linux-2.6.17-rc1/mm/migrate.c 2006-04-03 23:48:02.000000000 -0700
> @@ -151,27 +151,21 @@ int migrate_page_remove_references(struc
> * indicates that the page is in use or truncate has removed
> * the page.
> */
> - if (!mapping || page_mapcount(page) + nr_refs != page_count(page))
> - return -EAGAIN;
> + if (!page->mapping ||
> + page_mapcount(page) + nr_refs + !!mapping != page_count(page))
> + return -EAGAIN;
>
I think this hidden !!mapping refcnt is not easy to read.
How about modifying caller istead of callee ?
in migrate_page()
==
if (page->mapping)
rc = migrate_page_remove_reference(newpage, page, 2)
else
rc = migrate_page_remove_reference(newpage, page, 1);
==
If you dislike this 'if', plz do as you like.
Thanks,
--Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 2/6] Swapless V1: Add SWP_TYPE_MIGRATION
From: KAMEZAWA Hiroyuki @ 2006-04-04 11:04 UTC (permalink / raw)
To: Christoph Lameter
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <20060404065750.24532.67454.sendpatchset@schroedinger.engr.sgi.com>
On Mon, 3 Apr 2006 23:57:50 -0700 (PDT)
Christoph Lameter <clameter@sgi.com> wrote:
>
> #define MAX_SWAPFILES_SHIFT 5
> -#define MAX_SWAPFILES (1 << MAX_SWAPFILES_SHIFT)
> +#define MAX_SWAPFILES ((1 << MAX_SWAPFILES_SHIFT)-1)
> +
> +/* Use last entry for page migration swap entries */
> +#define SWP_TYPE_MIGRATION MAX_SWAPFILES
How about this ?
#ifdef CONFIG_MIGRATION
#define MAX_SWAPFILES ((1 << MAX_SWAPFILES_SHIFT) - 1)
#else
#define MAX_SWAPFILES (1 << MAX_SWAPFILES_SHIFT)
#endif
#define SWP_TYPE_MIGRATION (MAX_SWAPFILES + 1)
.....but I don't think there is a user who uses 32 swaps....
--Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 6/6] Swapless V1: Revise main migration logic
From: Christoph Lameter @ 2006-04-04 14:24 UTC (permalink / raw)
To: KAMEZAWA Hiroyuki
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <20060404195820.4adc09d7.kamezawa.hiroyu@jp.fujitsu.com>
On Tue, 4 Apr 2006, KAMEZAWA Hiroyuki wrote:
> > */
> > - if (!mapping || page_mapcount(page) + nr_refs != page_count(page))
> > - return -EAGAIN;
> > + if (!page->mapping ||
> > + page_mapcount(page) + nr_refs + !!mapping != page_count(page))
> > + return -EAGAIN;
> >
> I think this hidden !!mapping refcnt is not easy to read.
>
> How about modifying caller istead of callee ?
>
> in migrate_page()
> ==
> if (page->mapping)
> rc = migrate_page_remove_reference(newpage, page, 2)
> else
> rc = migrate_page_remove_reference(newpage, page, 1);
> ==
>
> If you dislike this 'if', plz do as you like.
Good idea.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 5/6] Swapless V1: Rip out swap migration code
From: Christoph Lameter @ 2006-04-04 15:06 UTC (permalink / raw)
To: KAMEZAWA Hiroyuki
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <20060404193714.2dfafa79.kamezawa.hiroyu@jp.fujitsu.com>
On Tue, 4 Apr 2006, KAMEZAWA Hiroyuki wrote:
> On Mon, 3 Apr 2006 23:58:05 -0700 (PDT)
> Christoph Lameter <clameter@sgi.com> wrote:
>
> > Rip the page migration logic out
> >
>
> Thank you. I like this removal, especially removing remove_from_swap() :)
Have a look at remove_migration_ptes(). Like remove_from_swap() it has the
requirement that the mmap_sem is held since that is the only secure way to
make sure that the anon_vma is not vanishing from under us. That may be a
problem if you are not coming from a process context. Any ideas on how to
fix that?
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [patch 2/3] mm: speculative get_page
From: Christoph Lameter @ 2006-04-04 15:20 UTC (permalink / raw)
To: Nick Piggin; +Cc: Andrew Morton, Linux Kernel, Linux Memory Management
In-Reply-To: <20060219020159.9923.94877.sendpatchset@linux.site>
Looks like the NoNewRefs flag is mostly ==
spin_is_locked(mapping->tree_lock)? Would it not be better to check the
tree_lock?
> --- linux-2.6.orig/mm/migrate.c
> +++ linux-2.6/mm/migrate.c
>
> + SetPageNoNewRefs(page);
> write_lock_irq(&mapping->tree_lock);
A dream come true! If this is really working as it sounds then we can
move the SetPageNoNewRefs up and avoid the final check under
mapping->tree_lock. Then keep SetPageNoNewRefs until the page has been
copied. It would basically play the same role as locking the page.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [patch 2/3] mm: speculative get_page
From: Christoph Lameter @ 2006-04-04 15:21 UTC (permalink / raw)
To: Nick Piggin; +Cc: Andrew Morton, Linux Kernel, Linux Memory Management
In-Reply-To: <20060219020159.9923.94877.sendpatchset@linux.site>
On Tue, 4 Apr 2006, Nick Piggin wrote:
> + /*
> + * PageNoNewRefs is set in order to prevent new references to the
> + * page (eg. before it gets removed from pagecache). Wait until it
> + * becomes clear (and checks below will ensure we still have the
> + * correct one).
> + */
> + while (unlikely(PageNoNewRefs(page)))
> + cpu_relax();
That part looks suspiciously like we need some sort of lock here.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [patch 2/3] mm: speculative get_page
From: Nick Piggin @ 2006-04-05 0:22 UTC (permalink / raw)
To: Christoph Lameter
Cc: Nick Piggin, Andrew Morton, Linux Kernel, Linux Memory Management
In-Reply-To: <Pine.LNX.4.64.0604040814140.26807@schroedinger.engr.sgi.com>
Christoph Lameter wrote:
> Looks like the NoNewRefs flag is mostly ==
> spin_is_locked(mapping->tree_lock)? Would it not be better to check the
> tree_lock?
>
Well there are other uses for the tree_lock (eg. tag operations)
which do not need the "no new references" guarantee.
>
>
>>--- linux-2.6.orig/mm/migrate.c
>>+++ linux-2.6/mm/migrate.c
>>
>>+ SetPageNoNewRefs(page);
>> write_lock_irq(&mapping->tree_lock);
>
>
> A dream come true! If this is really working as it sounds then we can
> move the SetPageNoNewRefs up and avoid the final check under
> mapping->tree_lock. Then keep SetPageNoNewRefs until the page has been
> copied. It would basically play the same role as locking the page.
>
Yes we could do that but at this stage I wouldn't like to seperate
SetPageNoNewRefs from tree_lock, as it is replacing a traditional
guarantee that tree_lock no longer provides.
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [patch 2/3] mm: speculative get_page
From: Nick Piggin @ 2006-04-05 0:27 UTC (permalink / raw)
To: Christoph Lameter
Cc: Nick Piggin, Andrew Morton, Linux Kernel, Linux Memory Management
In-Reply-To: <Pine.LNX.4.64.0604040820540.26807@schroedinger.engr.sgi.com>
Christoph Lameter wrote:
> On Tue, 4 Apr 2006, Nick Piggin wrote:
>
>
>>+ /*
>>+ * PageNoNewRefs is set in order to prevent new references to the
>>+ * page (eg. before it gets removed from pagecache). Wait until it
>>+ * becomes clear (and checks below will ensure we still have the
>>+ * correct one).
>>+ */
>>+ while (unlikely(PageNoNewRefs(page)))
>>+ cpu_relax();
>
>
> That part looks suspiciously like we need some sort of lock here.
>
It's very light-weight now. A lock of course would only be page local,
so it wouldn't really harm scalability, however it would slow down the
single threaded case. At the moment, single threaded performance of
find_get_page is anywhere from about 15-100% faster than before the
lockless patches.
I don't see why you think there needs to be a lock? Before the write
side clears PageNoNewRefs, they will have moved 'page' out of pagecache,
so when this loop breaks, the subsequent test will fail and this
function will be repeated.
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 5/6] Swapless V1: Rip out swap migration code
From: KAMEZAWA Hiroyuki @ 2006-04-05 1:06 UTC (permalink / raw)
To: Christoph Lameter
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <Pine.LNX.4.64.0604040804560.26787@schroedinger.engr.sgi.com>
On Tue, 4 Apr 2006 08:06:26 -0700 (PDT)
Christoph Lameter <clameter@sgi.com> wrote:
> On Tue, 4 Apr 2006, KAMEZAWA Hiroyuki wrote:
>
> > On Mon, 3 Apr 2006 23:58:05 -0700 (PDT)
> > Christoph Lameter <clameter@sgi.com> wrote:
> >
> > > Rip the page migration logic out
> > >
> >
> > Thank you. I like this removal, especially removing remove_from_swap() :)
>
> Have a look at remove_migration_ptes(). Like remove_from_swap() it has the
> requirement that the mmap_sem is held since that is the only secure way to
> make sure that the anon_vma is not vanishing from under us. That may be a
> problem if you are not coming from a process context. Any ideas on how to
> fix that?
>
I think adding SWP_TYPE_MIGRATION consideration to free_swap_and_cache() is
enough against anon_vma vanishing. Because remove_migration_ptes() compares
old pte entry with old page's pfn, a page cannot be remapped into old place
when anon_vma has gone. This is my first impression.
My concern is refcnt handling of SWP_TYPE_MIGRATION pages, but maybe no problem.
Note: unuse_vma() doesn't check what pte entry contains.
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 5/6] Swapless V1: Rip out swap migration code
From: Christoph Lameter @ 2006-04-05 2:45 UTC (permalink / raw)
To: KAMEZAWA Hiroyuki
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <20060405100614.97d2e422.kamezawa.hiroyu@jp.fujitsu.com>
On Wed, 5 Apr 2006, KAMEZAWA Hiroyuki wrote:
> I think adding SWP_TYPE_MIGRATION consideration to free_swap_and_cache() is
> enough against anon_vma vanishing. Because remove_migration_ptes() compares
> old pte entry with old page's pfn, a page cannot be remapped into old place
> when anon_vma has gone. This is my first impression.
However, the last process containing the page may terminate and free the
page, while we migrate. The SWAP_TYPE_MIGRATION pte will be rewoved
together with the anonvma if no lock is held on mmap_sem. Then
remove_migration_ptes() cannot obtain a anon_vma. So it would break
without holding mmap_sem. We could fix this if we could somehow know that
the last process mapping the page vanished and skip
remove_migration_ptes().
> My concern is refcnt handling of SWP_TYPE_MIGRATION pages, but maybe no problem.
What are the exact concerns?
> Note: unuse_vma() doesn't check what pte entry contains.
unuse_vma() relies on the mapping via swap space that will no longer exist
with the new code.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 5/6] Swapless V1: Rip out swap migration code
From: KAMEZAWA Hiroyuki @ 2006-04-05 3:33 UTC (permalink / raw)
To: Christoph Lameter
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <Pine.LNX.4.64.0604041940390.28908@schroedinger.engr.sgi.com>
On Tue, 4 Apr 2006 19:45:49 -0700 (PDT)
Christoph Lameter <clameter@sgi.com> wrote:
> > My concern is refcnt handling of SWP_TYPE_MIGRATION pages, but maybe no problem.
>
> What are the exact concerns?
>
When a page is converted into SWP_TYPE_MIGRATION, changed pte entry
implicitly points old page. This introduces the state 'a page is referred
but no refcnt'. if mmap_sem is held, this is maybe no problem.
but looks a bit dangerous.
> On Wed, 5 Apr 2006, KAMEZAWA Hiroyuki wrote:
>
> > I think adding SWP_TYPE_MIGRATION consideration to free_swap_and_cache() is
> > enough against anon_vma vanishing. Because remove_migration_ptes() compares
> > old pte entry with old page's pfn, a page cannot be remapped into old place
> > when anon_vma has gone. This is my first impression.
>
> However, the last process containing the page may terminate and free the
> page, while we migrate. The SWAP_TYPE_MIGRATION pte will be rewoved
> together with the anonvma if no lock is held on mmap_sem.
yes.
> Then remove_migration_ptes() cannot obtain a anon_vma. So it would break
> without holding mmap_sem. We could fix this if we could somehow know that
> the last process mapping the page vanished and skip
> remove_migration_ptes().
>
Hmm, I'm not sure but how about this way ?
1. don't drop refcnt in try_to_unmap_one() when changing a page to
SWP_TYPE_MIGRATION. because it is referred. (rmap should be removed ?)
2. drop refcnt of the old page and inc refcnt of the new page in
remove_migration_ptes()
like this.
==
in remove_migration_pte
+ ptep = page_check_address(old, mm, addr, &ptl);
+ if (!ptep)
+ return;
+
+ get_page(new);
+ set_pte_at(mm, addr, ptep, pte_mkold(mk_pte(new, vma->vm_page_prot)));
+ page_add_anon_rmap(new, vma, addr);
+ put_page(old); << add this
We can check old page's refcnt in remove_migration_ptes().
if page_count(oldpage)==1, this page's anon_vma is removed.
So we don't have to modify ptes, all of them are zapped..
(In this method, page's refcnt should be dropped when swp_entry
for SWP_TYPE_MIGRATION is freed.)
In page unmapping, each page's refcnt is dropped before zapping anon_vma.
So, I think this can work.
>
> > Note: unuse_vma() doesn't check what pte entry contains.
>
> unuse_vma() relies on the mapping via swap space that will no longer exist
> with the new code.
>
Yes. I know. just wrote about old code. sorry.
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 5/6] Swapless V1: Rip out swap migration code
From: Christoph Lameter @ 2006-04-05 3:47 UTC (permalink / raw)
To: KAMEZAWA Hiroyuki
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <20060405123341.52145bf5.kamezawa.hiroyu@jp.fujitsu.com>
On Wed, 5 Apr 2006, KAMEZAWA Hiroyuki wrote:
> On Tue, 4 Apr 2006 19:45:49 -0700 (PDT)
> Christoph Lameter <clameter@sgi.com> wrote:
>
> > > My concern is refcnt handling of SWP_TYPE_MIGRATION pages, but maybe no problem.
> >
> > What are the exact concerns?
> >
> When a page is converted into SWP_TYPE_MIGRATION, changed pte entry
> implicitly points old page. This introduces the state 'a page is referred
> but no refcnt'. if mmap_sem is held, this is maybe no problem.
> but looks a bit dangerous.
We have increased the refcnt on the page (see isolate_lru_page()) and the
page is locked when SWP_TYPE_MIGRATION is used. So there is a refcnt.
> > > I think adding SWP_TYPE_MIGRATION consideration to free_swap_and_cache() is
> > > enough against anon_vma vanishing. Because remove_migration_ptes() compares
> > > old pte entry with old page's pfn, a page cannot be remapped into old place
> > > when anon_vma has gone. This is my first impression.
> >
> > However, the last process containing the page may terminate and free the
> > page, while we migrate. The SWAP_TYPE_MIGRATION pte will be rewoved
> > together with the anonvma if no lock is held on mmap_sem.
> yes.
>
> > Then remove_migration_ptes() cannot obtain a anon_vma. So it would break
> > without holding mmap_sem. We could fix this if we could somehow know that
> > the last process mapping the page vanished and skip
> > remove_migration_ptes().
> >
>
> Hmm, I'm not sure but how about this way ?
> 1. don't drop refcnt in try_to_unmap_one() when changing a page to
> SWP_TYPE_MIGRATION. because it is referred. (rmap should be removed ?)
Then we would have a page with mapcounts but there are no real ptes
pointing to the page. It would be a strange condition for the page.
Moreover, a process may fork or terminate while we migrate. Forking may
increase the refcnt and termination may decrease it. We do not keep
refcnts for the SWP_TYPE_MIGRATION entry but rely on the reverse maps. So
we may end up with a messed up mapcount if we do not drop the refcnts.
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* Re: [RFC 5/6] Swapless V1: Rip out swap migration code
From: KAMEZAWA Hiroyuki @ 2006-04-05 4:07 UTC (permalink / raw)
To: Christoph Lameter
Cc: linux-mm, lee.schermerhorn, lhms-devel, taka, marcelo.tosatti
In-Reply-To: <Pine.LNX.4.64.0604042038370.31431@schroedinger.engr.sgi.com>
On Tue, 4 Apr 2006 20:47:58 -0700 (PDT)
Christoph Lameter <clameter@sgi.com> wrote:
> > When a page is converted into SWP_TYPE_MIGRATION, changed pte entry
> > implicitly points old page. This introduces the state 'a page is referred
> > but no refcnt'. if mmap_sem is held, this is maybe no problem.
> > but looks a bit dangerous.
>
> We have increased the refcnt on the page (see isolate_lru_page()) and the
> page is locked when SWP_TYPE_MIGRATION is used. So there is a refcnt.
>
yes. I just wrote about implicit refcnt.
> > > > I think adding SWP_TYPE_MIGRATION consideration to free_swap_and_cache() is
> > > > enough against anon_vma vanishing. Because remove_migration_ptes() compares
> > > > old pte entry with old page's pfn, a page cannot be remapped into old place
> > > > when anon_vma has gone. This is my first impression.
> > >
> > > However, the last process containing the page may terminate and free the
> > > page, while we migrate. The SWAP_TYPE_MIGRATION pte will be rewoved
> > > together with the anonvma if no lock is held on mmap_sem.
> > yes.
> >
> > > Then remove_migration_ptes() cannot obtain a anon_vma. So it would break
> > > without holding mmap_sem. We could fix this if we could somehow know that
> > > the last process mapping the page vanished and skip
> > > remove_migration_ptes().
> > >
> >
> > Hmm, I'm not sure but how about this way ?
> > 1. don't drop refcnt in try_to_unmap_one() when changing a page to
> > SWP_TYPE_MIGRATION. because it is referred. (rmap should be removed ?)
>
> Then we would have a page with mapcounts but there are no real ptes
> pointing to the page. It would be a strange condition for the page.
>
O.K. dropping mapcount is necessary. (migrate_page_remove_reference checks it,
anyway)
refcnt mentioned above is page_count(page).
> Moreover, a process may fork or terminate while we migrate. Forking may
> increase the refcnt and termination may decrease it. We do not keep
> refcnts for the SWP_TYPE_MIGRATION entry but rely on the reverse maps. So
> we may end up with a messed up mapcount if we do not drop the refcnts.
At fork, copy_one_pte() can manage swap entry.
Adding SWP_TYPE_MIGRATION consideration there is necessary and enough if
not holding mmap_sem. Hmm...maybe.
exit is the same case as zap_page_range(). modifing swap_entry_free() will be
necessary.
-Kame
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [Patch:000/004] wait_table and zonelist initializing for memory hotadd
From: Yasunori Goto @ 2006-04-05 10:57 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux Kernel ML, linux-mm, Yasunori Goto
Hi.
These are parts of patches for new nodes addition v4.
I picked them up because v4 might be a bit too many patches.
These patches can be used even when a new zone becomes available.
When empty zone becomes not empty, wait_table must be initialized,
and zonelists must be updated.
So, They are a good group for once post.
ex) x86-64 is good example of new zone addition.
- System boot up with memory under 4G address.
All of memory will be ZONE_DMA32.
- Then hot-add over 4G memory. It becomes ZONE_NORMAL. But,
wait table of zone normal is not initialized at this time.
This patch is for 2.6.17-rc1-mm1.
Please apply.
----------------------------
Change log from v4 of hot-add.
- update for 2.6.17-rc1-mm1.
- change allocation for wait_table from kmalloc() to vmalloc().
vmalloc() is enough for it.
V4 of post is here.
<description>
http://marc.theaimsgroup.com/?l=linux-mm&w=2&r=1&s=memory+hotplug+node+v.4&q=b
<patches>
http://marc.theaimsgroup.com/?l=linux-mm&w=2&r=1&s=memory+hotplug+node+v.4.&q=b
--
Yasunori Goto
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [Patch:001/004] wait_table and zonelist initializing for memory hotadd (change to meminit for build_zonelist)
From: Yasunori Goto @ 2006-04-05 11:01 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux Kernel ML, linux-mm
In-Reply-To: <20060405192737.3C3F.Y-GOTO@jp.fujitsu.com>
This is a patch to change definition of some functions and data
from __init to __meminit.
These functions and data can be used after bootup by this patch to
be used for hot-add codes.
Signed-off-by: Yasunori Goto <y-goto@jp.fujitsu.com>
include/linux/bootmem.h | 4 ++--
mm/page_alloc.c | 18 +++++++++---------
2 files changed, 11 insertions(+), 11 deletions(-)
Index: pgdat10/mm/page_alloc.c
===================================================================
--- pgdat10.orig/mm/page_alloc.c 2006-04-05 16:04:03.000000000 +0900
+++ pgdat10/mm/page_alloc.c 2006-04-05 16:04:12.000000000 +0900
@@ -81,8 +81,8 @@ EXPORT_SYMBOL(zone_table);
static char *zone_names[MAX_NR_ZONES] = { "DMA", "DMA32", "Normal", "HighMem" };
int min_free_kbytes = 1024;
-unsigned long __initdata nr_kernel_pages;
-unsigned long __initdata nr_all_pages;
+unsigned long __meminitdata nr_kernel_pages;
+unsigned long __meminitdata nr_all_pages;
#ifdef CONFIG_DEBUG_VM
static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
@@ -1575,7 +1575,7 @@ void show_free_areas(void)
*
* Add all populated zones of a node to the zonelist.
*/
-static int __init build_zonelists_node(pg_data_t *pgdat,
+static int __meminit build_zonelists_node(pg_data_t *pgdat,
struct zonelist *zonelist, int nr_zones, int zone_type)
{
struct zone *zone;
@@ -1611,7 +1611,7 @@ static inline int highest_zone(int zone_
#ifdef CONFIG_NUMA
#define MAX_NODE_LOAD (num_online_nodes())
-static int __initdata node_load[MAX_NUMNODES];
+static int __meminitdata node_load[MAX_NUMNODES];
/**
* find_next_best_node - find the next node that should appear in a given node's fallback list
* @node: node whose fallback list we're appending
@@ -1626,7 +1626,7 @@ static int __initdata node_load[MAX_NUMN
* on them otherwise.
* It returns -1 if no node is found.
*/
-static int __init find_next_best_node(int node, nodemask_t *used_node_mask)
+static int __meminit find_next_best_node(int node, nodemask_t *used_node_mask)
{
int n, val;
int min_val = INT_MAX;
@@ -1672,7 +1672,7 @@ static int __init find_next_best_node(in
return best_node;
}
-static void __init build_zonelists(pg_data_t *pgdat)
+static void __meminit build_zonelists(pg_data_t *pgdat)
{
int i, j, k, node, local_node;
int prev_node, load;
@@ -1724,7 +1724,7 @@ static void __init build_zonelists(pg_da
#else /* CONFIG_NUMA */
-static void __init build_zonelists(pg_data_t *pgdat)
+static void __meminit build_zonelists(pg_data_t *pgdat)
{
int i, j, k, node, local_node;
@@ -2130,7 +2130,7 @@ static __meminit void init_currently_emp
* - mark all memory queues empty
* - clear the memory bitmaps
*/
-static void __init free_area_init_core(struct pglist_data *pgdat,
+static void __meminit free_area_init_core(struct pglist_data *pgdat,
unsigned long *zones_size, unsigned long *zholes_size)
{
unsigned long j;
@@ -2210,7 +2210,7 @@ static void __init alloc_node_mem_map(st
#endif /* CONFIG_FLAT_NODE_MEM_MAP */
}
-void __init free_area_init_node(int nid, struct pglist_data *pgdat,
+void __meminit free_area_init_node(int nid, struct pglist_data *pgdat,
unsigned long *zones_size, unsigned long node_start_pfn,
unsigned long *zholes_size)
{
Index: pgdat10/include/linux/bootmem.h
===================================================================
--- pgdat10.orig/include/linux/bootmem.h 2006-04-05 16:04:03.000000000 +0900
+++ pgdat10/include/linux/bootmem.h 2006-04-05 16:04:12.000000000 +0900
@@ -91,8 +91,8 @@ static inline void *alloc_remap(int nid,
}
#endif
-extern unsigned long __initdata nr_kernel_pages;
-extern unsigned long __initdata nr_all_pages;
+extern unsigned long nr_kernel_pages;
+extern unsigned long nr_all_pages;
extern void *__init alloc_large_system_hash(const char *tablename,
unsigned long bucketsize,
--
Yasunori Goto
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
* [Patch:002/004] wait_table and zonelist initializing for memory hotadd (add return code for init_current_empty_zone)
From: Yasunori Goto @ 2006-04-05 11:01 UTC (permalink / raw)
To: Andrew Morton; +Cc: Linux Kernel ML, linux-mm
In-Reply-To: <20060405192737.3C3F.Y-GOTO@jp.fujitsu.com>
When add_zone() is called against empty zone (not populated zone),
we have to initialize the zone which didn't initialize at boot time.
But, init_currently_empty_zone() may fail due to allocation of
wait table. So, this patch is to catch its error code.
Changes against wait_table is in the next patch.
Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Signed-off-by: Yasunori Goto <y-goto@jp.fujitsu.com>
include/linux/mmzone.h | 3 +++
mm/memory_hotplug.c | 15 +++++++++++++--
mm/page_alloc.c | 11 ++++++++---
3 files changed, 24 insertions(+), 5 deletions(-)
Index: pgdat10/mm/page_alloc.c
===================================================================
--- pgdat10.orig/mm/page_alloc.c 2006-03-31 14:43:33.000000000 +0900
+++ pgdat10/mm/page_alloc.c 2006-03-31 15:50:08.000000000 +0900
@@ -2109,8 +2109,9 @@ static __meminit void zone_pcp_init(stru
zone->name, zone->present_pages, batch);
}
-static __meminit void init_currently_empty_zone(struct zone *zone,
- unsigned long zone_start_pfn, unsigned long size)
+__meminit int init_currently_empty_zone(struct zone *zone,
+ unsigned long zone_start_pfn,
+ unsigned long size)
{
struct pglist_data *pgdat = zone->zone_pgdat;
@@ -2122,6 +2123,8 @@ static __meminit void init_currently_emp
memmap_init(size, pgdat->node_id, zone_idx(zone), zone_start_pfn);
zone_init_free_lists(pgdat, zone, zone->spanned_pages);
+
+ return 0;
}
/*
@@ -2136,6 +2139,7 @@ static void __init free_area_init_core(s
unsigned long j;
int nid = pgdat->node_id;
unsigned long zone_start_pfn = pgdat->node_start_pfn;
+ int ret;
pgdat_resize_init(pgdat);
pgdat->nr_zones = 0;
@@ -2177,7 +2181,8 @@ static void __init free_area_init_core(s
continue;
zonetable_add(zone, nid, j, zone_start_pfn, size);
- init_currently_empty_zone(zone, zone_start_pfn, size);
+ ret = init_currently_empty_zone(zone, zone_start_pfn, size);
+ BUG_ON(ret);
zone_start_pfn += size;
}
}
Index: pgdat10/mm/memory_hotplug.c
===================================================================
--- pgdat10.orig/mm/memory_hotplug.c 2006-03-22 17:25:06.000000000 +0900
+++ pgdat10/mm/memory_hotplug.c 2006-03-31 15:50:08.000000000 +0900
@@ -26,7 +26,7 @@
extern void zonetable_add(struct zone *zone, int nid, int zid, unsigned long pfn,
unsigned long size);
-static void __add_zone(struct zone *zone, unsigned long phys_start_pfn)
+static int __add_zone(struct zone *zone, unsigned long phys_start_pfn)
{
struct pglist_data *pgdat = zone->zone_pgdat;
int nr_pages = PAGES_PER_SECTION;
@@ -34,8 +34,15 @@ static void __add_zone(struct zone *zone
int zone_type;
zone_type = zone - pgdat->node_zones;
+ if (!populated_zone(zone)) {
+ int ret = 0;
+ ret = init_currently_empty_zone(zone, phys_start_pfn, nr_pages);
+ if (ret < 0)
+ return ret;
+ }
memmap_init_zone(nr_pages, nid, zone_type, phys_start_pfn);
zonetable_add(zone, nid, zone_type, phys_start_pfn, nr_pages);
+ return 0;
}
extern int sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
@@ -50,7 +57,11 @@ static int __add_section(struct zone *zo
if (ret < 0)
return ret;
- __add_zone(zone, phys_start_pfn);
+ ret = __add_zone(zone, phys_start_pfn);
+
+ if (ret < 0)
+ return ret;
+
return register_new_memory(__pfn_to_section(phys_start_pfn));
}
Index: pgdat10/include/linux/mmzone.h
===================================================================
--- pgdat10.orig/include/linux/mmzone.h 2006-03-31 14:43:32.000000000 +0900
+++ pgdat10/include/linux/mmzone.h 2006-03-31 15:50:08.000000000 +0900
@@ -332,6 +332,9 @@ void wakeup_kswapd(struct zone *zone, in
int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
int classzone_idx, int alloc_flags);
+extern int init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
+ unsigned long size);
+
#ifdef CONFIG_HAVE_MEMORY_PRESENT
void memory_present(int nid, unsigned long start, unsigned long end);
#else
--
Yasunori Goto
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
^ permalink raw reply
page: next (older) | prev (newer) | latest
- recent:[subjects (threaded)|topics (new)|topics (active)]
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox