[PATCH/RFC 1/11] numa - Automatic-migration - preparation, cleanup

[Lee Schermerhorn <lee.schermerhorn@hp.com>]

AutoPage Migration -  Preparatory patch

Added AUTO_MIGRATION Kconfig option that depends on MIGRATION.
Conditionally compiled auto-migration features now controlled
by this option.

Define mempolicy.c internal flag for auto-migration.  This flag
will select auto-migration specific behavior in the existing
page migration functions.  Test this flag via helper function
is_auto_migration().  Can't be static inline in header because
flag is private to mempolicy.c.

Add auto_migrate_task_memory() to mempolicy.c.  This function sets up
to call migrate_to_node() with internal flags for auto-migration.

Modify vma_migratable() to skip VMAs that don't have local policy
when auto-migrating.  vma_migratable() now called from check_range()
in mempolicy.c and do_move_pages() in migrate.c.

Subsequent patches will arrange for auto_migrate_task_memory() to be
called when a task returns to user space after the scheduler migrates
it to a cpu on a node different from the node where it last executed.

Signed-off-by:  Lee Schermerhorn <lee.schermerhorn@hp.com>

 include/linux/auto-migrate.h |   25 +++++++++++++++
 include/linux/mempolicy.h    |   13 +++++++
 mm/Kconfig                   |    7 ++++
 mm/mempolicy.c               |   71 ++++++++++++++++++++++++++++++++++++++-----
 mm/migrate.c                 |    3 +
 5 files changed, 110 insertions(+), 9 deletions(-)

Index: linux-2.6.36-mmotm-101103-1217/mm/Kconfig
===================================================================
--- linux-2.6.36-mmotm-101103-1217.orig/mm/Kconfig
+++ linux-2.6.36-mmotm-101103-1217/mm/Kconfig
@@ -211,6 +211,13 @@ config MIGRATE_ON_FAULT
 	  page is not currently mapped by any tasks.  This allows a task to
 	  pull unmapped pages closer to itself when enabled for that task.
 
+config AUTO_MIGRATION
+	bool "Auto-migrate task memory"
+	depends on MIGRATION
+	help
+	  Allows tasks' private memory to follow that task itself across
+	  inter-node migrations.
+
 config PHYS_ADDR_T_64BIT
 	def_bool 64BIT || ARCH_PHYS_ADDR_T_64BIT
 
Index: linux-2.6.36-mmotm-101103-1217/include/linux/auto-migrate.h
===================================================================
--- /dev/null
+++ linux-2.6.36-mmotm-101103-1217/include/linux/auto-migrate.h
@@ -0,0 +1,25 @@
+#ifndef _LINUX_AUTO_MIGRATE_H
+#define _LINUX_AUTO_MIGRATE_H
+
+/*
+ * minimal memory migration definitions need by scheduler,
+ * sysctl, ..., so that they don't need to drag in the entire
+ * migrate.h and all that it depends on.
+ */
+
+#ifdef CONFIG_AUTO_MIGRATION
+
+extern int is_auto_migration(int flags);
+
+extern void auto_migrate_task_memory(void);
+
+#else	/* !CONFIG_AUTO_MIGRATION */
+
+static inline int is_auto_migration(int flags)
+{
+	return 0;
+}
+
+#endif	/* CONFIG_AUTO_MIGRATION */
+
+#endif
Index: linux-2.6.36-mmotm-101103-1217/mm/mempolicy.c
===================================================================
--- linux-2.6.36-mmotm-101103-1217.orig/mm/mempolicy.c
+++ linux-2.6.36-mmotm-101103-1217/mm/mempolicy.c
@@ -91,6 +91,7 @@
 #include <linux/syscalls.h>
 #include <linux/ctype.h>
 #include <linux/mm_inline.h>
+#include <linux/auto-migrate.h>
 
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
@@ -98,10 +99,16 @@
 #include "internal.h"
 
 /* Internal flags */
-#define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)	/* Skip checks for continuous vmas */
-#define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)		/* Invert check for nodemask */
-#define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2)		/* Gather statistics */
-#define MPOL_MF_MOVE_ANON_ONLY (MPOL_MF_INTERNAL << 3)
+#define MPOL_MF_DISCONTIG_OK \
+	(MPOL_MF_INTERNAL << 0)		/* Skip checks for continuous vmas */
+#define MPOL_MF_INVERT \
+	(MPOL_MF_INTERNAL << 1)		/* Invert check for nodemask */
+#define MPOL_MF_STATS \
+	(MPOL_MF_INTERNAL << 2)		/* Gather statistics */
+#define MPOL_MF_MOVE_ANON_ONLY \
+	(MPOL_MF_INTERNAL << 3)		/* migrate private, anon pages only */
+#define MPOL_MF_AUTOMIGRATE \
+	(MPOL_MF_INTERNAL << 4)		/* auto-migrating task memory */
 
 static struct kmem_cache *policy_cache;
 static struct kmem_cache *sp_cache;
@@ -467,8 +474,10 @@ static void migrate_page_add(struct page
 /*
  * Check whether a vma is migratable
  */
-int vma_migratable(struct vm_area_struct *vma)
+int vma_migratable(struct vm_area_struct *vma, int flags)
 {
+	int ret = 1;
+
 	if (vma->vm_flags & (VM_IO|VM_HUGETLB|VM_PFNMAP|VM_RESERVED))
 		return 0;
 	/*
@@ -480,7 +489,20 @@ int vma_migratable(struct vm_area_struct
 		gfp_zone(mapping_gfp_mask(vma->vm_file->f_mapping))
 								< policy_zone)
 			return 0;
-	return 1;
+
+	/*
+	 * Auto-migration:  only consider vmas with local allocation policy
+	 * NOTE:  we only query the start address of the vma.  For shared
+	 * segments with multiple policy ranges, this might lie, but we'll
+	 * live with that.
+	 */
+	if (is_auto_migration(flags)) {
+		struct mempolicy *pol =
+			get_vma_policy(current, vma, vma->vm_start);
+		ret = is_local_allocation(pol);
+		mpol_cond_put(pol);
+	}
+	return ret;
 }
 
 /* Scan through pages checking if pages follow certain conditions. */
@@ -627,7 +649,7 @@ check_range(struct mm_struct *mm, unsign
 				end = (end + HPAGE_MASK) & HPAGE_MASK;
 		} else if (((flags & MPOL_MF_STRICT) ||
 		     ((flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) &&
-				vma_migratable(vma)))) {
+				vma_migratable(vma, flags)))) {
 			unsigned long endvma = vma->vm_end;
 			unsigned long anononly = 0;
 
@@ -1190,6 +1212,41 @@ static struct page *new_vma_page(struct
 	 */
 	return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
 }
+
+#ifdef CONFIG_AUTO_MIGRATION
+
+int is_auto_migration(int flags)
+{
+	return !!(flags & MPOL_MF_AUTOMIGRATE);
+}
+
+/**
+ * auto_migrate_task_memory()
+ *
+ * Called just before returning to user state when a task has been
+ * migrated to a new node by the schedule and sched_migrate_memory
+ * is enabled.
+ */
+void auto_migrate_task_memory(void)
+{
+	struct mm_struct *mm = current->mm;
+	int dest = cpu_to_node(task_cpu(current));
+	int flags = MPOL_MF_MOVE | MPOL_MF_INVERT | MPOL_MF_AUTOMIGRATE;
+
+	/*
+	 * we're returning to user space, so mm must be non-NULL
+	 */
+	BUG_ON(!mm);
+
+	/*
+	 * Pass destination node as source node plus 'INVERT flag:
+	 *    Migrate all pages NOT on destination node.
+	 * 'AUTOMIGRATE flag selects only VMAs with default policy
+	 */
+	migrate_to_node(mm, dest, dest, flags);
+}
+#endif	/* _AUTO_MIGRATION */
+
 #else
 
 static void migrate_page_add(struct page *page, struct list_head *pagelist,
Index: linux-2.6.36-mmotm-101103-1217/mm/migrate.c
===================================================================
--- linux-2.6.36-mmotm-101103-1217.orig/mm/migrate.c
+++ linux-2.6.36-mmotm-101103-1217/mm/migrate.c
@@ -35,6 +35,7 @@
 #include <linux/hugetlb.h>
 #include <linux/gfp.h>
 #include <linux/vmstat.h>
+#include <linux/auto-migrate.h>
 
 #include "internal.h"
 
@@ -1156,7 +1157,7 @@ static int do_move_page_to_node_array(st
 
 		err = -EFAULT;
 		vma = find_vma(mm, pp->addr);
-		if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
+		if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma, 0))
 			goto set_status;
 
 		page = follow_page(vma, pp->addr, FOLL_GET);
Index: linux-2.6.36-mmotm-101103-1217/include/linux/mempolicy.h
===================================================================
--- linux-2.6.36-mmotm-101103-1217.orig/include/linux/mempolicy.h
+++ linux-2.6.36-mmotm-101103-1217/include/linux/mempolicy.h
@@ -234,7 +234,7 @@ extern int mpol_to_str(char *buffer, int
 			int no_context);
 #endif
 
-extern int vma_migratable(struct vm_area_struct *);
+extern int vma_migratable(struct vm_area_struct *, int);
 
 struct seq_file;
 extern int show_numa_map(struct seq_file *, void *);
@@ -249,6 +249,14 @@ extern struct mpol_range *get_numa_subma
 extern int mpol_misplaced(struct page *, struct vm_area_struct *,
 		unsigned long, int *);
 
+/*
+ * Does the argument mempolicy specify local allocation?
+ */
+static inline int is_local_allocation(struct mempolicy *mpol)
+{
+	return mpol->flags & MPOL_F_LOCAL;
+}
+
 #endif /* CONFIG_MIGRATE_ON_FAULT */
 
 #else
@@ -368,6 +376,9 @@ static inline int mpol_to_str(char *buff
 }
 #endif
 
+static inline int vma_migratable(struct vm_area_struct *vma, int flags)
+					{ return 0 };
+
 #endif /* CONFIG_NUMA */
 #endif /* __KERNEL__ */