[PATCH/RFC 3/11] numa - Automatic-migration - check notify migrate pending

[Lee Schermerhorn <lee.schermerhorn@hp.com>]

AutoPage Migration - generic check/notify internode migration

This patch adds the check for internode migration to be called
from scheduler load balancing functions, and the check for migration
pending to be called when a task returning to user space notices
NOTIFY_RESUME pending

Check for internode migration:  if automatic memory migration
is enabled [auto_migrate_enabled(task)] and this is a user task and
the destination cpu is on a different node from the task's current cpu,
the task will be marked for migration pending via member added to task
struct.  The TIF_NOTIFY_RESUME thread_info flag is set to cause the
task to enter do_notify_resume[_user]() to check for migration pending.

When a task is rescheduled to user space with TIF_NOTIFY_RESUME,
it will check for migration pending, unless SIGKILL is pending.
If the task notices migration pending, it will call
auto_migrate_task_memory() to migrate pages in vma's with default
policy.  Only default policy is affected by migration to a new node.

Note that we can't call auto_migrate_task_memory() with interrupts
disabled.  Temporarily enable interrupts around the call.
The check is last in line before returning to user space so this
should be safe here.

These checks become empty macros when 'AUTO_MIGRATION' is not
configured.

Signed-off-by:  Lee Schermerhorn <lee.schermerhorn@hp.com>

 include/linux/auto-migrate.h |   53 ++++++++++++++++++++++++++++++++++++++++++-
 include/linux/sched.h        |    4 +++
 mm/mempolicy.c               |   20 ++++++++++++++++
 3 files changed, 76 insertions(+), 1 deletion(-)

Index: linux-2.6.36-mmotm-101103-1217/include/linux/sched.h
===================================================================
--- linux-2.6.36-mmotm-101103-1217.orig/include/linux/sched.h
+++ linux-2.6.36-mmotm-101103-1217/include/linux/sched.h
@@ -1459,6 +1459,9 @@ struct task_struct {
 	short shared_file_policy_enabled:1;
 	short migrate_on_fault_enabled:1;
 	short auto_migrate_enabled:1;
+#ifdef CONFIG_AUTO_MIGRATION
+	short migrate_pending:1;	/* internode mem migration pending */
+#endif
 #endif
 	atomic_t fs_excl;	/* holding fs exclusive resources */
 	struct rcu_head rcu;
@@ -1928,6 +1931,7 @@ static inline int migrate_on_fault_enabl
 #endif
 
 #ifdef CONFIG_AUTO_MIGRATION
+#define SCHED_AUTO_MIGRATION 1
 static inline void set_auto_migrate_enabled(struct task_struct *tsk,
 							int val)
 {
Index: linux-2.6.36-mmotm-101103-1217/include/linux/auto-migrate.h
===================================================================
--- linux-2.6.36-mmotm-101103-1217.orig/include/linux/auto-migrate.h
+++ linux-2.6.36-mmotm-101103-1217/include/linux/auto-migrate.h
@@ -10,9 +10,49 @@
 #ifdef CONFIG_AUTO_MIGRATION
 
 extern int is_auto_migration(int flags);
-
 extern void auto_migrate_task_memory(void);
 
+#ifdef SCHED_AUTO_MIGRATION
+/* these need sched.h definition.  They're only where sched.h is
+ * already included.  Note we depend on sched.h being included
+ * first to see these functions.
+ */
+extern void __check_internode_migration(struct task_struct *, int);
+
+static inline void check_internode_migration(struct task_struct *task,
+			int dest_cpu)
+{
+	if (auto_migrate_enabled(task))
+		__check_internode_migration(task, dest_cpu);
+}
+
+/*
+ * called only by arch dependent code for architectures that
+ * support "migration work"
+ */
+static inline void check_migrate_pending(void)
+{
+	if (unlikely(current->migrate_pending)) {
+		int disable_irqs = 0;
+
+		if (irqs_disabled()) {
+			disable_irqs = 1;
+			local_irq_enable();
+		}
+
+		/*
+		 * can't be called in atomic context.
+		 */
+		auto_migrate_task_memory();
+
+		if (disable_irqs)
+			local_irq_disable();
+	}
+	current->migrate_pending = 0;
+	return;
+}
+#endif /* SCHED_AUTO_MIGRATION */
+
 #else	/* !CONFIG_AUTO_MIGRATION */
 
 static inline int is_auto_migration(int flags)
@@ -20,6 +60,17 @@ static inline int is_auto_migration(int
 	return 0;
 }
 
+static int is_auto_migration(int flags) { return 0; }
+
+static inline void check_internode_migration(struct task_struct *tsk, int cpu)
+{
+}
+
+static inline void check_migrate_pending(void)
+{
+	clear_thread_flag(TIF_NOTIFY_RESUME);
+}
+
 #endif	/* CONFIG_AUTO_MIGRATION */
 
 #endif
Index: linux-2.6.36-mmotm-101103-1217/mm/mempolicy.c
===================================================================
--- linux-2.6.36-mmotm-101103-1217.orig/mm/mempolicy.c
+++ linux-2.6.36-mmotm-101103-1217/mm/mempolicy.c
@@ -1215,6 +1215,26 @@ static struct page *new_vma_page(struct
 
 #ifdef CONFIG_AUTO_MIGRATION
 
+/*
+ * Check for task migration to a new node and flag to auto-migrate task memory.
+ * Only called if auto-migration is enabled for this task.
+ */
+void __check_internode_migration(struct task_struct *task,
+			int dest_cpu)
+{
+	if (task->mm) {
+		int node = cpu_to_node(task_cpu(task));
+		if ((node != cpu_to_node(dest_cpu))) {
+			/*
+			 * migrating a user task to a new node.
+			 * mark for memory migration on return to user space.
+			 */
+			task->migrate_pending = 1;
+			set_tsk_thread_flag(task, TIF_NOTIFY_RESUME);
+		}
+	}
+}
+
 int is_auto_migration(int flags)
 {
 	return !!(flags & MPOL_MF_AUTOMIGRATE);