Re: Add prefetch switch stack hook in scheduler function

[Ingo Molnar <mingo@elte.hu>]

* Ingo Molnar <mingo@elte.hu> wrote:

> [...] If yes then we want to have something like:
> 
> 	prefetch(kernel_stack(next));
> 
> to make it more generic. By default kernel_stack(next) could be 
> next->thread_info (to make sure we prefetch something real). On e.g. 
> x86/x64, kernel_stack(next) should be something like next->thread.esp.

i.e. like the patch below. Boot-tested on x86. x86, x64 and ia64 have a 
real kernel_stack() implementation, the other architectures all return 
'next'. (I've also cleaned up a couple of other things in the 
prefetch-next area, see the changelog below.)

Ken, would this patch generate a sufficient amount of prefetching on 
ia64?

	Ingo

-----
For architecture like ia64, the switch stack structure is fairly large
(currently 528 bytes).  For context switch intensive application, we
found that significant amount of cache misses occurs in switch_to()
function.  The following patch adds a hook in the schedule() function to
prefetch switch stack structure as soon as 'next' task is determined. 
This allows maximum overlap in prefetch cache lines for that structure.

Signed-off-by: Ken Chen <kenneth.w.chen@intel.com>

modifications:

- add a generic kernel_stack() function instead of a hook
- prefetch the next task's thread_info and kernel stack as well
- dont try to prefetch 'next' itself - we've already touched it
  so the cacheline is present.
- do the prefetching as early as possible

currently covered architectures: ia64, x86, x64. (Other architectures
will have to replace the default 'return task' in their kernel_stack()
implementations.)

Signed-off-by: Ingo Molnar <mingo@elte.hu>

 include/asm-alpha/mmu_context.h     |    9 +++++++++
 include/asm-arm/mmu_context.h       |    9 +++++++++
 include/asm-arm26/mmu_context.h     |    9 +++++++++
 include/asm-cris/mmu_context.h      |    9 +++++++++
 include/asm-frv/mmu_context.h       |    9 +++++++++
 include/asm-h8300/mmu_context.h     |    9 +++++++++
 include/asm-i386/mmu_context.h      |    5 +++++
 include/asm-ia64/mmu_context.h      |    9 +++++++++
 include/asm-m32r/mmu_context.h      |    9 +++++++++
 include/asm-m68k/mmu_context.h      |    9 +++++++++
 include/asm-m68knommu/mmu_context.h |    9 +++++++++
 include/asm-mips/mmu_context.h      |    9 +++++++++
 include/asm-parisc/mmu_context.h    |   10 ++++++++++
 include/asm-ppc/mmu_context.h       |    9 +++++++++
 include/asm-ppc64/mmu_context.h     |    9 +++++++++
 include/asm-s390/mmu_context.h      |    9 +++++++++
 include/asm-sh/mmu_context.h        |    9 +++++++++
 include/asm-sh64/mmu_context.h      |    9 +++++++++
 include/asm-sparc/mmu_context.h     |    9 +++++++++
 include/asm-sparc64/mmu_context.h   |    9 +++++++++
 include/asm-um/mmu_context.h        |    9 +++++++++
 include/asm-v850/mmu_context.h      |    9 +++++++++
 include/asm-x86_64/mmu_context.h    |    9 +++++++++
 include/asm-xtensa/mmu_context.h    |    9 +++++++++
 kernel/sched.c                      |    9 ++++++++-
 25 files changed, 221 insertions(+), 1 deletion(-)

Index: linux/include/asm-alpha/mmu_context.h
=================================--- linux.orig/include/asm-alpha/mmu_context.h
+++ linux/include/asm-alpha/mmu_context.h
@@ -258,4 +258,13 @@ enter_lazy_tlb(struct mm_struct *mm, str
 #undef __MMU_EXTERN_INLINE
 #endif
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* __ALPHA_MMU_CONTEXT_H */
Index: linux/include/asm-arm/mmu_context.h
=================================--- linux.orig/include/asm-arm/mmu_context.h
+++ linux/include/asm-arm/mmu_context.h
@@ -93,4 +93,13 @@ switch_mm(struct mm_struct *prev, struct
 #define deactivate_mm(tsk,mm)	do { } while (0)
 #define activate_mm(prev,next)	switch_mm(prev, next, NULL)
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
Index: linux/include/asm-arm26/mmu_context.h
=================================--- linux.orig/include/asm-arm26/mmu_context.h
+++ linux/include/asm-arm26/mmu_context.h
@@ -48,4 +48,13 @@ static inline void activate_mm(struct mm
 	cpu_switch_mm(next->pgd, next);
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
Index: linux/include/asm-cris/mmu_context.h
=================================--- linux.orig/include/asm-cris/mmu_context.h
+++ linux/include/asm-cris/mmu_context.h
@@ -21,4 +21,13 @@ static inline void enter_lazy_tlb(struct
 {
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
Index: linux/include/asm-frv/mmu_context.h
=================================--- linux.orig/include/asm-frv/mmu_context.h
+++ linux/include/asm-frv/mmu_context.h
@@ -47,4 +47,13 @@ do {									\
 do {						\
 } while(0)
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
Index: linux/include/asm-h8300/mmu_context.h
=================================--- linux.orig/include/asm-h8300/mmu_context.h
+++ linux/include/asm-h8300/mmu_context.h
@@ -29,4 +29,13 @@ extern inline void activate_mm(struct mm
 {
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
Index: linux/include/asm-i386/mmu_context.h
=================================--- linux.orig/include/asm-i386/mmu_context.h
+++ linux/include/asm-i386/mmu_context.h
@@ -69,4 +69,9 @@ static inline void switch_mm(struct mm_s
 #define activate_mm(prev, next) \
 	switch_mm((prev),(next),NULL)
 
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return (void *) task->thread.esp;
+}
+
 #endif
Index: linux/include/asm-ia64/mmu_context.h
=================================--- linux.orig/include/asm-ia64/mmu_context.h
+++ linux/include/asm-ia64/mmu_context.h
@@ -169,5 +169,14 @@ activate_mm (struct mm_struct *prev, str
 
 #define switch_mm(prev_mm,next_mm,next_task)	activate_mm(prev_mm, next_mm)
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return (void *) task->thread.ksp;
+}
+
 # endif /* ! __ASSEMBLY__ */
 #endif /* _ASM_IA64_MMU_CONTEXT_H */
Index: linux/include/asm-m32r/mmu_context.h
=================================--- linux.orig/include/asm-m32r/mmu_context.h
+++ linux/include/asm-m32r/mmu_context.h
@@ -167,4 +167,13 @@ static inline void switch_mm(struct mm_s
 
 #endif /* __KERNEL__ */
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* _ASM_M32R_MMU_CONTEXT_H */
Index: linux/include/asm-m68k/mmu_context.h
=================================--- linux.orig/include/asm-m68k/mmu_context.h
+++ linux/include/asm-m68k/mmu_context.h
@@ -150,5 +150,14 @@ static inline void activate_mm(struct mm
 	activate_context(next_mm);
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
 #endif
Index: linux/include/asm-m68knommu/mmu_context.h
=================================--- linux.orig/include/asm-m68knommu/mmu_context.h
+++ linux/include/asm-m68knommu/mmu_context.h
@@ -30,4 +30,13 @@ extern inline void activate_mm(struct mm
 {
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
Index: linux/include/asm-mips/mmu_context.h
=================================--- linux.orig/include/asm-mips/mmu_context.h
+++ linux/include/asm-mips/mmu_context.h
@@ -193,4 +193,13 @@ drop_mmu_context(struct mm_struct *mm, u
 	local_irq_restore(flags);
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* _ASM_MMU_CONTEXT_H */
Index: linux/include/asm-parisc/mmu_context.h
=================================--- linux.orig/include/asm-parisc/mmu_context.h
+++ linux/include/asm-parisc/mmu_context.h
@@ -70,4 +70,14 @@ static inline void activate_mm(struct mm
 
 	switch_mm(prev,next,current);
 }
+
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
Index: linux/include/asm-ppc/mmu_context.h
=================================--- linux.orig/include/asm-ppc/mmu_context.h
+++ linux/include/asm-ppc/mmu_context.h
@@ -195,5 +195,14 @@ static inline void switch_mm(struct mm_s
 
 extern void mmu_context_init(void);
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* __PPC_MMU_CONTEXT_H */
 #endif /* __KERNEL__ */
Index: linux/include/asm-ppc64/mmu_context.h
=================================--- linux.orig/include/asm-ppc64/mmu_context.h
+++ linux/include/asm-ppc64/mmu_context.h
@@ -84,4 +84,13 @@ static inline void activate_mm(struct mm
 	local_irq_restore(flags);
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* __PPC64_MMU_CONTEXT_H */
Index: linux/include/asm-s390/mmu_context.h
=================================--- linux.orig/include/asm-s390/mmu_context.h
+++ linux/include/asm-s390/mmu_context.h
@@ -51,4 +51,13 @@ extern inline void activate_mm(struct mm
 	set_fs(current->thread.mm_segment);
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
Index: linux/include/asm-sh/mmu_context.h
=================================--- linux.orig/include/asm-sh/mmu_context.h
+++ linux/include/asm-sh/mmu_context.h
@@ -202,5 +202,14 @@ static inline void disable_mmu(void)
 #define disable_mmu()	do { BUG(); } while (0)
 #endif
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* __KERNEL__ */
 #endif /* __ASM_SH_MMU_CONTEXT_H */
Index: linux/include/asm-sh64/mmu_context.h
=================================--- linux.orig/include/asm-sh64/mmu_context.h
+++ linux/include/asm-sh64/mmu_context.h
@@ -206,4 +206,13 @@ enter_lazy_tlb(struct mm_struct *mm, str
 
 #endif	/* __ASSEMBLY__ */
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* __ASM_SH64_MMU_CONTEXT_H */
Index: linux/include/asm-sparc/mmu_context.h
=================================--- linux.orig/include/asm-sparc/mmu_context.h
+++ linux/include/asm-sparc/mmu_context.h
@@ -37,4 +37,13 @@ BTFIXUPDEF_CALL(void, switch_mm, struct 
 
 #endif /* !(__ASSEMBLY__) */
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* !(__SPARC_MMU_CONTEXT_H) */
Index: linux/include/asm-sparc64/mmu_context.h
=================================--- linux.orig/include/asm-sparc64/mmu_context.h
+++ linux/include/asm-sparc64/mmu_context.h
@@ -142,4 +142,13 @@ static inline void activate_mm(struct mm
 
 #endif /* !(__ASSEMBLY__) */
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* !(__SPARC64_MMU_CONTEXT_H) */
Index: linux/include/asm-um/mmu_context.h
=================================--- linux.orig/include/asm-um/mmu_context.h
+++ linux/include/asm-um/mmu_context.h
@@ -66,6 +66,15 @@ static inline void destroy_context(struc
 	CHOOSE_MODE((void) 0, destroy_context_skas(mm));
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif
 
 /*
Index: linux/include/asm-v850/mmu_context.h
=================================--- linux.orig/include/asm-v850/mmu_context.h
+++ linux/include/asm-v850/mmu_context.h
@@ -8,4 +8,13 @@
 #define activate_mm(prev,next)		((void)0)
 #define enter_lazy_tlb(mm,tsk)		((void)0)
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* __V850_MMU_CONTEXT_H__ */
Index: linux/include/asm-x86_64/mmu_context.h
=================================--- linux.orig/include/asm-x86_64/mmu_context.h
+++ linux/include/asm-x86_64/mmu_context.h
@@ -76,4 +76,13 @@ static inline void switch_mm(struct mm_s
 	switch_mm((prev),(next),NULL)
 
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return (void *) task->thread.rsp;
+}
+
 #endif
Index: linux/include/asm-xtensa/mmu_context.h
=================================--- linux.orig/include/asm-xtensa/mmu_context.h
+++ linux/include/asm-xtensa/mmu_context.h
@@ -327,4 +327,13 @@ static inline void enter_lazy_tlb(struct
 
 }
 
+/*
+ * Returns the current bottom of a task's kernel stack. Used
+ * by the scheduler to prefetch it.
+ */
+static inline void * kernel_stack(struct task_struct *task)
+{
+	return task;
+}
+
 #endif /* _XTENSA_MMU_CONTEXT_H */
Index: linux/kernel/sched.c
=================================--- linux.orig/kernel/sched.c
+++ linux/kernel/sched.c
@@ -2864,6 +2864,13 @@ go_idle:
 	queue = array->queue + idx;
 	next = list_entry(queue->next, task_t, run_list);
 
+	/*
+	 * Cache-prefetch crutial memory areas of the next task,
+	 * its thread_info and its kernel stack:
+	 */
+	prefetch(next->thread_info);
+	prefetch(kernel_stack(next));
+
 	if (!rt_task(next) && next->activated > 0) {
 		unsigned long long delta = now - next->timestamp;
 		if (unlikely((long long)(now - next->timestamp) < 0))
@@ -2886,7 +2893,7 @@ go_idle:
 switch_tasks:
 	if (next = rq->idle)
 		schedstat_inc(rq, sched_goidle);
-	prefetch(next);
+
 	clear_tsk_need_resched(prev);
 	rcu_qsctr_inc(task_cpu(prev));