Re: [RFC PATCH 3/3] idle: store the idle state index in the struct rq

[Peter Zijlstra <peterz@infradead.org>]

On Mon, Feb 03, 2014 at 08:17:47AM -0800, Arjan van de Ven wrote:
> On 2/3/2014 6:56 AM, Peter Zijlstra wrote:
> if there's a simple api like
> 
> sched_cpu_cache_wiped(int llc)
> 
> that would be very nice for this; the menuidle side knows this
> for some cases and thus can just call it. This would be a very
> small and minimal change
> 
> * if you don't care about llc vs core local caches then that
>   parameter can go away
> 
> * I assume this is also called for the local cpu... if not then we
>   need to add a cpu number argument
> 
> * we can also call this from architecture code when wbinvd or the
>   arm equivalent is called etc

A little something like so?

---
 kernel/sched/core.c  | 21 +++++++++++++++++++++
 kernel/sched/fair.c  | 13 ++++++++++---
 kernel/sched/sched.h |  1 +
 3 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index fb9764fbc537..b06bcadc6d71 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5466,6 +5466,27 @@ static void update_top_cache_domain(int cpu)
 }
 
 /*
+ * Mark the current LLC as empty.
+ */
+void sched_llc_wipe_cache(void)
+{
+	struct sched_domain *sd;
+
+	rcu_read_lock();
+	sd = rcu_dereference(__get_cpu_var(sd_llc));
+	if (sd) {
+		int cpu;
+
+		for_each_cpu(cpu, sched_domain_span(sd)) {
+			struct rq *rq = cpu_rq(cpu);
+
+			rq->llc_wiped = sched_clock_cpu(cpu);
+		}
+	}
+	rcu_read_unlock();
+}
+
+/*
  * Attach the domain 'sd' to 'cpu' as its base domain. Callers must
  * hold the hotplug lock.
  */
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 235cfa7ad8fc..9f8ce98f8131 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -5025,9 +5025,10 @@ static void move_task(struct task_struct *p, struct lb_env *env)
 /*
  * Is this task likely cache-hot:
  */
-static int
-task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
+static int task_hot(struct task_struct *p, struct lb_env *env)
 {
+	u64 now = rq_clock_task(env->src_rq);
+	struct sched_domain *sd = env->sd;
 	s64 delta;
 
 	if (p->sched_class != &fair_sched_class)
@@ -5049,6 +5050,12 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
 	if (sysctl_sched_migration_cost == 0)
 		return 0;
 
+	/*
+	 * If its LLC got wiped after it ran last, we're as cold as it gets.
+	 */
+	if ((s64)(p->se.exec_start - env->src_rq->llc_wiped) < 0)
+		return 0;
+
 	delta = now - p->se.exec_start;
 
 	return delta < (s64)sysctl_sched_migration_cost;
@@ -5187,7 +5194,7 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
 	 * 2) task is cache cold, or
 	 * 3) too many balance attempts have failed.
 	 */
-	tsk_cache_hot = task_hot(p, rq_clock_task(env->src_rq), env->sd);
+	tsk_cache_hot = task_hot(p, env);
 	if (!tsk_cache_hot)
 		tsk_cache_hot = migrate_degrades_locality(p, env);
 
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 1bf34c257d3b..c98793112614 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -582,6 +582,7 @@ struct rq {
 
 	struct list_head cfs_tasks;
 
+	u64 llc_wiped;
 	u64 rt_avg;
 	u64 age_stamp;
 	u64 idle_stamp;