[PATCH 1/2] nohz: move NOHZ code bits out of io_schedule{,_timeout} into a helper

[PATCH 1/2] nohz: move NOHZ code bits out of io_schedule{,_timeout} into a helper

[Denys Vlasenko]

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Fernando Luis Vazquez Cao <fernando_b1@lab.ntt.co.jp>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Oleg Nesterov <oleg@redhat.com>
---
 kernel/sched/core.c | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index ffea757..3137980 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4208,6 +4208,21 @@ EXPORT_SYMBOL_GPL(yield_to);
  * This task is about to go to sleep on IO. Increment rq->nr_iowait so
  * that process accounting knows that this is a task in IO wait state.
  */
+#ifdef CONFIG_NO_HZ_COMMON
+static __sched void io_wait_end(struct rq *rq)
+{
+	if (atomic_dec_and_test(&rq->nr_iowait)) {
+		if (raw_smp_processor_id() != cpu_of(rq))
+			tick_nohz_iowait_to_idle(cpu_of(rq));
+	}
+}
+#else
+static inline void io_wait_end(struct rq *rq)
+{
+	atomic_dec(&rq->nr_iowait);
+}
+#endif
+
 void __sched io_schedule(void)
 {
 	struct rq *rq = raw_rq();
@@ -4218,14 +4233,7 @@ void __sched io_schedule(void)
 	current->in_iowait = 1;
 	schedule();
 	current->in_iowait = 0;
-#ifdef CONFIG_NO_HZ_COMMON
-	if (atomic_dec_and_test(&rq->nr_iowait)) {
-		if (raw_smp_processor_id() != cpu_of(rq))
-			tick_nohz_iowait_to_idle(cpu_of(rq));
-	}
-#else
-	atomic_dec(&rq->nr_iowait);
-#endif
+	io_wait_end(rq);
 	delayacct_blkio_end();
 }
 EXPORT_SYMBOL(io_schedule);
@@ -4241,14 +4249,7 @@ long __sched io_schedule_timeout(long timeout)
 	current->in_iowait = 1;
 	ret = schedule_timeout(timeout);
 	current->in_iowait = 0;
-#ifdef CONFIG_NO_HZ_COMMON
-	if (atomic_dec_and_test(&rq->nr_iowait)) {
-		if (raw_smp_processor_id() != cpu_of(rq))
-			tick_nohz_iowait_to_idle(cpu_of(rq));
-	}
-#else
-	atomic_dec(&rq->nr_iowait);
-#endif
+	io_wait_end(rq);
 	delayacct_blkio_end();
 	return ret;
 }
-- 
1.8.1.4

[PATCH 2/2] nohz: don't fetch per_cpu variable before we need it

[Denys Vlasenko]

In get_cpu_{idle,iowait}_time_us, we were fetching
per_cpu(tick_cpu_sched, cpu) upfront. But them we do
        if (!tick_nohz_active)
                return -1;
which may make that fetch pointless.
Moving it further down, closer to the usage point.

Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
Cc: Fernando Luis Vazquez Cao <fernando_b1@lab.ntt.co.jp>
Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Arjan van de Ven <arjan@linux.intel.com>
Cc: Oleg Nesterov <oleg@redhat.com>
---
 kernel/time/tick-sched.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index d78c942..4ffccd0 100644
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -476,7 +476,7 @@ void tick_nohz_iowait_to_idle(int cpu)
  */
 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
 {
-	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+	struct tick_sched *ts;
 	ktime_t now, idle;
 	unsigned int seq;
 
@@ -487,6 +487,8 @@ u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
 	if (last_update_time)
 		*last_update_time = ktime_to_us(now);
 
+	ts = &per_cpu(tick_cpu_sched, cpu);
+
 	do {
 		ktime_t start, delta, iowait_exit;
 
@@ -531,7 +533,7 @@ EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
  */
 u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 {
-	struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
+	struct tick_sched *ts;
 	ktime_t now, iowait;
 	unsigned int seq;
 
@@ -542,6 +544,8 @@ u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
 	if (last_update_time)
 		*last_update_time = ktime_to_us(now);
 
+	ts = &per_cpu(tick_cpu_sched, cpu);
+
 	do {
 		ktime_t delta, end;
 
-- 
1.8.1.4

Re: [PATCH 1/2] nohz: move NOHZ code bits out of io_schedule{,_timeout} into a helper

[Frederic Weisbecker]

On Fri, Apr 25, 2014 at 08:57:29PM +0200, Denys Vlasenko wrote:
> Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
> Cc: Frederic Weisbecker <fweisbec@gmail.com>
> Cc: Hidetoshi Seto <seto.hidetoshi@jp.fujitsu.com>
> Cc: Fernando Luis Vazquez Cao <fernando_b1@lab.ntt.co.jp>
> Cc: Tetsuo Handa <penguin-kernel@I-love.SAKURA.ne.jp>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Cc: Ingo Molnar <mingo@kernel.org>
> Cc: Peter Zijlstra <peterz@infradead.org>
> Cc: Andrew Morton <akpm@linux-foundation.org>
> Cc: Arjan van de Ven <arjan@linux.intel.com>
> Cc: Oleg Nesterov <oleg@redhat.com>
> ---
>  kernel/sched/core.c | 33 +++++++++++++++++----------------
>  1 file changed, 17 insertions(+), 16 deletions(-)
> 
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index ffea757..3137980 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -4208,6 +4208,21 @@ EXPORT_SYMBOL_GPL(yield_to);
>   * This task is about to go to sleep on IO. Increment rq->nr_iowait so
>   * that process accounting knows that this is a task in IO wait state.
>   */
> +#ifdef CONFIG_NO_HZ_COMMON
> +static __sched void io_wait_end(struct rq *rq)
> +{
> +	if (atomic_dec_and_test(&rq->nr_iowait)) {
> +		if (raw_smp_processor_id() != cpu_of(rq))
> +			tick_nohz_iowait_to_idle(cpu_of(rq));
> +	}
> +}
> +#else
> +static inline void io_wait_end(struct rq *rq)
> +{
> +	atomic_dec(&rq->nr_iowait);
> +}
> +#endif
> +
>  void __sched io_schedule(void)
>  {
>  	struct rq *rq = raw_rq();
> @@ -4218,14 +4233,7 @@ void __sched io_schedule(void)
>  	current->in_iowait = 1;
>  	schedule();
>  	current->in_iowait = 0;
> -#ifdef CONFIG_NO_HZ_COMMON
> -	if (atomic_dec_and_test(&rq->nr_iowait)) {
> -		if (raw_smp_processor_id() != cpu_of(rq))
> -			tick_nohz_iowait_to_idle(cpu_of(rq));
> -	}
> -#else
> -	atomic_dec(&rq->nr_iowait);
> -#endif
> +	io_wait_end(rq);
>  	delayacct_blkio_end();

There is much more to unify that the iowait accounting between all
the io_schedule() declensions.

Peterz I think you had a patch to unify that a few month ago?