Re: [PATCH 24/32] nohz/cpuset: Handle kernel entry/exit to account cputime

["Paul E. McKenney" <paulmck@linux.vnet.ibm.com>]

On Mon, Aug 15, 2011 at 05:52:21PM +0200, Frederic Weisbecker wrote:
> Provide a few APIs that archs can call to tell they are entering
> or exiting the kernel so that when we are in nohz adaptive mode
> we know precisely where we need to account the cputime.
> 
> The new APIs are:
> 
> - tick_nohz_enter_kernel() (called when we enter a syscall)
> - tick_nohz_exit_kernel() (called when we exit a syscall)
> - tick_nohz_enter_exception() (called when we enter any
>   exception, trap, faults...but not irqs)
> - tick_nohz_exit_exception() (called when we exit any exception)
> 
> Hooks into syscalls are typically driven by the TIF_NOHZ thread
> flag.
> 
> In addition, we use the value returned by user_mode(regs) from
> the timer interrupt to know where we are.
> Nonetheless, we can rely on user_mode(regs) != 0 to know
> we are in userspace, but we can't rely on user_mode(regs) == 0
> to know we are in the system.
> 
> Consider the following scenario: we stop the tick after syscall
> return, so we set TIF_NOHZ but the syscall exit hook is behind us.
> If we haven't yet returned to userspace, then we have
> user_mode(regs) == 0. If on top of that we consider we are in
> system mode, and later we issue a syscall but restart the tick
> right before reaching the syscall entry hook, then we have no clue
> that the whole elapsed cputime was not in the system but in the
> userspace.
> 
> The only way to fix this is to only start entering nohz mode once
> we know we are in userspace a first time, like when we reach the
> kernel exit hook or when a timer tick with user_mode(regs) == 1
> fires. Kernel threads don't have this worry.
> 
> This sucks but for now I have no better solution. Let's hope we
> can find better.
> 
> TODO: wrap operation on jiffies?

Hmmm...  Does the RCU dyntick-idle code need to know about exception
entry and exit?

							Thanx, Paul

> Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
> Cc: Andrew Morton <akpm@linux-foundation.org>
> Cc: Anton Blanchard <anton@au1.ibm.com>
> Cc: Avi Kivity <avi@redhat.com>
> Cc: Ingo Molnar <mingo@elte.hu>
> Cc: Lai Jiangshan <laijs@cn.fujitsu.com>
> Cc: Paul E . McKenney <paulmck@linux.vnet.ibm.com>
> Cc: Paul Menage <menage@google.com>
> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl>
> Cc: Stephen Hemminger <shemminger@vyatta.com>
> Cc: Thomas Gleixner <tglx@linutronix.de>
> Cc: Tim Pepper <lnxninja@linux.vnet.ibm.com>
> ---
>  include/linux/tick.h     |    8 +++
>  kernel/sched.c           |    1 +
>  kernel/time/tick-sched.c |  114 ++++++++++++++++++++++++++++++++++++++++------
>  3 files changed, 109 insertions(+), 14 deletions(-)
> 
> diff --git a/include/linux/tick.h b/include/linux/tick.h
> index ea6dfb7..3ad649f 100644
> --- a/include/linux/tick.h
> +++ b/include/linux/tick.h
> @@ -139,10 +139,18 @@ extern u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time);
>  #ifdef CONFIG_CPUSETS_NO_HZ
>  DECLARE_PER_CPU(int, task_nohz_mode);
> 
> +extern void tick_nohz_enter_kernel(void);
> +extern void tick_nohz_exit_kernel(void);
> +extern void tick_nohz_enter_exception(struct pt_regs *regs);
> +extern void tick_nohz_exit_exception(struct pt_regs *regs);
>  extern int tick_nohz_adaptive_mode(void);
>  extern bool tick_nohz_account_tick(void);
>  extern void tick_nohz_flush_current_times(void);
>  #else /* !CPUSETS_NO_HZ */
> +static inline void tick_nohz_enter_kernel(void) { }
> +static inline void tick_nohz_exit_kernel(void) { }
> +static inline void tick_nohz_enter_exception(struct pt_regs *regs) { }
> +static inline void tick_nohz_exit_exception(struct pt_regs *regs) { }
>  static inline int tick_nohz_adaptive_mode(void) { return 0; }
>  static inline bool tick_nohz_account_tick(void) { return false; }
>  #endif /* CPUSETS_NO_HZ */
> diff --git a/kernel/sched.c b/kernel/sched.c
> index a58f993..c49c1b1 100644
> --- a/kernel/sched.c
> +++ b/kernel/sched.c
> @@ -2503,6 +2503,7 @@ static void cpuset_nohz_restart_tick(void)
>  	tick_nohz_flush_current_times();
>  	__get_cpu_var(task_nohz_mode) = 0;
>  	tick_nohz_restart_sched_tick();
> +	clear_thread_flag(TIF_NOHZ);
>  }
> 
>  void cpuset_update_nohz(void)
> diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
> index c3a8f26..d8f01b8 100644
> --- a/kernel/time/tick-sched.c
> +++ b/kernel/time/tick-sched.c
> @@ -595,8 +595,9 @@ void tick_nohz_irq_exit(void)
>  	if (ts->inidle && !need_resched())
>  		__tick_nohz_enter_idle(ts, cpu);
>  	else if (tick_nohz_adaptive_mode() && !idle_cpu(cpu)) {
> -		if (tick_nohz_can_stop_tick(cpu, ts))
> -			tick_nohz_stop_sched_tick(ktime_get(), cpu, ts);
> +		if (ts->saved_jiffies_whence != JIFFIES_SAVED_NONE
> +		    && tick_nohz_can_stop_tick(cpu, ts))
> +				tick_nohz_stop_sched_tick(ktime_get(), cpu, ts);
>  	}
>  }
> 
> @@ -757,6 +758,74 @@ void tick_check_idle(int cpu)
> 
>  #ifdef CONFIG_CPUSETS_NO_HZ
> 
> +void tick_nohz_exit_kernel(void)
> +{
> +	unsigned long flags;
> +	struct tick_sched *ts;
> +	unsigned long delta_jiffies;
> +
> +	local_irq_save(flags);
> +
> +	if (!tick_nohz_adaptive_mode()) {
> +		local_irq_restore(flags);
> +		return;
> +	}
> +
> +	ts = &__get_cpu_var(tick_cpu_sched);
> +
> +	WARN_ON_ONCE(ts->saved_jiffies_whence == JIFFIES_SAVED_USER);
> +
> +	if (ts->saved_jiffies_whence == JIFFIES_SAVED_SYS) {
> +		delta_jiffies = jiffies - ts->saved_jiffies;
> +		account_system_jiffies(current, delta_jiffies);
> +	}
> +
> +	ts->saved_jiffies = jiffies;
> +	ts->saved_jiffies_whence = JIFFIES_SAVED_USER;
> +
> +	local_irq_restore(flags);
> +}
> +
> +void tick_nohz_enter_kernel(void)
> +{
> +	unsigned long flags;
> +	struct tick_sched *ts;
> +	unsigned long delta_jiffies;
> +
> +	local_irq_save(flags);
> +
> +	if (!tick_nohz_adaptive_mode()) {
> +		local_irq_restore(flags);
> +		return;
> +	}
> +
> +	ts = &__get_cpu_var(tick_cpu_sched);
> +
> +	WARN_ON_ONCE(ts->saved_jiffies_whence == JIFFIES_SAVED_SYS);
> +
> +	if (ts->saved_jiffies_whence == JIFFIES_SAVED_USER) {
> +		delta_jiffies = jiffies - ts->saved_jiffies;
> +		account_user_jiffies(current, delta_jiffies);
> +	}
> +
> +	ts->saved_jiffies = jiffies;
> +	ts->saved_jiffies_whence = JIFFIES_SAVED_SYS;
> +
> +	local_irq_restore(flags);
> +}
> +
> +void tick_nohz_enter_exception(struct pt_regs *regs)
> +{
> +	if (user_mode(regs))
> +		tick_nohz_enter_kernel();
> +}
> +
> +void tick_nohz_exit_exception(struct pt_regs *regs)
> +{
> +	if (user_mode(regs))
> +		tick_nohz_exit_kernel();
> +}
> +
>  int tick_nohz_adaptive_mode(void)
>  {
>  	return __get_cpu_var(task_nohz_mode);
> @@ -766,20 +835,33 @@ static void tick_nohz_cpuset_stop_tick(int user)
>  {
>  	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
> 
> -	if (!cpuset_adaptive_nohz() || tick_nohz_adaptive_mode())
> +	if (!cpuset_adaptive_nohz())
>  		return;
> 
> +	if (tick_nohz_adaptive_mode()) {
> +		if (user && ts->saved_jiffies_whence == JIFFIES_SAVED_NONE) {
> +			ts->saved_jiffies_whence = JIFFIES_SAVED_USER;
> +			ts->saved_jiffies = jiffies;
> +		}
> +
> +		return;
> +	}
> +
>  	if (cpuset_nohz_can_stop_tick()) {
>  		__get_cpu_var(task_nohz_mode) = 1;
>  		/* Nohz mode must be visible to wake_up_nohz_cpu() */
>  		smp_wmb();
> 
> +		set_thread_flag(TIF_NOHZ);
>  		WARN_ON_ONCE(ts->saved_jiffies_whence != JIFFIES_SAVED_NONE);
> -		ts->saved_jiffies = jiffies;
> -		if (user)
> +
> +		if (user) {
>  			ts->saved_jiffies_whence = JIFFIES_SAVED_USER;
> -		else
> +			ts->saved_jiffies = jiffies;
> +		} else if (!current->mm) {
>  			ts->saved_jiffies_whence = JIFFIES_SAVED_SYS;
> +			ts->saved_jiffies = jiffies;
> +		}
>  	}
>  }
> 
> @@ -803,7 +885,7 @@ static void tick_do_timer_check_handler(int cpu)
> 
>  bool tick_nohz_account_tick(void)
>  {
> -	struct tick_sched *ts;
> +	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
>  	unsigned long delta_jiffies;
> 
>  	if (!tick_nohz_adaptive_mode())
> @@ -811,11 +893,15 @@ bool tick_nohz_account_tick(void)
> 
>  	ts = &__get_cpu_var(tick_cpu_sched);
> 
> +	if (ts->saved_jiffies_whence == JIFFIES_SAVED_NONE)
> +		return false;
> +
>  	delta_jiffies = jiffies - ts->saved_jiffies;
> -	if (ts->saved_jiffies_whence == JIFFIES_SAVED_SYS)
> -		account_system_jiffies(current, delta_jiffies);
> -	else
> +
> +	if (ts->saved_jiffies_whence == JIFFIES_SAVED_USER)
>  		account_user_jiffies(current, delta_jiffies);
> +	else
> +		account_system_jiffies(current, delta_jiffies);
> 
>  	ts->saved_jiffies = jiffies;
> 
> @@ -825,12 +911,12 @@ bool tick_nohz_account_tick(void)
>  void tick_nohz_flush_current_times(void)
>  {
>  	struct tick_sched *ts = &__get_cpu_var(tick_cpu_sched);
> +	unsigned long delta_jiffies;
> +	struct pt_regs *regs;
> 
> -	tick_nohz_account_tick();
> -
> -	ts->saved_jiffies_whence = JIFFIES_SAVED_NONE;
> +	if (tick_nohz_account_tick())
> +		ts->saved_jiffies_whence = JIFFIES_SAVED_NONE;
>  }
> -
>  #else
> 
>  static void tick_nohz_cpuset_stop_tick(int user) { }
> -- 
> 1.7.5.4
>