[PATCH RFC tip/core/rcu 0/3] RT latency optimizations

[PATCH RFC tip/core/rcu 0/3] RT latency optimizations

[Paul E. McKenney]

Hello!

This series provides some minor latency optimizations for kernels that
build with CONFIG_RCU_NOCB_CPU_ALL=y:

1.	Create a static inline implementation of rcu_is_nocb_cpu() that
	unconditionally returns true when CONFIG_RCU_NOCB_CPU_ALL=y.

2.	Create a static inline implementation of rcu_needs_cpu() that
	unconditionally returns false when CONFIG_RCU_NOCB_CPU_ALL=y.

3.	Create static inline implementations of rcu_prepare_for_idle()
	and rcu_cleanup_after_idle() that are no-ops when
	CONFIG_RCU_NOCB_CPU_ALL=y.

							Thanx, Paul

------------------------------------------------------------------------

 b/include/linux/rcupdate.h |   13 +++++++++++++
 b/include/linux/rcutiny.h  |    6 ------
 b/include/linux/rcutree.h  |    2 ++
 b/kernel/rcu/tree.c        |    2 +-
 b/kernel/rcu/tree_plugin.h |   13 +++++++++++--
 5 files changed, 27 insertions(+), 9 deletions(-)

[PATCH tip/core/rcu 1/3] rcu: Optimize rcu_is_nocb_cpu() for RCU_NOCB_CPU_ALL

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_is_nocb_cpu() will always
return true, however, the current version nevertheless checks
rcu_nocb_mask.  This commit therefore creates a static inline
implementation of rcu_is_nocb_cpu() that unconditionally returns
true when CONFIG_RCU_NOCB_CPU_ALL=y.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 include/linux/rcupdate.h | 4 ++++
 kernel/rcu/tree_plugin.h | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 380910aa4e47..277d3697f2f3 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -1028,7 +1028,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
 	__kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
 
 #ifdef CONFIG_RCU_NOCB_CPU
+#ifdef CONFIG_RCU_NOCB_CPU_ALL
+static inline bool rcu_is_nocb_cpu(int cpu) { return true; }
+#else /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
 bool rcu_is_nocb_cpu(int cpu);
+#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU_ALL */
 #else
 static inline bool rcu_is_nocb_cpu(int cpu) { return false; }
 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 1aa33a59fadc..5e7305b40a27 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2089,6 +2089,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)
 	init_waitqueue_head(&rnp->nocb_gp_wq[1]);
 }
 
+#ifndef CONFIG_RCU_NOCB_CPU_ALL
 /* Is the specified CPU a no-CPUs CPU? */
 bool rcu_is_nocb_cpu(int cpu)
 {
@@ -2096,6 +2097,7 @@ bool rcu_is_nocb_cpu(int cpu)
 		return cpumask_test_cpu(cpu, rcu_nocb_mask);
 	return false;
 }
+#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
 
 /*
  * Enqueue the specified string of rcu_head structures onto the specified
-- 
1.8.1.5

[PATCH tip/core/rcu 2/3] rcu: Optimize rcu_needs_cpu() for RCU_NOCB_CPU_ALL

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_needs_cpu() will always
return false, however, the current version nevertheless checks
for RCU callbacks.  This commit therefore creates a static inline
implementation of rcu_needs_cpu() that unconditionally returns false
when CONFIG_RCU_NOCB_CPU_ALL=y.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 include/linux/rcupdate.h | 9 +++++++++
 include/linux/rcutiny.h  | 6 ------
 include/linux/rcutree.h  | 2 ++
 kernel/rcu/tree.c        | 2 +-
 kernel/rcu/tree_plugin.h | 4 ++++
 5 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 277d3697f2f3..e6331a6843c4 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -1027,6 +1027,15 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
 #define kfree_rcu(ptr, rcu_head)					\
 	__kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
 
+#if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL)
+
+static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
+{
+	*delta_jiffies = ULONG_MAX;
+	return 0;
+}
+#endif /* #if defined(CONFIG_TINY_RCU) || defined(CONFIG_RCU_NOCB_CPU_ALL) */
+
 #ifdef CONFIG_RCU_NOCB_CPU
 #ifdef CONFIG_RCU_NOCB_CPU_ALL
 static inline bool rcu_is_nocb_cpu(int cpu) { return true; }
diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index 6f01771b571c..9524903487d0 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -68,12 +68,6 @@ static inline void kfree_call_rcu(struct rcu_head *head,
 	call_rcu(head, func);
 }
 
-static inline int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
-{
-	*delta_jiffies = ULONG_MAX;
-	return 0;
-}
-
 static inline void rcu_note_context_switch(int cpu)
 {
 	rcu_sched_qs(cpu);
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index 72137ee8c603..81198c84e268 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -31,7 +31,9 @@
 #define __LINUX_RCUTREE_H
 
 void rcu_note_context_switch(int cpu);
+#ifndef CONFIG_RCU_NOCB_CPU_ALL
 int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies);
+#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
 void rcu_cpu_stall_reset(void);
 
 /*
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 7be5efd62fe5..ecca289351e0 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2851,7 +2851,7 @@ static int rcu_pending(int cpu)
  * non-NULL, store an indication of whether all callbacks are lazy.
  * (If there are no callbacks, all of them are deemed to be lazy.)
  */
-static int rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
+static int __maybe_unused rcu_cpu_has_callbacks(int cpu, bool *all_lazy)
 {
 	bool al = true;
 	bool hc = false;
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 5e7305b40a27..57d98aa2bd21 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1575,11 +1575,13 @@ static void rcu_prepare_kthreads(int cpu)
  * Because we not have RCU_FAST_NO_HZ, just check whether this CPU needs
  * any flavor of RCU.
  */
+#ifndef CONFIG_RCU_NOCB_CPU_ALL
 int rcu_needs_cpu(int cpu, unsigned long *delta_jiffies)
 {
 	*delta_jiffies = ULONG_MAX;
 	return rcu_cpu_has_callbacks(cpu, NULL);
 }
+#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
 
 /*
  * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up
@@ -1685,6 +1687,7 @@ static bool rcu_try_advance_all_cbs(void)
  *
  * The caller must have disabled interrupts.
  */
+#ifndef CONFIG_RCU_NOCB_CPU_ALL
 int rcu_needs_cpu(int cpu, unsigned long *dj)
 {
 	struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
@@ -1715,6 +1718,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj)
 	}
 	return 0;
 }
+#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
 
 /*
  * Prepare a CPU for idle from an RCU perspective.  The first major task
-- 
1.8.1.5

[PATCH tip/core/rcu 3/3] Optimize RCU_FAST_NO_HZ for RCU_NOCB_CPU_ALL

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

If CONFIG_RCU_NOCB_CPU_ALL=y, then no CPU will ever have RCU callbacks
because these callbacks will instead be handled by the rcuo kthreads.
However, the current version of RCU_FAST_NO_HZ nevertheless checks for RCU
callbacks.  This commit therefore creates static inline implementations
of rcu_prepare_for_idle() and rcu_cleanup_after_idle() that are no-ops
when CONFIG_RCU_NOCB_CPU_ALL=y.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree_plugin.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 57d98aa2bd21..d87493bf66e3 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1647,7 +1647,7 @@ extern int tick_nohz_enabled;
  * only if it has been awhile since the last time we did so.  Afterwards,
  * if there are any callbacks ready for immediate invocation, return true.
  */
-static bool rcu_try_advance_all_cbs(void)
+static bool __maybe_unused rcu_try_advance_all_cbs(void)
 {
 	bool cbs_ready = false;
 	struct rcu_data *rdp;
@@ -1732,6 +1732,7 @@ int rcu_needs_cpu(int cpu, unsigned long *dj)
  */
 static void rcu_prepare_for_idle(int cpu)
 {
+#ifndef CONFIG_RCU_NOCB_CPU_ALL
 	struct rcu_data *rdp;
 	struct rcu_dynticks *rdtp = &per_cpu(rcu_dynticks, cpu);
 	struct rcu_node *rnp;
@@ -1782,6 +1783,7 @@ static void rcu_prepare_for_idle(int cpu)
 		rcu_accelerate_cbs(rsp, rnp, rdp);
 		raw_spin_unlock(&rnp->lock); /* irqs remain disabled. */
 	}
+#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
 }
 
 /*
@@ -1791,11 +1793,12 @@ static void rcu_prepare_for_idle(int cpu)
  */
 static void rcu_cleanup_after_idle(int cpu)
 {
-
+#ifndef CONFIG_RCU_NOCB_CPU_ALL
 	if (rcu_is_nocb_cpu(cpu))
 		return;
 	if (rcu_try_advance_all_cbs())
 		invoke_rcu_core();
+#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
 }
 
 /*
-- 
1.8.1.5

[PATCH tip/core/rcu 1/3] rcu: Optimize rcu_is_nocb_cpu() for RCU_NOCB_CPU_ALL

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_is_nocb_cpu() will always
return true, however, the current version nevertheless checks
rcu_nocb_mask.  This commit therefore creates a static inline
implementation of rcu_is_nocb_cpu() that unconditionally returns
true when CONFIG_RCU_NOCB_CPU_ALL=y.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 include/linux/rcupdate.h | 4 ++++
 kernel/rcu/tree_plugin.h | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 72bf3a01a4ee..92365b1fede4 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -1016,7 +1016,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
 	__kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
 
 #ifdef CONFIG_RCU_NOCB_CPU
+#ifdef CONFIG_RCU_NOCB_CPU_ALL
+static inline bool rcu_is_nocb_cpu(int cpu) { return true; }
+#else /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
 bool rcu_is_nocb_cpu(int cpu);
+#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU_ALL */
 #else
 static inline bool rcu_is_nocb_cpu(int cpu) { return false; }
 #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 6e2ef4b2b920..39a50b918bff 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2101,6 +2101,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)
 	init_waitqueue_head(&rnp->nocb_gp_wq[1]);
 }
 
+#ifndef CONFIG_RCU_NOCB_CPU_ALL
 /* Is the specified CPU a no-CPUs CPU? */
 bool rcu_is_nocb_cpu(int cpu)
 {
@@ -2108,6 +2109,7 @@ bool rcu_is_nocb_cpu(int cpu)
 		return cpumask_test_cpu(cpu, rcu_nocb_mask);
 	return false;
 }
+#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
 
 /*
  * Enqueue the specified string of rcu_head structures onto the specified
-- 
1.8.1.5

Re: [PATCH tip/core/rcu 1/3] rcu: Optimize rcu_is_nocb_cpu() for RCU_NOCB_CPU_ALL

[Josh Triplett]

On Mon, Feb 17, 2014 at 01:40:25PM -0800, Paul E. McKenney wrote:
> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> 
> If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_is_nocb_cpu() will always
> return true, however, the current version nevertheless checks
> rcu_nocb_mask.  This commit therefore creates a static inline
> implementation of rcu_is_nocb_cpu() that unconditionally returns
> true when CONFIG_RCU_NOCB_CPU_ALL=y.
> 
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

One suggestion below.  With or without that change:
Reviewed-by: Josh Triplett <josh@joshtriplett.org>

>  include/linux/rcupdate.h | 4 ++++
>  kernel/rcu/tree_plugin.h | 2 ++
>  2 files changed, 6 insertions(+)
> 
> diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
> index 72bf3a01a4ee..92365b1fede4 100644
> --- a/include/linux/rcupdate.h
> +++ b/include/linux/rcupdate.h
> @@ -1016,7 +1016,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
>  	__kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
>  
>  #ifdef CONFIG_RCU_NOCB_CPU
> +#ifdef CONFIG_RCU_NOCB_CPU_ALL
> +static inline bool rcu_is_nocb_cpu(int cpu) { return true; }
> +#else /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
>  bool rcu_is_nocb_cpu(int cpu);
> +#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU_ALL */
>  #else
>  static inline bool rcu_is_nocb_cpu(int cpu) { return false; }
>  #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */

This seem clearer if written without the nesting:

#if defined(CONFIG_RCU_NOCB_CPU_ALL)
static inline bool rcu_is_nocb_cpu(int cpu) { return true; }
#elif defined(CONFIG_RCU_NOCB_CPU)
bool rcu_is_nocb_cpu(int cpu);
#else
static inline bool rcu_is_nocb_cpu(int cpu) { return false; }
#endif

That clearly distinguishes the three cases: all, some, none.

> diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
> index 6e2ef4b2b920..39a50b918bff 100644
> --- a/kernel/rcu/tree_plugin.h
> +++ b/kernel/rcu/tree_plugin.h
> @@ -2101,6 +2101,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)
>  	init_waitqueue_head(&rnp->nocb_gp_wq[1]);
>  }
>  
> +#ifndef CONFIG_RCU_NOCB_CPU_ALL
>  /* Is the specified CPU a no-CPUs CPU? */
>  bool rcu_is_nocb_cpu(int cpu)
>  {
> @@ -2108,6 +2109,7 @@ bool rcu_is_nocb_cpu(int cpu)
>  		return cpumask_test_cpu(cpu, rcu_nocb_mask);
>  	return false;
>  }
> +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
>  
>  /*
>   * Enqueue the specified string of rcu_head structures onto the specified
> -- 
> 1.8.1.5
> 

Re: [PATCH tip/core/rcu 1/3] rcu: Optimize rcu_is_nocb_cpu() for RCU_NOCB_CPU_ALL

[Paul E. McKenney]

On Mon, Feb 17, 2014 at 01:50:53PM -0800, Josh Triplett wrote:
> On Mon, Feb 17, 2014 at 01:40:25PM -0800, Paul E. McKenney wrote:
> > From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> > 
> > If CONFIG_RCU_NOCB_CPU_ALL=y, then rcu_is_nocb_cpu() will always
> > return true, however, the current version nevertheless checks
> > rcu_nocb_mask.  This commit therefore creates a static inline
> > implementation of rcu_is_nocb_cpu() that unconditionally returns
> > true when CONFIG_RCU_NOCB_CPU_ALL=y.
> > 
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> One suggestion below.  With or without that change:
> Reviewed-by: Josh Triplett <josh@joshtriplett.org>

I like the change -- applied both it and your Reviewed-by.

							Thanx, Paul

> >  include/linux/rcupdate.h | 4 ++++
> >  kernel/rcu/tree_plugin.h | 2 ++
> >  2 files changed, 6 insertions(+)
> > 
> > diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
> > index 72bf3a01a4ee..92365b1fede4 100644
> > --- a/include/linux/rcupdate.h
> > +++ b/include/linux/rcupdate.h
> > @@ -1016,7 +1016,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
> >  	__kfree_rcu(&((ptr)->rcu_head), offsetof(typeof(*(ptr)), rcu_head))
> >  
> >  #ifdef CONFIG_RCU_NOCB_CPU
> > +#ifdef CONFIG_RCU_NOCB_CPU_ALL
> > +static inline bool rcu_is_nocb_cpu(int cpu) { return true; }
> > +#else /* #ifdef CONFIG_RCU_NOCB_CPU_ALL */
> >  bool rcu_is_nocb_cpu(int cpu);
> > +#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU_ALL */
> >  #else
> >  static inline bool rcu_is_nocb_cpu(int cpu) { return false; }
> >  #endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
> 
> This seem clearer if written without the nesting:
> 
> #if defined(CONFIG_RCU_NOCB_CPU_ALL)
> static inline bool rcu_is_nocb_cpu(int cpu) { return true; }
> #elif defined(CONFIG_RCU_NOCB_CPU)
> bool rcu_is_nocb_cpu(int cpu);
> #else
> static inline bool rcu_is_nocb_cpu(int cpu) { return false; }
> #endif
> 
> That clearly distinguishes the three cases: all, some, none.
> 
> > diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
> > index 6e2ef4b2b920..39a50b918bff 100644
> > --- a/kernel/rcu/tree_plugin.h
> > +++ b/kernel/rcu/tree_plugin.h
> > @@ -2101,6 +2101,7 @@ static void rcu_init_one_nocb(struct rcu_node *rnp)
> >  	init_waitqueue_head(&rnp->nocb_gp_wq[1]);
> >  }
> >  
> > +#ifndef CONFIG_RCU_NOCB_CPU_ALL
> >  /* Is the specified CPU a no-CPUs CPU? */
> >  bool rcu_is_nocb_cpu(int cpu)
> >  {
> > @@ -2108,6 +2109,7 @@ bool rcu_is_nocb_cpu(int cpu)
> >  		return cpumask_test_cpu(cpu, rcu_nocb_mask);
> >  	return false;
> >  }
> > +#endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */
> >  
> >  /*
> >   * Enqueue the specified string of rcu_head structures onto the specified
> > -- 
> > 1.8.1.5
> > 
>