[PATCH] rcu: Add "expedite always" switch

[PATCH] rcu: Add "expedite always" switch

[Antti P Miettinen]

Add a module parameter for always using expedited RCU primitives.

Signed-off-by: Antti P Miettinen <amiettinen@nvidia.com>
---
 include/linux/rcupdate.h |    2 ++
 kernel/rcupdate.c        |    4 ++++
 kernel/rcutiny_plugin.h  |    5 ++++-
 kernel/rcutree.c         |   10 ++++++++--
 kernel/rcutree_plugin.h  |    5 ++++-
 kernel/srcu.c            |    4 +++-
 6 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 7c968e4..b37efae 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -45,6 +45,8 @@
 #include <linux/bug.h>
 #include <linux/compiler.h>
 
+extern int rcu_expedited;
+
 #ifdef CONFIG_RCU_TORTURE_TEST
 extern int rcutorture_runnable; /* for sysctl */
 #endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 29ca1c6..6057e58 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -46,12 +46,16 @@
 #include <linux/export.h>
 #include <linux/hardirq.h>
 #include <linux/delay.h>
+#include <linux/module.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/rcu.h>
 
 #include "rcu.h"
 
+int rcu_expedited;
+module_param(rcu_expedited, int, 0644);
+
 #ifdef CONFIG_PREEMPT_RCU
 
 /*
diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 3d01902..f85016a 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -706,7 +706,10 @@ void synchronize_rcu(void)
 		return;
 
 	/* Once we get past the fastpath checks, same code as rcu_barrier(). */
-	rcu_barrier();
+	if (rcu_expedited)
+		synchronize_rcu_expedited();
+	else
+		rcu_barrier();
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 4fb2376..744c117 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -2221,7 +2221,10 @@ void synchronize_sched(void)
 			   "Illegal synchronize_sched() in RCU-sched read-side critical section");
 	if (rcu_blocking_is_gp())
 		return;
-	wait_rcu_gp(call_rcu_sched);
+	if (rcu_expedited)
+		synchronize_sched_expedited();
+	else
+		wait_rcu_gp(call_rcu_sched);
 }
 EXPORT_SYMBOL_GPL(synchronize_sched);
 
@@ -2242,7 +2245,10 @@ void synchronize_rcu_bh(void)
 			   "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
 	if (rcu_blocking_is_gp())
 		return;
-	wait_rcu_gp(call_rcu_bh);
+	if (rcu_expedited)
+		synchronize_rcu_bh_expedited();
+	else
+		wait_rcu_gp(call_rcu_bh);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
 
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index f921154..0e9ca8b 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -679,7 +679,10 @@ void synchronize_rcu(void)
 			   "Illegal synchronize_rcu() in RCU read-side critical section");
 	if (!rcu_scheduler_active)
 		return;
-	wait_rcu_gp(call_rcu);
+	if (rcu_expedited)
+		synchronize_rcu_expedited();
+	else
+		wait_rcu_gp(call_rcu);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
diff --git a/kernel/srcu.c b/kernel/srcu.c
index 97c465e..00c4169 100644
--- a/kernel/srcu.c
+++ b/kernel/srcu.c
@@ -464,7 +464,9 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
  */
 void synchronize_srcu(struct srcu_struct *sp)
 {
-	__synchronize_srcu(sp, SYNCHRONIZE_SRCU_TRYCOUNT);
+	__synchronize_srcu(sp, rcu_expedited
+			   ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
+			   : SYNCHRONIZE_SRCU_TRYCOUNT);
 }
 EXPORT_SYMBOL_GPL(synchronize_srcu);
 
-- 
1.7.4.1

Re: [PATCH] rcu: Add "expedite always" switch

[Paul E. McKenney]

On Fri, Oct 05, 2012 at 12:00:53AM +0300, Antti P Miettinen wrote:
> Add a module parameter for always using expedited RCU primitives.

Cool, looks quite close!!!  Please see below for a couple of questions and
comments.

> Signed-off-by: Antti P Miettinen <amiettinen@nvidia.com>
> ---
>  include/linux/rcupdate.h |    2 ++
>  kernel/rcupdate.c        |    4 ++++
>  kernel/rcutiny_plugin.h  |    5 ++++-
>  kernel/rcutree.c         |   10 ++++++++--
>  kernel/rcutree_plugin.h  |    5 ++++-
>  kernel/srcu.c            |    4 +++-
>  6 files changed, 25 insertions(+), 5 deletions(-)
> 
> diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
> index 7c968e4..b37efae 100644
> --- a/include/linux/rcupdate.h
> +++ b/include/linux/rcupdate.h
> @@ -45,6 +45,8 @@
>  #include <linux/bug.h>
>  #include <linux/compiler.h>
> 
> +extern int rcu_expedited;
> +

This needs to move to kernel/rcu.h, otherwise it is being exported to
the entire kernel.  Don't get me wrong, if someone in the kernel needs
to be able switch to and from expedited mode, I am OK with their doing
that -- but via a proper API rather than a bare store to a variable.  ;-)

(That said, the networking guys are doing just fine by having
synchronize_net() call either synchronize_rcu_expedited() or
synchronize_rcu(), depending on internal networking locking state.
So I do not believe that we need an API, not yet, anyway.)

>  #ifdef CONFIG_RCU_TORTURE_TEST
>  extern int rcutorture_runnable; /* for sysctl */
>  #endif /* #ifdef CONFIG_RCU_TORTURE_TEST */
> diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
> index 29ca1c6..6057e58 100644
> --- a/kernel/rcupdate.c
> +++ b/kernel/rcupdate.c
> @@ -46,12 +46,16 @@
>  #include <linux/export.h>
>  #include <linux/hardirq.h>
>  #include <linux/delay.h>
> +#include <linux/module.h>
> 
>  #define CREATE_TRACE_POINTS
>  #include <trace/events/rcu.h>
> 
>  #include "rcu.h"
> 
> +int rcu_expedited;
> +module_param(rcu_expedited, int, 0644);

Ah, the joys of exported-to-userspace API design...  ;-)

Summary: Please s/0644/0/ and add a KERNEL_ATTR_RW()-style sysfs interface.

If you want the details, please read on!

This exports the ability to switch expedited mode on and off at boot
time and also via sysfs.  However, this approach to the sysfs interface
does not gracefully handle multiple things enabling and disabling
expedited mode concurrently.  There are a couple of possible approaches:

1.	Look at current use cases.  These seem to be networking
	applications such as wireshark that reportedly cause the kernel
	to do a large number of RCU grace periods when starting up.
	These would be best served by an enter/exit style that
	increments and decrements a counter.

2.	Of course, applications can abort without properly cleaning
	up, which would argue for some sort of open/close protocol
	relying on the fact that all the application's fds are
	closed on any sort of exit.  But that leaves scripts out.

3.	One way to handle the scripts is to use redirection tricks,
	opening (say) FD 9 to the device.

4.	Do the bare minimum initially, so that when/if other use
	cases appear, they can be accommodated -- but using real
	information about what they really need.

>From what I can see, #4 is the only approach with a decent chance.
So, what is the bare minimum?

a.	Boot-time setting, allowing small embedded systems to
	unconditionally speed up grace periods.  You have this with the
	module_param() above.  Setting the permissions to zero would
	disable the userspace direct-set ability that is likely to get
	us into backwards-compatibility hot water in the future.

b.	Maybe sysfs setting.  Initially, this can be simple "on" and "off",
	exported with root-only access (like you have above).  If more
	elaborate use cases appear, this might become the adminstrative
	override.  The key point is that this needs to go through some
	sort of separate function (for example, via the KERNEL_ATTR_RW()
	macro) so that if it needs to change in the future, we can do
	the change easily.

Or am I missing something?

The rest looks good!

							Thanx, Paul

> +
>  #ifdef CONFIG_PREEMPT_RCU
> 
>  /*
> diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
> index 3d01902..f85016a 100644
> --- a/kernel/rcutiny_plugin.h
> +++ b/kernel/rcutiny_plugin.h
> @@ -706,7 +706,10 @@ void synchronize_rcu(void)
>  		return;
> 
>  	/* Once we get past the fastpath checks, same code as rcu_barrier(). */
> -	rcu_barrier();
> +	if (rcu_expedited)
> +		synchronize_rcu_expedited();
> +	else
> +		rcu_barrier();
>  }
>  EXPORT_SYMBOL_GPL(synchronize_rcu);
> 
> diff --git a/kernel/rcutree.c b/kernel/rcutree.c
> index 4fb2376..744c117 100644
> --- a/kernel/rcutree.c
> +++ b/kernel/rcutree.c
> @@ -2221,7 +2221,10 @@ void synchronize_sched(void)
>  			   "Illegal synchronize_sched() in RCU-sched read-side critical section");
>  	if (rcu_blocking_is_gp())
>  		return;
> -	wait_rcu_gp(call_rcu_sched);
> +	if (rcu_expedited)
> +		synchronize_sched_expedited();
> +	else
> +		wait_rcu_gp(call_rcu_sched);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_sched);
> 
> @@ -2242,7 +2245,10 @@ void synchronize_rcu_bh(void)
>  			   "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
>  	if (rcu_blocking_is_gp())
>  		return;
> -	wait_rcu_gp(call_rcu_bh);
> +	if (rcu_expedited)
> +		synchronize_rcu_bh_expedited();
> +	else
> +		wait_rcu_gp(call_rcu_bh);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
> 
> diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
> index f921154..0e9ca8b 100644
> --- a/kernel/rcutree_plugin.h
> +++ b/kernel/rcutree_plugin.h
> @@ -679,7 +679,10 @@ void synchronize_rcu(void)
>  			   "Illegal synchronize_rcu() in RCU read-side critical section");
>  	if (!rcu_scheduler_active)
>  		return;
> -	wait_rcu_gp(call_rcu);
> +	if (rcu_expedited)
> +		synchronize_rcu_expedited();
> +	else
> +		wait_rcu_gp(call_rcu);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_rcu);
> 
> diff --git a/kernel/srcu.c b/kernel/srcu.c
> index 97c465e..00c4169 100644
> --- a/kernel/srcu.c
> +++ b/kernel/srcu.c
> @@ -464,7 +464,9 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
>   */
>  void synchronize_srcu(struct srcu_struct *sp)
>  {
> -	__synchronize_srcu(sp, SYNCHRONIZE_SRCU_TRYCOUNT);
> +	__synchronize_srcu(sp, rcu_expedited
> +			   ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
> +			   : SYNCHRONIZE_SRCU_TRYCOUNT);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_srcu);
> 
> -- 
> 1.7.4.1
> 

Re: [PATCH] rcu: Add "expedite always" switch

[Antti P Miettinen]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> b.	Maybe sysfs setting.  Initially, this can be simple "on" and "off",
> 	exported with root-only access (like you have above).  If more
> 	elaborate use cases appear, this might become the adminstrative
> 	override.  The key point is that this needs to go through some
> 	sort of separate function (for example, via the KERNEL_ATTR_RW()
> 	macro) so that if it needs to change in the future, we can do
> 	the change easily.

I did not really envision the module param to be an application
interface, more like an administrative interface. For dynamically
requesting things on/off you probably want something like PM QoS which
does proper book keeping and consolidation but PM QoS for this feels a
bit overkill. But I do not think sysfs node is appropriate either for
an application interface. I guess module_param_cb() would also achieve
the goal of "doing something" upon parameter change. Anyway - below
the 0-perm module param + sysfs node version.

	--Antti

Add a module parameter for always using expedited RCU primitives.

Signed-off-by: Antti P Miettinen <amiettinen@nvidia.com>
---
 kernel/ksysfs.c         |   18 ++++++++++++++++++
 kernel/rcu.h            |    2 ++
 kernel/rcupdate.c       |    3 +++
 kernel/rcutiny_plugin.h |    5 ++++-
 kernel/rcutree.c        |   10 ++++++++--
 kernel/rcutree_plugin.h |    5 ++++-
 kernel/srcu.c           |    6 +++++-
 7 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c
index 4e316e1..8715a79 100644
--- a/kernel/ksysfs.c
+++ b/kernel/ksysfs.c
@@ -141,6 +141,23 @@ static ssize_t fscaps_show(struct kobject *kobj,
 }
 KERNEL_ATTR_RO(fscaps);
 
+int rcu_expedited;
+static ssize_t rcu_expedited_show(struct kobject *kobj,
+				  struct kobj_attribute *attr, char *buf)
+{
+	return sprintf(buf, "%d\n", rcu_expedited);
+}
+static ssize_t rcu_expedited_store(struct kobject *kobj,
+				   struct kobj_attribute *attr,
+				   const char *buf, size_t count)
+{
+	if (kstrtoint(buf, 0, &rcu_expedited))
+		return -EINVAL;
+
+	return count;
+}
+KERNEL_ATTR_RW(rcu_expedited);
+
 /*
  * Make /sys/kernel/notes give the raw contents of our kernel .notes section.
  */
@@ -182,6 +199,7 @@ static struct attribute * kernel_attrs[] = {
 	&kexec_crash_size_attr.attr,
 	&vmcoreinfo_attr.attr,
 #endif
+	&rcu_expedited_attr.attr,
 	NULL
 };
 
diff --git a/kernel/rcu.h b/kernel/rcu.h
index 8ba99cd..20dfba5 100644
--- a/kernel/rcu.h
+++ b/kernel/rcu.h
@@ -109,4 +109,6 @@ static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)
 	}
 }
 
+extern int rcu_expedited;
+
 #endif /* __LINUX_RCU_H */
diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
index 29ca1c6..a2cf761 100644
--- a/kernel/rcupdate.c
+++ b/kernel/rcupdate.c
@@ -46,12 +46,15 @@
 #include <linux/export.h>
 #include <linux/hardirq.h>
 #include <linux/delay.h>
+#include <linux/module.h>
 
 #define CREATE_TRACE_POINTS
 #include <trace/events/rcu.h>
 
 #include "rcu.h"
 
+module_param(rcu_expedited, int, 0);
+
 #ifdef CONFIG_PREEMPT_RCU
 
 /*
diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
index 3d01902..f85016a 100644
--- a/kernel/rcutiny_plugin.h
+++ b/kernel/rcutiny_plugin.h
@@ -706,7 +706,10 @@ void synchronize_rcu(void)
 		return;
 
 	/* Once we get past the fastpath checks, same code as rcu_barrier(). */
-	rcu_barrier();
+	if (rcu_expedited)
+		synchronize_rcu_expedited();
+	else
+		rcu_barrier();
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
diff --git a/kernel/rcutree.c b/kernel/rcutree.c
index 4fb2376..744c117 100644
--- a/kernel/rcutree.c
+++ b/kernel/rcutree.c
@@ -2221,7 +2221,10 @@ void synchronize_sched(void)
 			   "Illegal synchronize_sched() in RCU-sched read-side critical section");
 	if (rcu_blocking_is_gp())
 		return;
-	wait_rcu_gp(call_rcu_sched);
+	if (rcu_expedited)
+		synchronize_sched_expedited();
+	else
+		wait_rcu_gp(call_rcu_sched);
 }
 EXPORT_SYMBOL_GPL(synchronize_sched);
 
@@ -2242,7 +2245,10 @@ void synchronize_rcu_bh(void)
 			   "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
 	if (rcu_blocking_is_gp())
 		return;
-	wait_rcu_gp(call_rcu_bh);
+	if (rcu_expedited)
+		synchronize_rcu_bh_expedited();
+	else
+		wait_rcu_gp(call_rcu_bh);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
 
diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
index f921154..0e9ca8b 100644
--- a/kernel/rcutree_plugin.h
+++ b/kernel/rcutree_plugin.h
@@ -679,7 +679,10 @@ void synchronize_rcu(void)
 			   "Illegal synchronize_rcu() in RCU read-side critical section");
 	if (!rcu_scheduler_active)
 		return;
-	wait_rcu_gp(call_rcu);
+	if (rcu_expedited)
+		synchronize_rcu_expedited();
+	else
+		wait_rcu_gp(call_rcu);
 }
 EXPORT_SYMBOL_GPL(synchronize_rcu);
 
diff --git a/kernel/srcu.c b/kernel/srcu.c
index 97c465e..d56873c 100644
--- a/kernel/srcu.c
+++ b/kernel/srcu.c
@@ -34,6 +34,8 @@
 #include <linux/delay.h>
 #include <linux/srcu.h>
 
+#include "rcu.h"
+
 /*
  * Initialize an rcu_batch structure to empty.
  */
@@ -464,7 +466,9 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
  */
 void synchronize_srcu(struct srcu_struct *sp)
 {
-	__synchronize_srcu(sp, SYNCHRONIZE_SRCU_TRYCOUNT);
+	__synchronize_srcu(sp, rcu_expedited
+			   ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
+			   : SYNCHRONIZE_SRCU_TRYCOUNT);
 }
 EXPORT_SYMBOL_GPL(synchronize_srcu);
 
-- 
1.7.4.1

Re: [PATCH] rcu: Add "expedite always" switch

[Paul E. McKenney]

On Fri, Oct 05, 2012 at 09:59:15AM +0300, Antti P Miettinen wrote:
> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> > b.	Maybe sysfs setting.  Initially, this can be simple "on" and "off",
> > 	exported with root-only access (like you have above).  If more
> > 	elaborate use cases appear, this might become the adminstrative
> > 	override.  The key point is that this needs to go through some
> > 	sort of separate function (for example, via the KERNEL_ATTR_RW()
> > 	macro) so that if it needs to change in the future, we can do
> > 	the change easily.
> 
> I did not really envision the module param to be an application
> interface, more like an administrative interface. For dynamically
> requesting things on/off you probably want something like PM QoS which
> does proper book keeping and consolidation but PM QoS for this feels a
> bit overkill. But I do not think sysfs node is appropriate either for
> an application interface. I guess module_param_cb() would also achieve
> the goal of "doing something" upon parameter change. Anyway - below
> the 0-perm module param + sysfs node version.

Queued, thank you!!!  I updated the commit log to read as follows:

    Add a module parameter to force use of expedited RCU primitives
    
    There have been some embedded applications that would benefit from
    use of expedited grace-period primitives.  In some ways, this is
    similar to synchronize_net() doing either a normal or an expedited
    grace period depending on lock state, but with control outside of
    the kernel.
    
    This commit therefore adds rcu_expedited boot and sysfs parameters
    that cause the kernel to substitute expedited primitives for the
    normal grace-period primitives.

							Thanx, Paul

> 	--Antti
> 
> Add a module parameter for always using expedited RCU primitives.
> 
> Signed-off-by: Antti P Miettinen <amiettinen@nvidia.com>
> ---
>  kernel/ksysfs.c         |   18 ++++++++++++++++++
>  kernel/rcu.h            |    2 ++
>  kernel/rcupdate.c       |    3 +++
>  kernel/rcutiny_plugin.h |    5 ++++-
>  kernel/rcutree.c        |   10 ++++++++--
>  kernel/rcutree_plugin.h |    5 ++++-
>  kernel/srcu.c           |    6 +++++-
>  7 files changed, 44 insertions(+), 5 deletions(-)
> 
> diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c
> index 4e316e1..8715a79 100644
> --- a/kernel/ksysfs.c
> +++ b/kernel/ksysfs.c
> @@ -141,6 +141,23 @@ static ssize_t fscaps_show(struct kobject *kobj,
>  }
>  KERNEL_ATTR_RO(fscaps);
> 
> +int rcu_expedited;
> +static ssize_t rcu_expedited_show(struct kobject *kobj,
> +				  struct kobj_attribute *attr, char *buf)
> +{
> +	return sprintf(buf, "%d\n", rcu_expedited);
> +}
> +static ssize_t rcu_expedited_store(struct kobject *kobj,
> +				   struct kobj_attribute *attr,
> +				   const char *buf, size_t count)
> +{
> +	if (kstrtoint(buf, 0, &rcu_expedited))
> +		return -EINVAL;
> +
> +	return count;
> +}
> +KERNEL_ATTR_RW(rcu_expedited);
> +
>  /*
>   * Make /sys/kernel/notes give the raw contents of our kernel .notes section.
>   */
> @@ -182,6 +199,7 @@ static struct attribute * kernel_attrs[] = {
>  	&kexec_crash_size_attr.attr,
>  	&vmcoreinfo_attr.attr,
>  #endif
> +	&rcu_expedited_attr.attr,
>  	NULL
>  };
> 
> diff --git a/kernel/rcu.h b/kernel/rcu.h
> index 8ba99cd..20dfba5 100644
> --- a/kernel/rcu.h
> +++ b/kernel/rcu.h
> @@ -109,4 +109,6 @@ static inline bool __rcu_reclaim(char *rn, struct rcu_head *head)
>  	}
>  }
> 
> +extern int rcu_expedited;
> +
>  #endif /* __LINUX_RCU_H */
> diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c
> index 29ca1c6..a2cf761 100644
> --- a/kernel/rcupdate.c
> +++ b/kernel/rcupdate.c
> @@ -46,12 +46,15 @@
>  #include <linux/export.h>
>  #include <linux/hardirq.h>
>  #include <linux/delay.h>
> +#include <linux/module.h>
> 
>  #define CREATE_TRACE_POINTS
>  #include <trace/events/rcu.h>
> 
>  #include "rcu.h"
> 
> +module_param(rcu_expedited, int, 0);
> +
>  #ifdef CONFIG_PREEMPT_RCU
> 
>  /*
> diff --git a/kernel/rcutiny_plugin.h b/kernel/rcutiny_plugin.h
> index 3d01902..f85016a 100644
> --- a/kernel/rcutiny_plugin.h
> +++ b/kernel/rcutiny_plugin.h
> @@ -706,7 +706,10 @@ void synchronize_rcu(void)
>  		return;
> 
>  	/* Once we get past the fastpath checks, same code as rcu_barrier(). */
> -	rcu_barrier();
> +	if (rcu_expedited)
> +		synchronize_rcu_expedited();
> +	else
> +		rcu_barrier();
>  }
>  EXPORT_SYMBOL_GPL(synchronize_rcu);
> 
> diff --git a/kernel/rcutree.c b/kernel/rcutree.c
> index 4fb2376..744c117 100644
> --- a/kernel/rcutree.c
> +++ b/kernel/rcutree.c
> @@ -2221,7 +2221,10 @@ void synchronize_sched(void)
>  			   "Illegal synchronize_sched() in RCU-sched read-side critical section");
>  	if (rcu_blocking_is_gp())
>  		return;
> -	wait_rcu_gp(call_rcu_sched);
> +	if (rcu_expedited)
> +		synchronize_sched_expedited();
> +	else
> +		wait_rcu_gp(call_rcu_sched);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_sched);
> 
> @@ -2242,7 +2245,10 @@ void synchronize_rcu_bh(void)
>  			   "Illegal synchronize_rcu_bh() in RCU-bh read-side critical section");
>  	if (rcu_blocking_is_gp())
>  		return;
> -	wait_rcu_gp(call_rcu_bh);
> +	if (rcu_expedited)
> +		synchronize_rcu_bh_expedited();
> +	else
> +		wait_rcu_gp(call_rcu_bh);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_rcu_bh);
> 
> diff --git a/kernel/rcutree_plugin.h b/kernel/rcutree_plugin.h
> index f921154..0e9ca8b 100644
> --- a/kernel/rcutree_plugin.h
> +++ b/kernel/rcutree_plugin.h
> @@ -679,7 +679,10 @@ void synchronize_rcu(void)
>  			   "Illegal synchronize_rcu() in RCU read-side critical section");
>  	if (!rcu_scheduler_active)
>  		return;
> -	wait_rcu_gp(call_rcu);
> +	if (rcu_expedited)
> +		synchronize_rcu_expedited();
> +	else
> +		wait_rcu_gp(call_rcu);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_rcu);
> 
> diff --git a/kernel/srcu.c b/kernel/srcu.c
> index 97c465e..d56873c 100644
> --- a/kernel/srcu.c
> +++ b/kernel/srcu.c
> @@ -34,6 +34,8 @@
>  #include <linux/delay.h>
>  #include <linux/srcu.h>
> 
> +#include "rcu.h"
> +
>  /*
>   * Initialize an rcu_batch structure to empty.
>   */
> @@ -464,7 +466,9 @@ static void __synchronize_srcu(struct srcu_struct *sp, int trycount)
>   */
>  void synchronize_srcu(struct srcu_struct *sp)
>  {
> -	__synchronize_srcu(sp, SYNCHRONIZE_SRCU_TRYCOUNT);
> +	__synchronize_srcu(sp, rcu_expedited
> +			   ? SYNCHRONIZE_SRCU_EXP_TRYCOUNT
> +			   : SYNCHRONIZE_SRCU_TRYCOUNT);
>  }
>  EXPORT_SYMBOL_GPL(synchronize_srcu);
> 
> -- 
> 1.7.4.1
>