[PATCH tip/core/rcu 0/7] Early-boot RCU callbacks for v4.1

[PATCH tip/core/rcu 0/7] Early-boot RCU callbacks for v4.1

[Paul E. McKenney]

Hello!

This series allows RCU callbacks to be posted very early at boot time,
even before rcu_init() is called.

1.	Abstract initialization of the default (non-no-CB) callbacks list
	from the init_callback_list() function into a new
	init_default_callback_list() function.

2.	Initialize rcu_state structures' ->rda pointers to per-CPU
	rcu_data structures at compile time, so that this linkage
	is established for early-boot usage.

3.	Remove diagnostics that currently splat in the presence of
	early-boot RCU callbacks.

4.	Update no-CBs diagnostics to allow for early-boot RCU callbacks.

5.	Make RCU's CPU-hotplug online code avoid initializing non-empty
	RCU callback lists in order to avoid leaking early-boot callbacks.

6.	When a given CPU is determined to be a no-CBs CPU, move any
	early-boot callbacks to its no-CBs callback list.

7.	Move the early-boot callback tests to precede rcu_init()'s
	initialization code, the better to test all the above.

							Thanx, Paul

------------------------------------------------------------------------

 b/kernel/rcu/tree.c        |   52 ++++++++++++++++++++++++++++++++-------------
 b/kernel/rcu/tree_plugin.h |   27 ++++++++++++-----------
 2 files changed, 51 insertions(+), 28 deletions(-)

[PATCH tip/core/rcu 1/7] rcu: Abstract default callback-list initialization from init_callback_list()

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

In preparation for early-boot posting of callbacks, this commit abstracts
initialization of the default (non-no-CB) callbacks list from the
init_callback_list() function into a new init_default_callback_list()
function.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 48d640ca1a05..f8cdb92da10b 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1328,20 +1328,30 @@ void rcu_cpu_stall_reset(void)
 }
 
 /*
- * Initialize the specified rcu_data structure's callback list to empty.
+ * Initialize the specified rcu_data structure's default callback list
+ * to empty.  The default callback list is the one that is not used by
+ * no-callbacks CPUs.
  */
-static void init_callback_list(struct rcu_data *rdp)
+static void init_default_callback_list(struct rcu_data *rdp)
 {
 	int i;
 
-	if (init_nocb_callback_list(rdp))
-		return;
 	rdp->nxtlist = NULL;
 	for (i = 0; i < RCU_NEXT_SIZE; i++)
 		rdp->nxttail[i] = &rdp->nxtlist;
 }
 
 /*
+ * Initialize the specified rcu_data structure's callback list to empty.
+ */
+static void init_callback_list(struct rcu_data *rdp)
+{
+	if (init_nocb_callback_list(rdp))
+		return;
+	init_default_callback_list(rdp);
+}
+
+/*
  * Determine the value that ->completed will have at the end of the
  * next subsequent grace period.  This is used to tag callbacks so that
  * a CPU can invoke callbacks in a timely fashion even if that CPU has
-- 
1.8.1.5

[PATCH tip/core/rcu 2/7] rcu: Wire ->rda pointers at compile time

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

This commit wires up the rcu_state structures' ->rda pointers to the
per-CPU rcu_data structures at compile time, thus ensuring that this
linkage is present at early boot, in turn allowing posting of callbacks
before rcu_init() is executed.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index f8cdb92da10b..d2fa95e4a268 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -91,8 +91,10 @@ static const char *tp_##sname##_varname __used __tracepoint_string = sname##_var
 
 #define RCU_STATE_INITIALIZER(sname, sabbr, cr) \
 DEFINE_RCU_TPS(sname) \
+DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data); \
 struct rcu_state sname##_state = { \
 	.level = { &sname##_state.node[0] }, \
+	.rda = &sname##_data, \
 	.call = cr, \
 	.fqs_state = RCU_GP_IDLE, \
 	.gpnum = 0UL - 300UL, \
@@ -104,8 +106,7 @@ struct rcu_state sname##_state = { \
 	.onoff_mutex = __MUTEX_INITIALIZER(sname##_state.onoff_mutex), \
 	.name = RCU_STATE_NAME(sname), \
 	.abbr = sabbr, \
-}; \
-DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_data, sname##_data)
+}
 
 RCU_STATE_INITIALIZER(rcu_sched, 's', call_rcu_sched);
 RCU_STATE_INITIALIZER(rcu_bh, 'b', call_rcu_bh);
@@ -3843,7 +3844,6 @@ static void __init rcu_init_one(struct rcu_state *rsp,
 		}
 	}
 
-	rsp->rda = rda;
 	init_waitqueue_head(&rsp->gp_wq);
 	rnp = rsp->level[rcu_num_lvls - 1];
 	for_each_possible_cpu(i) {
-- 
1.8.1.5

[PATCH tip/core/rcu 3/7] rcu: Prevent early-boot RCU callbacks from splatting

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

Currently, a call_rcu() that precedes rcu_init() will splat due to the
callback lists not having yet been initialized.  This commit causes the
first such callback to initialize the boot CPU's RCU callback list.

Note that this commit does not change rcu_init()-time initialization,
which means that the callback will be discarded at rcu_init() time.
Fixing this is the job of later commits.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index d2fa95e4a268..fcfdbe53bb70 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2838,11 +2838,21 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
 
 		if (cpu != -1)
 			rdp = per_cpu_ptr(rsp->rda, cpu);
-		offline = !__call_rcu_nocb(rdp, head, lazy, flags);
-		WARN_ON_ONCE(offline);
-		/* _call_rcu() is illegal on offline CPU; leak the callback. */
-		local_irq_restore(flags);
-		return;
+		if (likely(rdp->mynode)) {
+			/* Post-boot, so this should be for a no-CBs CPU. */
+			offline = !__call_rcu_nocb(rdp, head, lazy, flags);
+			WARN_ON_ONCE(offline);
+			/* Offline CPU, _call_rcu() illegal, leak callback.  */
+			local_irq_restore(flags);
+			return;
+		}
+		/*
+		 * Very early boot, before rcu_init().  Initialize if needed
+		 * and then drop through to queue the callback.
+		 */
+		BUG_ON(cpu != -1);
+		if (!likely(rdp->nxtlist))
+			init_default_callback_list(rdp);
 	}
 	ACCESS_ONCE(rdp->qlen) = rdp->qlen + 1;
 	if (lazy)
-- 
1.8.1.5

[PATCH tip/core/rcu 4/7] rcu: Refine diagnostics for lacking kthread for no-CBs callbacks

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

Some diagnostics under CONFIG_PROVE_RCU in rcu_nocb_cpu_needs_barrier()
assume that there can be no early-boot callbacks.  This commit therefore
qualifies the diagnostic with rcu_scheduler_fully_active to permit
early boot callbacks to avoid this splat.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree_plugin.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 0a571e9a0f1d..75d5f096bcb0 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -1945,7 +1945,8 @@ static bool rcu_nocb_cpu_needs_barrier(struct rcu_state *rsp, int cpu)
 		rhp = ACCESS_ONCE(rdp->nocb_follower_head);
 
 	/* Having no rcuo kthread but CBs after scheduler starts is bad! */
-	if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp) {
+	if (!ACCESS_ONCE(rdp->nocb_kthread) && rhp &&
+	    rcu_scheduler_fully_active) {
 		/* RCU callback enqueued before CPU first came online??? */
 		pr_err("RCU: Never-onlined no-CBs CPU %d has CB %p\n",
 		       cpu, rhp->func);
-- 
1.8.1.5

[PATCH tip/core/rcu 5/7] rcu: Avoid clobbering early boot callbacks

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

When a CPU comes online, it initializes its callback list.  This
is a bad thing if this is the first time that the CPU has come
online and if that CPU has early boot callbacks.  This commit therefore
avoid initializing the callback list if there are callbacks present,
in which case the initial call_rcu() did the initialization for us.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index fcfdbe53bb70..92fd3eab5823 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3583,7 +3583,8 @@ rcu_init_percpu_data(int cpu, struct rcu_state *rsp)
 	rdp->qlen_last_fqs_check = 0;
 	rdp->n_force_qs_snap = rsp->n_force_qs;
 	rdp->blimit = blimit;
-	init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
+	if (!rdp->nxtlist)
+		init_callback_list(rdp);  /* Re-enable callbacks on this CPU. */
 	rdp->dynticks->dynticks_nesting = DYNTICK_TASK_EXIT_IDLE;
 	rcu_sysidle_init_percpu_data(rdp->dynticks);
 	atomic_set(&rdp->dynticks->dynticks,
-- 
1.8.1.5

[PATCH tip/core/rcu 6/7] rcu: Move early-boot callbacks to no-CBs lists for no-CBs CPUs

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

When a CPU is first determined to be a no-CBs CPUs, this commit causes
any early boot callbacks to be moved to the no-CBs callback list,
allowing them t obe invoked.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c        |  1 +
 kernel/rcu/tree_plugin.h | 24 ++++++++++++------------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 92fd3eab5823..0317bf7d997f 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -2851,6 +2851,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
 		 * and then drop through to queue the callback.
 		 */
 		BUG_ON(cpu != -1);
+		WARN_ON_ONCE(!rcu_is_watching());
 		if (!likely(rdp->nxtlist))
 			init_default_callback_list(rdp);
 	}
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 75d5f096bcb0..afddd5641bea 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2393,18 +2393,8 @@ void __init rcu_init_nohz(void)
 		pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
 
 	for_each_rcu_flavor(rsp) {
-		for_each_cpu(cpu, rcu_nocb_mask) {
-			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
-
-			/*
-			 * If there are early callbacks, they will need
-			 * to be moved to the nocb lists.
-			 */
-			WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=
-				     &rdp->nxtlist &&
-				     rdp->nxttail[RCU_NEXT_TAIL] != NULL);
-			init_nocb_callback_list(rdp);
-		}
+		for_each_cpu(cpu, rcu_nocb_mask)
+			init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
 		rcu_organize_nocb_kthreads(rsp);
 	}
 }
@@ -2541,6 +2531,16 @@ static bool init_nocb_callback_list(struct rcu_data *rdp)
 	if (!rcu_is_nocb_cpu(rdp->cpu))
 		return false;
 
+	/* If there are early-boot callbacks, move them to nocb lists. */
+	if (rdp->nxtlist) {
+		rdp->nocb_head = rdp->nxtlist;
+		rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
+		atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
+		atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
+		rdp->nxtlist = NULL;
+		rdp->qlen = 0;
+		rdp->qlen_lazy = 0;
+	}
 	rdp->nxttail[RCU_NEXT_TAIL] = NULL;
 	return true;
 }
-- 
1.8.1.5

[PATCH tip/core/rcu 7/7] rcu: Move early boot callback tests earlier

[Paul E. McKenney]

From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>

Because callbacks can now be posted quite early in boot, move the
early boot callback tests to precede RCU initialization.

Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
---
 kernel/rcu/tree.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 0317bf7d997f..c8e6569c5fbd 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -3948,6 +3948,8 @@ void __init rcu_init(void)
 {
 	int cpu;
 
+	rcu_early_boot_tests();
+
 	rcu_bootup_announce();
 	rcu_init_geometry();
 	rcu_init_one(&rcu_bh_state, &rcu_bh_data);
@@ -3964,8 +3966,6 @@ void __init rcu_init(void)
 	pm_notifier(rcu_pm_notify, 0);
 	for_each_online_cpu(cpu)
 		rcu_cpu_notify(NULL, CPU_UP_PREPARE, (void *)(long)cpu);
-
-	rcu_early_boot_tests();
 }
 
 #include "tree_plugin.h"
-- 
1.8.1.5

Re: [PATCH tip/core/rcu 6/7] rcu: Move early-boot callbacks to no-CBs lists for no-CBs CPUs

[Mathieu Desnoyers]

----- Original Message -----
> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> To: linux-kernel@vger.kernel.org
> Cc: mingo@kernel.org, laijs@cn.fujitsu.com, dipankar@in.ibm.com, akpm@linux-foundation.org, "mathieu desnoyers"
> <mathieu.desnoyers@efficios.com>, josh@joshtriplett.org, tglx@linutronix.de, peterz@infradead.org,
> rostedt@goodmis.org, dhowells@redhat.com, edumazet@google.com, dvhart@linux.intel.com, fweisbec@gmail.com,
> oleg@redhat.com, "bobby prani" <bobby.prani@gmail.com>, "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> Sent: Tuesday, March 3, 2015 11:45:47 AM
> Subject: [PATCH tip/core/rcu 6/7] rcu: Move early-boot callbacks to no-CBs lists for no-CBs CPUs
> 
> From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> 
> When a CPU is first determined to be a no-CBs CPUs, this commit causes
> any early boot callbacks to be moved to the no-CBs callback list,
> allowing them t obe invoked.
                ^^^^^
out-of-order space ;)

Thanks,

Mathieu

> 
> Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> ---
>  kernel/rcu/tree.c        |  1 +
>  kernel/rcu/tree_plugin.h | 24 ++++++++++++------------
>  2 files changed, 13 insertions(+), 12 deletions(-)
> 
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index 92fd3eab5823..0317bf7d997f 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -2851,6 +2851,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct
> rcu_head *rcu),
>  		 * and then drop through to queue the callback.
>  		 */
>  		BUG_ON(cpu != -1);
> +		WARN_ON_ONCE(!rcu_is_watching());
>  		if (!likely(rdp->nxtlist))
>  			init_default_callback_list(rdp);
>  	}
> diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
> index 75d5f096bcb0..afddd5641bea 100644
> --- a/kernel/rcu/tree_plugin.h
> +++ b/kernel/rcu/tree_plugin.h
> @@ -2393,18 +2393,8 @@ void __init rcu_init_nohz(void)
>  		pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
>  
>  	for_each_rcu_flavor(rsp) {
> -		for_each_cpu(cpu, rcu_nocb_mask) {
> -			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
> -
> -			/*
> -			 * If there are early callbacks, they will need
> -			 * to be moved to the nocb lists.
> -			 */
> -			WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=
> -				     &rdp->nxtlist &&
> -				     rdp->nxttail[RCU_NEXT_TAIL] != NULL);
> -			init_nocb_callback_list(rdp);
> -		}
> +		for_each_cpu(cpu, rcu_nocb_mask)
> +			init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
>  		rcu_organize_nocb_kthreads(rsp);
>  	}
>  }
> @@ -2541,6 +2531,16 @@ static bool init_nocb_callback_list(struct rcu_data
> *rdp)
>  	if (!rcu_is_nocb_cpu(rdp->cpu))
>  		return false;
>  
> +	/* If there are early-boot callbacks, move them to nocb lists. */
> +	if (rdp->nxtlist) {
> +		rdp->nocb_head = rdp->nxtlist;
> +		rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
> +		atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
> +		atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
> +		rdp->nxtlist = NULL;
> +		rdp->qlen = 0;
> +		rdp->qlen_lazy = 0;
> +	}
>  	rdp->nxttail[RCU_NEXT_TAIL] = NULL;
>  	return true;
>  }
> --
> 1.8.1.5
> 
> 

-- 
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com

Re: [PATCH tip/core/rcu 6/7] rcu: Move early-boot callbacks to no-CBs lists for no-CBs CPUs

[Paul E. McKenney]

On Tue, Mar 03, 2015 at 04:57:16PM +0000, Mathieu Desnoyers wrote:
> ----- Original Message -----
> > From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> > To: linux-kernel@vger.kernel.org
> > Cc: mingo@kernel.org, laijs@cn.fujitsu.com, dipankar@in.ibm.com, akpm@linux-foundation.org, "mathieu desnoyers"
> > <mathieu.desnoyers@efficios.com>, josh@joshtriplett.org, tglx@linutronix.de, peterz@infradead.org,
> > rostedt@goodmis.org, dhowells@redhat.com, edumazet@google.com, dvhart@linux.intel.com, fweisbec@gmail.com,
> > oleg@redhat.com, "bobby prani" <bobby.prani@gmail.com>, "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> > Sent: Tuesday, March 3, 2015 11:45:47 AM
> > Subject: [PATCH tip/core/rcu 6/7] rcu: Move early-boot callbacks to no-CBs lists for no-CBs CPUs
> > 
> > From: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> > 
> > When a CPU is first determined to be a no-CBs CPUs, this commit causes
> > any early boot callbacks to be moved to the no-CBs callback list,
> > allowing them t obe invoked.
>                 ^^^^^
> out-of-order space ;)

I clearly needed a typography barrier!  ;-)

Good catch!

							Thanx, Paul

> Thanks,
> 
> Mathieu
> 
> > 
> > Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> > ---
> >  kernel/rcu/tree.c        |  1 +
> >  kernel/rcu/tree_plugin.h | 24 ++++++++++++------------
> >  2 files changed, 13 insertions(+), 12 deletions(-)
> > 
> > diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> > index 92fd3eab5823..0317bf7d997f 100644
> > --- a/kernel/rcu/tree.c
> > +++ b/kernel/rcu/tree.c
> > @@ -2851,6 +2851,7 @@ __call_rcu(struct rcu_head *head, void (*func)(struct
> > rcu_head *rcu),
> >  		 * and then drop through to queue the callback.
> >  		 */
> >  		BUG_ON(cpu != -1);
> > +		WARN_ON_ONCE(!rcu_is_watching());
> >  		if (!likely(rdp->nxtlist))
> >  			init_default_callback_list(rdp);
> >  	}
> > diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
> > index 75d5f096bcb0..afddd5641bea 100644
> > --- a/kernel/rcu/tree_plugin.h
> > +++ b/kernel/rcu/tree_plugin.h
> > @@ -2393,18 +2393,8 @@ void __init rcu_init_nohz(void)
> >  		pr_info("\tPoll for callbacks from no-CBs CPUs.\n");
> >  
> >  	for_each_rcu_flavor(rsp) {
> > -		for_each_cpu(cpu, rcu_nocb_mask) {
> > -			struct rcu_data *rdp = per_cpu_ptr(rsp->rda, cpu);
> > -
> > -			/*
> > -			 * If there are early callbacks, they will need
> > -			 * to be moved to the nocb lists.
> > -			 */
> > -			WARN_ON_ONCE(rdp->nxttail[RCU_NEXT_TAIL] !=
> > -				     &rdp->nxtlist &&
> > -				     rdp->nxttail[RCU_NEXT_TAIL] != NULL);
> > -			init_nocb_callback_list(rdp);
> > -		}
> > +		for_each_cpu(cpu, rcu_nocb_mask)
> > +			init_nocb_callback_list(per_cpu_ptr(rsp->rda, cpu));
> >  		rcu_organize_nocb_kthreads(rsp);
> >  	}
> >  }
> > @@ -2541,6 +2531,16 @@ static bool init_nocb_callback_list(struct rcu_data
> > *rdp)
> >  	if (!rcu_is_nocb_cpu(rdp->cpu))
> >  		return false;
> >  
> > +	/* If there are early-boot callbacks, move them to nocb lists. */
> > +	if (rdp->nxtlist) {
> > +		rdp->nocb_head = rdp->nxtlist;
> > +		rdp->nocb_tail = rdp->nxttail[RCU_NEXT_TAIL];
> > +		atomic_long_set(&rdp->nocb_q_count, rdp->qlen);
> > +		atomic_long_set(&rdp->nocb_q_count_lazy, rdp->qlen_lazy);
> > +		rdp->nxtlist = NULL;
> > +		rdp->qlen = 0;
> > +		rdp->qlen_lazy = 0;
> > +	}
> >  	rdp->nxttail[RCU_NEXT_TAIL] = NULL;
> >  	return true;
> >  }
> > --
> > 1.8.1.5
> > 
> > 
> 
> -- 
> Mathieu Desnoyers
> EfficiOS Inc.
> http://www.efficios.com
>