[PATCH] sched: Teach might_sleep about preemptable rcu

[PATCH] sched: Teach might_sleep about preemptable rcu

[Frederic Weisbecker]

In practice, it is harmless to voluntarily sleep in a rcu_read_lock()
section if we are running under preempt rcu, but it is illegal because
if we build a kernel running non-preemptable rcu.

Currently, might_sleep() doesn't notice sleepable operations under
rcu_read_lock() sections if we are running under preemptable rcu
because preempt_count() is left untouched after rcu_read_lock() in
this case. But we want developers who test their changes under such
config to notice the "sleeping while atomic" issues.

Then we add rcu_read_lock_nesting to prempt_count() in might_sleep()
checks.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
---
 include/linux/rcutree.h |   11 +++++++++++
 kernel/sched.c          |    2 +-
 2 files changed, 12 insertions(+), 1 deletions(-)

diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index c93eee5..8044b1b 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -45,6 +45,12 @@ extern void __rcu_read_unlock(void);
 extern void synchronize_rcu(void);
 extern void exit_rcu(void);
 
+/*
+ * Defined as macro as it is a very low level header
+ * included from areas that don't even know about current
+ */
+#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
+
 #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static inline void __rcu_read_lock(void)
@@ -63,6 +69,11 @@ static inline void exit_rcu(void)
 {
 }
 
+static inline int rcu_preempt_depth(void)
+{
+	return 0;
+}
+
 #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static inline void __rcu_read_lock_bh(void)
diff --git a/kernel/sched.c b/kernel/sched.c
index ab42754..586c82c 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -9658,7 +9658,7 @@ void __init sched_init(void)
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
 static inline int preempt_count_equals(int preempt_offset)
 {
-	int nested = preempt_count() & ~PREEMPT_ACTIVE;
+	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
 
 	return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
 }
-- 
1.6.2.3

Re: [PATCH] sched: Teach might_sleep about preemptable rcu

[Paul E. McKenney]

On Mon, Dec 14, 2009 at 11:44:32PM +0100, Frederic Weisbecker wrote:
> In practice, it is harmless to voluntarily sleep in a rcu_read_lock()
> section if we are running under preempt rcu, but it is illegal because
> if we build a kernel running non-preemptable rcu.
> 
> Currently, might_sleep() doesn't notice sleepable operations under
> rcu_read_lock() sections if we are running under preemptable rcu
> because preempt_count() is left untouched after rcu_read_lock() in
> this case. But we want developers who test their changes under such
> config to notice the "sleeping while atomic" issues.
> 
> Then we add rcu_read_lock_nesting to prempt_count() in might_sleep()
> checks.

Cute!!!

Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>

> Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com>
> Cc: Peter Zijlstra <peterz@infradead.org>
> ---
>  include/linux/rcutree.h |   11 +++++++++++
>  kernel/sched.c          |    2 +-
>  2 files changed, 12 insertions(+), 1 deletions(-)
> 
> diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
> index c93eee5..8044b1b 100644
> --- a/include/linux/rcutree.h
> +++ b/include/linux/rcutree.h
> @@ -45,6 +45,12 @@ extern void __rcu_read_unlock(void);
>  extern void synchronize_rcu(void);
>  extern void exit_rcu(void);
> 
> +/*
> + * Defined as macro as it is a very low level header
> + * included from areas that don't even know about current
> + */
> +#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
> +
>  #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
> 
>  static inline void __rcu_read_lock(void)
> @@ -63,6 +69,11 @@ static inline void exit_rcu(void)
>  {
>  }
> 
> +static inline int rcu_preempt_depth(void)
> +{
> +	return 0;
> +}
> +
>  #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
> 
>  static inline void __rcu_read_lock_bh(void)
> diff --git a/kernel/sched.c b/kernel/sched.c
> index ab42754..586c82c 100644
> --- a/kernel/sched.c
> +++ b/kernel/sched.c
> @@ -9658,7 +9658,7 @@ void __init sched_init(void)
>  #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
>  static inline int preempt_count_equals(int preempt_offset)
>  {
> -	int nested = preempt_count() & ~PREEMPT_ACTIVE;
> +	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
> 
>  	return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
>  }
> -- 
> 1.6.2.3
> 

[tip:sched/urgent] sched: Teach might_sleep() about preemptible RCU

[tip-bot for Frederic Weisbecker]

Commit-ID:  ba1b1cbcc9b458c4b9ebb28c9cf6a2ccf64ba1cd
Gitweb:     http://git.kernel.org/tip/ba1b1cbcc9b458c4b9ebb28c9cf6a2ccf64ba1cd
Author:     Frederic Weisbecker <fweisbec@gmail.com>
AuthorDate: Mon, 14 Dec 2009 23:44:32 +0100
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Wed, 16 Dec 2009 08:43:16 +0100

sched: Teach might_sleep() about preemptible RCU

In practice, it is harmless to voluntarily sleep in a
rcu_read_lock() section if we are running under preempt rcu, but
it is illegal if we build a kernel running non-preemptable rcu.

Currently, might_sleep() doesn't notice sleepable operations
under rcu_read_lock() sections if we are running under
preemptable rcu because preempt_count() is left untouched after
rcu_read_lock() in this case. But we want developers who test
their changes under such config to notice the "sleeping while
atomic" issues.

So we add rcu_read_lock_nesting to prempt_count() in
might_sleep() checks.

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
LKML-Reference: <1260830672-7166-1-git-send-regression-fweisbec@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 include/linux/rcutree.h |   11 +++++++++++
 kernel/sched.c          |    2 +-
 2 files changed, 12 insertions(+), 1 deletions(-)

diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index c93eee5..8044b1b 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -45,6 +45,12 @@ extern void __rcu_read_unlock(void);
 extern void synchronize_rcu(void);
 extern void exit_rcu(void);
 
+/*
+ * Defined as macro as it is a very low level header
+ * included from areas that don't even know about current
+ */
+#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
+
 #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static inline void __rcu_read_lock(void)
@@ -63,6 +69,11 @@ static inline void exit_rcu(void)
 {
 }
 
+static inline int rcu_preempt_depth(void)
+{
+	return 0;
+}
+
 #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static inline void __rcu_read_lock_bh(void)
diff --git a/kernel/sched.c b/kernel/sched.c
index db5c266..d76c790 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -9655,7 +9655,7 @@ void __init sched_init(void)
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
 static inline int preempt_count_equals(int preempt_offset)
 {
-	int nested = preempt_count() & ~PREEMPT_ACTIVE;
+	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
 
 	return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
 }

Re: [tip:sched/urgent] sched: Teach might_sleep() about preemptible RCU

[Ingo Molnar]

* tip-bot for Frederic Weisbecker <fweisbec@gmail.com> wrote:

> Commit-ID:  ba1b1cbcc9b458c4b9ebb28c9cf6a2ccf64ba1cd
> Gitweb:     http://git.kernel.org/tip/ba1b1cbcc9b458c4b9ebb28c9cf6a2ccf64ba1cd
> Author:     Frederic Weisbecker <fweisbec@gmail.com>
> AuthorDate: Mon, 14 Dec 2009 23:44:32 +0100
> Committer:  Ingo Molnar <mingo@elte.hu>
> CommitDate: Wed, 16 Dec 2009 08:43:16 +0100
> 
> sched: Teach might_sleep() about preemptible RCU

Causes a build failure, so skipped this patch for now:

kernel/sched.c:9661: error: implicit declaration of function 'rcu_preempt_depth'

	Ingo

Re: [tip:sched/urgent] sched: Teach might_sleep() about preemptible RCU

[Paul E. McKenney]

On Wed, Dec 16, 2009 at 03:25:36PM +0100, Ingo Molnar wrote:
> 
> * tip-bot for Frederic Weisbecker <fweisbec@gmail.com> wrote:
> 
> > Commit-ID:  ba1b1cbcc9b458c4b9ebb28c9cf6a2ccf64ba1cd
> > Gitweb:     http://git.kernel.org/tip/ba1b1cbcc9b458c4b9ebb28c9cf6a2ccf64ba1cd
> > Author:     Frederic Weisbecker <fweisbec@gmail.com>
> > AuthorDate: Mon, 14 Dec 2009 23:44:32 +0100
> > Committer:  Ingo Molnar <mingo@elte.hu>
> > CommitDate: Wed, 16 Dec 2009 08:43:16 +0100
> > 
> > sched: Teach might_sleep() about preemptible RCU
> 
> Causes a build failure, so skipped this patch for now:
> 
> kernel/sched.c:9661: error: implicit declaration of function 'rcu_preempt_depth'

I guess my testing missed something.  :-/  Could you please send the
.config file that caused this?

							Thanx, Paul

Re: [tip:sched/urgent] sched: Teach might_sleep() about preemptible RCU

[Frederic Weisbecker]

On Wed, Dec 16, 2009 at 07:30:20AM -0800, Paul E. McKenney wrote:
> On Wed, Dec 16, 2009 at 03:25:36PM +0100, Ingo Molnar wrote:
> > 
> > * tip-bot for Frederic Weisbecker <fweisbec@gmail.com> wrote:
> > 
> > > Commit-ID:  ba1b1cbcc9b458c4b9ebb28c9cf6a2ccf64ba1cd
> > > Gitweb:     http://git.kernel.org/tip/ba1b1cbcc9b458c4b9ebb28c9cf6a2ccf64ba1cd
> > > Author:     Frederic Weisbecker <fweisbec@gmail.com>
> > > AuthorDate: Mon, 14 Dec 2009 23:44:32 +0100
> > > Committer:  Ingo Molnar <mingo@elte.hu>
> > > CommitDate: Wed, 16 Dec 2009 08:43:16 +0100
> > > 
> > > sched: Teach might_sleep() about preemptible RCU
> > 
> > Causes a build failure, so skipped this patch for now:
> > 
> > kernel/sched.c:9661: error: implicit declaration of function 'rcu_preempt_depth'
> 
> I guess my testing missed something.  :-/  Could you please send the
> .config file that caused this?
> 
> 							Thanx, Paul


Looks like the problem is here:

rcupdate.h:

	#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
	#include <linux/rcutree.h>
	#elif defined(CONFIG_TINY_RCU)
	#include <linux/rcutiny.h>

I guess it barfs in rcutiny configs.

I should define these helpers in rcupdate.

Re: [tip:sched/urgent] sched: Teach might_sleep() about preemptible RCU

[Frederic Weisbecker]

On Wed, Dec 16, 2009 at 07:57:32PM +0100, Frederic Weisbecker wrote:
> Looks like the problem is here:
> 
> rcupdate.h:
> 
> 	#if defined(CONFIG_TREE_RCU) || defined(CONFIG_TREE_PREEMPT_RCU)
> 	#include <linux/rcutree.h>
> 	#elif defined(CONFIG_TINY_RCU)
> 	#include <linux/rcutiny.h>
> 
> I guess it barfs in rcutiny configs.
> 
> I should define these helpers in rcupdate.
> 


Actually I'm going to define the rcu tiny off-case from rcutiny.h
It looks like rcupdate.h should really keep only the generic
bits.

[PATCH] sched: Teach might_sleep() about preemptible RCU

[Frederic Weisbecker]

From: tip-bot for Frederic Weisbecker <fweisbec@gmail.com>

In practice, it is harmless to voluntarily sleep in a
rcu_read_lock() section if we are running under preempt rcu, but
it is illegal if we build a kernel running non-preemptable rcu.

Currently, might_sleep() doesn't notice sleepable operations
under rcu_read_lock() sections if we are running under
preemptable rcu because preempt_count() is left untouched after
rcu_read_lock() in this case. But we want developers who test
their changes under such config to notice the "sleeping while
atomic" issues.

So we add rcu_read_lock_nesting to prempt_count() in
might_sleep() checks.

v2: Handle rcu tiny

Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
---
 include/linux/rcutiny.h |    5 +++++
 include/linux/rcutree.h |   11 +++++++++++
 kernel/sched.c          |    2 +-
 3 files changed, 17 insertions(+), 1 deletions(-)

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index c4ba9a7..96cc307 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -101,4 +101,9 @@ static inline void exit_rcu(void)
 {
 }
 
+static inline int rcu_preempt_depth(void)
+{
+	return 0;
+}
+
 #endif /* __LINUX_RCUTINY_H */
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index c93eee5..8044b1b 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -45,6 +45,12 @@ extern void __rcu_read_unlock(void);
 extern void synchronize_rcu(void);
 extern void exit_rcu(void);
 
+/*
+ * Defined as macro as it is a very low level header
+ * included from areas that don't even know about current
+ */
+#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
+
 #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static inline void __rcu_read_lock(void)
@@ -63,6 +69,11 @@ static inline void exit_rcu(void)
 {
 }
 
+static inline int rcu_preempt_depth(void)
+{
+	return 0;
+}
+
 #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static inline void __rcu_read_lock_bh(void)
diff --git a/kernel/sched.c b/kernel/sched.c
index 8a2bfd3..418e638 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -9682,7 +9682,7 @@ void __init sched_init(void)
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
 static inline int preempt_count_equals(int preempt_offset)
 {
-	int nested = preempt_count() & ~PREEMPT_ACTIVE;
+	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
 
 	return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
 }
-- 
1.6.2.3

Re: [PATCH] sched: Teach might_sleep() about preemptible RCU

[Frederic Weisbecker]

On Wed, Dec 16, 2009 at 08:21:05PM +0100, Frederic Weisbecker wrote:
> From: tip-bot for Frederic Weisbecker <fweisbec@gmail.com>


Crap... I think I'm really becoming this cyborg...

> Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
> Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>


Although I've happily put Paul's Reviewed-by tag in this v2,
I'd feel more comfortable if he could confirm it :)

Thanks!

Re: [PATCH] sched: Teach might_sleep() about preemptible RCU

[Paul E. McKenney]

On Wed, Dec 16, 2009 at 08:23:57PM +0100, Frederic Weisbecker wrote:
> On Wed, Dec 16, 2009 at 08:21:05PM +0100, Frederic Weisbecker wrote:
> > From: tip-bot for Frederic Weisbecker <fweisbec@gmail.com>
> 
> Crap... I think I'm really becoming this cyborg...
> 
> > Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
> > Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> 
> Although I've happily put Paul's Reviewed-by tag in this v2,
> I'd feel more comfortable if he could confirm it :)

Given that I missed the TINY_RCU problem, I am not sure how much my
Reviewed-by tag is worth, but I have no problem attaching it to this
updated patch.  ;-)

						Thanx, Paul

Re: [PATCH] sched: Teach might_sleep() about preemptible RCU

[Ingo Molnar]

* Paul E. McKenney <paulmck@linux.vnet.ibm.com> wrote:

> On Wed, Dec 16, 2009 at 08:23:57PM +0100, Frederic Weisbecker wrote:
> > On Wed, Dec 16, 2009 at 08:21:05PM +0100, Frederic Weisbecker wrote:
> > > From: tip-bot for Frederic Weisbecker <fweisbec@gmail.com>
> > 
> > Crap... I think I'm really becoming this cyborg...
> > 
> > > Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
> > > Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> > 
> > Although I've happily put Paul's Reviewed-by tag in this v2,
> > I'd feel more comfortable if he could confirm it :)
> 
> Given that I missed the TINY_RCU problem, I am not sure how much my 
> Reviewed-by tag is worth, but I have no problem attaching it to this updated 
> patch.  ;-)

Reviewed-by is valuable in terms of you both acking the patch and expressing 
that you thought it through and like it. It does not mean you have done a 
mental kernel build test of all 2^3000 kernel .config variants and have proven 
the patch correct under circumstances! :-)

So yes, i've added it - thanks Paul!

	Ingo

Re: [PATCH] sched: Teach might_sleep() about preemptible RCU

[Paul E. McKenney]

On Thu, Dec 17, 2009 at 09:49:12AM +0100, Ingo Molnar wrote:
> 
> * Paul E. McKenney <paulmck@linux.vnet.ibm.com> wrote:
> 
> > On Wed, Dec 16, 2009 at 08:23:57PM +0100, Frederic Weisbecker wrote:
> > > On Wed, Dec 16, 2009 at 08:21:05PM +0100, Frederic Weisbecker wrote:
> > > > From: tip-bot for Frederic Weisbecker <fweisbec@gmail.com>
> > > 
> > > Crap... I think I'm really becoming this cyborg...
> > > 
> > > > Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
> > > > Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
> > > 
> > > Although I've happily put Paul's Reviewed-by tag in this v2,
> > > I'd feel more comfortable if he could confirm it :)
> > 
> > Given that I missed the TINY_RCU problem, I am not sure how much my 
> > Reviewed-by tag is worth, but I have no problem attaching it to this updated 
> > patch.  ;-)
> 
> Reviewed-by is valuable in terms of you both acking the patch and expressing 
> that you thought it through and like it. It does not mean you have done a 
> mental kernel build test of all 2^3000 kernel .config variants and have proven 
> the patch correct under circumstances! :-)
> 
> So yes, i've added it - thanks Paul!

The one misgiving I have is that addition of threaded interrupts and
sleeping spinlocks might break this, but those changes will be large
enough that this is the least of the worries.  We might then need to
split might_sleep() into might_sleep() and might_sleep_rt(), but hard
to say before the fact.  And this check will be very worthwhile as-is
in the meantime!  ;-)

							Thanx, Paul

[tip:sched/urgent] sched: Teach might_sleep() about preemptible RCU

[tip-bot for Frederic Weisbecker]

Commit-ID:  234da7bcdc7aaa935846534c3b726dbc79a9cdd5
Gitweb:     http://git.kernel.org/tip/234da7bcdc7aaa935846534c3b726dbc79a9cdd5
Author:     Frederic Weisbecker <fweisbec@gmail.com>
AuthorDate: Wed, 16 Dec 2009 20:21:05 +0100
Committer:  Ingo Molnar <mingo@elte.hu>
CommitDate: Thu, 17 Dec 2009 09:46:44 +0100

sched: Teach might_sleep() about preemptible RCU

In practice, it is harmless to voluntarily sleep in a
rcu_read_lock() section if we are running under preempt rcu, but
it is illegal if we build a kernel running non-preemptable rcu.

Currently, might_sleep() doesn't notice sleepable operations
under rcu_read_lock() sections if we are running under
preemptable rcu because preempt_count() is left untouched after
rcu_read_lock() in this case. But we want developers who test
their changes under such config to notice the "sleeping while
atomic" issues.

So we add rcu_read_lock_nesting to prempt_count() in
might_sleep() checks.

[ v2: Handle rcu-tiny ]
Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com>
Reviewed-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Peter Zijlstra <peterz@infradead.org>
LKML-Reference: <1260991265-8451-1-git-send-regression-fweisbec@gmail.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>
---
 include/linux/rcutiny.h |    5 +++++
 include/linux/rcutree.h |   11 +++++++++++
 kernel/sched.c          |    2 +-
 3 files changed, 17 insertions(+), 1 deletions(-)

diff --git a/include/linux/rcutiny.h b/include/linux/rcutiny.h
index c4ba9a7..96cc307 100644
--- a/include/linux/rcutiny.h
+++ b/include/linux/rcutiny.h
@@ -101,4 +101,9 @@ static inline void exit_rcu(void)
 {
 }
 
+static inline int rcu_preempt_depth(void)
+{
+	return 0;
+}
+
 #endif /* __LINUX_RCUTINY_H */
diff --git a/include/linux/rcutree.h b/include/linux/rcutree.h
index c93eee5..8044b1b 100644
--- a/include/linux/rcutree.h
+++ b/include/linux/rcutree.h
@@ -45,6 +45,12 @@ extern void __rcu_read_unlock(void);
 extern void synchronize_rcu(void);
 extern void exit_rcu(void);
 
+/*
+ * Defined as macro as it is a very low level header
+ * included from areas that don't even know about current
+ */
+#define rcu_preempt_depth() (current->rcu_read_lock_nesting)
+
 #else /* #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static inline void __rcu_read_lock(void)
@@ -63,6 +69,11 @@ static inline void exit_rcu(void)
 {
 }
 
+static inline int rcu_preempt_depth(void)
+{
+	return 0;
+}
+
 #endif /* #else #ifdef CONFIG_TREE_PREEMPT_RCU */
 
 static inline void __rcu_read_lock_bh(void)
diff --git a/kernel/sched.c b/kernel/sched.c
index af7dfa7..7be88a7 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -9682,7 +9682,7 @@ void __init sched_init(void)
 #ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
 static inline int preempt_count_equals(int preempt_offset)
 {
-	int nested = preempt_count() & ~PREEMPT_ACTIVE;
+	int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth();
 
 	return (nested == PREEMPT_INATOMIC_BASE + preempt_offset);
 }