* [PATCH v4 01/17] preempt: Track NMI nesting to separate per-CPU counter
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Joel Fernandes
2026-08-04 16:14 ` [PATCH v4 02/17] preempt: Introduce HARDIRQ_DISABLE_BITS Boqun Feng
` (15 subsequent siblings)
16 siblings, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux,
Joel Fernandes
From: Joel Fernandes <joelagnelf@nvidia.com>
Move NMI nesting tracking from the preempt_count bits to a separate
per-CPU counter (nmi_nesting). This is to free up the NMI bits in the
preempt_count, allowing those bits to be repurposed for other uses.
Reduce NMI_BITS from 4 to 1, using it only to detect if we're in an NMI.
The per-CPU counter currently caps nesting at 15.
[boqun: Address Steven Rostedt's comment on the BUG_ON() condition]
[boqun: Use preempt_count_set() in __nmi_exit() to avoid underflow]
Suggested-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Joel Fernandes <joelagnelf@nvidia.com>
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
include/linux/hardirq.h | 17 +++++++++++++----
include/linux/preempt.h | 9 +++++++--
kernel/softirq.c | 2 ++
tools/testing/selftests/bpf/bpf_experimental.h | 2 +-
4 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index d57cab4d4c06..8d4895531a45 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -10,6 +10,8 @@
#include <linux/vtime.h>
#include <asm/hardirq.h>
+DECLARE_PER_CPU(unsigned int, nmi_nesting);
+
extern void synchronize_irq(unsigned int irq);
extern bool synchronize_hardirq(unsigned int irq);
@@ -102,14 +104,17 @@ void irq_exit_rcu(void);
*/
/*
- * nmi_enter() can nest up to 15 times; see NMI_BITS.
+ * nmi_enter() can nest - nesting is tracked in a per-CPU counter.
*/
#define __nmi_enter() \
do { \
lockdep_off(); \
arch_nmi_enter(); \
- BUG_ON(in_nmi() == NMI_MASK); \
- __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
+ /* Maximum NMI nesting is 15. */ \
+ BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
+ __this_cpu_inc(nmi_nesting); \
+ __preempt_count_add(HARDIRQ_OFFSET); \
+ preempt_count_set(preempt_count() | NMI_MASK); \
} while (0)
#define nmi_enter() \
@@ -124,8 +129,12 @@ void irq_exit_rcu(void);
#define __nmi_exit() \
do { \
+ unsigned int nesting; \
BUG_ON(!in_nmi()); \
- __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
+ __preempt_count_sub(HARDIRQ_OFFSET); \
+ nesting = __this_cpu_dec_return(nmi_nesting); \
+ if (!nesting) \
+ preempt_count_set(preempt_count() & ~NMI_MASK); \
arch_nmi_exit(); \
lockdep_on(); \
} while (0)
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index d964f965c8ff..586f96688325 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -17,6 +17,8 @@
*
* - bits 0-7 are the preemption count (max preemption depth: 256)
* - bits 8-15 are the softirq count (max # of softirqs: 256)
+ * - bits 16-19 are the hardirq count (max # of hardirqs: 16)
+ * - bit 20 is the NMI flag (no nesting count, tracked separately)
*
* The hardirq count could in theory be the same as the number of
* interrupts in the system, but we run all interrupt handlers with
@@ -24,16 +26,19 @@
* there are a few palaeontologic drivers which reenable interrupts in
* the handler, so we need more than one bit here.
*
+ * NMI nesting depth is tracked in a separate per-CPU variable
+ * (nmi_nesting) to save bits in preempt_count.
+ *
* PREEMPT_MASK: 0x000000ff
* SOFTIRQ_MASK: 0x0000ff00
* HARDIRQ_MASK: 0x000f0000
- * NMI_MASK: 0x00f00000
+ * NMI_MASK: 0x00100000
* PREEMPT_NEED_RESCHED: 0x80000000
*/
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
#define HARDIRQ_BITS 4
-#define NMI_BITS 4
+#define NMI_BITS 1
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 4425d8dce44b..10af5ed859e7 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -88,6 +88,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
#endif
+DEFINE_PER_CPU(unsigned int, nmi_nesting);
+
/*
* SOFTIRQ_OFFSET usage:
*
diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h
index 67ff7882299e..e4e12001fce9 100644
--- a/tools/testing/selftests/bpf/bpf_experimental.h
+++ b/tools/testing/selftests/bpf/bpf_experimental.h
@@ -367,7 +367,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
#define HARDIRQ_BITS 4
-#define NMI_BITS 4
+#define NMI_BITS 1
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] preempt: Track NMI nesting to separate per-CPU counter
2026-08-04 16:14 ` [PATCH v4 01/17] preempt: Track NMI nesting to separate per-CPU counter Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Joel Fernandes
0 siblings, 0 replies; 66+ messages in thread
From: tip-bot2 for Joel Fernandes @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits
Cc: Boqun Feng, Joel Fernandes, Lyude Paul, Peter Zijlstra (Intel),
x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: b54aa0edf0594a6e1138e626bfd8a352ed582ae9
Gitweb: https://git.kernel.org/tip/b54aa0edf0594a6e1138e626bfd8a352ed582ae9
Author: Joel Fernandes <joelagnelf@nvidia.com>
AuthorDate: Tue, 04 Aug 2026 09:14:23 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:05 +02:00
preempt: Track NMI nesting to separate per-CPU counter
Move NMI nesting tracking from the preempt_count bits to a separate
per-CPU counter (nmi_nesting). This is to free up the NMI bits in the
preempt_count, allowing those bits to be repurposed for other uses.
Reduce NMI_BITS from 4 to 1, using it only to detect if we're in an NMI.
The per-CPU counter currently caps nesting at 15.
[boqun: Address Steven Rostedt's comment on the BUG_ON() condition]
[boqun: Use preempt_count_set() in __nmi_exit() to avoid underflow]
Suggested-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Joel Fernandes <joelagnelf@nvidia.com>
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260121223933.1568682-3-lyude@redhat.com
Link: https://patch.msgid.link/20260804161447.84806-2-boqun@kernel.org
---
include/linux/hardirq.h | 17 +++++++++++++----
include/linux/preempt.h | 9 +++++++--
kernel/softirq.c | 2 ++-
tools/testing/selftests/bpf/bpf_experimental.h | 2 +-
4 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index d57cab4..8d48955 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -10,6 +10,8 @@
#include <linux/vtime.h>
#include <asm/hardirq.h>
+DECLARE_PER_CPU(unsigned int, nmi_nesting);
+
extern void synchronize_irq(unsigned int irq);
extern bool synchronize_hardirq(unsigned int irq);
@@ -102,14 +104,17 @@ void irq_exit_rcu(void);
*/
/*
- * nmi_enter() can nest up to 15 times; see NMI_BITS.
+ * nmi_enter() can nest - nesting is tracked in a per-CPU counter.
*/
#define __nmi_enter() \
do { \
lockdep_off(); \
arch_nmi_enter(); \
- BUG_ON(in_nmi() == NMI_MASK); \
- __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
+ /* Maximum NMI nesting is 15. */ \
+ BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
+ __this_cpu_inc(nmi_nesting); \
+ __preempt_count_add(HARDIRQ_OFFSET); \
+ preempt_count_set(preempt_count() | NMI_MASK); \
} while (0)
#define nmi_enter() \
@@ -124,8 +129,12 @@ void irq_exit_rcu(void);
#define __nmi_exit() \
do { \
+ unsigned int nesting; \
BUG_ON(!in_nmi()); \
- __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
+ __preempt_count_sub(HARDIRQ_OFFSET); \
+ nesting = __this_cpu_dec_return(nmi_nesting); \
+ if (!nesting) \
+ preempt_count_set(preempt_count() & ~NMI_MASK); \
arch_nmi_exit(); \
lockdep_on(); \
} while (0)
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index d964f96..586f966 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -17,6 +17,8 @@
*
* - bits 0-7 are the preemption count (max preemption depth: 256)
* - bits 8-15 are the softirq count (max # of softirqs: 256)
+ * - bits 16-19 are the hardirq count (max # of hardirqs: 16)
+ * - bit 20 is the NMI flag (no nesting count, tracked separately)
*
* The hardirq count could in theory be the same as the number of
* interrupts in the system, but we run all interrupt handlers with
@@ -24,16 +26,19 @@
* there are a few palaeontologic drivers which reenable interrupts in
* the handler, so we need more than one bit here.
*
+ * NMI nesting depth is tracked in a separate per-CPU variable
+ * (nmi_nesting) to save bits in preempt_count.
+ *
* PREEMPT_MASK: 0x000000ff
* SOFTIRQ_MASK: 0x0000ff00
* HARDIRQ_MASK: 0x000f0000
- * NMI_MASK: 0x00f00000
+ * NMI_MASK: 0x00100000
* PREEMPT_NEED_RESCHED: 0x80000000
*/
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
#define HARDIRQ_BITS 4
-#define NMI_BITS 4
+#define NMI_BITS 1
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 4425d8d..10af5ed 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -88,6 +88,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
#endif
+DEFINE_PER_CPU(unsigned int, nmi_nesting);
+
/*
* SOFTIRQ_OFFSET usage:
*
diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h
index 67ff788..e4e1200 100644
--- a/tools/testing/selftests/bpf/bpf_experimental.h
+++ b/tools/testing/selftests/bpf/bpf_experimental.h
@@ -367,7 +367,7 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
#define HARDIRQ_BITS 4
-#define NMI_BITS 4
+#define NMI_BITS 1
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 02/17] preempt: Introduce HARDIRQ_DISABLE_BITS
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
2026-08-04 16:14 ` [PATCH v4 01/17] preempt: Track NMI nesting to separate per-CPU counter Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-05 6:31 ` Peter Zijlstra
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
2026-08-04 16:14 ` [PATCH v4 03/17] preempt: Introduce __preempt_count_{sub,add}_return() Boqun Feng
` (14 subsequent siblings)
16 siblings, 2 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
In order to support preempt_disable()-like interrupt disabling, that is,
using part of preempt_count() to track interrupt disabling nesting
level, change the preempt_count() layout to contain 8-bit
HARDIRQ_DISABLE count.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
include/linux/preempt.h | 16 +++++++++++-----
tools/testing/selftests/bpf/bpf_experimental.h | 5 ++++-
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 586f96688325..e2d3079d3f5f 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -17,8 +17,9 @@
*
* - bits 0-7 are the preemption count (max preemption depth: 256)
* - bits 8-15 are the softirq count (max # of softirqs: 256)
- * - bits 16-19 are the hardirq count (max # of hardirqs: 16)
- * - bit 20 is the NMI flag (no nesting count, tracked separately)
+ * - bits 16-23 are the hardirq disable count (max # of hardirq disable: 256)
+ * - bits 24-27 are the hardirq count (max # of hardirqs: 16)
+ * - bit 28 is the NMI flag (no nesting count, tracked separately)
*
* The hardirq count could in theory be the same as the number of
* interrupts in the system, but we run all interrupt handlers with
@@ -31,29 +32,34 @@
*
* PREEMPT_MASK: 0x000000ff
* SOFTIRQ_MASK: 0x0000ff00
- * HARDIRQ_MASK: 0x000f0000
- * NMI_MASK: 0x00100000
+ * HARDIRQ_DISABLE_MASK: 0x00ff0000
+ * HARDIRQ_MASK: 0x0f000000
+ * NMI_MASK: 0x10000000
* PREEMPT_NEED_RESCHED: 0x80000000
*/
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
+#define HARDIRQ_DISABLE_BITS 8
#define HARDIRQ_BITS 4
#define NMI_BITS 1
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
-#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
+#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
+#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS)
#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
#define __IRQ_MASK(x) ((1UL << (x))-1)
#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
+#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT)
#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)
#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
+#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT)
#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
#define NMI_OFFSET (1UL << NMI_SHIFT)
diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h
index e4e12001fce9..0159a3d365c8 100644
--- a/tools/testing/selftests/bpf/bpf_experimental.h
+++ b/tools/testing/selftests/bpf/bpf_experimental.h
@@ -366,17 +366,20 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
+#define HARDIRQ_DISABLE_BITS 8
#define HARDIRQ_BITS 4
#define NMI_BITS 1
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
-#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
+#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
+#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS)
#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
#define __IRQ_MASK(x) ((1UL << (x))-1)
#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
+#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT)
#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* Re: [PATCH v4 02/17] preempt: Introduce HARDIRQ_DISABLE_BITS
2026-08-04 16:14 ` [PATCH v4 02/17] preempt: Introduce HARDIRQ_DISABLE_BITS Boqun Feng
@ 2026-08-05 6:31 ` Peter Zijlstra
2026-08-05 6:59 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
1 sibling, 1 reply; 66+ messages in thread
From: Peter Zijlstra @ 2026-08-05 6:31 UTC (permalink / raw)
To: Boqun Feng
Cc: Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux
On Tue, Aug 04, 2026 at 09:14:24AM -0700, Boqun Feng wrote:
> In order to support preempt_disable()-like interrupt disabling, that is,
> using part of preempt_count() to track interrupt disabling nesting
> level, change the preempt_count() layout to contain 8-bit
> HARDIRQ_DISABLE count.
>
> Signed-off-by: Lyude Paul <lyude@redhat.com>
> Signed-off-by: Boqun Feng <boqun@kernel.org>
This still isn't right. Either it needs a: 'From: Lyude ....' or so
Co-developed-by or however you spell that.
Since current from is Boqun, first SoB should be Boqun.
^ permalink raw reply [flat|nested] 66+ messages in thread
* Re: [PATCH v4 02/17] preempt: Introduce HARDIRQ_DISABLE_BITS
2026-08-05 6:31 ` Peter Zijlstra
@ 2026-08-05 6:59 ` Boqun Feng
0 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 6:59 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 08:31:28AM +0200, Peter Zijlstra wrote:
> On Tue, Aug 04, 2026 at 09:14:24AM -0700, Boqun Feng wrote:
> > In order to support preempt_disable()-like interrupt disabling, that is,
> > using part of preempt_count() to track interrupt disabling nesting
> > level, change the preempt_count() layout to contain 8-bit
> > HARDIRQ_DISABLE count.
> >
> > Signed-off-by: Lyude Paul <lyude@redhat.com>
> > Signed-off-by: Boqun Feng <boqun@kernel.org>
>
> This still isn't right. Either it needs a: 'From: Lyude ....' or so
> Co-developed-by or however you spell that.
>
> Since current from is Boqun, first SoB should be Boqun.
Got it, it happened when I switched mail addresses, so it used to be
From: Boqun Feng <boqun.feng@gmail.com>
...
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
;-)
Will fix this.
Regards,
Boqun
^ permalink raw reply [flat|nested] 66+ messages in thread
* [tip: locking/core] preempt: Introduce HARDIRQ_DISABLE_BITS
2026-08-04 16:14 ` [PATCH v4 02/17] preempt: Introduce HARDIRQ_DISABLE_BITS Boqun Feng
2026-08-05 6:31 ` Peter Zijlstra
@ 2026-08-08 20:48 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits
Cc: Boqun Feng, Lyude Paul, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 9634d860ac15dad51a62955566424c9d2996b15c
Gitweb: https://git.kernel.org/tip/9634d860ac15dad51a62955566424c9d2996b15c
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:24 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:06 +02:00
preempt: Introduce HARDIRQ_DISABLE_BITS
In order to support preempt_disable()-like interrupt disabling, that is,
using part of preempt_count() to track interrupt disabling nesting
level, change the preempt_count() layout to contain 8-bit
HARDIRQ_DISABLE count.
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260121223933.1568682-2-lyude@redhat.com
Link: https://patch.msgid.link/20260804161447.84806-3-boqun@kernel.org
---
include/linux/preempt.h | 16 +++++++++++-----
tools/testing/selftests/bpf/bpf_experimental.h | 5 ++++-
2 files changed, 15 insertions(+), 6 deletions(-)
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 586f966..e2d3079 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -17,8 +17,9 @@
*
* - bits 0-7 are the preemption count (max preemption depth: 256)
* - bits 8-15 are the softirq count (max # of softirqs: 256)
- * - bits 16-19 are the hardirq count (max # of hardirqs: 16)
- * - bit 20 is the NMI flag (no nesting count, tracked separately)
+ * - bits 16-23 are the hardirq disable count (max # of hardirq disable: 256)
+ * - bits 24-27 are the hardirq count (max # of hardirqs: 16)
+ * - bit 28 is the NMI flag (no nesting count, tracked separately)
*
* The hardirq count could in theory be the same as the number of
* interrupts in the system, but we run all interrupt handlers with
@@ -31,29 +32,34 @@
*
* PREEMPT_MASK: 0x000000ff
* SOFTIRQ_MASK: 0x0000ff00
- * HARDIRQ_MASK: 0x000f0000
- * NMI_MASK: 0x00100000
+ * HARDIRQ_DISABLE_MASK: 0x00ff0000
+ * HARDIRQ_MASK: 0x0f000000
+ * NMI_MASK: 0x10000000
* PREEMPT_NEED_RESCHED: 0x80000000
*/
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
+#define HARDIRQ_DISABLE_BITS 8
#define HARDIRQ_BITS 4
#define NMI_BITS 1
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
-#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
+#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
+#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS)
#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
#define __IRQ_MASK(x) ((1UL << (x))-1)
#define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT)
#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
+#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT)
#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
#define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT)
#define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT)
+#define HARDIRQ_DISABLE_OFFSET (1UL << HARDIRQ_DISABLE_SHIFT)
#define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT)
#define NMI_OFFSET (1UL << NMI_SHIFT)
diff --git a/tools/testing/selftests/bpf/bpf_experimental.h b/tools/testing/selftests/bpf/bpf_experimental.h
index e4e1200..0159a3d 100644
--- a/tools/testing/selftests/bpf/bpf_experimental.h
+++ b/tools/testing/selftests/bpf/bpf_experimental.h
@@ -366,17 +366,20 @@ extern int bpf_cgroup_read_xattr(struct cgroup *cgroup, const char *name__str,
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
+#define HARDIRQ_DISABLE_BITS 8
#define HARDIRQ_BITS 4
#define NMI_BITS 1
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
-#define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
+#define HARDIRQ_DISABLE_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS)
+#define HARDIRQ_SHIFT (HARDIRQ_DISABLE_SHIFT + HARDIRQ_DISABLE_BITS)
#define NMI_SHIFT (HARDIRQ_SHIFT + HARDIRQ_BITS)
#define __IRQ_MASK(x) ((1UL << (x))-1)
#define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT)
+#define HARDIRQ_DISABLE_MASK (__IRQ_MASK(HARDIRQ_DISABLE_BITS) << HARDIRQ_DISABLE_SHIFT)
#define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT)
#define NMI_MASK (__IRQ_MASK(NMI_BITS) << NMI_SHIFT)
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 03/17] preempt: Introduce __preempt_count_{sub,add}_return()
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
2026-08-04 16:14 ` [PATCH v4 01/17] preempt: Track NMI nesting to separate per-CPU counter Boqun Feng
2026-08-04 16:14 ` [PATCH v4 02/17] preempt: Introduce HARDIRQ_DISABLE_BITS Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
2026-08-04 16:14 ` [PATCH v4 04/17] openrisc: Include <linux/cpumask.h> in smp.h Boqun Feng
` (13 subsequent siblings)
16 siblings, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux,
Heiko Carstens
In order to use preempt_count() to track the interrupt disable nesting
level, __preempt_count_{add,sub}_return() are introduced, as their names
suggest, these primitives return the new value of the preempt_count()
after changing it. The following example shows the usage of it in
local_interrupt_disable():
// increase the HARDIRQ_DISABLE bit
new_count = __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET);
// if it's the first-time increment, then disable the interrupt
// at hardware level.
if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) {
local_irq_save(flags);
raw_cpu_write(local_interrupt_disable_state, flags);
}
Having these primitives will avoid a read of preempt_count() after
changing preempt_count() on certain architectures.
Acked-by: Heiko Carstens <hca@linux.ibm.com> # s390
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
arch/arm64/include/asm/preempt.h | 20 ++++++++++++++++++++
arch/s390/include/asm/preempt.h | 10 ++++++++++
arch/x86/include/asm/preempt.h | 10 ++++++++++
include/asm-generic/preempt.h | 14 ++++++++++++++
4 files changed, 54 insertions(+)
diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h
index 932ea4b62042..9ecc2766a9f2 100644
--- a/arch/arm64/include/asm/preempt.h
+++ b/arch/arm64/include/asm/preempt.h
@@ -55,6 +55,26 @@ static inline void __preempt_count_sub(int val)
WRITE_ONCE(current_thread_info()->preempt.count, pc);
}
+static inline int __preempt_count_add_return(int val)
+{
+ u32 pc = READ_ONCE(current_thread_info()->preempt.count);
+
+ pc += val;
+ WRITE_ONCE(current_thread_info()->preempt.count, pc);
+
+ return pc;
+}
+
+static inline int __preempt_count_sub_return(int val)
+{
+ u32 pc = READ_ONCE(current_thread_info()->preempt.count);
+
+ pc -= val;
+ WRITE_ONCE(current_thread_info()->preempt.count, pc);
+
+ return pc;
+}
+
static inline bool __preempt_count_dec_and_test(void)
{
struct thread_info *ti = current_thread_info();
diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h
index 6e5821bb047e..0a25d4648b4c 100644
--- a/arch/s390/include/asm/preempt.h
+++ b/arch/s390/include/asm/preempt.h
@@ -139,6 +139,16 @@ static __always_inline bool should_resched(int preempt_offset)
return unlikely(READ_ONCE(get_lowcore()->preempt_count) == preempt_offset);
}
+static __always_inline int __preempt_count_add_return(int val)
+{
+ return val + __atomic_add(val, &get_lowcore()->preempt_count);
+}
+
+static __always_inline int __preempt_count_sub_return(int val)
+{
+ return __preempt_count_add_return(-val);
+}
+
#define init_task_preempt_count(p) do { } while (0)
/* Deferred to CPU bringup time */
#define init_idle_preempt_count(p, cpu) do { } while (0)
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 578441db09f0..1220656f3370 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -85,6 +85,16 @@ static __always_inline void __preempt_count_sub(int val)
raw_cpu_add_4(__preempt_count, -val);
}
+static __always_inline int __preempt_count_add_return(int val)
+{
+ return raw_cpu_add_return_4(__preempt_count, val);
+}
+
+static __always_inline int __preempt_count_sub_return(int val)
+{
+ return raw_cpu_add_return_4(__preempt_count, -val);
+}
+
/*
* Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
* a decrement which hits zero means we have no preempt_count and should
diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h
index 51f8f3881523..c8683c046615 100644
--- a/include/asm-generic/preempt.h
+++ b/include/asm-generic/preempt.h
@@ -59,6 +59,20 @@ static __always_inline void __preempt_count_sub(int val)
*preempt_count_ptr() -= val;
}
+static __always_inline int __preempt_count_add_return(int val)
+{
+ *preempt_count_ptr() += val;
+
+ return *preempt_count_ptr();
+}
+
+static __always_inline int __preempt_count_sub_return(int val)
+{
+ *preempt_count_ptr() -= val;
+
+ return *preempt_count_ptr();
+}
+
static __always_inline bool __preempt_count_dec_and_test(void)
{
/*
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] preempt: Introduce __preempt_count_{sub,add}_return()
2026-08-04 16:14 ` [PATCH v4 03/17] preempt: Introduce __preempt_count_{sub,add}_return() Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Boqun Feng
0 siblings, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits
Cc: Boqun Feng, Peter Zijlstra (Intel), Heiko Carstens, x86,
linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 4ad87eee546ef97e3a5ad80414327f41f48a4e70
Gitweb: https://git.kernel.org/tip/4ad87eee546ef97e3a5ad80414327f41f48a4e70
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:25 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:06 +02:00
preempt: Introduce __preempt_count_{sub,add}_return()
In order to use preempt_count() to track the interrupt disable nesting
level, __preempt_count_{add,sub}_return() are introduced, as their names
suggest, these primitives return the new value of the preempt_count()
after changing it. The following example shows the usage of it in
local_interrupt_disable():
// increase the HARDIRQ_DISABLE bit
new_count = __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET);
// if it's the first-time increment, then disable the interrupt
// at hardware level.
if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET) {
local_irq_save(flags);
raw_cpu_write(local_interrupt_disable_state, flags);
}
Having these primitives will avoid a read of preempt_count() after
changing preempt_count() on certain architectures.
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Heiko Carstens <hca@linux.ibm.com> # s390
Link: https://patch.msgid.link/20260804161447.84806-4-boqun@kernel.org
---
arch/arm64/include/asm/preempt.h | 20 ++++++++++++++++++++
arch/s390/include/asm/preempt.h | 10 ++++++++++
arch/x86/include/asm/preempt.h | 10 ++++++++++
include/asm-generic/preempt.h | 14 ++++++++++++++
4 files changed, 54 insertions(+)
diff --git a/arch/arm64/include/asm/preempt.h b/arch/arm64/include/asm/preempt.h
index 932ea4b..9ecc276 100644
--- a/arch/arm64/include/asm/preempt.h
+++ b/arch/arm64/include/asm/preempt.h
@@ -55,6 +55,26 @@ static inline void __preempt_count_sub(int val)
WRITE_ONCE(current_thread_info()->preempt.count, pc);
}
+static inline int __preempt_count_add_return(int val)
+{
+ u32 pc = READ_ONCE(current_thread_info()->preempt.count);
+
+ pc += val;
+ WRITE_ONCE(current_thread_info()->preempt.count, pc);
+
+ return pc;
+}
+
+static inline int __preempt_count_sub_return(int val)
+{
+ u32 pc = READ_ONCE(current_thread_info()->preempt.count);
+
+ pc -= val;
+ WRITE_ONCE(current_thread_info()->preempt.count, pc);
+
+ return pc;
+}
+
static inline bool __preempt_count_dec_and_test(void)
{
struct thread_info *ti = current_thread_info();
diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h
index 6e5821b..0a25d46 100644
--- a/arch/s390/include/asm/preempt.h
+++ b/arch/s390/include/asm/preempt.h
@@ -139,6 +139,16 @@ static __always_inline bool should_resched(int preempt_offset)
return unlikely(READ_ONCE(get_lowcore()->preempt_count) == preempt_offset);
}
+static __always_inline int __preempt_count_add_return(int val)
+{
+ return val + __atomic_add(val, &get_lowcore()->preempt_count);
+}
+
+static __always_inline int __preempt_count_sub_return(int val)
+{
+ return __preempt_count_add_return(-val);
+}
+
#define init_task_preempt_count(p) do { } while (0)
/* Deferred to CPU bringup time */
#define init_idle_preempt_count(p, cpu) do { } while (0)
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 578441d..1220656 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -85,6 +85,16 @@ static __always_inline void __preempt_count_sub(int val)
raw_cpu_add_4(__preempt_count, -val);
}
+static __always_inline int __preempt_count_add_return(int val)
+{
+ return raw_cpu_add_return_4(__preempt_count, val);
+}
+
+static __always_inline int __preempt_count_sub_return(int val)
+{
+ return raw_cpu_add_return_4(__preempt_count, -val);
+}
+
/*
* Because we keep PREEMPT_NEED_RESCHED set when we do _not_ need to reschedule
* a decrement which hits zero means we have no preempt_count and should
diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h
index 51f8f38..c8683c0 100644
--- a/include/asm-generic/preempt.h
+++ b/include/asm-generic/preempt.h
@@ -59,6 +59,20 @@ static __always_inline void __preempt_count_sub(int val)
*preempt_count_ptr() -= val;
}
+static __always_inline int __preempt_count_add_return(int val)
+{
+ *preempt_count_ptr() += val;
+
+ return *preempt_count_ptr();
+}
+
+static __always_inline int __preempt_count_sub_return(int val)
+{
+ *preempt_count_ptr() -= val;
+
+ return *preempt_count_ptr();
+}
+
static __always_inline bool __preempt_count_dec_and_test(void)
{
/*
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 04/17] openrisc: Include <linux/cpumask.h> in smp.h
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (2 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 03/17] preempt: Introduce __preempt_count_{sub,add}_return() Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Lyude Paul
2026-08-04 16:14 ` [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling Boqun Feng
` (12 subsequent siblings)
16 siblings, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux,
Stafford Horne
From: Lyude Paul <lyude@redhat.com>
While OpenRISC currently doesn't fail to build upstream, it appears that
including <asm/smp.h> in the right headers is enough to break that -
primarily because OpenRISC's asm/smp.h header doesn't actually provide
any definition for struct cpumask. Which means the only reason we aren't
failing to build the kernel is because we've been lucky enough that
every spot including asm/smp.h already has definitions for struct
cpumask pulled in.
This became evident when trying to work on a patch series for adding
ref-counted interrupt enable/disable to the kernel, where introducing a
new interrupt_rc.h header suddenly introduced a build error on OpenRISC:
In file included from include/linux/interrupt_rc.h:17,
from include/linux/spinlock.h:60,
from include/linux/mmzone.h:8,
from include/linux/gfp.h:7,
from include/linux/mm.h:7,
from arch/openrisc/include/asm/pgalloc.h:20,
from arch/openrisc/include/asm/io.h:18,
from include/linux/io.h:12,
from drivers/irqchip/irq-ompic.c:61:
arch/openrisc/include/asm/smp.h:21:59: warning: 'struct cpumask'
declared inside parameter list will not be visible outside of this
definition or declaration
21 | extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
| ^~~~~~~
arch/openrisc/include/asm/smp.h:23:54: warning: 'struct cpumask'
declared inside parameter list will not be visible outside of this
definition or declaration
23 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
| ^~~~~~~
drivers/irqchip/irq-ompic.c: In function 'ompic_of_init':
>> drivers/irqchip/irq-ompic.c:191:28: error: passing argument 1 of
'set_smp_cross_call' from incompatible pointer type
[-Werror=incompatible-pointer-types]
191 | set_smp_cross_call(ompic_raise_softirq);
| ^~~~~~~~~~~~~~~~~~~
| |
| void (*)(const struct cpumask *, unsigned int)
arch/openrisc/include/asm/smp.h:23:32: note: expected 'void (*)(const
struct cpumask *, unsigned int)' but argument is of type 'void
(*)(const struct cpumask *, unsigned int)'
23 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
To fix this, let's take an example from the smp.h headers of other
architectures (x86, hexagon, arm64, probably more): just include
linux/cpumask.h at the top.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Acked-by: Stafford Horne <shorne@gmail.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
arch/openrisc/include/asm/smp.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/openrisc/include/asm/smp.h b/arch/openrisc/include/asm/smp.h
index 007296f160ef..84653aaffa96 100644
--- a/arch/openrisc/include/asm/smp.h
+++ b/arch/openrisc/include/asm/smp.h
@@ -9,6 +9,8 @@
#ifndef __ASM_OPENRISC_SMP_H
#define __ASM_OPENRISC_SMP_H
+#include <linux/cpumask.h>
+
#include <asm/spr.h>
#include <asm/spr_defs.h>
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] openrisc: Include <linux/cpumask.h> in smp.h
2026-08-04 16:14 ` [PATCH v4 04/17] openrisc: Include <linux/cpumask.h> in smp.h Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Lyude Paul
0 siblings, 0 replies; 66+ messages in thread
From: tip-bot2 for Lyude Paul @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits
Cc: Lyude Paul, Boqun Feng, Peter Zijlstra (Intel), Stafford Horne,
x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: e35e7700029b936356d736d6378cf596019e4cc8
Gitweb: https://git.kernel.org/tip/e35e7700029b936356d736d6378cf596019e4cc8
Author: Lyude Paul <lyude@redhat.com>
AuthorDate: Tue, 04 Aug 2026 09:14:26 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:06 +02:00
openrisc: Include <linux/cpumask.h> in smp.h
While OpenRISC currently doesn't fail to build upstream, it appears that
including <asm/smp.h> in the right headers is enough to break that -
primarily because OpenRISC's asm/smp.h header doesn't actually provide
any definition for struct cpumask. Which means the only reason we aren't
failing to build the kernel is because we've been lucky enough that
every spot including asm/smp.h already has definitions for struct
cpumask pulled in.
This became evident when trying to work on a patch series for adding
ref-counted interrupt enable/disable to the kernel, where introducing a
new interrupt_rc.h header suddenly introduced a build error on OpenRISC:
In file included from include/linux/interrupt_rc.h:17,
from include/linux/spinlock.h:60,
from include/linux/mmzone.h:8,
from include/linux/gfp.h:7,
from include/linux/mm.h:7,
from arch/openrisc/include/asm/pgalloc.h:20,
from arch/openrisc/include/asm/io.h:18,
from include/linux/io.h:12,
from drivers/irqchip/irq-ompic.c:61:
arch/openrisc/include/asm/smp.h:21:59: warning: 'struct cpumask'
declared inside parameter list will not be visible outside of this
definition or declaration
21 | extern void arch_send_call_function_ipi_mask(const struct cpumask *mask);
| ^~~~~~~
arch/openrisc/include/asm/smp.h:23:54: warning: 'struct cpumask'
declared inside parameter list will not be visible outside of this
definition or declaration
23 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
| ^~~~~~~
drivers/irqchip/irq-ompic.c: In function 'ompic_of_init':
>> drivers/irqchip/irq-ompic.c:191:28: error: passing argument 1 of
'set_smp_cross_call' from incompatible pointer type
[-Werror=incompatible-pointer-types]
191 | set_smp_cross_call(ompic_raise_softirq);
| ^~~~~~~~~~~~~~~~~~~
| |
| void (*)(const struct cpumask *, unsigned int)
arch/openrisc/include/asm/smp.h:23:32: note: expected 'void (*)(const
struct cpumask *, unsigned int)' but argument is of type 'void
(*)(const struct cpumask *, unsigned int)'
23 | extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
To fix this, let's take an example from the smp.h headers of other
architectures (x86, hexagon, arm64, probably more): just include
linux/cpumask.h at the top.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Stafford Horne <shorne@gmail.com>
Link: https://patch.msgid.link/20260804161447.84806-5-boqun@kernel.org
---
arch/openrisc/include/asm/smp.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/openrisc/include/asm/smp.h b/arch/openrisc/include/asm/smp.h
index 007296f..84653aa 100644
--- a/arch/openrisc/include/asm/smp.h
+++ b/arch/openrisc/include/asm/smp.h
@@ -9,6 +9,8 @@
#ifndef __ASM_OPENRISC_SMP_H
#define __ASM_OPENRISC_SMP_H
+#include <linux/cpumask.h>
+
#include <asm/spr.h>
#include <asm/spr_defs.h>
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (3 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 04/17] openrisc: Include <linux/cpumask.h> in smp.h Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-04 18:20 ` Boqun Feng
` (2 more replies)
2026-08-04 16:14 ` [PATCH v4 06/17] irq: Add KUnit test for refcounted interrupt enable/disable Boqun Feng
` (11 subsequent siblings)
16 siblings, 3 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
Currently the nested interrupt disabling and enabling is represented by
_irqsave() and _irqrestore() APIs, which are relatively unsafe, for
example:
<interrupts are enabled as beginning>
spin_lock_irqsave(l1, flag1);
spin_lock_irqsave(l2, flag2);
spin_unlock_irqrestore(l1, flags1);
<l2 is still held but interrupts are enabled>
// accesses to interrupt-disable protected data will cause races
This is even easier to trigger with guard facilities:
unsigned long flag2;
scoped_guard(spin_lock_irqsave, l1) {
spin_lock_irqsave(l2, flag2);
}
// l2 locked but interrupts are enabled.
spin_unlock_irqrestore(l2, flag2);
(Hand-to-hand locking critical sections are not uncommon for a
fine-grained lock design)
And because of this unsafety, Rust cannot easily wrap the
interrupt-disabling locks in a safe API, which complicates the design.
To resolve this, introduce a new set of interrupt disabling APIs:
* local_interrupt_disable();
* local_interrupt_enable();
They work like local_irq_save() and local_irq_restore() except that 1)
the outermost local_interrupt_disable() call saves the interrupt state
into a per-CPU variable, so that the outermost local_interrupt_enable()
can restore the state, and 2) a per-CPU counter is added to record the
nest level of these calls, so that interrupts are not accidentally
enabled inside the outermost critical section.
Also add the corresponding spin_lock primitives: spin_lock_irq_disable()
and spin_unlock_irq_enable(), as a result, code as follows:
spin_lock_irq_disable(l1);
spin_lock_irq_disable(l2);
spin_unlock_irq_enable(l1);
// Interrupts are still disabled.
spin_unlock_irq_enable(l2);
doesn't have the issue that interrupts are accidentally enabled.
This also makes the wrapper of interrupt-disabling locks on Rust easier
to design.
Signed-off-by: Lyude Paul <lyude@redhat.com>
[boqun: Apply Peter's feedback and fix spell errors reported by Ingo]
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
include/linux/interrupt_rc.h | 82 ++++++++++++++++++++++++++++++++
include/linux/preempt.h | 4 ++
include/linux/spinlock.h | 23 +++++++++
include/linux/spinlock_api_smp.h | 43 +++++++++++++++++
include/linux/spinlock_api_up.h | 15 ++++++
include/linux/spinlock_rt.h | 18 +++++++
kernel/locking/spinlock.c | 31 ++++++++++++
kernel/softirq.c | 28 ++++++++++-
8 files changed, 242 insertions(+), 2 deletions(-)
create mode 100644 include/linux/interrupt_rc.h
diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
new file mode 100644
index 000000000000..b9a7f05ecf42
--- /dev/null
+++ b/include/linux/interrupt_rc.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_INTERRUPT_RC_H
+#define __LINUX_INTERRUPT_RC_H
+
+/*
+ * include/linux/interrupt_rc.h - refcounted local processor interrupt
+ * management.
+ *
+ * Since the implementation of this API currently depends on
+ * local_irq_save()/local_irq_restore(), we split this into its own header to
+ * make it easier to include without hitting circular header dependencies.
+ */
+
+#include <linux/irqflags.h>
+#include <linux/preempt.h>
+#include <linux/processor.h>
+#include <linux/smp.h>
+
+#ifndef MODULE
+/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */
+DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);
+
+static __always_inline void __local_interrupt_disable(void)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ raw_cpu_write(local_interrupt_disable_state, flags);
+}
+
+static __always_inline void __local_interrupt_enable(void)
+{
+ unsigned long flags = raw_cpu_read(local_interrupt_disable_state);
+
+ local_irq_restore(flags);
+}
+
+#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
+static __always_inline void _local_interrupt_disable(void)
+{
+ __local_interrupt_disable();
+}
+
+static __always_inline void _local_interrupt_enable(void)
+{
+ __local_interrupt_enable();
+}
+#else
+extern void _local_interrupt_disable(void);
+extern void _local_interrupt_enable(void);
+#endif
+
+#else /* !MODULE */
+extern void _local_interrupt_disable(void);
+extern void _local_interrupt_enable(void);
+#endif /* !MODULE */
+
+static inline void local_interrupt_disable(void)
+{
+ int new_count;
+
+ WARN_ON_ONCE(in_nmi());
+
+ new_count = hardirq_disable_enter();
+
+ /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
+
+ if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
+ _local_interrupt_disable();
+}
+
+static inline void local_interrupt_enable(void)
+{
+ int new_count;
+
+ new_count = hardirq_disable_exit();
+
+ if ((new_count & HARDIRQ_DISABLE_MASK) == 0)
+ _local_interrupt_enable();
+}
+
+#endif /* !__LINUX_INTERRUPT_RC_H */
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index e2d3079d3f5f..33fc4c814a9f 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context_level(void)
#define in_softirq() (softirq_count())
#define in_interrupt() (irq_count())
+#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT)
+#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET)
+#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET)
+
/*
* The preempt_count offset after preempt_disable();
*/
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 241277cd34cf..3d405cc4c121 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -57,6 +57,7 @@
#include <linux/linkage.h>
#include <linux/compiler.h>
#include <linux/irqflags.h>
+#include <linux/interrupt_rc.h>
#include <linux/thread_info.h>
#include <linux/stringify.h>
#include <linux/bottom_half.h>
@@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#endif
#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
+#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock)
#define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
#define raw_spin_unlock(lock) _raw_spin_unlock(lock)
#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
+#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock)
#define raw_spin_unlock_irqrestore(lock, flags) \
do { \
@@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags))
+#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(lock)
+
#ifndef CONFIG_PREEMPT_RT
/* Include rwlock functions for !RT */
#include <linux/rwlock.h>
@@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)
raw_spin_lock_irq(&lock->rlock);
}
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+ __acquires(lock) __no_context_analysis
+{
+ raw_spin_lock_irq_disable(&lock->rlock);
+}
+
#define spin_lock_irqsave(lock, flags) \
do { \
raw_spin_lock_irqsave(spinlock_check(lock), flags); \
@@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)
raw_spin_unlock_irq(&lock->rlock);
}
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+ __releases(lock) __no_context_analysis
+{
+ raw_spin_unlock_irq_enable(&lock->rlock);
+}
+
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
__releases(lock) __no_context_analysis
{
@@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned lon
}
#define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags))
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+ __cond_acquires(true, lock) __no_context_analysis
+{
+ return raw_spin_trylock_irq_disable(&lock->rlock);
+}
+
/**
* spin_is_locked() - Check whether a spinlock is locked.
* @lock: Pointer to the spinlock.
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index bda5e7a390cd..a5d164ac9612 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map)
void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock);
void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
__acquires(lock);
+void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+ __acquires(lock);
unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
__acquires(lock);
@@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(true,
void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock);
+void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releases(lock);
void __lockfunc
_raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
__releases(lock);
@@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
#define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock)
#endif
+/* Use the same config as spin_lock_irq() temporarily. */
+#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ
+#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock)
+#endif
+
#ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
#define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock)
#endif
@@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
#define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock)
#endif
+/* Use the same config as spin_unlock_irq() temporarily. */
+#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(lock)
+#endif
+
#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
#define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrestore(lock, flags)
#endif
@@ -105,6 +118,18 @@ static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock)
return false;
}
+static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ local_interrupt_disable();
+ if (_raw_spin_trylock(lock)) {
+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
+ return true;
+ }
+ local_interrupt_enable();
+ return false;
+}
+
static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
__cond_acquires(true, lock)
{
@@ -143,6 +168,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock)
LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
}
+static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+ __acquires(lock) __no_context_analysis
+{
+ local_interrupt_disable();
+ preempt_disable();
+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+ LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+}
+
static inline void __raw_spin_lock_bh(raw_spinlock_t *lock)
__acquires(lock) __no_context_analysis
{
@@ -188,6 +222,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock)
preempt_enable();
}
+static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+ __releases(lock)
+{
+ spin_release(&lock->dep_map, _RET_IP_);
+ do_raw_spin_unlock(lock);
+ local_interrupt_enable();
+ preempt_enable();
+}
+
static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock)
__releases(lock)
{
diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h
index a9d5c7c66e03..d03d3065ee04 100644
--- a/include/linux/spinlock_api_up.h
+++ b/include/linux/spinlock_api_up.h
@@ -42,6 +42,9 @@
#define __LOCK_IRQSAVE(lock, flags, ...) \
do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0)
+#define __LOCK_IRQ_DISABLE(lock, ...) \
+ do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0)
+
#define ___UNLOCK_(lock) \
do { __release(lock); (void)(lock); } while (0)
@@ -61,6 +64,9 @@
#define __UNLOCK_IRQRESTORE(lock, flags, ...) \
do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0)
+#define __UNLOCK_IRQ_ENABLE(lock, ...) \
+ do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0)
+
#define _raw_spin_lock(lock) __LOCK(lock)
#define _raw_spin_lock_nested(lock, subclass) __LOCK(lock)
#define _raw_read_lock(lock) __LOCK(lock, shared)
@@ -70,6 +76,7 @@
#define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared)
#define _raw_write_lock_bh(lock) __LOCK_BH(lock)
#define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock)
+#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock)
#define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared)
#define _raw_write_lock_irq(lock) __LOCK_IRQ(lock)
#define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags)
@@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock)
return 1;
}
+static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ __LOCK_IRQ_DISABLE(lock);
+ return 1;
+}
+
static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
__cond_acquires(true, lock)
{
@@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned l
#define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock)
#define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared)
#define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock)
+#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock)
#define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared)
#define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock)
#define _raw_spin_unlock_irqrestore(lock, flags) \
diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h
index 373618a4243c..560d06384e0c 100644
--- a/include/linux/spinlock_rt.h
+++ b/include/linux/spinlock_rt.h
@@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)
rt_spin_lock(lock);
}
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+ __acquires(lock)
+{
+ rt_spin_lock(lock);
+}
+
#define spin_lock_irqsave(lock, flags) \
do { \
typecheck(unsigned long, flags); \
@@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)
rt_spin_unlock(lock);
}
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+ __releases(lock)
+{
+ rt_spin_unlock(lock);
+}
+
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
unsigned long flags)
__releases(lock)
@@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
#define spin_trylock(lock) rt_spin_trylock(lock)
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ return rt_spin_trylock(lock);
+}
+
#define spin_trylock_bh(lock) rt_spin_trylock_bh(lock)
#define spin_trylock_irq(lock) rt_spin_trylock(lock)
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index b42d293da38b..83a17eaf5717 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
*/
BUILD_LOCK_OPS(spin, raw_spinlock, __acquires);
+/* No rwlock_t variants for now, so just build this function by hand */
+static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ for (;;) {
+ preempt_disable();
+ local_interrupt_disable();
+ if (likely(do_raw_spin_trylock(lock)))
+ break;
+ local_interrupt_enable();
+ preempt_enable();
+
+ arch_spin_relax(&lock->raw_lock);
+ }
+}
+
#ifndef CONFIG_PREEMPT_RT
BUILD_LOCK_OPS(read, rwlock, __acquires_shared);
BUILD_LOCK_OPS(write, rwlock, __acquires);
@@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_lock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
+noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ __raw_spin_lock_irq_disable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_LOCK_BH
noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
{
@@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_unlock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+{
+ __raw_spin_unlock_irq_enable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
{
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 10af5ed859e7..0c9b2269a8d6 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -9,6 +9,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
#include <linux/export.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
@@ -88,6 +89,20 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
#endif
+DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state);
+
+void _local_interrupt_disable(void)
+{
+ __local_interrupt_disable();
+}
+EXPORT_SYMBOL(_local_interrupt_disable);
+
+void _local_interrupt_enable(void)
+{
+ __local_interrupt_enable();
+}
+EXPORT_SYMBOL(_local_interrupt_enable);
+
DEFINE_PER_CPU(unsigned int, nmi_nesting);
/*
@@ -728,10 +743,19 @@ static inline void __irq_exit_rcu(void)
#endif
account_hardirq_exit(current);
preempt_count_sub(HARDIRQ_OFFSET);
- if (!in_interrupt() && local_softirq_pending()) {
+ /*
+ * Interrupts may happen between hardirq_disable_enter() and
+ * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes
+ * softirq here, we may have a softirq handler calling
+ * local_interrupt_disable() but it won't disable the IRQ because
+ * hardirq disabling count is already 1, hence we need to prevent
+ * invoking softirq when a local_interrupt_disable() is ongoing.
+ */
+ if (!in_interrupt() && !hardirq_disable_count() &&
+ local_softirq_pending()) {
/*
* If we left hrtimers unarmed, make sure to arm them now,
- * before enabling interrupts to run SoftIRQ.
+ * before enabling interrupts to run softirq.
*/
hrtimer_rearm_deferred();
invoke_softirq();
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-04 16:14 ` [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling Boqun Feng
@ 2026-08-04 18:20 ` Boqun Feng
2026-08-04 18:26 ` [PATCH v4.1 " Boqun Feng
2026-08-04 20:51 ` [PATCH v4 05/17] irq & spin_lock: " Shrikanth Hegde
2 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 18:20 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux
On Tue, Aug 04, 2026 at 09:14:27AM -0700, Boqun Feng wrote:
[...]
> +static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
> + __cond_acquires(true, lock)
> +{
> + local_interrupt_disable();
> + if (_raw_spin_trylock(lock)) {
> + spin_acquire(&lock->dep_map, 0, 1, _RET_IP_);
This spin_acquire() needs to be removed as spotted by sashiko.
Regards,
Boqun
> + return true;
> + }
> + local_interrupt_enable();
> + return false;
> +}
> +
[...]
^ permalink raw reply [flat|nested] 66+ messages in thread* [PATCH v4.1 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-04 16:14 ` [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling Boqun Feng
2026-08-04 18:20 ` Boqun Feng
@ 2026-08-04 18:26 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
2026-08-10 8:57 ` [tip: locking/core] irq,spin_lock: " tip-bot2 for Boqun Feng
2026-08-04 20:51 ` [PATCH v4 05/17] irq & spin_lock: " Shrikanth Hegde
2 siblings, 2 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 18:26 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
Currently the nested interrupt disabling and enabling is represented by
_irqsave() and _irqrestore() APIs, which are relatively unsafe, for
example:
<interrupts are enabled as beginning>
spin_lock_irqsave(l1, flag1);
spin_lock_irqsave(l2, flag2);
spin_unlock_irqrestore(l1, flags1);
<l2 is still held but interrupts are enabled>
// accesses to interrupt-disable protected data will cause races
This is even easier to trigger with guard facilities:
unsigned long flag2;
scoped_guard(spin_lock_irqsave, l1) {
spin_lock_irqsave(l2, flag2);
}
// l2 locked but interrupts are enabled.
spin_unlock_irqrestore(l2, flag2);
(Hand-to-hand locking critical sections are not uncommon for a
fine-grained lock design)
And because of this unsafety, Rust cannot easily wrap the
interrupt-disabling locks in a safe API, which complicates the design.
To resolve this, introduce a new set of interrupt disabling APIs:
* local_interrupt_disable();
* local_interrupt_enable();
They work like local_irq_save() and local_irq_restore() except that 1)
the outermost local_interrupt_disable() call saves the interrupt state
into a per-CPU variable, so that the outermost local_interrupt_enable()
can restore the state, and 2) a per-CPU counter is added to record the
nest level of these calls, so that interrupts are not accidentally
enabled inside the outermost critical section.
Also add the corresponding spin_lock primitives: spin_lock_irq_disable()
and spin_unlock_irq_enable(), as a result, code as follows:
spin_lock_irq_disable(l1);
spin_lock_irq_disable(l2);
spin_unlock_irq_enable(l1);
// Interrupts are still disabled.
spin_unlock_irq_enable(l2);
doesn't have the issue that interrupts are accidentally enabled.
This also makes the wrapper of interrupt-disabling locks on Rust easier
to design.
Signed-off-by: Lyude Paul <lyude@redhat.com>
[boqun: Apply Peter's feedback and fix spell errors reported by Ingo]
[boqun: Address the duplicate spin_acquire() spotted by sashiko]
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
include/linux/interrupt_rc.h | 82 ++++++++++++++++++++++++++++++++
include/linux/preempt.h | 4 ++
include/linux/spinlock.h | 23 +++++++++
include/linux/spinlock_api_smp.h | 41 ++++++++++++++++
include/linux/spinlock_api_up.h | 15 ++++++
include/linux/spinlock_rt.h | 18 +++++++
kernel/locking/spinlock.c | 31 ++++++++++++
kernel/softirq.c | 28 ++++++++++-
8 files changed, 240 insertions(+), 2 deletions(-)
create mode 100644 include/linux/interrupt_rc.h
diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
new file mode 100644
index 000000000000..b9a7f05ecf42
--- /dev/null
+++ b/include/linux/interrupt_rc.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_INTERRUPT_RC_H
+#define __LINUX_INTERRUPT_RC_H
+
+/*
+ * include/linux/interrupt_rc.h - refcounted local processor interrupt
+ * management.
+ *
+ * Since the implementation of this API currently depends on
+ * local_irq_save()/local_irq_restore(), we split this into its own header to
+ * make it easier to include without hitting circular header dependencies.
+ */
+
+#include <linux/irqflags.h>
+#include <linux/preempt.h>
+#include <linux/processor.h>
+#include <linux/smp.h>
+
+#ifndef MODULE
+/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */
+DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);
+
+static __always_inline void __local_interrupt_disable(void)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ raw_cpu_write(local_interrupt_disable_state, flags);
+}
+
+static __always_inline void __local_interrupt_enable(void)
+{
+ unsigned long flags = raw_cpu_read(local_interrupt_disable_state);
+
+ local_irq_restore(flags);
+}
+
+#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
+static __always_inline void _local_interrupt_disable(void)
+{
+ __local_interrupt_disable();
+}
+
+static __always_inline void _local_interrupt_enable(void)
+{
+ __local_interrupt_enable();
+}
+#else
+extern void _local_interrupt_disable(void);
+extern void _local_interrupt_enable(void);
+#endif
+
+#else /* !MODULE */
+extern void _local_interrupt_disable(void);
+extern void _local_interrupt_enable(void);
+#endif /* !MODULE */
+
+static inline void local_interrupt_disable(void)
+{
+ int new_count;
+
+ WARN_ON_ONCE(in_nmi());
+
+ new_count = hardirq_disable_enter();
+
+ /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
+
+ if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
+ _local_interrupt_disable();
+}
+
+static inline void local_interrupt_enable(void)
+{
+ int new_count;
+
+ new_count = hardirq_disable_exit();
+
+ if ((new_count & HARDIRQ_DISABLE_MASK) == 0)
+ _local_interrupt_enable();
+}
+
+#endif /* !__LINUX_INTERRUPT_RC_H */
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index e2d3079d3f5f..33fc4c814a9f 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context_level(void)
#define in_softirq() (softirq_count())
#define in_interrupt() (irq_count())
+#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT)
+#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET)
+#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET)
+
/*
* The preempt_count offset after preempt_disable();
*/
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 241277cd34cf..3d405cc4c121 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -57,6 +57,7 @@
#include <linux/linkage.h>
#include <linux/compiler.h>
#include <linux/irqflags.h>
+#include <linux/interrupt_rc.h>
#include <linux/thread_info.h>
#include <linux/stringify.h>
#include <linux/bottom_half.h>
@@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#endif
#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
+#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock)
#define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
#define raw_spin_unlock(lock) _raw_spin_unlock(lock)
#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
+#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock)
#define raw_spin_unlock_irqrestore(lock, flags) \
do { \
@@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags))
+#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(lock)
+
#ifndef CONFIG_PREEMPT_RT
/* Include rwlock functions for !RT */
#include <linux/rwlock.h>
@@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)
raw_spin_lock_irq(&lock->rlock);
}
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+ __acquires(lock) __no_context_analysis
+{
+ raw_spin_lock_irq_disable(&lock->rlock);
+}
+
#define spin_lock_irqsave(lock, flags) \
do { \
raw_spin_lock_irqsave(spinlock_check(lock), flags); \
@@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)
raw_spin_unlock_irq(&lock->rlock);
}
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+ __releases(lock) __no_context_analysis
+{
+ raw_spin_unlock_irq_enable(&lock->rlock);
+}
+
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
__releases(lock) __no_context_analysis
{
@@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned lon
}
#define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags))
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+ __cond_acquires(true, lock) __no_context_analysis
+{
+ return raw_spin_trylock_irq_disable(&lock->rlock);
+}
+
/**
* spin_is_locked() - Check whether a spinlock is locked.
* @lock: Pointer to the spinlock.
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index bda5e7a390cd..90909d933dab 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map)
void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock);
void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
__acquires(lock);
+void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+ __acquires(lock);
unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
__acquires(lock);
@@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(true,
void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock);
+void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releases(lock);
void __lockfunc
_raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
__releases(lock);
@@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
#define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock)
#endif
+/* Use the same config as spin_lock_irq() temporarily. */
+#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ
+#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock)
+#endif
+
#ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
#define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock)
#endif
@@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
#define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock)
#endif
+/* Use the same config as spin_unlock_irq() temporarily. */
+#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(lock)
+#endif
+
#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
#define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrestore(lock, flags)
#endif
@@ -105,6 +118,16 @@ static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock)
return false;
}
+static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ local_interrupt_disable();
+ if (_raw_spin_trylock(lock))
+ return true;
+ local_interrupt_enable();
+ return false;
+}
+
static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
__cond_acquires(true, lock)
{
@@ -143,6 +166,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock)
LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
}
+static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+ __acquires(lock) __no_context_analysis
+{
+ local_interrupt_disable();
+ preempt_disable();
+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+ LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+}
+
static inline void __raw_spin_lock_bh(raw_spinlock_t *lock)
__acquires(lock) __no_context_analysis
{
@@ -188,6 +220,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock)
preempt_enable();
}
+static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+ __releases(lock)
+{
+ spin_release(&lock->dep_map, _RET_IP_);
+ do_raw_spin_unlock(lock);
+ local_interrupt_enable();
+ preempt_enable();
+}
+
static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock)
__releases(lock)
{
diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h
index a9d5c7c66e03..d03d3065ee04 100644
--- a/include/linux/spinlock_api_up.h
+++ b/include/linux/spinlock_api_up.h
@@ -42,6 +42,9 @@
#define __LOCK_IRQSAVE(lock, flags, ...) \
do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0)
+#define __LOCK_IRQ_DISABLE(lock, ...) \
+ do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0)
+
#define ___UNLOCK_(lock) \
do { __release(lock); (void)(lock); } while (0)
@@ -61,6 +64,9 @@
#define __UNLOCK_IRQRESTORE(lock, flags, ...) \
do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0)
+#define __UNLOCK_IRQ_ENABLE(lock, ...) \
+ do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0)
+
#define _raw_spin_lock(lock) __LOCK(lock)
#define _raw_spin_lock_nested(lock, subclass) __LOCK(lock)
#define _raw_read_lock(lock) __LOCK(lock, shared)
@@ -70,6 +76,7 @@
#define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared)
#define _raw_write_lock_bh(lock) __LOCK_BH(lock)
#define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock)
+#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock)
#define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared)
#define _raw_write_lock_irq(lock) __LOCK_IRQ(lock)
#define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags)
@@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock)
return 1;
}
+static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ __LOCK_IRQ_DISABLE(lock);
+ return 1;
+}
+
static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
__cond_acquires(true, lock)
{
@@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned l
#define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock)
#define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared)
#define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock)
+#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock)
#define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared)
#define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock)
#define _raw_spin_unlock_irqrestore(lock, flags) \
diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h
index 373618a4243c..560d06384e0c 100644
--- a/include/linux/spinlock_rt.h
+++ b/include/linux/spinlock_rt.h
@@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)
rt_spin_lock(lock);
}
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+ __acquires(lock)
+{
+ rt_spin_lock(lock);
+}
+
#define spin_lock_irqsave(lock, flags) \
do { \
typecheck(unsigned long, flags); \
@@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)
rt_spin_unlock(lock);
}
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+ __releases(lock)
+{
+ rt_spin_unlock(lock);
+}
+
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
unsigned long flags)
__releases(lock)
@@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
#define spin_trylock(lock) rt_spin_trylock(lock)
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ return rt_spin_trylock(lock);
+}
+
#define spin_trylock_bh(lock) rt_spin_trylock_bh(lock)
#define spin_trylock_irq(lock) rt_spin_trylock(lock)
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index b42d293da38b..83a17eaf5717 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
*/
BUILD_LOCK_OPS(spin, raw_spinlock, __acquires);
+/* No rwlock_t variants for now, so just build this function by hand */
+static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ for (;;) {
+ preempt_disable();
+ local_interrupt_disable();
+ if (likely(do_raw_spin_trylock(lock)))
+ break;
+ local_interrupt_enable();
+ preempt_enable();
+
+ arch_spin_relax(&lock->raw_lock);
+ }
+}
+
#ifndef CONFIG_PREEMPT_RT
BUILD_LOCK_OPS(read, rwlock, __acquires_shared);
BUILD_LOCK_OPS(write, rwlock, __acquires);
@@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_lock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
+noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ __raw_spin_lock_irq_disable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_LOCK_BH
noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
{
@@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_unlock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+{
+ __raw_spin_unlock_irq_enable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
{
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 10af5ed859e7..0c9b2269a8d6 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -9,6 +9,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
#include <linux/export.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
@@ -88,6 +89,20 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
#endif
+DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state);
+
+void _local_interrupt_disable(void)
+{
+ __local_interrupt_disable();
+}
+EXPORT_SYMBOL(_local_interrupt_disable);
+
+void _local_interrupt_enable(void)
+{
+ __local_interrupt_enable();
+}
+EXPORT_SYMBOL(_local_interrupt_enable);
+
DEFINE_PER_CPU(unsigned int, nmi_nesting);
/*
@@ -728,10 +743,19 @@ static inline void __irq_exit_rcu(void)
#endif
account_hardirq_exit(current);
preempt_count_sub(HARDIRQ_OFFSET);
- if (!in_interrupt() && local_softirq_pending()) {
+ /*
+ * Interrupts may happen between hardirq_disable_enter() and
+ * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes
+ * softirq here, we may have a softirq handler calling
+ * local_interrupt_disable() but it won't disable the IRQ because
+ * hardirq disabling count is already 1, hence we need to prevent
+ * invoking softirq when a local_interrupt_disable() is ongoing.
+ */
+ if (!in_interrupt() && !hardirq_disable_count() &&
+ local_softirq_pending()) {
/*
* If we left hrtimers unarmed, make sure to arm them now,
- * before enabling interrupts to run SoftIRQ.
+ * before enabling interrupts to run softirq.
*/
hrtimer_rearm_deferred();
invoke_softirq();
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-04 18:26 ` [PATCH v4.1 " Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Boqun Feng
2026-08-10 8:57 ` [tip: locking/core] irq,spin_lock: " tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits
Cc: Lyude Paul, Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: a58d7a2f53e80422ea9156e7fcda93dc331ccc45
Gitweb: https://git.kernel.org/tip/a58d7a2f53e80422ea9156e7fcda93dc331ccc45
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 11:26:57 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:06 +02:00
irq & spin_lock: Add counted interrupt disabling/enabling
Currently the nested interrupt disabling and enabling is represented by
_irqsave() and _irqrestore() APIs, which are relatively unsafe, for
example:
<interrupts are enabled as beginning>
spin_lock_irqsave(l1, flag1);
spin_lock_irqsave(l2, flag2);
spin_unlock_irqrestore(l1, flags1);
<l2 is still held but interrupts are enabled>
// accesses to interrupt-disable protected data will cause races
This is even easier to trigger with guard facilities:
unsigned long flag2;
scoped_guard(spin_lock_irqsave, l1) {
spin_lock_irqsave(l2, flag2);
}
// l2 locked but interrupts are enabled.
spin_unlock_irqrestore(l2, flag2);
(Hand-to-hand locking critical sections are not uncommon for a
fine-grained lock design)
And because of this unsafety, Rust cannot easily wrap the
interrupt-disabling locks in a safe API, which complicates the design.
To resolve this, introduce a new set of interrupt disabling APIs:
* local_interrupt_disable();
* local_interrupt_enable();
They work like local_irq_save() and local_irq_restore() except that 1)
the outermost local_interrupt_disable() call saves the interrupt state
into a per-CPU variable, so that the outermost local_interrupt_enable()
can restore the state, and 2) a per-CPU counter is added to record the
nest level of these calls, so that interrupts are not accidentally
enabled inside the outermost critical section.
Also add the corresponding spin_lock primitives: spin_lock_irq_disable()
and spin_unlock_irq_enable(), as a result, code as follows:
spin_lock_irq_disable(l1);
spin_lock_irq_disable(l2);
spin_unlock_irq_enable(l1);
// Interrupts are still disabled.
spin_unlock_irq_enable(l2);
doesn't have the issue that interrupts are accidentally enabled.
This also makes the wrapper of interrupt-disabling locks on Rust easier
to design.
[boqun: Apply Peter's feedback and fix spell errors reported by Ingo]
[boqun: Address the duplicate spin_acquire() spotted by sashiko]
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804182657.87716-1-boqun@kernel.org
---
include/linux/interrupt_rc.h | 82 +++++++++++++++++++++++++++++++-
include/linux/preempt.h | 4 ++-
include/linux/spinlock.h | 23 +++++++++-
include/linux/spinlock_api_smp.h | 41 ++++++++++++++++-
include/linux/spinlock_api_up.h | 15 ++++++-
include/linux/spinlock_rt.h | 18 +++++++-
kernel/locking/spinlock.c | 31 ++++++++++++-
kernel/softirq.c | 28 ++++++++++-
8 files changed, 240 insertions(+), 2 deletions(-)
create mode 100644 include/linux/interrupt_rc.h
diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
new file mode 100644
index 0000000..b9a7f05
--- /dev/null
+++ b/include/linux/interrupt_rc.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_INTERRUPT_RC_H
+#define __LINUX_INTERRUPT_RC_H
+
+/*
+ * include/linux/interrupt_rc.h - refcounted local processor interrupt
+ * management.
+ *
+ * Since the implementation of this API currently depends on
+ * local_irq_save()/local_irq_restore(), we split this into its own header to
+ * make it easier to include without hitting circular header dependencies.
+ */
+
+#include <linux/irqflags.h>
+#include <linux/preempt.h>
+#include <linux/processor.h>
+#include <linux/smp.h>
+
+#ifndef MODULE
+/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */
+DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);
+
+static __always_inline void __local_interrupt_disable(void)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ raw_cpu_write(local_interrupt_disable_state, flags);
+}
+
+static __always_inline void __local_interrupt_enable(void)
+{
+ unsigned long flags = raw_cpu_read(local_interrupt_disable_state);
+
+ local_irq_restore(flags);
+}
+
+#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
+static __always_inline void _local_interrupt_disable(void)
+{
+ __local_interrupt_disable();
+}
+
+static __always_inline void _local_interrupt_enable(void)
+{
+ __local_interrupt_enable();
+}
+#else
+extern void _local_interrupt_disable(void);
+extern void _local_interrupt_enable(void);
+#endif
+
+#else /* !MODULE */
+extern void _local_interrupt_disable(void);
+extern void _local_interrupt_enable(void);
+#endif /* !MODULE */
+
+static inline void local_interrupt_disable(void)
+{
+ int new_count;
+
+ WARN_ON_ONCE(in_nmi());
+
+ new_count = hardirq_disable_enter();
+
+ /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
+
+ if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
+ _local_interrupt_disable();
+}
+
+static inline void local_interrupt_enable(void)
+{
+ int new_count;
+
+ new_count = hardirq_disable_exit();
+
+ if ((new_count & HARDIRQ_DISABLE_MASK) == 0)
+ _local_interrupt_enable();
+}
+
+#endif /* !__LINUX_INTERRUPT_RC_H */
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index e2d3079..33fc4c8 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context_level(void)
#define in_softirq() (softirq_count())
#define in_interrupt() (irq_count())
+#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT)
+#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET)
+#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET)
+
/*
* The preempt_count offset after preempt_disable();
*/
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 241277c..3d405cc 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -57,6 +57,7 @@
#include <linux/linkage.h>
#include <linux/compiler.h>
#include <linux/irqflags.h>
+#include <linux/interrupt_rc.h>
#include <linux/thread_info.h>
#include <linux/stringify.h>
#include <linux/bottom_half.h>
@@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#endif
#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
+#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock)
#define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
#define raw_spin_unlock(lock) _raw_spin_unlock(lock)
#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
+#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock)
#define raw_spin_unlock_irqrestore(lock, flags) \
do { \
@@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags))
+#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(lock)
+
#ifndef CONFIG_PREEMPT_RT
/* Include rwlock functions for !RT */
#include <linux/rwlock.h>
@@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)
raw_spin_lock_irq(&lock->rlock);
}
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+ __acquires(lock) __no_context_analysis
+{
+ raw_spin_lock_irq_disable(&lock->rlock);
+}
+
#define spin_lock_irqsave(lock, flags) \
do { \
raw_spin_lock_irqsave(spinlock_check(lock), flags); \
@@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)
raw_spin_unlock_irq(&lock->rlock);
}
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+ __releases(lock) __no_context_analysis
+{
+ raw_spin_unlock_irq_enable(&lock->rlock);
+}
+
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
__releases(lock) __no_context_analysis
{
@@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned lon
}
#define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags))
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+ __cond_acquires(true, lock) __no_context_analysis
+{
+ return raw_spin_trylock_irq_disable(&lock->rlock);
+}
+
/**
* spin_is_locked() - Check whether a spinlock is locked.
* @lock: Pointer to the spinlock.
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index bda5e7a..90909d9 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map)
void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock);
void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
__acquires(lock);
+void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+ __acquires(lock);
unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
__acquires(lock);
@@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(true,
void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock);
+void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releases(lock);
void __lockfunc
_raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
__releases(lock);
@@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
#define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock)
#endif
+/* Use the same config as spin_lock_irq() temporarily. */
+#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ
+#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock)
+#endif
+
#ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
#define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock)
#endif
@@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
#define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock)
#endif
+/* Use the same config as spin_unlock_irq() temporarily. */
+#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(lock)
+#endif
+
#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
#define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrestore(lock, flags)
#endif
@@ -105,6 +118,16 @@ static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock)
return false;
}
+static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ local_interrupt_disable();
+ if (_raw_spin_trylock(lock))
+ return true;
+ local_interrupt_enable();
+ return false;
+}
+
static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
__cond_acquires(true, lock)
{
@@ -143,6 +166,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock)
LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
}
+static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+ __acquires(lock) __no_context_analysis
+{
+ local_interrupt_disable();
+ preempt_disable();
+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+ LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+}
+
static inline void __raw_spin_lock_bh(raw_spinlock_t *lock)
__acquires(lock) __no_context_analysis
{
@@ -188,6 +220,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock)
preempt_enable();
}
+static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+ __releases(lock)
+{
+ spin_release(&lock->dep_map, _RET_IP_);
+ do_raw_spin_unlock(lock);
+ local_interrupt_enable();
+ preempt_enable();
+}
+
static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock)
__releases(lock)
{
diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h
index a9d5c7c..d03d306 100644
--- a/include/linux/spinlock_api_up.h
+++ b/include/linux/spinlock_api_up.h
@@ -42,6 +42,9 @@
#define __LOCK_IRQSAVE(lock, flags, ...) \
do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0)
+#define __LOCK_IRQ_DISABLE(lock, ...) \
+ do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0)
+
#define ___UNLOCK_(lock) \
do { __release(lock); (void)(lock); } while (0)
@@ -61,6 +64,9 @@
#define __UNLOCK_IRQRESTORE(lock, flags, ...) \
do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0)
+#define __UNLOCK_IRQ_ENABLE(lock, ...) \
+ do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0)
+
#define _raw_spin_lock(lock) __LOCK(lock)
#define _raw_spin_lock_nested(lock, subclass) __LOCK(lock)
#define _raw_read_lock(lock) __LOCK(lock, shared)
@@ -70,6 +76,7 @@
#define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared)
#define _raw_write_lock_bh(lock) __LOCK_BH(lock)
#define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock)
+#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock)
#define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared)
#define _raw_write_lock_irq(lock) __LOCK_IRQ(lock)
#define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags)
@@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock)
return 1;
}
+static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ __LOCK_IRQ_DISABLE(lock);
+ return 1;
+}
+
static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
__cond_acquires(true, lock)
{
@@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned l
#define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock)
#define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared)
#define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock)
+#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock)
#define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared)
#define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock)
#define _raw_spin_unlock_irqrestore(lock, flags) \
diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h
index 373618a..560d063 100644
--- a/include/linux/spinlock_rt.h
+++ b/include/linux/spinlock_rt.h
@@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)
rt_spin_lock(lock);
}
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+ __acquires(lock)
+{
+ rt_spin_lock(lock);
+}
+
#define spin_lock_irqsave(lock, flags) \
do { \
typecheck(unsigned long, flags); \
@@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)
rt_spin_unlock(lock);
}
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+ __releases(lock)
+{
+ rt_spin_unlock(lock);
+}
+
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
unsigned long flags)
__releases(lock)
@@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
#define spin_trylock(lock) rt_spin_trylock(lock)
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ return rt_spin_trylock(lock);
+}
+
#define spin_trylock_bh(lock) rt_spin_trylock_bh(lock)
#define spin_trylock_irq(lock) rt_spin_trylock(lock)
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index b42d293..83a17ea 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
*/
BUILD_LOCK_OPS(spin, raw_spinlock, __acquires);
+/* No rwlock_t variants for now, so just build this function by hand */
+static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ for (;;) {
+ preempt_disable();
+ local_interrupt_disable();
+ if (likely(do_raw_spin_trylock(lock)))
+ break;
+ local_interrupt_enable();
+ preempt_enable();
+
+ arch_spin_relax(&lock->raw_lock);
+ }
+}
+
#ifndef CONFIG_PREEMPT_RT
BUILD_LOCK_OPS(read, rwlock, __acquires_shared);
BUILD_LOCK_OPS(write, rwlock, __acquires);
@@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_lock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
+noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ __raw_spin_lock_irq_disable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_LOCK_BH
noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
{
@@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_unlock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+{
+ __raw_spin_unlock_irq_enable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
{
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 10af5ed..0c9b226 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -9,6 +9,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
#include <linux/export.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
@@ -88,6 +89,20 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
#endif
+DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state);
+
+void _local_interrupt_disable(void)
+{
+ __local_interrupt_disable();
+}
+EXPORT_SYMBOL(_local_interrupt_disable);
+
+void _local_interrupt_enable(void)
+{
+ __local_interrupt_enable();
+}
+EXPORT_SYMBOL(_local_interrupt_enable);
+
DEFINE_PER_CPU(unsigned int, nmi_nesting);
/*
@@ -728,10 +743,19 @@ static inline void __irq_exit_rcu(void)
#endif
account_hardirq_exit(current);
preempt_count_sub(HARDIRQ_OFFSET);
- if (!in_interrupt() && local_softirq_pending()) {
+ /*
+ * Interrupts may happen between hardirq_disable_enter() and
+ * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes
+ * softirq here, we may have a softirq handler calling
+ * local_interrupt_disable() but it won't disable the IRQ because
+ * hardirq disabling count is already 1, hence we need to prevent
+ * invoking softirq when a local_interrupt_disable() is ongoing.
+ */
+ if (!in_interrupt() && !hardirq_disable_count() &&
+ local_softirq_pending()) {
/*
* If we left hrtimers unarmed, make sure to arm them now,
- * before enabling interrupts to run SoftIRQ.
+ * before enabling interrupts to run softirq.
*/
hrtimer_rearm_deferred();
invoke_softirq();
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] irq,spin_lock: Add counted interrupt disabling/enabling
2026-08-04 18:26 ` [PATCH v4.1 " Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
@ 2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-10 8:57 UTC (permalink / raw)
To: linux-tip-commits
Cc: Lyude Paul, Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: e901c1510e24726dcbd6340ee927b3ac8b992043
Gitweb: https://git.kernel.org/tip/e901c1510e24726dcbd6340ee927b3ac8b992043
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 11:26:57 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Mon, 10 Aug 2026 10:50:18 +02:00
irq,spin_lock: Add counted interrupt disabling/enabling
Currently the nested interrupt disabling and enabling is represented by
_irqsave() and _irqrestore() APIs, which are relatively unsafe, for
example:
<interrupts are enabled as beginning>
spin_lock_irqsave(l1, flag1);
spin_lock_irqsave(l2, flag2);
spin_unlock_irqrestore(l1, flags1);
<l2 is still held but interrupts are enabled>
// accesses to interrupt-disable protected data will cause races
This is even easier to trigger with guard facilities:
unsigned long flag2;
scoped_guard(spin_lock_irqsave, l1) {
spin_lock_irqsave(l2, flag2);
}
// l2 locked but interrupts are enabled.
spin_unlock_irqrestore(l2, flag2);
(Hand-to-hand locking critical sections are not uncommon for a
fine-grained lock design)
And because of this unsafety, Rust cannot easily wrap the
interrupt-disabling locks in a safe API, which complicates the design.
To resolve this, introduce a new set of interrupt disabling APIs:
* local_interrupt_disable();
* local_interrupt_enable();
They work like local_irq_save() and local_irq_restore() except that 1)
the outermost local_interrupt_disable() call saves the interrupt state
into a per-CPU variable, so that the outermost local_interrupt_enable()
can restore the state, and 2) a per-CPU counter is added to record the
nest level of these calls, so that interrupts are not accidentally
enabled inside the outermost critical section.
Also add the corresponding spin_lock primitives: spin_lock_irq_disable()
and spin_unlock_irq_enable(), as a result, code as follows:
spin_lock_irq_disable(l1);
spin_lock_irq_disable(l2);
spin_unlock_irq_enable(l1);
// Interrupts are still disabled.
spin_unlock_irq_enable(l2);
doesn't have the issue that interrupts are accidentally enabled.
This also makes the wrapper of interrupt-disabling locks on Rust easier
to design.
[boqun: Apply Peter's feedback and fix spell errors reported by Ingo]
[boqun: Address the duplicate spin_acquire() spotted by sashiko]
Co-developed-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804182657.87716-1-boqun@kernel.org
---
include/linux/interrupt_rc.h | 82 +++++++++++++++++++++++++++++++-
include/linux/preempt.h | 4 ++-
include/linux/spinlock.h | 23 +++++++++-
include/linux/spinlock_api_smp.h | 41 ++++++++++++++++-
include/linux/spinlock_api_up.h | 15 ++++++-
include/linux/spinlock_rt.h | 18 +++++++-
kernel/locking/spinlock.c | 31 ++++++++++++-
kernel/softirq.c | 28 ++++++++++-
8 files changed, 240 insertions(+), 2 deletions(-)
create mode 100644 include/linux/interrupt_rc.h
diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
new file mode 100644
index 0000000..b9a7f05
--- /dev/null
+++ b/include/linux/interrupt_rc.h
@@ -0,0 +1,82 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __LINUX_INTERRUPT_RC_H
+#define __LINUX_INTERRUPT_RC_H
+
+/*
+ * include/linux/interrupt_rc.h - refcounted local processor interrupt
+ * management.
+ *
+ * Since the implementation of this API currently depends on
+ * local_irq_save()/local_irq_restore(), we split this into its own header to
+ * make it easier to include without hitting circular header dependencies.
+ */
+
+#include <linux/irqflags.h>
+#include <linux/preempt.h>
+#include <linux/processor.h>
+#include <linux/smp.h>
+
+#ifndef MODULE
+/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */
+DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);
+
+static __always_inline void __local_interrupt_disable(void)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ raw_cpu_write(local_interrupt_disable_state, flags);
+}
+
+static __always_inline void __local_interrupt_enable(void)
+{
+ unsigned long flags = raw_cpu_read(local_interrupt_disable_state);
+
+ local_irq_restore(flags);
+}
+
+#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
+static __always_inline void _local_interrupt_disable(void)
+{
+ __local_interrupt_disable();
+}
+
+static __always_inline void _local_interrupt_enable(void)
+{
+ __local_interrupt_enable();
+}
+#else
+extern void _local_interrupt_disable(void);
+extern void _local_interrupt_enable(void);
+#endif
+
+#else /* !MODULE */
+extern void _local_interrupt_disable(void);
+extern void _local_interrupt_enable(void);
+#endif /* !MODULE */
+
+static inline void local_interrupt_disable(void)
+{
+ int new_count;
+
+ WARN_ON_ONCE(in_nmi());
+
+ new_count = hardirq_disable_enter();
+
+ /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
+
+ if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
+ _local_interrupt_disable();
+}
+
+static inline void local_interrupt_enable(void)
+{
+ int new_count;
+
+ new_count = hardirq_disable_exit();
+
+ if ((new_count & HARDIRQ_DISABLE_MASK) == 0)
+ _local_interrupt_enable();
+}
+
+#endif /* !__LINUX_INTERRUPT_RC_H */
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index e2d3079..33fc4c8 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -151,6 +151,10 @@ static __always_inline unsigned char interrupt_context_level(void)
#define in_softirq() (softirq_count())
#define in_interrupt() (irq_count())
+#define hardirq_disable_count() ((preempt_count() & HARDIRQ_DISABLE_MASK) >> HARDIRQ_DISABLE_SHIFT)
+#define hardirq_disable_enter() __preempt_count_add_return(HARDIRQ_DISABLE_OFFSET)
+#define hardirq_disable_exit() __preempt_count_sub_return(HARDIRQ_DISABLE_OFFSET)
+
/*
* The preempt_count offset after preempt_disable();
*/
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 241277c..3d405cc 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -57,6 +57,7 @@
#include <linux/linkage.h>
#include <linux/compiler.h>
#include <linux/irqflags.h>
+#include <linux/interrupt_rc.h>
#include <linux/thread_info.h>
#include <linux/stringify.h>
#include <linux/bottom_half.h>
@@ -273,9 +274,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#endif
#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock)
+#define raw_spin_lock_irq_disable(lock) _raw_spin_lock_irq_disable(lock)
#define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock)
#define raw_spin_unlock(lock) _raw_spin_unlock(lock)
#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock)
+#define raw_spin_unlock_irq_enable(lock) _raw_spin_unlock_irq_enable(lock)
#define raw_spin_unlock_irqrestore(lock, flags) \
do { \
@@ -290,6 +293,8 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock)
#define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags))
+#define raw_spin_trylock_irq_disable(lock) _raw_spin_trylock_irq_disable(lock)
+
#ifndef CONFIG_PREEMPT_RT
/* Include rwlock functions for !RT */
#include <linux/rwlock.h>
@@ -372,6 +377,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)
raw_spin_lock_irq(&lock->rlock);
}
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+ __acquires(lock) __no_context_analysis
+{
+ raw_spin_lock_irq_disable(&lock->rlock);
+}
+
#define spin_lock_irqsave(lock, flags) \
do { \
raw_spin_lock_irqsave(spinlock_check(lock), flags); \
@@ -402,6 +413,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)
raw_spin_unlock_irq(&lock->rlock);
}
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+ __releases(lock) __no_context_analysis
+{
+ raw_spin_unlock_irq_enable(&lock->rlock);
+}
+
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
__releases(lock) __no_context_analysis
{
@@ -427,6 +444,12 @@ static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned lon
}
#define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags))
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+ __cond_acquires(true, lock) __no_context_analysis
+{
+ return raw_spin_trylock_irq_disable(&lock->rlock);
+}
+
/**
* spin_is_locked() - Check whether a spinlock is locked.
* @lock: Pointer to the spinlock.
diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h
index bda5e7a..90909d9 100644
--- a/include/linux/spinlock_api_smp.h
+++ b/include/linux/spinlock_api_smp.h
@@ -28,6 +28,8 @@ _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map)
void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) __acquires(lock);
void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
__acquires(lock);
+void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+ __acquires(lock);
unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock)
__acquires(lock);
@@ -39,6 +41,7 @@ int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(true,
void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock);
void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock);
+void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock) __releases(lock);
void __lockfunc
_raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
__releases(lock);
@@ -55,6 +58,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
#define _raw_spin_lock_irq(lock) __raw_spin_lock_irq(lock)
#endif
+/* Use the same config as spin_lock_irq() temporarily. */
+#ifdef CONFIG_INLINE_SPIN_LOCK_IRQ
+#define _raw_spin_lock_irq_disable(lock) __raw_spin_lock_irq_disable(lock)
+#endif
+
#ifdef CONFIG_INLINE_SPIN_LOCK_IRQSAVE
#define _raw_spin_lock_irqsave(lock) __raw_spin_lock_irqsave(lock)
#endif
@@ -79,6 +87,11 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags)
#define _raw_spin_unlock_irq(lock) __raw_spin_unlock_irq(lock)
#endif
+/* Use the same config as spin_unlock_irq() temporarily. */
+#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+#define _raw_spin_unlock_irq_enable(lock) __raw_spin_unlock_irq_enable(lock)
+#endif
+
#ifdef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE
#define _raw_spin_unlock_irqrestore(lock, flags) __raw_spin_unlock_irqrestore(lock, flags)
#endif
@@ -105,6 +118,16 @@ static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock)
return false;
}
+static __always_inline bool _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ local_interrupt_disable();
+ if (_raw_spin_trylock(lock))
+ return true;
+ local_interrupt_enable();
+ return false;
+}
+
static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
__cond_acquires(true, lock)
{
@@ -143,6 +166,15 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock)
LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
}
+static inline void __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+ __acquires(lock) __no_context_analysis
+{
+ local_interrupt_disable();
+ preempt_disable();
+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_);
+ LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock);
+}
+
static inline void __raw_spin_lock_bh(raw_spinlock_t *lock)
__acquires(lock) __no_context_analysis
{
@@ -188,6 +220,15 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock)
preempt_enable();
}
+static inline void __raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+ __releases(lock)
+{
+ spin_release(&lock->dep_map, _RET_IP_);
+ do_raw_spin_unlock(lock);
+ local_interrupt_enable();
+ preempt_enable();
+}
+
static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock)
__releases(lock)
{
diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h
index a9d5c7c..d03d306 100644
--- a/include/linux/spinlock_api_up.h
+++ b/include/linux/spinlock_api_up.h
@@ -42,6 +42,9 @@
#define __LOCK_IRQSAVE(lock, flags, ...) \
do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0)
+#define __LOCK_IRQ_DISABLE(lock, ...) \
+ do { local_interrupt_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0)
+
#define ___UNLOCK_(lock) \
do { __release(lock); (void)(lock); } while (0)
@@ -61,6 +64,9 @@
#define __UNLOCK_IRQRESTORE(lock, flags, ...) \
do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0)
+#define __UNLOCK_IRQ_ENABLE(lock, ...) \
+ do { __UNLOCK(lock, ##__VA_ARGS__); local_interrupt_enable(); } while (0)
+
#define _raw_spin_lock(lock) __LOCK(lock)
#define _raw_spin_lock_nested(lock, subclass) __LOCK(lock)
#define _raw_read_lock(lock) __LOCK(lock, shared)
@@ -70,6 +76,7 @@
#define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared)
#define _raw_write_lock_bh(lock) __LOCK_BH(lock)
#define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock)
+#define _raw_spin_lock_irq_disable(lock) __LOCK_IRQ_DISABLE(lock)
#define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared)
#define _raw_write_lock_irq(lock) __LOCK_IRQ(lock)
#define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags)
@@ -97,6 +104,13 @@ static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock)
return 1;
}
+static __always_inline int _raw_spin_trylock_irq_disable(raw_spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ __LOCK_IRQ_DISABLE(lock);
+ return 1;
+}
+
static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags)
__cond_acquires(true, lock)
{
@@ -132,6 +146,7 @@ static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned l
#define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock)
#define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared)
#define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock)
+#define _raw_spin_unlock_irq_enable(lock) __UNLOCK_IRQ_ENABLE(lock)
#define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared)
#define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock)
#define _raw_spin_unlock_irqrestore(lock, flags) \
diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h
index 373618a..560d063 100644
--- a/include/linux/spinlock_rt.h
+++ b/include/linux/spinlock_rt.h
@@ -96,6 +96,12 @@ static __always_inline void spin_lock_irq(spinlock_t *lock)
rt_spin_lock(lock);
}
+static __always_inline void spin_lock_irq_disable(spinlock_t *lock)
+ __acquires(lock)
+{
+ rt_spin_lock(lock);
+}
+
#define spin_lock_irqsave(lock, flags) \
do { \
typecheck(unsigned long, flags); \
@@ -122,6 +128,12 @@ static __always_inline void spin_unlock_irq(spinlock_t *lock)
rt_spin_unlock(lock);
}
+static __always_inline void spin_unlock_irq_enable(spinlock_t *lock)
+ __releases(lock)
+{
+ rt_spin_unlock(lock);
+}
+
static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
unsigned long flags)
__releases(lock)
@@ -131,6 +143,12 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock,
#define spin_trylock(lock) rt_spin_trylock(lock)
+static __always_inline int spin_trylock_irq_disable(spinlock_t *lock)
+ __cond_acquires(true, lock)
+{
+ return rt_spin_trylock(lock);
+}
+
#define spin_trylock_bh(lock) rt_spin_trylock_bh(lock)
#define spin_trylock_irq(lock) rt_spin_trylock(lock)
diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c
index b42d293..83a17ea 100644
--- a/kernel/locking/spinlock.c
+++ b/kernel/locking/spinlock.c
@@ -129,6 +129,21 @@ static void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \
*/
BUILD_LOCK_OPS(spin, raw_spinlock, __acquires);
+/* No rwlock_t variants for now, so just build this function by hand */
+static void __lockfunc __raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ for (;;) {
+ preempt_disable();
+ local_interrupt_disable();
+ if (likely(do_raw_spin_trylock(lock)))
+ break;
+ local_interrupt_enable();
+ preempt_enable();
+
+ arch_spin_relax(&lock->raw_lock);
+ }
+}
+
#ifndef CONFIG_PREEMPT_RT
BUILD_LOCK_OPS(read, rwlock, __acquires_shared);
BUILD_LOCK_OPS(write, rwlock, __acquires);
@@ -176,6 +191,14 @@ noinline void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_lock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_LOCK_IRQ
+noinline void __lockfunc _raw_spin_lock_irq_disable(raw_spinlock_t *lock)
+{
+ __raw_spin_lock_irq_disable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_lock_irq_disable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_LOCK_BH
noinline void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock)
{
@@ -208,6 +231,14 @@ noinline void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock)
EXPORT_SYMBOL(_raw_spin_unlock_irq);
#endif
+#ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ
+noinline void __lockfunc _raw_spin_unlock_irq_enable(raw_spinlock_t *lock)
+{
+ __raw_spin_unlock_irq_enable(lock);
+}
+EXPORT_SYMBOL_GPL(_raw_spin_unlock_irq_enable);
+#endif
+
#ifndef CONFIG_INLINE_SPIN_UNLOCK_BH
noinline void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock)
{
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 10af5ed..0c9b226 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -9,6 +9,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#define INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
#include <linux/export.h>
#include <linux/kernel_stat.h>
#include <linux/interrupt.h>
@@ -88,6 +89,20 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
#endif
+DEFINE_PER_CPU(unsigned long, local_interrupt_disable_state);
+
+void _local_interrupt_disable(void)
+{
+ __local_interrupt_disable();
+}
+EXPORT_SYMBOL(_local_interrupt_disable);
+
+void _local_interrupt_enable(void)
+{
+ __local_interrupt_enable();
+}
+EXPORT_SYMBOL(_local_interrupt_enable);
+
DEFINE_PER_CPU(unsigned int, nmi_nesting);
/*
@@ -728,10 +743,19 @@ static inline void __irq_exit_rcu(void)
#endif
account_hardirq_exit(current);
preempt_count_sub(HARDIRQ_OFFSET);
- if (!in_interrupt() && local_softirq_pending()) {
+ /*
+ * Interrupts may happen between hardirq_disable_enter() and
+ * local_irq_save() in local_interrupt_disable(), if irq_exit() invokes
+ * softirq here, we may have a softirq handler calling
+ * local_interrupt_disable() but it won't disable the IRQ because
+ * hardirq disabling count is already 1, hence we need to prevent
+ * invoking softirq when a local_interrupt_disable() is ongoing.
+ */
+ if (!in_interrupt() && !hardirq_disable_count() &&
+ local_softirq_pending()) {
/*
* If we left hrtimers unarmed, make sure to arm them now,
- * before enabling interrupts to run SoftIRQ.
+ * before enabling interrupts to run softirq.
*/
hrtimer_rearm_deferred();
invoke_softirq();
^ permalink raw reply related [flat|nested] 66+ messages in thread
* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-04 16:14 ` [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling Boqun Feng
2026-08-04 18:20 ` Boqun Feng
2026-08-04 18:26 ` [PATCH v4.1 " Boqun Feng
@ 2026-08-04 20:51 ` Shrikanth Hegde
2026-08-04 21:08 ` Boqun Feng
2 siblings, 1 reply; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-04 20:51 UTC (permalink / raw)
To: Boqun Feng, Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux
On 8/4/26 9:44 PM, Boqun Feng wrote:
> Currently the nested interrupt disabling and enabling is represented by
> _irqsave() and _irqrestore() APIs, which are relatively unsafe, for
> example:
>
> <interrupts are enabled as beginning>
> spin_lock_irqsave(l1, flag1);
> spin_lock_irqsave(l2, flag2);
> spin_unlock_irqrestore(l1, flags1);
> <l2 is still held but interrupts are enabled>
> // accesses to interrupt-disable protected data will cause races
>
> This is even easier to trigger with guard facilities:
>
> unsigned long flag2;
>
> scoped_guard(spin_lock_irqsave, l1) {
> spin_lock_irqsave(l2, flag2);
> }
> // l2 locked but interrupts are enabled.
> spin_unlock_irqrestore(l2, flag2);
>
> (Hand-to-hand locking critical sections are not uncommon for a
> fine-grained lock design)
>
> And because of this unsafety, Rust cannot easily wrap the
> interrupt-disabling locks in a safe API, which complicates the design.
>
> To resolve this, introduce a new set of interrupt disabling APIs:
>
> * local_interrupt_disable();
> * local_interrupt_enable();
>
> They work like local_irq_save() and local_irq_restore() except that 1)
> the outermost local_interrupt_disable() call saves the interrupt state
> into a per-CPU variable, so that the outermost local_interrupt_enable()
> can restore the state, and 2) a per-CPU counter is added to record the
> nest level of these calls, so that interrupts are not accidentally
> enabled inside the outermost critical section.
>
> Also add the corresponding spin_lock primitives: spin_lock_irq_disable()
> and spin_unlock_irq_enable(), as a result, code as follows:
>
> spin_lock_irq_disable(l1);
> spin_lock_irq_disable(l2);
> spin_unlock_irq_enable(l1);
> // Interrupts are still disabled.
> spin_unlock_irq_enable(l2);
>
> doesn't have the issue that interrupts are accidentally enabled.
>
> This also makes the wrapper of interrupt-disabling locks on Rust easier
> to design.
>
> Signed-off-by: Lyude Paul <lyude@redhat.com>
> [boqun: Apply Peter's feedback and fix spell errors reported by Ingo]
> Signed-off-by: Boqun Feng <boqun@kernel.org>
> ---
> include/linux/interrupt_rc.h | 82 ++++++++++++++++++++++++++++++++
> include/linux/preempt.h | 4 ++
> include/linux/spinlock.h | 23 +++++++++
> include/linux/spinlock_api_smp.h | 43 +++++++++++++++++
> include/linux/spinlock_api_up.h | 15 ++++++
> include/linux/spinlock_rt.h | 18 +++++++
> kernel/locking/spinlock.c | 31 ++++++++++++
> kernel/softirq.c | 28 ++++++++++-
> 8 files changed, 242 insertions(+), 2 deletions(-)
> create mode 100644 include/linux/interrupt_rc.h
>
> diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
> new file mode 100644
> index 000000000000..b9a7f05ecf42
> --- /dev/null
> +++ b/include/linux/interrupt_rc.h
> @@ -0,0 +1,82 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +#ifndef __LINUX_INTERRUPT_RC_H
> +#define __LINUX_INTERRUPT_RC_H
> +
> +/*
> + * include/linux/interrupt_rc.h - refcounted local processor interrupt
> + * management.
> + *
> + * Since the implementation of this API currently depends on
> + * local_irq_save()/local_irq_restore(), we split this into its own header to
> + * make it easier to include without hitting circular header dependencies.
> + */
> +
> +#include <linux/irqflags.h>
> +#include <linux/preempt.h>
> +#include <linux/processor.h>
> +#include <linux/smp.h>
> +
> +#ifndef MODULE
> +/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */
> +DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);
> +
> +static __always_inline void __local_interrupt_disable(void)
> +{
> + unsigned long flags;
> +
> + local_irq_save(flags);
> + raw_cpu_write(local_interrupt_disable_state, flags);
> +}
> +
> +static __always_inline void __local_interrupt_enable(void)
> +{
> + unsigned long flags = raw_cpu_read(local_interrupt_disable_state);
> +
> + local_irq_restore(flags);
> +}
> +
> +#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
> +static __always_inline void _local_interrupt_disable(void)
> +{
> + __local_interrupt_disable();
> +}
> +
> +static __always_inline void _local_interrupt_enable(void)
> +{
> + __local_interrupt_enable();
> +}
> +#else
> +extern void _local_interrupt_disable(void);
> +extern void _local_interrupt_enable(void);
> +#endif
> +
> +#else /* !MODULE */
> +extern void _local_interrupt_disable(void);
> +extern void _local_interrupt_enable(void);
> +#endif /* !MODULE */
> +
> +static inline void local_interrupt_disable(void)
> +{
> + int new_count;
> +
> + WARN_ON_ONCE(in_nmi());
> +
> + new_count = hardirq_disable_enter();
> +
> + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> +
> + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> + _local_interrupt_disable();
> +}
Maximum nesting possible is 256 right? Whats is stopping here to do more than that?
Should there be a warn_on?
> +
> +static inline void local_interrupt_enable(void)
> +{
> + int new_count;
> +
> + new_count = hardirq_disable_exit();
> +
> + if ((new_count & HARDIRQ_DISABLE_MASK) == 0)
> + _local_interrupt_enable();
> +}
> +
> +#endif /* !__LINUX_INTERRUPT_RC_H */
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-04 20:51 ` [PATCH v4 05/17] irq & spin_lock: " Shrikanth Hegde
@ 2026-08-04 21:08 ` Boqun Feng
2026-08-05 6:36 ` Peter Zijlstra
0 siblings, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 21:08 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 02:21:49AM +0530, Shrikanth Hegde wrote:
>
>
> On 8/4/26 9:44 PM, Boqun Feng wrote:
> > Currently the nested interrupt disabling and enabling is represented by
> > _irqsave() and _irqrestore() APIs, which are relatively unsafe, for
> > example:
> >
> > <interrupts are enabled as beginning>
> > spin_lock_irqsave(l1, flag1);
> > spin_lock_irqsave(l2, flag2);
> > spin_unlock_irqrestore(l1, flags1);
> > <l2 is still held but interrupts are enabled>
> > // accesses to interrupt-disable protected data will cause races
> >
> > This is even easier to trigger with guard facilities:
> >
> > unsigned long flag2;
> >
> > scoped_guard(spin_lock_irqsave, l1) {
> > spin_lock_irqsave(l2, flag2);
> > }
> > // l2 locked but interrupts are enabled.
> > spin_unlock_irqrestore(l2, flag2);
> >
> > (Hand-to-hand locking critical sections are not uncommon for a
> > fine-grained lock design)
> >
> > And because of this unsafety, Rust cannot easily wrap the
> > interrupt-disabling locks in a safe API, which complicates the design.
> >
> > To resolve this, introduce a new set of interrupt disabling APIs:
> >
> > * local_interrupt_disable();
> > * local_interrupt_enable();
> >
> > They work like local_irq_save() and local_irq_restore() except that 1)
> > the outermost local_interrupt_disable() call saves the interrupt state
> > into a per-CPU variable, so that the outermost local_interrupt_enable()
> > can restore the state, and 2) a per-CPU counter is added to record the
> > nest level of these calls, so that interrupts are not accidentally
> > enabled inside the outermost critical section.
> >
> > Also add the corresponding spin_lock primitives: spin_lock_irq_disable()
> > and spin_unlock_irq_enable(), as a result, code as follows:
> >
> > spin_lock_irq_disable(l1);
> > spin_lock_irq_disable(l2);
> > spin_unlock_irq_enable(l1);
> > // Interrupts are still disabled.
> > spin_unlock_irq_enable(l2);
> >
> > doesn't have the issue that interrupts are accidentally enabled.
> >
> > This also makes the wrapper of interrupt-disabling locks on Rust easier
> > to design.
> >
> > Signed-off-by: Lyude Paul <lyude@redhat.com>
> > [boqun: Apply Peter's feedback and fix spell errors reported by Ingo]
> > Signed-off-by: Boqun Feng <boqun@kernel.org>
> > ---
> > include/linux/interrupt_rc.h | 82 ++++++++++++++++++++++++++++++++
> > include/linux/preempt.h | 4 ++
> > include/linux/spinlock.h | 23 +++++++++
> > include/linux/spinlock_api_smp.h | 43 +++++++++++++++++
> > include/linux/spinlock_api_up.h | 15 ++++++
> > include/linux/spinlock_rt.h | 18 +++++++
> > kernel/locking/spinlock.c | 31 ++++++++++++
> > kernel/softirq.c | 28 ++++++++++-
> > 8 files changed, 242 insertions(+), 2 deletions(-)
> > create mode 100644 include/linux/interrupt_rc.h
> >
> > diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
> > new file mode 100644
> > index 000000000000..b9a7f05ecf42
> > --- /dev/null
> > +++ b/include/linux/interrupt_rc.h
> > @@ -0,0 +1,82 @@
> > +/* SPDX-License-Identifier: GPL-2.0 */
> > +#ifndef __LINUX_INTERRUPT_RC_H
> > +#define __LINUX_INTERRUPT_RC_H
> > +
> > +/*
> > + * include/linux/interrupt_rc.h - refcounted local processor interrupt
> > + * management.
> > + *
> > + * Since the implementation of this API currently depends on
> > + * local_irq_save()/local_irq_restore(), we split this into its own header to
> > + * make it easier to include without hitting circular header dependencies.
> > + */
> > +
> > +#include <linux/irqflags.h>
> > +#include <linux/preempt.h>
> > +#include <linux/processor.h>
> > +#include <linux/smp.h>
> > +
> > +#ifndef MODULE
> > +/* Per-CPU interrupt disabling state for local_interrupt_{disable,enable}(). */
> > +DECLARE_PER_CPU(unsigned long, local_interrupt_disable_state);
> > +
> > +static __always_inline void __local_interrupt_disable(void)
> > +{
> > + unsigned long flags;
> > +
> > + local_irq_save(flags);
> > + raw_cpu_write(local_interrupt_disable_state, flags);
> > +}
> > +
> > +static __always_inline void __local_interrupt_enable(void)
> > +{
> > + unsigned long flags = raw_cpu_read(local_interrupt_disable_state);
> > +
> > + local_irq_restore(flags);
> > +}
> > +
> > +#ifndef INSTANTIATE_EXPORTED_INTERRUPT_DISABLE
> > +static __always_inline void _local_interrupt_disable(void)
> > +{
> > + __local_interrupt_disable();
> > +}
> > +
> > +static __always_inline void _local_interrupt_enable(void)
> > +{
> > + __local_interrupt_enable();
> > +}
> > +#else
> > +extern void _local_interrupt_disable(void);
> > +extern void _local_interrupt_enable(void);
> > +#endif
> > +
> > +#else /* !MODULE */
> > +extern void _local_interrupt_disable(void);
> > +extern void _local_interrupt_enable(void);
> > +#endif /* !MODULE */
> > +
> > +static inline void local_interrupt_disable(void)
> > +{
> > + int new_count;
> > +
> > + WARN_ON_ONCE(in_nmi());
> > +
> > + new_count = hardirq_disable_enter();
> > +
> > + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> > +
> > + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> > + _local_interrupt_disable();
> > +}
>
> Maximum nesting possible is 256 right? Whats is stopping here to do more than that?
Yes. Currently similar as softirq, we don't detect the overflow.
> Should there be a warn_on?
A simple warn_on could be problematic because warn_on() itself may take
an irq-disabling lock, and that may trigger another overflow on top of
the existing overflow. It's a bit tricky to do a proper detection. But
I'm open to ideas.
Regards,
Boqun
>
> > +
> > +static inline void local_interrupt_enable(void)
> > +{
> > + int new_count;
> > +
> > + new_count = hardirq_disable_exit();
> > +
> > + if ((new_count & HARDIRQ_DISABLE_MASK) == 0)
> > + _local_interrupt_enable();
> > +}
> > +
> > +#endif /* !__LINUX_INTERRUPT_RC_H */
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-04 21:08 ` Boqun Feng
@ 2026-08-05 6:36 ` Peter Zijlstra
2026-08-05 7:07 ` Boqun Feng
0 siblings, 1 reply; 66+ messages in thread
From: Peter Zijlstra @ 2026-08-05 6:36 UTC (permalink / raw)
To: Boqun Feng
Cc: Shrikanth Hegde, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Tue, Aug 04, 2026 at 02:08:11PM -0700, Boqun Feng wrote:
> > > + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> > > +
> > > + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> > > + _local_interrupt_disable();
> > > +}
> >
> > Maximum nesting possible is 256 right? Whats is stopping here to do more than that?
>
> Yes. Currently similar as softirq, we don't detect the overflow.
>
> > Should there be a warn_on?
>
> A simple warn_on could be problematic because warn_on() itself may take
> an irq-disabling lock, and that may trigger another overflow on top of
> the existing overflow. It's a bit tricky to do a proper detection. But
> I'm open to ideas.
DEBUG_PREEMPT's preempt_count_add() does:
DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK - 10);
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 6:36 ` Peter Zijlstra
@ 2026-08-05 7:07 ` Boqun Feng
2026-08-05 7:09 ` Shrikanth Hegde
0 siblings, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 7:07 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Shrikanth Hegde, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 08:36:45AM +0200, Peter Zijlstra wrote:
> On Tue, Aug 04, 2026 at 02:08:11PM -0700, Boqun Feng wrote:
>
> > > > + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> > > > +
> > > > + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> > > > + _local_interrupt_disable();
> > > > +}
> > >
> > > Maximum nesting possible is 256 right? Whats is stopping here to do more than that?
> >
> > Yes. Currently similar as softirq, we don't detect the overflow.
> >
> > > Should there be a warn_on?
> >
> > A simple warn_on could be problematic because warn_on() itself may take
> > an irq-disabling lock, and that may trigger another overflow on top of
> > the existing overflow. It's a bit tricky to do a proper detection. But
> > I'm open to ideas.
>
> DEBUG_PREEMPT's preempt_count_add() does:
>
> DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK - 10);
>
Yeah, but this is behind a kconfig (DEBUG_PREEMPT), so not sure whether
it's what Shrikanth asked here.
Regards,
Boqun
>
^ permalink raw reply [flat|nested] 66+ messages in thread
* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 7:07 ` Boqun Feng
@ 2026-08-05 7:09 ` Shrikanth Hegde
2026-08-05 7:19 ` Boqun Feng
0 siblings, 1 reply; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-05 7:09 UTC (permalink / raw)
To: Boqun Feng, Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux
On 8/5/26 12:37 PM, Boqun Feng wrote:
> On Wed, Aug 05, 2026 at 08:36:45AM +0200, Peter Zijlstra wrote:
>> On Tue, Aug 04, 2026 at 02:08:11PM -0700, Boqun Feng wrote:
>>
>>>>> + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
>>>>> +
>>>>> + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
>>>>> + _local_interrupt_disable();
>>>>> +}
>>>>
>>>> Maximum nesting possible is 256 right? Whats is stopping here to do more than that?
>>>
>>> Yes. Currently similar as softirq, we don't detect the overflow.
>>>
>>>> Should there be a warn_on?
>>>
>>> A simple warn_on could be problematic because warn_on() itself may take
>>> an irq-disabling lock, and that may trigger another overflow on top of
>>> the existing overflow. It's a bit tricky to do a proper detection. But
>>> I'm open to ideas.
>>
>> DEBUG_PREEMPT's preempt_count_add() does:
>>
>> DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK - 10);
>>
>
> Yeah, but this is behind a kconfig (DEBUG_PREEMPT), so not sure whether
> it's what Shrikanth asked here.
>
I was suggesting to have a mechanism which allows to debug it/
find the callers.
> Regards,
> Boqun
>
>>
^ permalink raw reply [flat|nested] 66+ messages in thread
* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 7:09 ` Shrikanth Hegde
@ 2026-08-05 7:19 ` Boqun Feng
2026-08-05 13:53 ` Boqun Feng
0 siblings, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 7:19 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 12:39:36PM +0530, Shrikanth Hegde wrote:
>
>
> On 8/5/26 12:37 PM, Boqun Feng wrote:
> > On Wed, Aug 05, 2026 at 08:36:45AM +0200, Peter Zijlstra wrote:
> > > On Tue, Aug 04, 2026 at 02:08:11PM -0700, Boqun Feng wrote:
> > >
> > > > > > + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> > > > > > +
> > > > > > + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> > > > > > + _local_interrupt_disable();
> > > > > > +}
> > > > >
> > > > > Maximum nesting possible is 256 right? Whats is stopping here to do more than that?
> > > >
> > > > Yes. Currently similar as softirq, we don't detect the overflow.
> > > >
> > > > > Should there be a warn_on?
> > > >
> > > > A simple warn_on could be problematic because warn_on() itself may take
> > > > an irq-disabling lock, and that may trigger another overflow on top of
> > > > the existing overflow. It's a bit tricky to do a proper detection. But
> > > > I'm open to ideas.
> > >
> > > DEBUG_PREEMPT's preempt_count_add() does:
> > >
> > > DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK - 10);
> > >
> >
> > Yeah, but this is behind a kconfig (DEBUG_PREEMPT), so not sure whether
> > it's what Shrikanth asked here.
> >
>
> I was suggesting to have a mechanism which allows to debug it/
> find the callers.
>
Ok, I can reuse the DEBUG_PREEMPT kconfig and the DEBUG_LOCKS_WARN_ON()
here, but it'll be similar to the detection here, not a 256 maximum
nesting but a 256 - 10 value (and I'm not going to explain why we think
10 is a good buffer, unless you think we should have a discussion about
it ;-) )
(But still who is going to add the detection for BH count over? ;-) )
Regards,
Boqun
> > Regards,
> > Boqun
> >
> > >
>
^ permalink raw reply [flat|nested] 66+ messages in thread
* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 7:19 ` Boqun Feng
@ 2026-08-05 13:53 ` Boqun Feng
2026-08-05 14:10 ` Shrikanth Hegde
0 siblings, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 13:53 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 12:19:25AM -0700, Boqun Feng wrote:
> On Wed, Aug 05, 2026 at 12:39:36PM +0530, Shrikanth Hegde wrote:
> >
> >
> > On 8/5/26 12:37 PM, Boqun Feng wrote:
> > > On Wed, Aug 05, 2026 at 08:36:45AM +0200, Peter Zijlstra wrote:
> > > > On Tue, Aug 04, 2026 at 02:08:11PM -0700, Boqun Feng wrote:
> > > >
> > > > > > > + /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> > > > > > > +
> > > > > > > + if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> > > > > > > + _local_interrupt_disable();
> > > > > > > +}
> > > > > >
> > > > > > Maximum nesting possible is 256 right? Whats is stopping here to do more than that?
> > > > >
> > > > > Yes. Currently similar as softirq, we don't detect the overflow.
> > > > >
> > > > > > Should there be a warn_on?
> > > > >
> > > > > A simple warn_on could be problematic because warn_on() itself may take
> > > > > an irq-disabling lock, and that may trigger another overflow on top of
> > > > > the existing overflow. It's a bit tricky to do a proper detection. But
> > > > > I'm open to ideas.
> > > >
> > > > DEBUG_PREEMPT's preempt_count_add() does:
> > > >
> > > > DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= PREEMPT_MASK - 10);
> > > >
> > >
> > > Yeah, but this is behind a kconfig (DEBUG_PREEMPT), so not sure whether
> > > it's what Shrikanth asked here.
> > >
> >
> > I was suggesting to have a mechanism which allows to debug it/
> > find the callers.
> >
>
> Ok, I can reuse the DEBUG_PREEMPT kconfig and the DEBUG_LOCKS_WARN_ON()
> here, but it'll be similar to the detection here, not a 256 maximum
> nesting but a 256 - 10 value (and I'm not going to explain why we think
> 10 is a good buffer, unless you think we should have a discussion about
> it ;-) )
>
Something as below? Going to send it to kernel build bot and see if it
works for all configs.
----------------->8
diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
index b9a7f05ecf42..39f30bc65548 100644
--- a/include/linux/interrupt_rc.h
+++ b/include/linux/interrupt_rc.h
@@ -12,6 +12,7 @@
*/
#include <linux/irqflags.h>
+#include <linux/debug_locks.h>
#include <linux/preempt.h>
#include <linux/processor.h>
#include <linux/smp.h>
@@ -63,6 +64,12 @@ static inline void local_interrupt_disable(void)
new_count = hardirq_disable_enter();
+ /* Is hardirq disable count overflow soon? */
+ if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
+ DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) +
+ (10 << HARDIRQ_DISABLE_SHIFT) >
+ HARDIRQ_DISABLE_MASK);
+
/* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
@@ -73,6 +80,11 @@ static inline void local_interrupt_enable(void)
{
int new_count;
+ /* Unpaired local_interrupt_enable()? Warn and abort. */
+ if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
+ DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0))
+ return;
+
new_count = hardirq_disable_exit();
if ((new_count & HARDIRQ_DISABLE_MASK) == 0) >
^ permalink raw reply related [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 13:53 ` Boqun Feng
@ 2026-08-05 14:10 ` Shrikanth Hegde
2026-08-05 14:20 ` Boqun Feng
0 siblings, 1 reply; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-05 14:10 UTC (permalink / raw)
To: Boqun Feng
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
Hi Boqun,
>
> Something as below? Going to send it to kernel build bot and see if it
> works for all configs.
>
> ----------------->8
> diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
> index b9a7f05ecf42..39f30bc65548 100644
> --- a/include/linux/interrupt_rc.h
> +++ b/include/linux/interrupt_rc.h
> @@ -12,6 +12,7 @@
> */
>
> #include <linux/irqflags.h>
> +#include <linux/debug_locks.h>
> #include <linux/preempt.h>
> #include <linux/processor.h>
> #include <linux/smp.h>
> @@ -63,6 +64,12 @@ static inline void local_interrupt_disable(void)
>
> new_count = hardirq_disable_enter();
>
> + /* Is hardirq disable count overflow soon? */
> + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
> + DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) +
> + (10 << HARDIRQ_DISABLE_SHIFT) >
> + HARDIRQ_DISABLE_MASK);
> +
This needs a return here right? Else we will see warning for 10 times
and then overflow happens and we will call _local_interrupt_disable. No?
Not sure, if below is any better? (Igore whitespace mangling)
if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) >=
HARDIRQ_DISABLE_MASK - (10 << HARDIRQ_DISABLE_SHIFT)))
return;
> /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
>
> if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> @@ -73,6 +80,11 @@ static inline void local_interrupt_enable(void)
> {
> int new_count;
>
> + /* Unpaired local_interrupt_enable()? Warn and abort. */
> + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
> + DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0))
> + return;
> +
> new_count = hardirq_disable_exit();
>
> if ((new_count & HARDIRQ_DISABLE_MASK) == 0) >
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 14:10 ` Shrikanth Hegde
@ 2026-08-05 14:20 ` Boqun Feng
2026-08-05 14:56 ` Shrikanth Hegde
0 siblings, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 14:20 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 07:40:18PM +0530, Shrikanth Hegde wrote:
>
> Hi Boqun,
>
> >
> > Something as below? Going to send it to kernel build bot and see if it
> > works for all configs.
> >
> > ----------------->8
> > diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
> > index b9a7f05ecf42..39f30bc65548 100644
> > --- a/include/linux/interrupt_rc.h
> > +++ b/include/linux/interrupt_rc.h
> > @@ -12,6 +12,7 @@
> > */
> >
> > #include <linux/irqflags.h>
> > +#include <linux/debug_locks.h>
> > #include <linux/preempt.h>
> > #include <linux/processor.h>
> > #include <linux/smp.h>
> > @@ -63,6 +64,12 @@ static inline void local_interrupt_disable(void)
> >
> > new_count = hardirq_disable_enter();
> >
> > + /* Is hardirq disable count overflow soon? */
> > + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
> > + DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) +
> > + (10 << HARDIRQ_DISABLE_SHIFT) >
> > + HARDIRQ_DISABLE_MASK);
> > +
>
> This needs a return here right? Else we will see warning for 10 times
> and then overflow happens and we will call _local_interrupt_disable. No?
>
DEBUG_LOCKS_WARN_ON() uses debug_locks_off() to avoid this, so we won't
see it 10 times. The reason not using return here, because we would
introduce unpaired local_interrupt_disable() if we returned:
// hardirq disable count is n
local_interrupt_disable(); // hardirq disable count is n + 1
local_interrupt_disable(); <- trigger the warning, if we return
// hardirq disable count is n + 1
local_interrupt_enable(); // hardirq disable count is n
local_interrupt_enable(); // hardirq disable count is n - 1
Regards,
Boqun
> Not sure, if below is any better? (Igore whitespace mangling)
>
> if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
> DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) >=
> HARDIRQ_DISABLE_MASK - (10 << HARDIRQ_DISABLE_SHIFT)))
> return;
>
> > /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> >
> > if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> > @@ -73,6 +80,11 @@ static inline void local_interrupt_enable(void)
> > {
> > int new_count;
> >
> > + /* Unpaired local_interrupt_enable()? Warn and abort. */
> > + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
> > + DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0))
> > + return;
> > +
> > new_count = hardirq_disable_exit();
> >
> > if ((new_count & HARDIRQ_DISABLE_MASK) == 0) >
>
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 14:20 ` Boqun Feng
@ 2026-08-05 14:56 ` Shrikanth Hegde
2026-08-05 15:11 ` Boqun Feng
2026-08-05 18:07 ` Boqun Feng
0 siblings, 2 replies; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-05 14:56 UTC (permalink / raw)
To: Boqun Feng
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On 8/5/26 7:50 PM, Boqun Feng wrote:
> On Wed, Aug 05, 2026 at 07:40:18PM +0530, Shrikanth Hegde wrote:
>>
>> Hi Boqun,
>>
>>>
>>> Something as below? Going to send it to kernel build bot and see if it
>>> works for all configs.
>>>
>>> ----------------->8
>>> diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
>>> index b9a7f05ecf42..39f30bc65548 100644
>>> --- a/include/linux/interrupt_rc.h
>>> +++ b/include/linux/interrupt_rc.h
>>> @@ -12,6 +12,7 @@
>>> */
>>>
>>> #include <linux/irqflags.h>
>>> +#include <linux/debug_locks.h>
>>> #include <linux/preempt.h>
>>> #include <linux/processor.h>
>>> #include <linux/smp.h>
>>> @@ -63,6 +64,12 @@ static inline void local_interrupt_disable(void)
>>>
>>> new_count = hardirq_disable_enter();
>>>
>>> + /* Is hardirq disable count overflow soon? */
>>> + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
>>> + DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) +
>>> + (10 << HARDIRQ_DISABLE_SHIFT) >
>>> + HARDIRQ_DISABLE_MASK);
>>> +
>>
>> This needs a return here right? Else we will see warning for 10 times
>> and then overflow happens and we will call _local_interrupt_disable. No?
>>
>
> DEBUG_LOCKS_WARN_ON() uses debug_locks_off() to avoid this, so we won't
> see it 10 times. The reason not using return here, because we would
> introduce unpaired local_interrupt_disable() if we returned:
Yes, it could be a weird case if the overflow actually happens.
So just warning maybe enough to catch such callers.
Maybe your kunit test can actually help test the behavior with the loop
count.
>
> // hardirq disable count is n
> local_interrupt_disable(); // hardirq disable count is n + 1
>
> local_interrupt_disable(); <- trigger the warning, if we return
> // hardirq disable count is n + 1
Likely i am missing to understand.
Isn't the count incremented earlier than return?
I.e even if return happens it should be n + 2 right?
>
> local_interrupt_enable(); // hardirq disable count is n
>
> local_interrupt_enable(); // hardirq disable count is n - 1
>
> Regards,
> Boqun
>
>> Not sure, if below is any better? (Igore whitespace mangling)
>>
>> if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
>> DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) >=
>> HARDIRQ_DISABLE_MASK - (10 << HARDIRQ_DISABLE_SHIFT)))
>> return;
>>
>>> /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
>>>
>>> if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
>>> @@ -73,6 +80,11 @@ static inline void local_interrupt_enable(void)
>>> {
>>> int new_count;
>>>
>>> + /* Unpaired local_interrupt_enable()? Warn and abort. */
>>> + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
>>> + DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0))
>>> + return;
>>> +
>>> new_count = hardirq_disable_exit();
>>>
>>> if ((new_count & HARDIRQ_DISABLE_MASK) == 0) >
>>
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 14:56 ` Shrikanth Hegde
@ 2026-08-05 15:11 ` Boqun Feng
2026-08-05 16:53 ` Shrikanth Hegde
2026-08-05 18:07 ` Boqun Feng
1 sibling, 1 reply; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 15:11 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 08:26:49PM +0530, Shrikanth Hegde wrote:
>
>
> On 8/5/26 7:50 PM, Boqun Feng wrote:
> > On Wed, Aug 05, 2026 at 07:40:18PM +0530, Shrikanth Hegde wrote:
> > >
> > > Hi Boqun,
> > >
> > > >
> > > > Something as below? Going to send it to kernel build bot and see if it
> > > > works for all configs.
> > > >
> > > > ----------------->8
> > > > diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
> > > > index b9a7f05ecf42..39f30bc65548 100644
> > > > --- a/include/linux/interrupt_rc.h
> > > > +++ b/include/linux/interrupt_rc.h
> > > > @@ -12,6 +12,7 @@
> > > > */
> > > >
> > > > #include <linux/irqflags.h>
> > > > +#include <linux/debug_locks.h>
> > > > #include <linux/preempt.h>
> > > > #include <linux/processor.h>
> > > > #include <linux/smp.h>
> > > > @@ -63,6 +64,12 @@ static inline void local_interrupt_disable(void)
> > > >
> > > > new_count = hardirq_disable_enter();
> > > >
> > > > + /* Is hardirq disable count overflow soon? */
> > > > + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
> > > > + DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) +
> > > > + (10 << HARDIRQ_DISABLE_SHIFT) >
> > > > + HARDIRQ_DISABLE_MASK);
> > > > +
> > >
> > > This needs a return here right? Else we will see warning for 10 times
> > > and then overflow happens and we will call _local_interrupt_disable. No?
Oh, seems I overlooked something... could you elaborate on this? What's
the scenario in your mind? You said we hit 10 times warning and *then*
overflow?
> > >
> >
> > DEBUG_LOCKS_WARN_ON() uses debug_locks_off() to avoid this, so we won't
> > see it 10 times. The reason not using return here, because we would
> > introduce unpaired local_interrupt_disable() if we returned:
>
> Yes, it could be a weird case if the overflow actually happens.
> So just warning maybe enough to catch such callers.
>
> Maybe your kunit test can actually help test the behavior with the loop
> count.
>
> >
> > // hardirq disable count is n
> > local_interrupt_disable(); // hardirq disable count is n + 1
> >
> > local_interrupt_disable(); <- trigger the warning, if we return
> > // hardirq disable count is n + 1
>
> Likely i am missing to understand.
>
No, it was me who misunderstand ;-)
> Isn't the count incremented earlier than return?
> I.e even if return happens it should be n + 2 right?
>
Yeah, you're right, but then why do we want to return earlier? Since the
following if:
if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
will be false, and we will just return from the function, no?
Regards,
Boqun
> >
> > local_interrupt_enable(); // hardirq disable count is n
> >
> > local_interrupt_enable(); // hardirq disable count is n - 1
> >
> > Regards,
> > Boqun
> >
> > > Not sure, if below is any better? (Igore whitespace mangling)
> > >
> > > if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
> > > DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) >=
> > > HARDIRQ_DISABLE_MASK - (10 << HARDIRQ_DISABLE_SHIFT)))
> > > return;
> > >
> > > > /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> > > >
> > > > if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> > > > @@ -73,6 +80,11 @@ static inline void local_interrupt_enable(void)
> > > > {
> > > > int new_count;
> > > >
> > > > + /* Unpaired local_interrupt_enable()? Warn and abort. */
> > > > + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
> > > > + DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0))
> > > > + return;
> > > > +
> > > > new_count = hardirq_disable_exit();
> > > >
> > > > if ((new_count & HARDIRQ_DISABLE_MASK) == 0) >
> > >
>
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 15:11 ` Boqun Feng
@ 2026-08-05 16:53 ` Shrikanth Hegde
2026-08-05 17:38 ` Boqun Feng
0 siblings, 1 reply; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-05 16:53 UTC (permalink / raw)
To: Boqun Feng
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
Hi Boqun.
On 8/5/26 8:41 PM, Boqun Feng wrote:
> On Wed, Aug 05, 2026 at 08:26:49PM +0530, Shrikanth Hegde wrote:
>>
>>
>> On 8/5/26 7:50 PM, Boqun Feng wrote:
>>> On Wed, Aug 05, 2026 at 07:40:18PM +0530, Shrikanth Hegde wrote:
>>>>
>>>> Hi Boqun,
>>>>
>>>>>
>>>>> Something as below? Going to send it to kernel build bot and see if it
>>>>> works for all configs.
>>>>>
>>>>> ----------------->8
>>>>> diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
>>>>> index b9a7f05ecf42..39f30bc65548 100644
>>>>> --- a/include/linux/interrupt_rc.h
>>>>> +++ b/include/linux/interrupt_rc.h
>>>>> @@ -12,6 +12,7 @@
>>>>> */
>>>>>
>>>>> #include <linux/irqflags.h>
>>>>> +#include <linux/debug_locks.h>
>>>>> #include <linux/preempt.h>
>>>>> #include <linux/processor.h>
>>>>> #include <linux/smp.h>
>>>>> @@ -63,6 +64,12 @@ static inline void local_interrupt_disable(void)
>>>>>
>>>>> new_count = hardirq_disable_enter();
>>>>>
>>>>> + /* Is hardirq disable count overflow soon? */
>>>>> + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
>>>>> + DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) +
>>>>> + (10 << HARDIRQ_DISABLE_SHIFT) >
>>>>> + HARDIRQ_DISABLE_MASK);
>>>>> +
>>>>
>>>> This needs a return here right? Else we will see warning for 10 times
>>>> and then overflow happens and we will call _local_interrupt_disable. No?
>
> Oh, seems I overlooked something... could you elaborate on this? What's
> the scenario in your mind? You said we hit 10 times warning and *then*
> overflow?
>
(the “10 times” wording was for the possible wraparound.
I didn't know DEBUG_LOCKS_WARN_ON() will turn debug_locks
off after the first warning, I thought it will print 10 times.)
Main concern is the wraparound itself. If the HARDIRQ_DISABLE field reaches
0xff and we increment it once more, the field becomes zero after masking
(ff + 1 -> 100 in the shifted field). Then local_interrupt_enable() can
observe:
(new_count & HARDIRQ_DISABLE_MASK) == 0
and treat it as the outermost enable, potentially calling
_local_interrupt_enable() while there are still outstanding logical
local_interrupt_disable() users.
So I think the useful debug check is one that catches the count before it
gets close enough to wrap. Thing I wanted to avoid is silently making the
HARDIRQ_DISABLE field look like zero after overflow.
Whether it returns or just warns, I am not sure. As recovery may
not be easy. It is meant more to be a damage contol than recovery.
My line of thought is,
If we return after debug checks in local_interrupt_disable(), we wont advance the preempt count
further, so it wont wrap around. Now, there will be corresponding local_interrupt_enable(),
at some point it will reach 0, we enable the interrupts. There will be some more
local_interrupt_enable() still, but they will be caught with your debug check in
local_interrupt_enable() and preempt count won't be decremented further.
So if we put return we may have a damage control.
>>>>
>>>
>>> DEBUG_LOCKS_WARN_ON() uses debug_locks_off() to avoid this, so we won't
>>> see it 10 times. The reason not using return here, because we would
>>> introduce unpaired local_interrupt_disable() if we returned:
>>
>> Yes, it could be a weird case if the overflow actually happens.
>> So just warning maybe enough to catch such callers.
>>
>> Maybe your kunit test can actually help test the behavior with the loop
>> count.
>>
>>>
>>> // hardirq disable count is n
>>> local_interrupt_disable(); // hardirq disable count is n + 1
>>>
>>> local_interrupt_disable(); <- trigger the warning, if we return
>>> // hardirq disable count is n + 1
>>
>> Likely i am missing to understand.
>>
>
> No, it was me who misunderstand ;-)
>
>> Isn't the count incremented earlier than return?
>> I.e even if return happens it should be n + 2 right?
>>
>
> Yeah, you're right, but then why do we want to return earlier? Since the
> following if:
>
> if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
>
> will be false, and we will just return from the function, no?
>
> Regards,
> Boqun
>
>>>
>>> local_interrupt_enable(); // hardirq disable count is n
>>>
>>> local_interrupt_enable(); // hardirq disable count is n - 1
>>>
>>> Regards,
>>> Boqun
>>>
>>>> Not sure, if below is any better? (Igore whitespace mangling)
>>>>
>>>> if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
>>>> DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) >=
>>>> HARDIRQ_DISABLE_MASK - (10 << HARDIRQ_DISABLE_SHIFT)))
>>>> return;
>>>>
>>>>> /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
>>>>>
>>>>> if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
>>>>> @@ -73,6 +80,11 @@ static inline void local_interrupt_enable(void)
>>>>> {
>>>>> int new_count;
>>>>>
>>>>> + /* Unpaired local_interrupt_enable()? Warn and abort. */
>>>>> + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
>>>>> + DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0))
>>>>> + return;
>>>>> +
>>>>> new_count = hardirq_disable_exit();
>>>>>
>>>>> if ((new_count & HARDIRQ_DISABLE_MASK) == 0) >
>>>>
>>
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 16:53 ` Shrikanth Hegde
@ 2026-08-05 17:38 ` Boqun Feng
0 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 17:38 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 10:23:20PM +0530, Shrikanth Hegde wrote:
> Hi Boqun.
>
> On 8/5/26 8:41 PM, Boqun Feng wrote:
> > On Wed, Aug 05, 2026 at 08:26:49PM +0530, Shrikanth Hegde wrote:
> > >
> > >
> > > On 8/5/26 7:50 PM, Boqun Feng wrote:
> > > > On Wed, Aug 05, 2026 at 07:40:18PM +0530, Shrikanth Hegde wrote:
> > > > >
> > > > > Hi Boqun,
> > > > >
> > > > > >
> > > > > > Something as below? Going to send it to kernel build bot and see if it
> > > > > > works for all configs.
> > > > > >
> > > > > > ----------------->8
> > > > > > diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
> > > > > > index b9a7f05ecf42..39f30bc65548 100644
> > > > > > --- a/include/linux/interrupt_rc.h
> > > > > > +++ b/include/linux/interrupt_rc.h
> > > > > > @@ -12,6 +12,7 @@
> > > > > > */
> > > > > >
> > > > > > #include <linux/irqflags.h>
> > > > > > +#include <linux/debug_locks.h>
> > > > > > #include <linux/preempt.h>
> > > > > > #include <linux/processor.h>
> > > > > > #include <linux/smp.h>
> > > > > > @@ -63,6 +64,12 @@ static inline void local_interrupt_disable(void)
> > > > > >
> > > > > > new_count = hardirq_disable_enter();
> > > > > >
> > > > > > + /* Is hardirq disable count overflow soon? */
> > > > > > + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
> > > > > > + DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) +
> > > > > > + (10 << HARDIRQ_DISABLE_SHIFT) >
> > > > > > + HARDIRQ_DISABLE_MASK);
> > > > > > +
> > > > >
> > > > > This needs a return here right? Else we will see warning for 10 times
> > > > > and then overflow happens and we will call _local_interrupt_disable. No?
> >
> > Oh, seems I overlooked something... could you elaborate on this? What's
> > the scenario in your mind? You said we hit 10 times warning and *then*
> > overflow?
> >
>
>
> (the "10 times" wording was for the possible wraparound.
> I didn't know DEBUG_LOCKS_WARN_ON() will turn debug_locks
> off after the first warning, I thought it will print 10 times.)
>
> Main concern is the wraparound itself. If the HARDIRQ_DISABLE field reaches
> 0xff and we increment it once more, the field becomes zero after masking
> (ff + 1 -> 100 in the shifted field). Then local_interrupt_enable() can
> observe:
>
> (new_count & HARDIRQ_DISABLE_MASK) == 0
>
> and treat it as the outermost enable, potentially calling
> _local_interrupt_enable() while there are still outstanding logical
> local_interrupt_disable() users.
>
> So I think the useful debug check is one that catches the count before it
> gets close enough to wrap. Thing I wanted to avoid is silently making the
> HARDIRQ_DISABLE field look like zero after overflow.
>
Given that the detection is only on debug kernel, I think the damage of
keeping increment after warn triggered is low (this is assuming that we
can detect all the issues with the debug kernel, similar treatment as
the preempt_count nowadays). However, your suggestion does look better
for the corner cases and it's a better damage control, so I will add the
return part, thank you!
For the future, we need better detection and report for these counters
for sure.
Regards,
Boqun
> Whether it returns or just warns, I am not sure. As recovery may
> not be easy. It is meant more to be a damage contol than recovery.
>
>
> My line of thought is,
>
> If we return after debug checks in local_interrupt_disable(), we wont advance the preempt count
> further, so it wont wrap around. Now, there will be corresponding local_interrupt_enable(),
> at some point it will reach 0, we enable the interrupts. There will be some more
> local_interrupt_enable() still, but they will be caught with your debug check in
> local_interrupt_enable() and preempt count won't be decremented further.
> So if we put return we may have a damage control.
>
> > > > >
> > > >
> > > > DEBUG_LOCKS_WARN_ON() uses debug_locks_off() to avoid this, so we won't
> > > > see it 10 times. The reason not using return here, because we would
> > > > introduce unpaired local_interrupt_disable() if we returned:
> > >
> > > Yes, it could be a weird case if the overflow actually happens.
> > > So just warning maybe enough to catch such callers.
> > >
> > > Maybe your kunit test can actually help test the behavior with the loop
> > > count.
> > >
> > > >
> > > > // hardirq disable count is n
> > > > local_interrupt_disable(); // hardirq disable count is n + 1
> > > >
> > > > local_interrupt_disable(); <- trigger the warning, if we return
> > > > // hardirq disable count is n + 1
> > >
> > > Likely i am missing to understand.
> > >
> >
> > No, it was me who misunderstand ;-)
> >
> > > Isn't the count incremented earlier than return?
> > > I.e even if return happens it should be n + 2 right?
> > >
> >
> > Yeah, you're right, but then why do we want to return earlier? Since the
> > following if:
> >
> > if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> >
> > will be false, and we will just return from the function, no?
> >
> > Regards,
> > Boqun
> >
> > > >
> > > > local_interrupt_enable(); // hardirq disable count is n
> > > >
> > > > local_interrupt_enable(); // hardirq disable count is n - 1
> > > >
> > > > Regards,
> > > > Boqun
> > > >
> > > > > Not sure, if below is any better? (Igore whitespace mangling)
> > > > >
> > > > > if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
> > > > > DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) >=
> > > > > HARDIRQ_DISABLE_MASK - (10 << HARDIRQ_DISABLE_SHIFT)))
> > > > > return;
> > > > >
> > > > > > /* Interrupts can happen here, but it's OK, see __irq_exit_rcu(). */
> > > > > >
> > > > > > if ((new_count & HARDIRQ_DISABLE_MASK) == HARDIRQ_DISABLE_OFFSET)
> > > > > > @@ -73,6 +80,11 @@ static inline void local_interrupt_enable(void)
> > > > > > {
> > > > > > int new_count;
> > > > > >
> > > > > > + /* Unpaired local_interrupt_enable()? Warn and abort. */
> > > > > > + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT) &&
> > > > > > + DEBUG_LOCKS_WARN_ON((preempt_count() & HARDIRQ_DISABLE_MASK) == 0))
> > > > > > + return;
> > > > > > +
> > > > > > new_count = hardirq_disable_exit();
> > > > > >
> > > > > > if ((new_count & HARDIRQ_DISABLE_MASK) == 0) >
> > > > >
> > >
>
^ permalink raw reply [flat|nested] 66+ messages in thread
* Re: [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling
2026-08-05 14:56 ` Shrikanth Hegde
2026-08-05 15:11 ` Boqun Feng
@ 2026-08-05 18:07 ` Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 18:07 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 08:26:49PM +0530, Shrikanth Hegde wrote:
>
>
> On 8/5/26 7:50 PM, Boqun Feng wrote:
> > On Wed, Aug 05, 2026 at 07:40:18PM +0530, Shrikanth Hegde wrote:
> > >
> > > Hi Boqun,
> > >
> > > >
> > > > Something as below? Going to send it to kernel build bot and see if it
> > > > works for all configs.
> > > >
> > > > ----------------->8
> > > > diff --git a/include/linux/interrupt_rc.h b/include/linux/interrupt_rc.h
> > > > index b9a7f05ecf42..39f30bc65548 100644
> > > > --- a/include/linux/interrupt_rc.h
> > > > +++ b/include/linux/interrupt_rc.h
> > > > @@ -12,6 +12,7 @@
> > > > */
> > > >
> > > > #include <linux/irqflags.h>
> > > > +#include <linux/debug_locks.h>
> > > > #include <linux/preempt.h>
> > > > #include <linux/processor.h>
> > > > #include <linux/smp.h>
> > > > @@ -63,6 +64,12 @@ static inline void local_interrupt_disable(void)
> > > >
> > > > new_count = hardirq_disable_enter();
> > > >
> > > > + /* Is hardirq disable count overflow soon? */
> > > > + if (IS_ENABLED(CONFIG_DEBUG_PREEMPT))
> > > > + DEBUG_LOCKS_WARN_ON((new_count & HARDIRQ_DISABLE_MASK) +
> > > > + (10 << HARDIRQ_DISABLE_SHIFT) >
> > > > + HARDIRQ_DISABLE_MASK);
> > > > +
> > >
> > > This needs a return here right? Else we will see warning for 10 times
> > > and then overflow happens and we will call _local_interrupt_disable. No?
> > >
> >
> > DEBUG_LOCKS_WARN_ON() uses debug_locks_off() to avoid this, so we won't
> > see it 10 times. The reason not using return here, because we would
> > introduce unpaired local_interrupt_disable() if we returned:
>
> Yes, it could be a weird case if the overflow actually happens.
> So just warning maybe enough to catch such callers.
>
> Maybe your kunit test can actually help test the behavior with the loop
> count.
>
I'm sure that we can have more tests, but this is a good start, thank
you!
Regards,
Boqun
------------->8
Subject: [PATCH] irq: Add max local_interrupt_disable() nesting level kunit
test case
To confirm the max nesting level of local_interrupt_disable() works, a
kunit test is added to the whole test suite.
Note that when DEBUG_PREEMPT=y, it'll generate a warning which is
expected.
Suggested-by: Shrikanth Hegde <sshegde@linux.ibm.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
kernel/irq/refcount_interrupt_test.c | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_interrupt_test.c
index ca904dba24b9..38dfccbaa4d4 100644
--- a/kernel/irq/refcount_interrupt_test.c
+++ b/kernel/irq/refcount_interrupt_test.c
@@ -52,6 +52,27 @@ static void test_multiple_irq_change(struct kunit *test)
TEST_IRQ_ON();
}
+static void test_max_nesting_irq_change(struct kunit *test)
+{
+ for (int i = 0; i < __IRQ_MASK(HARDIRQ_DISABLE_BITS); i++) {
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ }
+
+
+ for (int i = 0; i < __IRQ_MASK(HARDIRQ_DISABLE_BITS); i++) {
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ }
+
+ TEST_IRQ_ON();
+
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
static void test_irq_save(struct kunit *test)
{
unsigned long flags;
@@ -79,6 +100,7 @@ static struct kunit_case test_cases[] = {
KUNIT_CASE(test_single_irq_change),
KUNIT_CASE(test_nested_irq_change),
KUNIT_CASE(test_multiple_irq_change),
+ KUNIT_CASE(test_max_nesting_irq_change),
KUNIT_CASE(test_irq_save),
{},
};
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 06/17] irq: Add KUnit test for refcounted interrupt enable/disable
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (4 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 05/17] irq & spin_lock: Add counted interrupt disabling/enabling Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Lyude Paul
2026-08-10 8:57 ` tip-bot2 for Lyude Paul
2026-08-04 16:14 ` [PATCH v4 07/17] locking: Switch to _irq_{disable,enable}() variants in cleanup guards Boqun Feng
` (10 subsequent siblings)
16 siblings, 2 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
From: Lyude Paul <lyude@redhat.com>
While making changes to the refcounted interrupt patch series, at some
point on my local branch I broke something and ended up writing some kunit
tests for testing refcounted interrupts as a result. So, let's include
these tests now that we have refcounted interrupts.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
kernel/irq/Makefile | 1 +
kernel/irq/refcount_interrupt_test.c | 109 +++++++++++++++++++++++++++
2 files changed, 110 insertions(+)
create mode 100644 kernel/irq/refcount_interrupt_test.c
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile
index 86a2e5ae08f9..44c4d6fc502a 100644
--- a/kernel/irq/Makefile
+++ b/kernel/irq/Makefile
@@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o
obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o
obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o
obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o
+obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o
diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_interrupt_test.c
new file mode 100644
index 000000000000..ca904dba24b9
--- /dev/null
+++ b/kernel/irq/refcount_interrupt_test.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit test for refcounted interrupt enable/disables.
+ */
+
+#include <kunit/test.h>
+#include <linux/interrupt_rc.h>
+
+#define TEST_IRQ_ON() KUNIT_EXPECT_FALSE(test, irqs_disabled())
+#define TEST_IRQ_OFF() KUNIT_EXPECT_TRUE(test, irqs_disabled())
+
+/* ===== Test cases ===== */
+static void test_single_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+}
+
+static void test_nested_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static void test_multiple_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static void test_irq_save(struct kunit *test)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_irq_restore(flags);
+ TEST_IRQ_ON();
+
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_irq_save(flags);
+ TEST_IRQ_OFF();
+ local_irq_restore(flags);
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static struct kunit_case test_cases[] = {
+ KUNIT_CASE(test_single_irq_change),
+ KUNIT_CASE(test_nested_irq_change),
+ KUNIT_CASE(test_multiple_irq_change),
+ KUNIT_CASE(test_irq_save),
+ {},
+};
+
+/* init and exit are the same. */
+static int test_init(struct kunit *test)
+{
+ TEST_IRQ_ON();
+
+ return 0;
+}
+
+static void test_exit(struct kunit *test)
+{
+ TEST_IRQ_ON();
+}
+
+static struct kunit_suite refcount_interrupt_test_suite = {
+ .name = "refcount_interrupt",
+ .test_cases = test_cases,
+ .init = test_init,
+ .exit = test_exit,
+};
+
+kunit_test_suite(refcount_interrupt_test_suite);
+MODULE_AUTHOR("Lyude Paul <lyude@redhat.com>");
+MODULE_DESCRIPTION("Refcounted interrupt unit test suite");
+MODULE_LICENSE("GPL");
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] irq: Add KUnit test for refcounted interrupt enable/disable
2026-08-04 16:14 ` [PATCH v4 06/17] irq: Add KUnit test for refcounted interrupt enable/disable Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Lyude Paul
2026-08-10 8:57 ` tip-bot2 for Lyude Paul
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Lyude Paul @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits
Cc: Lyude Paul, Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 417f9684716b6577c717b09337e14040a1a067f2
Gitweb: https://git.kernel.org/tip/417f9684716b6577c717b09337e14040a1a067f2
Author: Lyude Paul <lyude@redhat.com>
AuthorDate: Tue, 04 Aug 2026 09:14:28 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:07 +02:00
irq: Add KUnit test for refcounted interrupt enable/disable
While making changes to the refcounted interrupt patch series, at some
point on my local branch I broke something and ended up writing some kunit
tests for testing refcounted interrupts as a result. So, let's include
these tests now that we have refcounted interrupts.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-7-boqun@kernel.org
---
kernel/irq/Makefile | 1 +-
kernel/irq/refcount_interrupt_test.c | 109 ++++++++++++++++++++++++++-
2 files changed, 110 insertions(+)
create mode 100644 kernel/irq/refcount_interrupt_test.c
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile
index 86a2e5a..44c4d6f 100644
--- a/kernel/irq/Makefile
+++ b/kernel/irq/Makefile
@@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o
obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o
obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o
obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o
+obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o
diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_interrupt_test.c
new file mode 100644
index 0000000..ca904db
--- /dev/null
+++ b/kernel/irq/refcount_interrupt_test.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit test for refcounted interrupt enable/disables.
+ */
+
+#include <kunit/test.h>
+#include <linux/interrupt_rc.h>
+
+#define TEST_IRQ_ON() KUNIT_EXPECT_FALSE(test, irqs_disabled())
+#define TEST_IRQ_OFF() KUNIT_EXPECT_TRUE(test, irqs_disabled())
+
+/* ===== Test cases ===== */
+static void test_single_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+}
+
+static void test_nested_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static void test_multiple_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static void test_irq_save(struct kunit *test)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_irq_restore(flags);
+ TEST_IRQ_ON();
+
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_irq_save(flags);
+ TEST_IRQ_OFF();
+ local_irq_restore(flags);
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static struct kunit_case test_cases[] = {
+ KUNIT_CASE(test_single_irq_change),
+ KUNIT_CASE(test_nested_irq_change),
+ KUNIT_CASE(test_multiple_irq_change),
+ KUNIT_CASE(test_irq_save),
+ {},
+};
+
+/* init and exit are the same. */
+static int test_init(struct kunit *test)
+{
+ TEST_IRQ_ON();
+
+ return 0;
+}
+
+static void test_exit(struct kunit *test)
+{
+ TEST_IRQ_ON();
+}
+
+static struct kunit_suite refcount_interrupt_test_suite = {
+ .name = "refcount_interrupt",
+ .test_cases = test_cases,
+ .init = test_init,
+ .exit = test_exit,
+};
+
+kunit_test_suite(refcount_interrupt_test_suite);
+MODULE_AUTHOR("Lyude Paul <lyude@redhat.com>");
+MODULE_DESCRIPTION("Refcounted interrupt unit test suite");
+MODULE_LICENSE("GPL");
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] irq: Add KUnit test for refcounted interrupt enable/disable
2026-08-04 16:14 ` [PATCH v4 06/17] irq: Add KUnit test for refcounted interrupt enable/disable Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Lyude Paul
@ 2026-08-10 8:57 ` tip-bot2 for Lyude Paul
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Lyude Paul @ 2026-08-10 8:57 UTC (permalink / raw)
To: linux-tip-commits
Cc: Lyude Paul, Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 07a88e2bcd5b5bd7881b2e12b6aad1897a7ee1de
Gitweb: https://git.kernel.org/tip/07a88e2bcd5b5bd7881b2e12b6aad1897a7ee1de
Author: Lyude Paul <lyude@redhat.com>
AuthorDate: Tue, 04 Aug 2026 09:14:28 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Mon, 10 Aug 2026 10:50:18 +02:00
irq: Add KUnit test for refcounted interrupt enable/disable
While making changes to the refcounted interrupt patch series, at some
point on my local branch I broke something and ended up writing some kunit
tests for testing refcounted interrupts as a result. So, let's include
these tests now that we have refcounted interrupts.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-7-boqun@kernel.org
---
kernel/irq/Makefile | 1 +-
kernel/irq/refcount_interrupt_test.c | 109 ++++++++++++++++++++++++++-
2 files changed, 110 insertions(+)
create mode 100644 kernel/irq/refcount_interrupt_test.c
diff --git a/kernel/irq/Makefile b/kernel/irq/Makefile
index 86a2e5a..44c4d6f 100644
--- a/kernel/irq/Makefile
+++ b/kernel/irq/Makefile
@@ -16,3 +16,4 @@ obj-$(CONFIG_SMP) += affinity.o
obj-$(CONFIG_GENERIC_IRQ_DEBUGFS) += debugfs.o
obj-$(CONFIG_GENERIC_IRQ_MATRIX_ALLOCATOR) += matrix.o
obj-$(CONFIG_IRQ_KUNIT_TEST) += irq_test.o
+obj-$(CONFIG_KUNIT) += refcount_interrupt_test.o
diff --git a/kernel/irq/refcount_interrupt_test.c b/kernel/irq/refcount_interrupt_test.c
new file mode 100644
index 0000000..ca904db
--- /dev/null
+++ b/kernel/irq/refcount_interrupt_test.c
@@ -0,0 +1,109 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * KUnit test for refcounted interrupt enable/disables.
+ */
+
+#include <kunit/test.h>
+#include <linux/interrupt_rc.h>
+
+#define TEST_IRQ_ON() KUNIT_EXPECT_FALSE(test, irqs_disabled())
+#define TEST_IRQ_OFF() KUNIT_EXPECT_TRUE(test, irqs_disabled())
+
+/* ===== Test cases ===== */
+static void test_single_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+}
+
+static void test_nested_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static void test_multiple_irq_change(struct kunit *test)
+{
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static void test_irq_save(struct kunit *test)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ TEST_IRQ_OFF();
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_OFF();
+ local_irq_restore(flags);
+ TEST_IRQ_ON();
+
+ local_interrupt_disable();
+ TEST_IRQ_OFF();
+ local_irq_save(flags);
+ TEST_IRQ_OFF();
+ local_irq_restore(flags);
+ TEST_IRQ_OFF();
+ local_interrupt_enable();
+ TEST_IRQ_ON();
+}
+
+static struct kunit_case test_cases[] = {
+ KUNIT_CASE(test_single_irq_change),
+ KUNIT_CASE(test_nested_irq_change),
+ KUNIT_CASE(test_multiple_irq_change),
+ KUNIT_CASE(test_irq_save),
+ {},
+};
+
+/* init and exit are the same. */
+static int test_init(struct kunit *test)
+{
+ TEST_IRQ_ON();
+
+ return 0;
+}
+
+static void test_exit(struct kunit *test)
+{
+ TEST_IRQ_ON();
+}
+
+static struct kunit_suite refcount_interrupt_test_suite = {
+ .name = "refcount_interrupt",
+ .test_cases = test_cases,
+ .init = test_init,
+ .exit = test_exit,
+};
+
+kunit_test_suite(refcount_interrupt_test_suite);
+MODULE_AUTHOR("Lyude Paul <lyude@redhat.com>");
+MODULE_DESCRIPTION("Refcounted interrupt unit test suite");
+MODULE_LICENSE("GPL");
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 07/17] locking: Switch to _irq_{disable,enable}() variants in cleanup guards
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (5 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 06/17] irq: Add KUnit test for refcounted interrupt enable/disable Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
2026-08-04 16:14 ` [PATCH v4 08/17] sched: Remove the unused preempt_offset parameter of __cant_sleep() Boqun Feng
` (9 subsequent siblings)
16 siblings, 2 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
The semantics of various IRQ disabling guards match what
*_irq_{disable,enable}() provide, i.e. the interrupt disabling is
properly nested, therefore it's OK to switch to use
*_irq_{disable,enable}() primitives.
[boqun: Adjust the user-side changes in do_sched_cfs_*_timer() provided
by Peter and Lyude]
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
include/linux/spinlock.h | 26 ++++++++++++--------------
kernel/sched/fair.c | 12 ++++++------
2 files changed, 18 insertions(+), 20 deletions(-)
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 3d405cc4c121..799a8f7d2741 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -572,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acquires(_T), __releases(*(raw
#define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, _T)
DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t,
- raw_spin_lock_irq(_T->lock),
- raw_spin_unlock_irq(_T->lock))
+ raw_spin_lock_irq_disable(_T->lock),
+ raw_spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, _T)
-DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock))
+DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, _T)
@@ -592,14 +592,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acquires(_T), __releases(*(raw
#define class_raw_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, _T)
DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t,
- raw_spin_lock_irqsave(_T->lock, _T->flags),
- raw_spin_unlock_irqrestore(_T->lock, _T->flags),
- unsigned long flags)
+ raw_spin_lock_irq_disable(_T->lock),
+ raw_spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, _T)
DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try,
- raw_spin_trylock_irqsave(_T->lock, _T->flags))
+ raw_spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, _T)
@@ -618,13 +617,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_T), __releases(*(spinlock_t
#define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_try, _T)
DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t,
- spin_lock_irq(_T->lock),
- spin_unlock_irq(_T->lock))
+ spin_lock_irq_disable(_T->lock),
+ spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq, _T)
DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try,
- spin_trylock_irq(_T->lock))
+ spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq_try, _T)
@@ -640,14 +639,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquires(_T), __releases(*(spinloc
#define class_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_bh_try, _T)
DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t,
- spin_lock_irqsave(_T->lock, _T->flags),
- spin_unlock_irqrestore(_T->lock, _T->flags),
- unsigned long flags)
+ spin_lock_irq_disable(_T->lock),
+ spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave, _T)
DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try,
- spin_trylock_irqsave(_T->lock, _T->flags))
+ spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, _T)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d78467ec6ee1..a46c4ff49f74 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7120,7 +7120,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
* period the timer is deactivated until scheduling resumes; cfs_b->idle is
* used to track this state.
*/
-static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
+static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
__must_hold(&cfs_b->lock)
{
int throttled;
@@ -7155,10 +7155,10 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, u
* This check is repeated as we release cfs_b->lock while we unthrottle.
*/
while (throttled && cfs_b->runtime > 0) {
- raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
+ raw_spin_unlock_irq_enable(&cfs_b->lock);
/* we can't nest cfs_b->lock while distributing bandwidth */
throttled = distribute_cfs_runtime(cfs_b);
- raw_spin_lock_irqsave(&cfs_b->lock, flags);
+ raw_spin_lock_irq_disable(&cfs_b->lock);
}
/*
@@ -7266,7 +7266,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
{
/* confirm we're still not at a refresh boundary */
- scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) {
+ scoped_guard(raw_spinlock_irq, &cfs_b->lock) {
u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
cfs_b->slack_started = false;
@@ -7351,14 +7351,14 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
int idle = 0;
int count = 0;
- CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock);
+ guard(raw_spinlock_irq)(&cfs_b->lock);
for (;;) {
overrun = hrtimer_forward_now(timer, cfs_b->period);
if (!overrun)
break;
- idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags);
+ idle = do_sched_cfs_period_timer(cfs_b, overrun);
if (++count > 3) {
u64 new, old = ktime_to_ns(cfs_b->period);
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] locking: Switch to _irq_{disable,enable}() variants in cleanup guards
2026-08-04 16:14 ` [PATCH v4 07/17] locking: Switch to _irq_{disable,enable}() variants in cleanup guards Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: d512f73c83c3a74073833e7d29584e890f90db36
Gitweb: https://git.kernel.org/tip/d512f73c83c3a74073833e7d29584e890f90db36
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:29 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:07 +02:00
locking: Switch to _irq_{disable,enable}() variants in cleanup guards
The semantics of various IRQ disabling guards match what
*_irq_{disable,enable}() provide, i.e. the interrupt disabling is
properly nested, therefore it's OK to switch to use
*_irq_{disable,enable}() primitives.
[boqun: Adjust the user-side changes in do_sched_cfs_*_timer() provided
by Peter and Lyude]
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-8-boqun@kernel.org
---
include/linux/spinlock.h | 26 ++++++++++++--------------
kernel/sched/fair.c | 12 ++++++------
2 files changed, 18 insertions(+), 20 deletions(-)
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 3d405cc..799a8f7 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -572,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acquires(_T), __releases(*(raw
#define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, _T)
DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t,
- raw_spin_lock_irq(_T->lock),
- raw_spin_unlock_irq(_T->lock))
+ raw_spin_lock_irq_disable(_T->lock),
+ raw_spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, _T)
-DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock))
+DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, _T)
@@ -592,14 +592,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acquires(_T), __releases(*(raw
#define class_raw_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, _T)
DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t,
- raw_spin_lock_irqsave(_T->lock, _T->flags),
- raw_spin_unlock_irqrestore(_T->lock, _T->flags),
- unsigned long flags)
+ raw_spin_lock_irq_disable(_T->lock),
+ raw_spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, _T)
DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try,
- raw_spin_trylock_irqsave(_T->lock, _T->flags))
+ raw_spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, _T)
@@ -618,13 +617,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_T), __releases(*(spinlock_t
#define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_try, _T)
DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t,
- spin_lock_irq(_T->lock),
- spin_unlock_irq(_T->lock))
+ spin_lock_irq_disable(_T->lock),
+ spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq, _T)
DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try,
- spin_trylock_irq(_T->lock))
+ spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq_try, _T)
@@ -640,14 +639,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquires(_T), __releases(*(spinloc
#define class_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_bh_try, _T)
DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t,
- spin_lock_irqsave(_T->lock, _T->flags),
- spin_unlock_irqrestore(_T->lock, _T->flags),
- unsigned long flags)
+ spin_lock_irq_disable(_T->lock),
+ spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave, _T)
DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try,
- spin_trylock_irqsave(_T->lock, _T->flags))
+ spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, _T)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d78467e..a46c4ff 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7120,7 +7120,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
* period the timer is deactivated until scheduling resumes; cfs_b->idle is
* used to track this state.
*/
-static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
+static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
__must_hold(&cfs_b->lock)
{
int throttled;
@@ -7155,10 +7155,10 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, u
* This check is repeated as we release cfs_b->lock while we unthrottle.
*/
while (throttled && cfs_b->runtime > 0) {
- raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
+ raw_spin_unlock_irq_enable(&cfs_b->lock);
/* we can't nest cfs_b->lock while distributing bandwidth */
throttled = distribute_cfs_runtime(cfs_b);
- raw_spin_lock_irqsave(&cfs_b->lock, flags);
+ raw_spin_lock_irq_disable(&cfs_b->lock);
}
/*
@@ -7266,7 +7266,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
{
/* confirm we're still not at a refresh boundary */
- scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) {
+ scoped_guard(raw_spinlock_irq, &cfs_b->lock) {
u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
cfs_b->slack_started = false;
@@ -7351,14 +7351,14 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
int idle = 0;
int count = 0;
- CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock);
+ guard(raw_spinlock_irq)(&cfs_b->lock);
for (;;) {
overrun = hrtimer_forward_now(timer, cfs_b->period);
if (!overrun)
break;
- idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags);
+ idle = do_sched_cfs_period_timer(cfs_b, overrun);
if (++count > 3) {
u64 new, old = ktime_to_ns(cfs_b->period);
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] locking: Switch to _irq_{disable,enable}() variants in cleanup guards
2026-08-04 16:14 ` [PATCH v4 07/17] locking: Switch to _irq_{disable,enable}() variants in cleanup guards Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
@ 2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-10 8:57 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 1b086687483371621e684e2a05b2bcd2cf07b634
Gitweb: https://git.kernel.org/tip/1b086687483371621e684e2a05b2bcd2cf07b634
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:29 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Mon, 10 Aug 2026 10:50:18 +02:00
locking: Switch to _irq_{disable,enable}() variants in cleanup guards
The semantics of various IRQ disabling guards match what
*_irq_{disable,enable}() provide, i.e. the interrupt disabling is
properly nested, therefore it's OK to switch to use
*_irq_{disable,enable}() primitives.
[boqun: Adjust the user-side changes in do_sched_cfs_*_timer() provided
by Peter and Lyude]
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-8-boqun@kernel.org
---
include/linux/spinlock.h | 26 ++++++++++++--------------
kernel/sched/fair.c | 12 ++++++------
2 files changed, 18 insertions(+), 20 deletions(-)
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
index 3d405cc..799a8f7 100644
--- a/include/linux/spinlock.h
+++ b/include/linux/spinlock.h
@@ -572,12 +572,12 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acquires(_T), __releases(*(raw
#define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, _T)
DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t,
- raw_spin_lock_irq(_T->lock),
- raw_spin_unlock_irq(_T->lock))
+ raw_spin_lock_irq_disable(_T->lock),
+ raw_spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, _T)
-DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock))
+DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, _T)
@@ -592,14 +592,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acquires(_T), __releases(*(raw
#define class_raw_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, _T)
DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t,
- raw_spin_lock_irqsave(_T->lock, _T->flags),
- raw_spin_unlock_irqrestore(_T->lock, _T->flags),
- unsigned long flags)
+ raw_spin_lock_irq_disable(_T->lock),
+ raw_spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, _T)
DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try,
- raw_spin_trylock_irqsave(_T->lock, _T->flags))
+ raw_spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T))
#define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, _T)
@@ -618,13 +617,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_T), __releases(*(spinlock_t
#define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_try, _T)
DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t,
- spin_lock_irq(_T->lock),
- spin_unlock_irq(_T->lock))
+ spin_lock_irq_disable(_T->lock),
+ spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq, _T)
DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try,
- spin_trylock_irq(_T->lock))
+ spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq_try, _T)
@@ -640,14 +639,13 @@ DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquires(_T), __releases(*(spinloc
#define class_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_bh_try, _T)
DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t,
- spin_lock_irqsave(_T->lock, _T->flags),
- spin_unlock_irqrestore(_T->lock, _T->flags),
- unsigned long flags)
+ spin_lock_irq_disable(_T->lock),
+ spin_unlock_irq_enable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave, _T)
DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try,
- spin_trylock_irqsave(_T->lock, _T->flags))
+ spin_trylock_irq_disable(_T->lock))
DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __releases(*(spinlock_t **)_T))
#define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, _T)
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index d78467e..a46c4ff 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7120,7 +7120,7 @@ static bool distribute_cfs_runtime(struct cfs_bandwidth *cfs_b)
* period the timer is deactivated until scheduling resumes; cfs_b->idle is
* used to track this state.
*/
-static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags)
+static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
__must_hold(&cfs_b->lock)
{
int throttled;
@@ -7155,10 +7155,10 @@ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, u
* This check is repeated as we release cfs_b->lock while we unthrottle.
*/
while (throttled && cfs_b->runtime > 0) {
- raw_spin_unlock_irqrestore(&cfs_b->lock, flags);
+ raw_spin_unlock_irq_enable(&cfs_b->lock);
/* we can't nest cfs_b->lock while distributing bandwidth */
throttled = distribute_cfs_runtime(cfs_b);
- raw_spin_lock_irqsave(&cfs_b->lock, flags);
+ raw_spin_lock_irq_disable(&cfs_b->lock);
}
/*
@@ -7266,7 +7266,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq)
static void do_sched_cfs_slack_timer(struct cfs_bandwidth *cfs_b)
{
/* confirm we're still not at a refresh boundary */
- scoped_guard(raw_spinlock_irqsave, &cfs_b->lock) {
+ scoped_guard(raw_spinlock_irq, &cfs_b->lock) {
u64 runtime = 0, slice = sched_cfs_bandwidth_slice();
cfs_b->slack_started = false;
@@ -7351,14 +7351,14 @@ static enum hrtimer_restart sched_cfs_period_timer(struct hrtimer *timer)
int idle = 0;
int count = 0;
- CLASS(raw_spinlock_irqsave, cfsb_guard)(&cfs_b->lock);
+ guard(raw_spinlock_irq)(&cfs_b->lock);
for (;;) {
overrun = hrtimer_forward_now(timer, cfs_b->period);
if (!overrun)
break;
- idle = do_sched_cfs_period_timer(cfs_b, overrun, cfsb_guard.flags);
+ idle = do_sched_cfs_period_timer(cfs_b, overrun);
if (++count > 3) {
u64 new, old = ktime_to_ns(cfs_b->period);
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 08/17] sched: Remove the unused preempt_offset parameter of __cant_sleep()
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (6 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 07/17] locking: Switch to _irq_{disable,enable}() variants in cleanup guards Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
2026-08-04 16:14 ` [PATCH v4 09/17] sched: Avoid signed comparison of preempt_count() in __cant_migrate() Boqun Feng
` (8 subsequent siblings)
16 siblings, 2 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
The preempt_offset is always 0 in all the callsites of __cant_sleep(),
hence remove it. It also allows us to clear up the code a bit by
no longer using a "preempt_count() > .." comparison.
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
include/linux/kernel.h | 4 ++--
kernel/sched/core.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index e5570a16cbb1..24414c79e59a 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -72,7 +72,7 @@ extern int dynamic_might_resched(void);
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
extern void __might_resched(const char *file, int line, unsigned int offsets);
extern void __might_sleep(const char *file, int line);
-extern void __cant_sleep(const char *file, int line, int preempt_offset);
+extern void __cant_sleep(const char *file, int line);
extern void __cant_migrate(const char *file, int line);
/**
@@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line);
* this macro will print a stack trace if it is executed with preemption enabled
*/
# define cant_sleep() \
- do { __cant_sleep(__FILE__, __LINE__, 0); } while (0)
+ do { __cant_sleep(__FILE__, __LINE__); } while (0)
# define sched_annotate_sleep() (current->task_state_change = 0)
/**
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 96226707c2f6..aa116daf21bd 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -9199,7 +9199,7 @@ void __might_resched(const char *file, int line, unsigned int offsets)
}
EXPORT_SYMBOL(__might_resched);
-void __cant_sleep(const char *file, int line, int preempt_offset)
+void __cant_sleep(const char *file, int line)
{
static unsigned long prev_jiffy;
@@ -9209,7 +9209,7 @@ void __cant_sleep(const char *file, int line, int preempt_offset)
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
- if (preempt_count() > preempt_offset)
+ if (preempt_count())
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] sched: Remove the unused preempt_offset parameter of __cant_sleep()
2026-08-04 16:14 ` [PATCH v4 08/17] sched: Remove the unused preempt_offset parameter of __cant_sleep() Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 1cda40c48e295f8ad16bd5401591d7e6e5bcb591
Gitweb: https://git.kernel.org/tip/1cda40c48e295f8ad16bd5401591d7e6e5bcb591
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:30 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:07 +02:00
sched: Remove the unused preempt_offset parameter of __cant_sleep()
The preempt_offset is always 0 in all the callsites of __cant_sleep(),
hence remove it. It also allows us to clear up the code a bit by
no longer using a "preempt_count() > .." comparison.
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-9-boqun@kernel.org
---
include/linux/kernel.h | 4 ++--
kernel/sched/core.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index e5570a1..24414c7 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -72,7 +72,7 @@ extern int dynamic_might_resched(void);
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
extern void __might_resched(const char *file, int line, unsigned int offsets);
extern void __might_sleep(const char *file, int line);
-extern void __cant_sleep(const char *file, int line, int preempt_offset);
+extern void __cant_sleep(const char *file, int line);
extern void __cant_migrate(const char *file, int line);
/**
@@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line);
* this macro will print a stack trace if it is executed with preemption enabled
*/
# define cant_sleep() \
- do { __cant_sleep(__FILE__, __LINE__, 0); } while (0)
+ do { __cant_sleep(__FILE__, __LINE__); } while (0)
# define sched_annotate_sleep() (current->task_state_change = 0)
/**
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9622670..aa116da 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -9199,7 +9199,7 @@ void __might_resched(const char *file, int line, unsigned int offsets)
}
EXPORT_SYMBOL(__might_resched);
-void __cant_sleep(const char *file, int line, int preempt_offset)
+void __cant_sleep(const char *file, int line)
{
static unsigned long prev_jiffy;
@@ -9209,7 +9209,7 @@ void __cant_sleep(const char *file, int line, int preempt_offset)
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
- if (preempt_count() > preempt_offset)
+ if (preempt_count())
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] sched: Remove the unused preempt_offset parameter of __cant_sleep()
2026-08-04 16:14 ` [PATCH v4 08/17] sched: Remove the unused preempt_offset parameter of __cant_sleep() Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
@ 2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-10 8:57 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: ac4231a77973fc20808ed84c4af343eca2342d4b
Gitweb: https://git.kernel.org/tip/ac4231a77973fc20808ed84c4af343eca2342d4b
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:30 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Mon, 10 Aug 2026 10:50:19 +02:00
sched: Remove the unused preempt_offset parameter of __cant_sleep()
The preempt_offset is always 0 in all the callsites of __cant_sleep(),
hence remove it. It also allows us to clear up the code a bit by
no longer using a "preempt_count() > .." comparison.
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-9-boqun@kernel.org
---
include/linux/kernel.h | 4 ++--
kernel/sched/core.c | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
index e5570a1..24414c7 100644
--- a/include/linux/kernel.h
+++ b/include/linux/kernel.h
@@ -72,7 +72,7 @@ extern int dynamic_might_resched(void);
#ifdef CONFIG_DEBUG_ATOMIC_SLEEP
extern void __might_resched(const char *file, int line, unsigned int offsets);
extern void __might_sleep(const char *file, int line);
-extern void __cant_sleep(const char *file, int line, int preempt_offset);
+extern void __cant_sleep(const char *file, int line);
extern void __cant_migrate(const char *file, int line);
/**
@@ -95,7 +95,7 @@ extern void __cant_migrate(const char *file, int line);
* this macro will print a stack trace if it is executed with preemption enabled
*/
# define cant_sleep() \
- do { __cant_sleep(__FILE__, __LINE__, 0); } while (0)
+ do { __cant_sleep(__FILE__, __LINE__); } while (0)
# define sched_annotate_sleep() (current->task_state_change = 0)
/**
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9622670..aa116da 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -9199,7 +9199,7 @@ void __might_resched(const char *file, int line, unsigned int offsets)
}
EXPORT_SYMBOL(__might_resched);
-void __cant_sleep(const char *file, int line, int preempt_offset)
+void __cant_sleep(const char *file, int line)
{
static unsigned long prev_jiffy;
@@ -9209,7 +9209,7 @@ void __cant_sleep(const char *file, int line, int preempt_offset)
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
- if (preempt_count() > preempt_offset)
+ if (preempt_count())
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 09/17] sched: Avoid signed comparison of preempt_count() in __cant_migrate()
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (7 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 08/17] sched: Remove the unused preempt_offset parameter of __cant_sleep() Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
2026-08-04 16:14 ` [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
` (7 subsequent siblings)
16 siblings, 2 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
Currently preempt_count() is always a non-negative int on all archs
(PREEMPT_NEED_RESCHED archs will mask out the MSB when returning
preempt_count()), hence the checking in __cant_migrate() is in fact just
checking whether preempt_count() is 0 or not. In a future change, we are
going to use all the 32 bits of preempt_count(), which would make
negative int values possible from preempt_count(). Therefore convert the
"> 0" comparison into a zero check to prepare for the future change.
No functional changes are intended.
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
kernel/sched/core.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index aa116daf21bd..9b3f1764fa9e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -9241,7 +9241,7 @@ void __cant_migrate(const char *file, int line)
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
- if (preempt_count() > 0)
+ if (preempt_count())
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] sched: Avoid signed comparison of preempt_count() in __cant_migrate()
2026-08-04 16:14 ` [PATCH v4 09/17] sched: Avoid signed comparison of preempt_count() in __cant_migrate() Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 31f35d04a29a7554bb3b9ad827dd37c2942e31d4
Gitweb: https://git.kernel.org/tip/31f35d04a29a7554bb3b9ad827dd37c2942e31d4
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:31 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:08 +02:00
sched: Avoid signed comparison of preempt_count() in __cant_migrate()
Currently preempt_count() is always a non-negative int on all archs
(PREEMPT_NEED_RESCHED archs will mask out the MSB when returning
preempt_count()), hence the checking in __cant_migrate() is in fact just
checking whether preempt_count() is 0 or not. In a future change, we are
going to use all the 32 bits of preempt_count(), which would make
negative int values possible from preempt_count(). Therefore convert the
"> 0" comparison into a zero check to prepare for the future change.
No functional changes are intended.
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-10-boqun@kernel.org
---
kernel/sched/core.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index aa116da..9b3f176 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -9241,7 +9241,7 @@ void __cant_migrate(const char *file, int line)
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
- if (preempt_count() > 0)
+ if (preempt_count())
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [tip: locking/core] sched: Avoid signed comparison of preempt_count() in __cant_migrate()
2026-08-04 16:14 ` [PATCH v4 09/17] sched: Avoid signed comparison of preempt_count() in __cant_migrate() Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
@ 2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-10 8:57 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 560fcaa92ef983315023eb4ebfc1ebae1132fb2a
Gitweb: https://git.kernel.org/tip/560fcaa92ef983315023eb4ebfc1ebae1132fb2a
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:31 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Mon, 10 Aug 2026 10:50:19 +02:00
sched: Avoid signed comparison of preempt_count() in __cant_migrate()
Currently preempt_count() is always a non-negative int on all archs
(PREEMPT_NEED_RESCHED archs will mask out the MSB when returning
preempt_count()), hence the checking in __cant_migrate() is in fact just
checking whether preempt_count() is 0 or not. In a future change, we are
going to use all the 32 bits of preempt_count(), which would make
negative int values possible from preempt_count(). Therefore convert the
"> 0" comparison into a zero check to prepare for the future change.
No functional changes are intended.
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-10-boqun@kernel.org
---
kernel/sched/core.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index aa116da..9b3f176 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -9241,7 +9241,7 @@ void __cant_migrate(const char *file, int line)
if (!IS_ENABLED(CONFIG_PREEMPT_COUNT))
return;
- if (preempt_count() > 0)
+ if (preempt_count())
return;
if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy)
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (8 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 09/17] sched: Avoid signed comparison of preempt_count() in __cant_migrate() Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-04 20:11 ` Shrikanth Hegde
` (3 more replies)
2026-08-04 16:14 ` [PATCH v4 11/17] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
` (6 subsequent siblings)
16 siblings, 4 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
With the changes that enable preempt count to track IRQ disabling
nesting, we don't have enough bits in 32-bit preempt count
implementation, as a result we move NMI nesting bits out of the 32-bit
preempt count. However on the architectures that can support 64-bit
preempt count implementation, we can keep the NMI nesting bits in the
32-bit preempt count and avoid maintaining NMI nesting bits outside of
the same cache line.
Therefore HAS_SEPARATE_PREEMPT_RESCHED_BITS is introduced to allow
architectures to select this. Note that under this Kconfig, preempt
count is maintained in a 64-bit word however preempt_count() still
remains as an int because all the effective bits still fit in
(previously we mask out NEED_RESCHED bit in preempt_count()). This
should make no functional changes for existing preempt_count() users.
Enable this for x86_64 along with the introduction of the Kconfig.
[boqun: Undo the __preempt_count_{add,sub}() optimization in 32-bit
preempt count since it may introduce {over,under}flow]
Originally-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
arch/x86/Kconfig | 1 +
arch/x86/include/asm/preempt.h | 55 +++++++++++++++++++++++-----------
arch/x86/kernel/cpu/common.c | 2 +-
include/linux/hardirq.h | 47 +++++++++++++++++++++--------
include/linux/preempt.h | 23 ++++++++++++--
kernel/Kconfig.preempt | 4 +++
kernel/sched/core.c | 12 ++++++--
kernel/softirq.c | 6 ++++
lib/locking-selftest.c | 2 +-
9 files changed, 115 insertions(+), 37 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index bdad90f210e4..6a7067d20a6a 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -326,6 +326,7 @@ config X86
select USER_STACKTRACE_SUPPORT
select HAVE_ARCH_KCSAN if X86_64
select PROC_PID_ARCH_STATUS if PROC_FS
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS if X86_64 && PREEMPT_COUNT
select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX
select FUNCTION_ALIGNMENT_16B if X86_64 || X86_ALIGNMENT_16
select FUNCTION_ALIGNMENT_4B
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 1220656f3370..12353eeebc52 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -7,10 +7,20 @@
#include <linux/static_call_types.h>
-DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);
+DECLARE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count);
-/* We use the MSB mostly because its available */
-#define PREEMPT_NEED_RESCHED 0x80000000
+/*
+ * We use the MSB for PREEMPT_NEED_RESCHED mostly because it is available.
+ */
+#define PREEMPT_NEED_RESCHED (~(((unsigned long)-1L) >> 1))
+
+#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+#define __pc_dec "decq"
+#define __pc_op(op, ...) raw_cpu_##op##_8(__VA_ARGS__)
+#else
+#define __pc_dec "decl"
+#define __pc_op(op, ...) raw_cpu_##op##_4(__VA_ARGS__)
+#endif
/*
* We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
@@ -24,18 +34,26 @@ DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);
*/
static __always_inline int preempt_count(void)
{
- return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
+ return __pc_op(read, __preempt_count) & ~PREEMPT_NEED_RESCHED;
}
-static __always_inline void preempt_count_set(int pc)
+/*
+ * unsigned long preempt count parameter works for both 32bit and 64bit cases:
+ *
+ * - For 32bit, "int" (the return of preempt_count()) and "unsigned long" have
+ * the same size.
+ * - For 64bit, the effective bits of a preempt count sits in 32bit, and we
+ * reserve the NEED_RESCHED bit from the old count.
+ */
+static __always_inline void preempt_count_set(unsigned long pc)
{
- int old, new;
+ unsigned long old, new;
- old = raw_cpu_read_4(__preempt_count);
+ old = __pc_op(read, __preempt_count);
do {
new = (old & PREEMPT_NEED_RESCHED) |
(pc & ~PREEMPT_NEED_RESCHED);
- } while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new));
+ } while (!__pc_op(try_cmpxchg, __preempt_count, &old, new));
}
/*
@@ -58,17 +76,17 @@ static __always_inline void preempt_count_set(int pc)
static __always_inline void set_preempt_need_resched(void)
{
- raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
+ __pc_op(and, __preempt_count, ~PREEMPT_NEED_RESCHED);
}
static __always_inline void clear_preempt_need_resched(void)
{
- raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
+ __pc_op(or, __preempt_count, PREEMPT_NEED_RESCHED);
}
static __always_inline bool test_preempt_need_resched(void)
{
- return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
+ return !(__pc_op(read, __preempt_count) & PREEMPT_NEED_RESCHED);
}
/*
@@ -77,22 +95,22 @@ static __always_inline bool test_preempt_need_resched(void)
static __always_inline void __preempt_count_add(int val)
{
- raw_cpu_add_4(__preempt_count, val);
+ __pc_op(add, __preempt_count, val);
}
static __always_inline void __preempt_count_sub(int val)
{
- raw_cpu_add_4(__preempt_count, -val);
+ __pc_op(add, __preempt_count, -val);
}
static __always_inline int __preempt_count_add_return(int val)
{
- return raw_cpu_add_return_4(__preempt_count, val);
+ return __pc_op(add_return, __preempt_count, val);
}
static __always_inline int __preempt_count_sub_return(int val)
{
- return raw_cpu_add_return_4(__preempt_count, -val);
+ return __pc_op(add_return, __preempt_count, -val);
}
/*
@@ -102,7 +120,7 @@ static __always_inline int __preempt_count_sub_return(int val)
*/
static __always_inline bool __preempt_count_dec_and_test(void)
{
- return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e,
+ return GEN_UNARY_RMWcc(__pc_dec, __my_cpu_var(__preempt_count), e,
__percpu_arg([var]));
}
@@ -111,7 +129,7 @@ static __always_inline bool __preempt_count_dec_and_test(void)
*/
static __always_inline bool should_resched(int preempt_offset)
{
- return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
+ return unlikely(__pc_op(read, __preempt_count) == preempt_offset);
}
#ifdef CONFIG_PREEMPTION
@@ -158,4 +176,7 @@ do { \
#endif /* PREEMPTION */
+#undef __pc_op
+#undef __pc_dec
+
#endif /* __ASM_PREEMPT_H */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a3df21d26460..73a6d9f6a78e 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -2236,7 +2236,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
EXPORT_PER_CPU_SYMBOL(const_current_task);
-DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;
+DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 8d4895531a45..73b48dd9f135 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -10,8 +10,6 @@
#include <linux/vtime.h>
#include <asm/hardirq.h>
-DECLARE_PER_CPU(unsigned int, nmi_nesting);
-
extern void synchronize_irq(unsigned int irq);
extern bool synchronize_hardirq(unsigned int irq);
@@ -94,6 +92,37 @@ void irq_exit_rcu(void);
#define arch_nmi_exit() do { } while (0)
#endif
+#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+static __always_inline void __preempt_count_nmi_enter(void)
+{
+ __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);
+}
+
+static __always_inline void __preempt_count_nmi_exit(void)
+{
+ __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);
+}
+#else
+DECLARE_PER_CPU(unsigned int, nmi_nesting);
+
+#define __preempt_count_nmi_enter() \
+ do { \
+ __preempt_count_add(HARDIRQ_OFFSET); \
+ /* Maximum NMI nesting is 15. */ \
+ BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
+ __this_cpu_inc(nmi_nesting); \
+ preempt_count_set(preempt_count() | NMI_MASK); \
+ } while (0)
+
+#define __preempt_count_nmi_exit() \
+ do { \
+ __preempt_count_sub(HARDIRQ_OFFSET); \
+ if (!__this_cpu_dec_return(nmi_nesting)) \
+ preempt_count_set(preempt_count() & ~NMI_MASK); \
+ } while (0)
+
+#endif
+
/*
* NMI vs Tracing
* --------------
@@ -110,18 +139,14 @@ void irq_exit_rcu(void);
do { \
lockdep_off(); \
arch_nmi_enter(); \
- /* Maximum NMI nesting is 15. */ \
- BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
- __this_cpu_inc(nmi_nesting); \
- __preempt_count_add(HARDIRQ_OFFSET); \
- preempt_count_set(preempt_count() | NMI_MASK); \
+ __preempt_count_nmi_enter(); \
} while (0)
#define nmi_enter() \
do { \
__nmi_enter(); \
lockdep_hardirq_enter(); \
- ct_nmi_enter(); \
+ ct_nmi_enter(); \
instrumentation_begin(); \
ftrace_nmi_enter(); \
instrumentation_end(); \
@@ -129,12 +154,8 @@ void irq_exit_rcu(void);
#define __nmi_exit() \
do { \
- unsigned int nesting; \
BUG_ON(!in_nmi()); \
- __preempt_count_sub(HARDIRQ_OFFSET); \
- nesting = __this_cpu_dec_return(nmi_nesting); \
- if (!nesting) \
- preempt_count_set(preempt_count() & ~NMI_MASK); \
+ __preempt_count_nmi_exit(); \
arch_nmi_exit(); \
lockdep_on(); \
} while (0)
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 33fc4c814a9f..8299657f0f86 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -34,14 +34,31 @@
* SOFTIRQ_MASK: 0x0000ff00
* HARDIRQ_DISABLE_MASK: 0x00ff0000
* HARDIRQ_MASK: 0x0f000000
+ *
+ * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a
+ * separate word and that allows 64bit load-store architectures to 'set'
+ * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric
+ * modifications used on preempt_count and still load the whole thing
+ * (single-copy) atomically, without having to resort to full atomic
+ * operations.
+ *
+ * Because of the above, NMI_MASK bits are different depending on
+ * HAS_SEPARATE_PREEMPT_RESCHED_BITS:
+ *
+ * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n:
+ *
* NMI_MASK: 0x10000000
* PREEMPT_NEED_RESCHED: 0x80000000
+ *
+ * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y:
+ * NMI_MASK: 0xf0000000
+ * (PREEMPT_NEED_RESCHED is in a different word)
*/
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
#define HARDIRQ_DISABLE_BITS 8
#define HARDIRQ_BITS 4
-#define NMI_BITS 1
+#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS))
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
@@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void)
* preempt_count() is commonly implemented with READ_ONCE().
*/
-#define nmi_count() (preempt_count() & NMI_MASK)
-#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
+#define nmi_count() (preempt_count() & NMI_MASK)
+#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
#ifdef CONFIG_PREEMPT_RT
# define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK)
# define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index 88c594c6d7fc..35f546a042b1 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK
config PREEMPT_COUNT
bool
+config HAS_SEPARATE_PREEMPT_RESCHED_BITS
+ bool
+ depends on PREEMPT_COUNT && 64BIT
+
config PREEMPTION
bool
select PREEMPT_COUNT
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9b3f1764fa9e..6d88343c3bad 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5973,8 +5973,13 @@ void preempt_count_add(int val)
#ifdef CONFIG_DEBUG_PREEMPT
/*
* Underflow?
+ *
+ * Cannot detect underflow based on the current preempt_count() value
+ * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32
+ * bits.
*/
- if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
+ if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) &&
+ DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
return;
#endif
__preempt_count_add(val);
@@ -6006,7 +6011,10 @@ void preempt_count_sub(int val)
/*
* Underflow?
*/
- if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
+ unsigned int uval = val;
+ unsigned int pc = preempt_count();
+
+ if (DEBUG_LOCKS_WARN_ON(pc - uval > pc))
return;
/*
* Is the spinlock portion underflowing?
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 0c9b2269a8d6..7980a4a232f9 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -103,7 +103,13 @@ void _local_interrupt_enable(void)
}
EXPORT_SYMBOL(_local_interrupt_enable);
+#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+/*
+ * Any 32bit architecture that still cares about performance should
+ * probably ensure this is near preempt_count.
+ */
DEFINE_PER_CPU(unsigned int, nmi_nesting);
+#endif
/*
* SOFTIRQ_OFFSET usage:
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index bfafe1204c7b..c3d976c801bb 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures;
static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
{
- int saved_preempt_count = preempt_count();
+ long saved_preempt_count = preempt_count();
#ifdef CONFIG_PREEMPT_RT
int saved_mgd_count = current->migration_disabled;
int saved_rcu_count = current->rcu_read_lock_nesting;
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* Re: [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
@ 2026-08-04 20:11 ` Shrikanth Hegde
2026-08-05 6:54 ` Boqun Feng
2026-08-04 21:09 ` Shrikanth Hegde
` (2 subsequent siblings)
3 siblings, 1 reply; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-04 20:11 UTC (permalink / raw)
To: Boqun Feng, Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux
Hi Boqun,
On 8/4/26 9:44 PM, Boqun Feng wrote:
> With the changes that enable preempt count to track IRQ disabling
> nesting, we don't have enough bits in 32-bit preempt count
> implementation, as a result we move NMI nesting bits out of the 32-bit
> preempt count. However on the architectures that can support 64-bit
> preempt count implementation, we can keep the NMI nesting bits in the
> 32-bit preempt count and avoid maintaining NMI nesting bits outside of
> the same cache line.
>
[...]
> --- a/include/linux/hardirq.h
> +++ b/include/linux/hardirq.h
> @@ -10,8 +10,6 @@
> #include <linux/vtime.h>
> #include <asm/hardirq.h>
>
> -DECLARE_PER_CPU(unsigned int, nmi_nesting);
> -
> extern void synchronize_irq(unsigned int irq);
> extern bool synchronize_hardirq(unsigned int irq);
>
> @@ -94,6 +92,37 @@ void irq_exit_rcu(void);
> #define arch_nmi_exit() do { } while (0)
> #endif
>
> +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
> +static __always_inline void __preempt_count_nmi_enter(void)
> +{
> + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);
> +}
> +
> +static __always_inline void __preempt_count_nmi_exit(void)
> +{
> + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);
> +}
> +#else
> +DECLARE_PER_CPU(unsigned int, nmi_nesting);
> +
> +#define __preempt_count_nmi_enter() \
> + do { \
> + __preempt_count_add(HARDIRQ_OFFSET); \
nit: This limit is because to have the same behavior as other case when
NMI_BITS=4 right?
It is not easy to infer that from comment.
> + /* Maximum NMI nesting is 15. */ \
> + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
> + __this_cpu_inc(nmi_nesting); \
> + preempt_count_set(preempt_count() | NMI_MASK); \
Is there a reason preempt count updates are split rather than
folded into a single preempt_count update?
> + } while (0)
> +
> +#define __preempt_count_nmi_exit() \
> + do { \
> + __preempt_count_sub(HARDIRQ_OFFSET); \
> + if (!__this_cpu_dec_return(nmi_nesting)) \
> + preempt_count_set(preempt_count() & ~NMI_MASK); \
> + } while (0)
> +
> +#endif
> +
> /*
> * NMI vs Tracing
> * --------------
> @@ -110,18 +139,14 @@ void irq_exit_rcu(void);
> do { \
> lockdep_off(); \
> arch_nmi_enter(); \
> - /* Maximum NMI nesting is 15. */ \
> - BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
> - __this_cpu_inc(nmi_nesting); \
> - __preempt_count_add(HARDIRQ_OFFSET); \
> - preempt_count_set(preempt_count() | NMI_MASK); \
> + __preempt_count_nmi_enter(); \
> } while (0)
>
> #define nmi_enter() \
> do { \
> __nmi_enter(); \
> lockdep_hardirq_enter(); \
> - ct_nmi_enter(); \
> + ct_nmi_enter(); \
> instrumentation_begin(); \
> ftrace_nmi_enter(); \
> instrumentation_end(); \
> @@ -129,12 +154,8 @@ void irq_exit_rcu(void);
>
> #define __nmi_exit() \
> do { \
> - unsigned int nesting; \
> BUG_ON(!in_nmi()); \
> - __preempt_count_sub(HARDIRQ_OFFSET); \
> - nesting = __this_cpu_dec_return(nmi_nesting); \
> - if (!nesting) \
> - preempt_count_set(preempt_count() & ~NMI_MASK); \
> + __preempt_count_nmi_exit(); \
> arch_nmi_exit(); \
> lockdep_on(); \
> } while (0)
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 20:11 ` Shrikanth Hegde
@ 2026-08-05 6:54 ` Boqun Feng
2026-08-05 7:15 ` Shrikanth Hegde
2026-08-06 0:58 ` Boqun Feng
0 siblings, 2 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 6:54 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 01:41:48AM +0530, Shrikanth Hegde wrote:
> Hi Boqun,
>
> On 8/4/26 9:44 PM, Boqun Feng wrote:
> > With the changes that enable preempt count to track IRQ disabling
> > nesting, we don't have enough bits in 32-bit preempt count
> > implementation, as a result we move NMI nesting bits out of the 32-bit
> > preempt count. However on the architectures that can support 64-bit
> > preempt count implementation, we can keep the NMI nesting bits in the
> > 32-bit preempt count and avoid maintaining NMI nesting bits outside of
> > the same cache line.
> >
>
> [...]
>
> > --- a/include/linux/hardirq.h
> > +++ b/include/linux/hardirq.h
> > @@ -10,8 +10,6 @@
> > #include <linux/vtime.h>
> > #include <asm/hardirq.h>
> > -DECLARE_PER_CPU(unsigned int, nmi_nesting);
> > -
> > extern void synchronize_irq(unsigned int irq);
> > extern bool synchronize_hardirq(unsigned int irq);
> > @@ -94,6 +92,37 @@ void irq_exit_rcu(void);
> > #define arch_nmi_exit() do { } while (0)
> > #endif
> > +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
> > +static __always_inline void __preempt_count_nmi_enter(void)
> > +{
> > + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);
> > +}
> > +
> > +static __always_inline void __preempt_count_nmi_exit(void)
> > +{
> > + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);
> > +}
> > +#else
> > +DECLARE_PER_CPU(unsigned int, nmi_nesting);
> > +
> > +#define __preempt_count_nmi_enter() \
> > + do { \
> > + __preempt_count_add(HARDIRQ_OFFSET); \
>
> nit: This limit is because to have the same behavior as other case when
> NMI_BITS=4 right?
> It is not easy to infer that from comment.
>
It's sort of design by implementation IIUC, previously because of
NMI_BITS=4, we could only support nesting level being 15. And here we
just want to keep the same behavior here.
If your question is why 15 was a good number before this change, I guess
would be it's just a number that is neither too big or too small.
> > + /* Maximum NMI nesting is 15. */ \
> > + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
> > + __this_cpu_inc(nmi_nesting); \
> > + preempt_count_set(preempt_count() | NMI_MASK); \
>
>
> Is there a reason preempt count updates are split rather than
> folded into a single preempt_count update?
>
Keeping the implementation simple is one reason, most architectures
could utilize the HAS_SEPARATE_PREEMPT_RESCHED_BITS for better
performance. So that reduces the importance of having something
complicated but saves one access here. But if you see an optimization
that can be done here, please do share!
Peter had proposed one optimization here:
#define __preempt_count_nmi_enter() \
do { \
unsigned int _o = NMI_MASK + HARDIRQ_OFFSET; \
/* Maximum NMI nesting is 15. */ \
BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
__this_cpu_inc(nmi_nesting); \
_o -= (preempt_count() & NMI_MASK); \
__preempt_count_add(_o); \
} while (0)
#define __preempt_count_nmi_exit() \
do { \
unsigned int _o = HARDIRQ_OFFSET; \
if (!__this_cpu_dec_return(nmi_nesting)) \
_o += NMI_MASK; \
__preempt_count_sub(_o); \
} while (0)
but it has a problem considering this:
// outermost NMI handler
// nmi_nesting == 0
nmi_enter();
// ^ nmi_nesting == 1 and NMI_MASK is set.
...
nmi_exit():
if (!__this_cpu_dec_return(nmi_nesting)) // return true
_o += NMI_MASK;
<NMI start>
nmi_enter();
// ^ nmi_nesting == 1 and NMI_MASK is set.
nmi_exit();
// ^ nmi_nesting == 0 and NMI_MASK is *unset*.
<NMI end>
preempt_count_sub(_o); // _o == HARDIRQ_OFFSET + NMI_MASK,
// underflow
(Now think about this, the __preempt_count_nmi_enter() does seems
fine, maybe we can keep that, too tired to remember whether there is any
subtly here... will take another look tomorrow)
Regards,
Boqun
> > + } while (0)
> > +
> > +#define __preempt_count_nmi_exit() \
> > + do { \
> > + __preempt_count_sub(HARDIRQ_OFFSET); \
> > + if (!__this_cpu_dec_return(nmi_nesting)) \
> > + preempt_count_set(preempt_count() & ~NMI_MASK); \
> > + } while (0)
> > +
> > +#endif
> > +
> > /*
> > * NMI vs Tracing
> > * --------------
> > @@ -110,18 +139,14 @@ void irq_exit_rcu(void);
> > do { \
> > lockdep_off(); \
> > arch_nmi_enter(); \
> > - /* Maximum NMI nesting is 15. */ \
> > - BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
> > - __this_cpu_inc(nmi_nesting); \
> > - __preempt_count_add(HARDIRQ_OFFSET); \
> > - preempt_count_set(preempt_count() | NMI_MASK); \
> > + __preempt_count_nmi_enter(); \
> > } while (0)
> > #define nmi_enter() \
> > do { \
> > __nmi_enter(); \
> > lockdep_hardirq_enter(); \
> > - ct_nmi_enter(); \
> > + ct_nmi_enter(); \
> > instrumentation_begin(); \
> > ftrace_nmi_enter(); \
> > instrumentation_end(); \
> > @@ -129,12 +154,8 @@ void irq_exit_rcu(void);
> > #define __nmi_exit() \
> > do { \
> > - unsigned int nesting; \
> > BUG_ON(!in_nmi()); \
> > - __preempt_count_sub(HARDIRQ_OFFSET); \
> > - nesting = __this_cpu_dec_return(nmi_nesting); \
> > - if (!nesting) \
> > - preempt_count_set(preempt_count() & ~NMI_MASK); \
> > + __preempt_count_nmi_exit(); \
> > arch_nmi_exit(); \
> > lockdep_on(); \
> > } while (0)
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-05 6:54 ` Boqun Feng
@ 2026-08-05 7:15 ` Shrikanth Hegde
2026-08-05 7:27 ` Boqun Feng
2026-08-06 0:58 ` Boqun Feng
1 sibling, 1 reply; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-05 7:15 UTC (permalink / raw)
To: Boqun Feng
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
Hi Boqun.
>> nit: This limit is because to have the same behavior as other case when
>> NMI_BITS=4 right?
>> It is not easy to infer that from comment.
>>
>
> It's sort of design by implementation IIUC, previously because of
> NMI_BITS=4, we could only support nesting level being 15. And here we
> just want to keep the same behavior here.
That I understood.
>
> If your question is why 15 was a good number before this change, I guess
> would be it's just a number that is neither too big or too small.
Its more about below comment. Something like below is better?
/* NMI nesting is represented in 4 bits. */
>
>>> + /* Maximum NMI nesting is 15. */ \
>>> + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
>>> + __this_cpu_inc(nmi_nesting); \
>>> + preempt_count_set(preempt_count() | NMI_MASK); \
^ permalink raw reply [flat|nested] 66+ messages in thread
* Re: [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-05 7:15 ` Shrikanth Hegde
@ 2026-08-05 7:27 ` Boqun Feng
0 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-05 7:27 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 12:45:04PM +0530, Shrikanth Hegde wrote:
>
> Hi Boqun.
>
> > > nit: This limit is because to have the same behavior as other case when
> > > NMI_BITS=4 right?
> > > It is not easy to infer that from comment.
> > >
> >
> > It's sort of design by implementation IIUC, previously because of
> > NMI_BITS=4, we could only support nesting level being 15. And here we
> > just want to keep the same behavior here.
>
> That I understood.
>
> >
> > If your question is why 15 was a good number before this change, I guess
> > would be it's just a number that is neither too big or too small.
>
> Its more about below comment. Something like below is better?
>
> /* NMI nesting is represented in 4 bits. */
>
Sounds good, I will apply this, thank you!
Regards,
Boqun
> >
> > > > + /* Maximum NMI nesting is 15. */ \
> > > > + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
> > > > + __this_cpu_inc(nmi_nesting); \
> > > > + preempt_count_set(preempt_count() | NMI_MASK); \
^ permalink raw reply [flat|nested] 66+ messages in thread
* Re: [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-05 6:54 ` Boqun Feng
2026-08-05 7:15 ` Shrikanth Hegde
@ 2026-08-06 0:58 ` Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-06 0:58 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Tue, Aug 04, 2026 at 11:54:34PM -0700, Boqun Feng wrote:
> On Wed, Aug 05, 2026 at 01:41:48AM +0530, Shrikanth Hegde wrote:
> > Hi Boqun,
> >
> > On 8/4/26 9:44 PM, Boqun Feng wrote:
> > > With the changes that enable preempt count to track IRQ disabling
> > > nesting, we don't have enough bits in 32-bit preempt count
> > > implementation, as a result we move NMI nesting bits out of the 32-bit
> > > preempt count. However on the architectures that can support 64-bit
> > > preempt count implementation, we can keep the NMI nesting bits in the
> > > 32-bit preempt count and avoid maintaining NMI nesting bits outside of
> > > the same cache line.
> > >
> >
> > [...]
> >
> > > --- a/include/linux/hardirq.h
> > > +++ b/include/linux/hardirq.h
> > > @@ -10,8 +10,6 @@
> > > #include <linux/vtime.h>
> > > #include <asm/hardirq.h>
> > > -DECLARE_PER_CPU(unsigned int, nmi_nesting);
> > > -
> > > extern void synchronize_irq(unsigned int irq);
> > > extern bool synchronize_hardirq(unsigned int irq);
> > > @@ -94,6 +92,37 @@ void irq_exit_rcu(void);
> > > #define arch_nmi_exit() do { } while (0)
> > > #endif
> > > +#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
> > > +static __always_inline void __preempt_count_nmi_enter(void)
> > > +{
> > > + __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);
> > > +}
> > > +
> > > +static __always_inline void __preempt_count_nmi_exit(void)
> > > +{
> > > + __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);
> > > +}
> > > +#else
> > > +DECLARE_PER_CPU(unsigned int, nmi_nesting);
> > > +
> > > +#define __preempt_count_nmi_enter() \
> > > + do { \
> > > + __preempt_count_add(HARDIRQ_OFFSET); \
> >
> > nit: This limit is because to have the same behavior as other case when
> > NMI_BITS=4 right?
> > It is not easy to infer that from comment.
> >
>
> It's sort of design by implementation IIUC, previously because of
> NMI_BITS=4, we could only support nesting level being 15. And here we
> just want to keep the same behavior here.
>
> If your question is why 15 was a good number before this change, I guess
> would be it's just a number that is neither too big or too small.
>
> > > + /* Maximum NMI nesting is 15. */ \
> > > + BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
> > > + __this_cpu_inc(nmi_nesting); \
> > > + preempt_count_set(preempt_count() | NMI_MASK); \
> >
> >
> > Is there a reason preempt count updates are split rather than
> > folded into a single preempt_count update?
> >
>
> Keeping the implementation simple is one reason, most architectures
> could utilize the HAS_SEPARATE_PREEMPT_RESCHED_BITS for better
> performance. So that reduces the importance of having something
> complicated but saves one access here. But if you see an optimization
> that can be done here, please do share!
>
> Peter had proposed one optimization here:
>
> #define __preempt_count_nmi_enter() \
> do { \
> unsigned int _o = NMI_MASK + HARDIRQ_OFFSET; \
> /* Maximum NMI nesting is 15. */ \
> BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
> __this_cpu_inc(nmi_nesting); \
> _o -= (preempt_count() & NMI_MASK); \
> __preempt_count_add(_o); \
> } while (0)
>
> #define __preempt_count_nmi_exit() \
> do { \
> unsigned int _o = HARDIRQ_OFFSET; \
> if (!__this_cpu_dec_return(nmi_nesting)) \
> _o += NMI_MASK; \
> __preempt_count_sub(_o); \
> } while (0)
>
> but it has a problem considering this:
>
> // outermost NMI handler
> // nmi_nesting == 0
>
> nmi_enter();
> // ^ nmi_nesting == 1 and NMI_MASK is set.
> ...
> nmi_exit():
> if (!__this_cpu_dec_return(nmi_nesting)) // return true
> _o += NMI_MASK;
> <NMI start>
> nmi_enter();
> // ^ nmi_nesting == 1 and NMI_MASK is set.
> nmi_exit();
> // ^ nmi_nesting == 0 and NMI_MASK is *unset*.
> <NMI end>
>
> preempt_count_sub(_o); // _o == HARDIRQ_OFFSET + NMI_MASK,
> // underflow
>
> (Now think about this, the __preempt_count_nmi_enter() does seems
> fine, maybe we can keep that, too tired to remember whether there is any
> subtly here... will take another look tomorrow)
>
Ok, now I remember the issue of the preempt_count_add() implemented
__preempt_count_nmi_enter(), considering this:
// outermost NMI handler
// nmi_nesting == 0
nmi_enter():
__preempt_count_nmi_enter():
unsigned int _o = NMI_MASK + HARDIRQ_OFFSET;
...
__this_cpu_inc(nmi_nesting);
// ^ nmi_nesting == 1
_o -= (preempt_count() & NMI_MASK);
// ^ _o == NMI_MASK + HARDIRQ_OFFSET because the NMI_MASK bit was not set
<NMI start>
nmi_enter();
// ^ nmi_nesting == 2 and NMI_MASK is set.
nmi_exti();
// ^ nmi_nesting == 1 so NMI_MASK is *still* set
<NMI end>
__preempt_count_add(_o);
// ^ NMI_MASK overflows because of the addition.
Make sense?
But maybe there are clever ways that I don't know. Please do tell!
Regards,
Boqun
> Regards,
> Boqun
>
>
> > > + } while (0)
> > > +
> > > +#define __preempt_count_nmi_exit() \
> > > + do { \
> > > + __preempt_count_sub(HARDIRQ_OFFSET); \
> > > + if (!__this_cpu_dec_return(nmi_nesting)) \
> > > + preempt_count_set(preempt_count() & ~NMI_MASK); \
> > > + } while (0)
> > > +
> > > +#endif
> > > +
> > > /*
> > > * NMI vs Tracing
> > > * --------------
> > > @@ -110,18 +139,14 @@ void irq_exit_rcu(void);
> > > do { \
> > > lockdep_off(); \
> > > arch_nmi_enter(); \
> > > - /* Maximum NMI nesting is 15. */ \
> > > - BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
> > > - __this_cpu_inc(nmi_nesting); \
> > > - __preempt_count_add(HARDIRQ_OFFSET); \
> > > - preempt_count_set(preempt_count() | NMI_MASK); \
> > > + __preempt_count_nmi_enter(); \
> > > } while (0)
> > > #define nmi_enter() \
> > > do { \
> > > __nmi_enter(); \
> > > lockdep_hardirq_enter(); \
> > > - ct_nmi_enter(); \
> > > + ct_nmi_enter(); \
> > > instrumentation_begin(); \
> > > ftrace_nmi_enter(); \
> > > instrumentation_end(); \
> > > @@ -129,12 +154,8 @@ void irq_exit_rcu(void);
> > > #define __nmi_exit() \
> > > do { \
> > > - unsigned int nesting; \
> > > BUG_ON(!in_nmi()); \
> > > - __preempt_count_sub(HARDIRQ_OFFSET); \
> > > - nesting = __this_cpu_dec_return(nmi_nesting); \
> > > - if (!nesting) \
> > > - preempt_count_set(preempt_count() & ~NMI_MASK); \
> > > + __preempt_count_nmi_exit(); \
> > > arch_nmi_exit(); \
> > > lockdep_on(); \
> > > } while (0)
^ permalink raw reply [flat|nested] 66+ messages in thread
* Re: [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
2026-08-04 20:11 ` Shrikanth Hegde
@ 2026-08-04 21:09 ` Shrikanth Hegde
2026-08-04 23:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
3 siblings, 1 reply; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-04 21:09 UTC (permalink / raw)
To: Boqun Feng, Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux
Hi.
> diff --git a/include/linux/preempt.h b/include/linux/preempt.h
> index 33fc4c814a9f..8299657f0f86 100644
> --- a/include/linux/preempt.h
> +++ b/include/linux/preempt.h
> @@ -34,14 +34,31 @@
> * SOFTIRQ_MASK: 0x0000ff00
> * HARDIRQ_DISABLE_MASK: 0x00ff0000
> * HARDIRQ_MASK: 0x0f000000
> + *
> + * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a
> + * separate word and that allows 64bit load-store architectures to 'set'
> + * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric
> + * modifications used on preempt_count and still load the whole thing
> + * (single-copy) atomically, without having to resort to full atomic
> + * operations.
> + *
> + * Because of the above, NMI_MASK bits are different depending on
> + * HAS_SEPARATE_PREEMPT_RESCHED_BITS:
> + *
> + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n:
> + *
> * NMI_MASK: 0x10000000
> * PREEMPT_NEED_RESCHED: 0x80000000
> + *
> + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y:
> + * NMI_MASK: 0xf0000000
> + * (PREEMPT_NEED_RESCHED is in a different word)
> */
> #define PREEMPT_BITS 8
> #define SOFTIRQ_BITS 8
> #define HARDIRQ_DISABLE_BITS 8
> #define HARDIRQ_BITS 4
> -#define NMI_BITS 1
> +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS))
>
Shouldn't testing/selftests/bpf/bpf_experimental.h also be updated with
same?
> #define PREEMPT_SHIFT 0
> #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
> @@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void)
> * preempt_count() is commonly implemented with READ_ONCE().
> */
>
> -#define nmi_count() (preempt_count() & NMI_MASK)
> -#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
> +#define nmi_count() (preempt_count() & NMI_MASK)
> +#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
> #ifdef CONFIG_PREEMPT_RT
> # define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK)
> # define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
> diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
> index 88c594c6d7fc..35f546a042b1 100644
> --- a/kernel/Kconfig.preempt
> +++ b/kernel/Kconfig.preempt
> @@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK
> config PREEMPT_COUNT
> bool
>
> +config HAS_SEPARATE_PREEMPT_RESCHED_BITS
> + bool
> + depends on PREEMPT_COUNT && 64BIT
> +
> config PREEMPTION
> bool
> select PREEMPT_COUNT
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 9b3f1764fa9e..6d88343c3bad 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -5973,8 +5973,13 @@ void preempt_count_add(int val)
> #ifdef CONFIG_DEBUG_PREEMPT
> /*
> * Underflow?
> + *
> + * Cannot detect underflow based on the current preempt_count() value
> + * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32
> + * bits.
> */
> - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
> + if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) &&
> + DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
> return;
> #endif
> __preempt_count_add(val);
> @@ -6006,7 +6011,10 @@ void preempt_count_sub(int val)
> /*
> * Underflow?
> */
> - if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
> + unsigned int uval = val;
> + unsigned int pc = preempt_count();
> +
> + if (DEBUG_LOCKS_WARN_ON(pc - uval > pc))
> return;
> /*
> * Is the spinlock portion underflowing?
> diff --git a/kernel/softirq.c b/kernel/softirq.c
> index 0c9b2269a8d6..7980a4a232f9 100644
> --- a/kernel/softirq.c
> +++ b/kernel/softirq.c
> @@ -103,7 +103,13 @@ void _local_interrupt_enable(void)
> }
> EXPORT_SYMBOL(_local_interrupt_enable);
>
> +#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
> +/*
> + * Any 32bit architecture that still cares about performance should
> + * probably ensure this is near preempt_count.
> + */
> DEFINE_PER_CPU(unsigned int, nmi_nesting);
> +#endif
>
> /*
> * SOFTIRQ_OFFSET usage:
> diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
> index bfafe1204c7b..c3d976c801bb 100644
> --- a/lib/locking-selftest.c
> +++ b/lib/locking-selftest.c
> @@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures;
>
> static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
> {
> - int saved_preempt_count = preempt_count();
> + long saved_preempt_count = preempt_count();
nit: Isn't preempt_count still returns int?
> #ifdef CONFIG_PREEMPT_RT
> int saved_mgd_count = current->migration_disabled;
> int saved_rcu_count = current->rcu_read_lock_nesting;
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 21:09 ` Shrikanth Hegde
@ 2026-08-04 23:14 ` Boqun Feng
0 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 23:14 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Peter Zijlstra, Ingo Molnar, Will Deacon, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
On Wed, Aug 05, 2026 at 02:39:20AM +0530, Shrikanth Hegde wrote:
> Hi.
>
> > diff --git a/include/linux/preempt.h b/include/linux/preempt.h
> > index 33fc4c814a9f..8299657f0f86 100644
> > --- a/include/linux/preempt.h
> > +++ b/include/linux/preempt.h
> > @@ -34,14 +34,31 @@
> > * SOFTIRQ_MASK: 0x0000ff00
> > * HARDIRQ_DISABLE_MASK: 0x00ff0000
> > * HARDIRQ_MASK: 0x0f000000
> > + *
> > + * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a
> > + * separate word and that allows 64bit load-store architectures to 'set'
> > + * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric
> > + * modifications used on preempt_count and still load the whole thing
> > + * (single-copy) atomically, without having to resort to full atomic
> > + * operations.
> > + *
> > + * Because of the above, NMI_MASK bits are different depending on
> > + * HAS_SEPARATE_PREEMPT_RESCHED_BITS:
> > + *
> > + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n:
> > + *
> > * NMI_MASK: 0x10000000
> > * PREEMPT_NEED_RESCHED: 0x80000000
> > + *
> > + * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y:
> > + * NMI_MASK: 0xf0000000
> > + * (PREEMPT_NEED_RESCHED is in a different word)
> > */
> > #define PREEMPT_BITS 8
> > #define SOFTIRQ_BITS 8
> > #define HARDIRQ_DISABLE_BITS 8
> > #define HARDIRQ_BITS 4
> > -#define NMI_BITS 1
> > +#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS))
>
>
> Shouldn't testing/selftests/bpf/bpf_experimental.h also be updated with
> same?
>
Yeah, sashiko found out the same thing.
>
> > #define PREEMPT_SHIFT 0
> > #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
> > @@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void)
> > * preempt_count() is commonly implemented with READ_ONCE().
> > */
> > -#define nmi_count() (preempt_count() & NMI_MASK)
> > -#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
> > +#define nmi_count() (preempt_count() & NMI_MASK)
> > +#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
> > #ifdef CONFIG_PREEMPT_RT
> > # define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK)
> > # define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
> > diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
> > index 88c594c6d7fc..35f546a042b1 100644
> > --- a/kernel/Kconfig.preempt
> > +++ b/kernel/Kconfig.preempt
> > @@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK
> > config PREEMPT_COUNT
> > bool
> > +config HAS_SEPARATE_PREEMPT_RESCHED_BITS
> > + bool
> > + depends on PREEMPT_COUNT && 64BIT
> > +
> > config PREEMPTION
> > bool
> > select PREEMPT_COUNT
> > diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> > index 9b3f1764fa9e..6d88343c3bad 100644
> > --- a/kernel/sched/core.c
> > +++ b/kernel/sched/core.c
> > @@ -5973,8 +5973,13 @@ void preempt_count_add(int val)
> > #ifdef CONFIG_DEBUG_PREEMPT
> > /*
> > * Underflow?
> > + *
> > + * Cannot detect underflow based on the current preempt_count() value
> > + * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32
> > + * bits.
> > */
> > - if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
> > + if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) &&
> > + DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
> > return;
> > #endif
> > __preempt_count_add(val);
> > @@ -6006,7 +6011,10 @@ void preempt_count_sub(int val)
> > /*
> > * Underflow?
> > */
> > - if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
> > + unsigned int uval = val;
> > + unsigned int pc = preempt_count();
> > +
> > + if (DEBUG_LOCKS_WARN_ON(pc - uval > pc))
> > return;
> > /*
> > * Is the spinlock portion underflowing?
> > diff --git a/kernel/softirq.c b/kernel/softirq.c
> > index 0c9b2269a8d6..7980a4a232f9 100644
> > --- a/kernel/softirq.c
> > +++ b/kernel/softirq.c
> > @@ -103,7 +103,13 @@ void _local_interrupt_enable(void)
> > }
> > EXPORT_SYMBOL(_local_interrupt_enable);
> > +#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
> > +/*
> > + * Any 32bit architecture that still cares about performance should
> > + * probably ensure this is near preempt_count.
> > + */
> > DEFINE_PER_CPU(unsigned int, nmi_nesting);
> > +#endif
> > /*
> > * SOFTIRQ_OFFSET usage:
> > diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
> > index bfafe1204c7b..c3d976c801bb 100644
> > --- a/lib/locking-selftest.c
> > +++ b/lib/locking-selftest.c
> > @@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures;
> > static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
> > {
> > - int saved_preempt_count = preempt_count();
> > + long saved_preempt_count = preempt_count();
>
> nit: Isn't preempt_count still returns int?
>
Right, let me fix that. Thank you!
Regards,
Boqun
> > #ifdef CONFIG_PREEMPT_RT
> > int saved_mgd_count = current->migration_disabled;
> > int saved_rcu_count = current->rcu_read_lock_nesting;
>
^ permalink raw reply [flat|nested] 66+ messages in thread
* [tip: locking/core] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
2026-08-04 20:11 ` Shrikanth Hegde
2026-08-04 21:09 ` Shrikanth Hegde
@ 2026-08-08 20:48 ` tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
3 siblings, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Peter Zijlstra, Boqun Feng, x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: a1ad9bd8e22a26a15f57100e9feef92386808ebe
Gitweb: https://git.kernel.org/tip/a1ad9bd8e22a26a15f57100e9feef92386808ebe
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:32 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:08 +02:00
preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
With the changes that enable preempt count to track IRQ disabling
nesting, we don't have enough bits in 32-bit preempt count
implementation, as a result we move NMI nesting bits out of the 32-bit
preempt count. However on the architectures that can support 64-bit
preempt count implementation, we can keep the NMI nesting bits in the
32-bit preempt count and avoid maintaining NMI nesting bits outside of
the same cache line.
Therefore HAS_SEPARATE_PREEMPT_RESCHED_BITS is introduced to allow
architectures to select this. Note that under this Kconfig, preempt
count is maintained in a 64-bit word however preempt_count() still
remains as an int because all the effective bits still fit in
(previously we mask out NEED_RESCHED bit in preempt_count()). This
should make no functional changes for existing preempt_count() users.
Enable this for x86_64 along with the introduction of the Kconfig.
[boqun: Undo the __preempt_count_{add,sub}() optimization in 32-bit
preempt count since it may introduce {over,under}flow]
Originally-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-11-boqun@kernel.org
---
arch/x86/Kconfig | 1 +-
arch/x86/include/asm/preempt.h | 55 ++++++++++++++++++++++-----------
arch/x86/kernel/cpu/common.c | 2 +-
include/linux/hardirq.h | 47 ++++++++++++++++++++--------
include/linux/preempt.h | 23 ++++++++++++--
kernel/Kconfig.preempt | 4 ++-
kernel/sched/core.c | 12 +++++--
kernel/softirq.c | 6 ++++-
lib/locking-selftest.c | 2 +-
9 files changed, 115 insertions(+), 37 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index bdad90f..6a7067d 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -326,6 +326,7 @@ config X86
select USER_STACKTRACE_SUPPORT
select HAVE_ARCH_KCSAN if X86_64
select PROC_PID_ARCH_STATUS if PROC_FS
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS if X86_64 && PREEMPT_COUNT
select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX
select FUNCTION_ALIGNMENT_16B if X86_64 || X86_ALIGNMENT_16
select FUNCTION_ALIGNMENT_4B
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 1220656..fafb6f8 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -7,10 +7,20 @@
#include <linux/static_call_types.h>
-DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);
+DECLARE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count);
-/* We use the MSB mostly because its available */
-#define PREEMPT_NEED_RESCHED 0x80000000
+/*
+ * We use the MSB for PREEMPT_NEED_RESCHED mostly because it is available.
+ */
+#define PREEMPT_NEED_RESCHED (~(((unsigned long)-1L) >> 1))
+
+#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+#define __pc_dec "decq"
+#define __pc_op(op, ...) raw_cpu_##op##_8(__VA_ARGS__)
+#else
+#define __pc_dec "decl"
+#define __pc_op(op, ...) raw_cpu_##op##_4(__VA_ARGS__)
+#endif
/*
* We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
@@ -24,18 +34,26 @@ DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);
*/
static __always_inline int preempt_count(void)
{
- return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
+ return __pc_op(read, __preempt_count) & ~PREEMPT_NEED_RESCHED;
}
-static __always_inline void preempt_count_set(int pc)
+/*
+ * unsigned long preempt count parameter works for both 32bit and 64bit cases:
+ *
+ * - For 32bit, "int" (the return of preempt_count()) and "unsigned long" have
+ * the same size.
+ * - For 64bit, the effective bits of a preempt count sit in 32bit, and we
+ * preserve the NEED_RESCHED bit from the old count.
+ */
+static __always_inline void preempt_count_set(unsigned long pc)
{
- int old, new;
+ unsigned long old, new;
- old = raw_cpu_read_4(__preempt_count);
+ old = __pc_op(read, __preempt_count);
do {
new = (old & PREEMPT_NEED_RESCHED) |
(pc & ~PREEMPT_NEED_RESCHED);
- } while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new));
+ } while (!__pc_op(try_cmpxchg, __preempt_count, &old, new));
}
/*
@@ -58,17 +76,17 @@ static __always_inline void preempt_count_set(int pc)
static __always_inline void set_preempt_need_resched(void)
{
- raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
+ __pc_op(and, __preempt_count, ~PREEMPT_NEED_RESCHED);
}
static __always_inline void clear_preempt_need_resched(void)
{
- raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
+ __pc_op(or, __preempt_count, PREEMPT_NEED_RESCHED);
}
static __always_inline bool test_preempt_need_resched(void)
{
- return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
+ return !(__pc_op(read, __preempt_count) & PREEMPT_NEED_RESCHED);
}
/*
@@ -77,22 +95,22 @@ static __always_inline bool test_preempt_need_resched(void)
static __always_inline void __preempt_count_add(int val)
{
- raw_cpu_add_4(__preempt_count, val);
+ __pc_op(add, __preempt_count, val);
}
static __always_inline void __preempt_count_sub(int val)
{
- raw_cpu_add_4(__preempt_count, -val);
+ __pc_op(add, __preempt_count, -val);
}
static __always_inline int __preempt_count_add_return(int val)
{
- return raw_cpu_add_return_4(__preempt_count, val);
+ return __pc_op(add_return, __preempt_count, val);
}
static __always_inline int __preempt_count_sub_return(int val)
{
- return raw_cpu_add_return_4(__preempt_count, -val);
+ return __pc_op(add_return, __preempt_count, -val);
}
/*
@@ -102,7 +120,7 @@ static __always_inline int __preempt_count_sub_return(int val)
*/
static __always_inline bool __preempt_count_dec_and_test(void)
{
- return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e,
+ return GEN_UNARY_RMWcc(__pc_dec, __my_cpu_var(__preempt_count), e,
__percpu_arg([var]));
}
@@ -111,7 +129,7 @@ static __always_inline bool __preempt_count_dec_and_test(void)
*/
static __always_inline bool should_resched(int preempt_offset)
{
- return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
+ return unlikely(__pc_op(read, __preempt_count) == preempt_offset);
}
#ifdef CONFIG_PREEMPTION
@@ -158,4 +176,7 @@ do { \
#endif /* PREEMPTION */
+#undef __pc_op
+#undef __pc_dec
+
#endif /* __ASM_PREEMPT_H */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a3df21d..73a6d9f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -2236,7 +2236,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
EXPORT_PER_CPU_SYMBOL(const_current_task);
-DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;
+DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 8d48955..73b48dd 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -10,8 +10,6 @@
#include <linux/vtime.h>
#include <asm/hardirq.h>
-DECLARE_PER_CPU(unsigned int, nmi_nesting);
-
extern void synchronize_irq(unsigned int irq);
extern bool synchronize_hardirq(unsigned int irq);
@@ -94,6 +92,37 @@ void irq_exit_rcu(void);
#define arch_nmi_exit() do { } while (0)
#endif
+#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+static __always_inline void __preempt_count_nmi_enter(void)
+{
+ __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);
+}
+
+static __always_inline void __preempt_count_nmi_exit(void)
+{
+ __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);
+}
+#else
+DECLARE_PER_CPU(unsigned int, nmi_nesting);
+
+#define __preempt_count_nmi_enter() \
+ do { \
+ __preempt_count_add(HARDIRQ_OFFSET); \
+ /* Maximum NMI nesting is 15. */ \
+ BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
+ __this_cpu_inc(nmi_nesting); \
+ preempt_count_set(preempt_count() | NMI_MASK); \
+ } while (0)
+
+#define __preempt_count_nmi_exit() \
+ do { \
+ __preempt_count_sub(HARDIRQ_OFFSET); \
+ if (!__this_cpu_dec_return(nmi_nesting)) \
+ preempt_count_set(preempt_count() & ~NMI_MASK); \
+ } while (0)
+
+#endif
+
/*
* NMI vs Tracing
* --------------
@@ -110,18 +139,14 @@ void irq_exit_rcu(void);
do { \
lockdep_off(); \
arch_nmi_enter(); \
- /* Maximum NMI nesting is 15. */ \
- BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
- __this_cpu_inc(nmi_nesting); \
- __preempt_count_add(HARDIRQ_OFFSET); \
- preempt_count_set(preempt_count() | NMI_MASK); \
+ __preempt_count_nmi_enter(); \
} while (0)
#define nmi_enter() \
do { \
__nmi_enter(); \
lockdep_hardirq_enter(); \
- ct_nmi_enter(); \
+ ct_nmi_enter(); \
instrumentation_begin(); \
ftrace_nmi_enter(); \
instrumentation_end(); \
@@ -129,12 +154,8 @@ void irq_exit_rcu(void);
#define __nmi_exit() \
do { \
- unsigned int nesting; \
BUG_ON(!in_nmi()); \
- __preempt_count_sub(HARDIRQ_OFFSET); \
- nesting = __this_cpu_dec_return(nmi_nesting); \
- if (!nesting) \
- preempt_count_set(preempt_count() & ~NMI_MASK); \
+ __preempt_count_nmi_exit(); \
arch_nmi_exit(); \
lockdep_on(); \
} while (0)
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 33fc4c8..8299657 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -34,14 +34,31 @@
* SOFTIRQ_MASK: 0x0000ff00
* HARDIRQ_DISABLE_MASK: 0x00ff0000
* HARDIRQ_MASK: 0x0f000000
+ *
+ * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a
+ * separate word and that allows 64bit load-store architectures to 'set'
+ * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric
+ * modifications used on preempt_count and still load the whole thing
+ * (single-copy) atomically, without having to resort to full atomic
+ * operations.
+ *
+ * Because of the above, NMI_MASK bits are different depending on
+ * HAS_SEPARATE_PREEMPT_RESCHED_BITS:
+ *
+ * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n:
+ *
* NMI_MASK: 0x10000000
* PREEMPT_NEED_RESCHED: 0x80000000
+ *
+ * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y:
+ * NMI_MASK: 0xf0000000
+ * (PREEMPT_NEED_RESCHED is in a different word)
*/
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
#define HARDIRQ_DISABLE_BITS 8
#define HARDIRQ_BITS 4
-#define NMI_BITS 1
+#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS))
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
@@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void)
* preempt_count() is commonly implemented with READ_ONCE().
*/
-#define nmi_count() (preempt_count() & NMI_MASK)
-#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
+#define nmi_count() (preempt_count() & NMI_MASK)
+#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
#ifdef CONFIG_PREEMPT_RT
# define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK)
# define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index 88c594c..35f546a 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK
config PREEMPT_COUNT
bool
+config HAS_SEPARATE_PREEMPT_RESCHED_BITS
+ bool
+ depends on PREEMPT_COUNT && 64BIT
+
config PREEMPTION
bool
select PREEMPT_COUNT
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9b3f176..6d88343 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5973,8 +5973,13 @@ void preempt_count_add(int val)
#ifdef CONFIG_DEBUG_PREEMPT
/*
* Underflow?
+ *
+ * Cannot detect underflow based on the current preempt_count() value
+ * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32
+ * bits.
*/
- if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
+ if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) &&
+ DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
return;
#endif
__preempt_count_add(val);
@@ -6006,7 +6011,10 @@ void preempt_count_sub(int val)
/*
* Underflow?
*/
- if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
+ unsigned int uval = val;
+ unsigned int pc = preempt_count();
+
+ if (DEBUG_LOCKS_WARN_ON(pc - uval > pc))
return;
/*
* Is the spinlock portion underflowing?
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 0c9b226..7980a4a 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -103,7 +103,13 @@ void _local_interrupt_enable(void)
}
EXPORT_SYMBOL(_local_interrupt_enable);
+#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+/*
+ * Any 32bit architecture that still cares about performance should
+ * probably ensure this is near preempt_count.
+ */
DEFINE_PER_CPU(unsigned int, nmi_nesting);
+#endif
/*
* SOFTIRQ_OFFSET usage:
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index bfafe12..c3d976c 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures;
static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
{
- int saved_preempt_count = preempt_count();
+ long saved_preempt_count = preempt_count();
#ifdef CONFIG_PREEMPT_RT
int saved_mgd_count = current->migration_disabled;
int saved_rcu_count = current->rcu_read_lock_nesting;
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
` (2 preceding siblings ...)
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
@ 2026-08-10 8:57 ` tip-bot2 for Boqun Feng
3 siblings, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-10 8:57 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Peter Zijlstra, Boqun Feng, x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 3b0e2a22d4086ed7c1294584c4417f7d19f1ac67
Gitweb: https://git.kernel.org/tip/3b0e2a22d4086ed7c1294584c4417f7d19f1ac67
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:32 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Mon, 10 Aug 2026 10:50:19 +02:00
preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS
With the changes that enable preempt count to track IRQ disabling
nesting, we don't have enough bits in 32-bit preempt count
implementation, as a result we move NMI nesting bits out of the 32-bit
preempt count. However on the architectures that can support 64-bit
preempt count implementation, we can keep the NMI nesting bits in the
32-bit preempt count and avoid maintaining NMI nesting bits outside of
the same cache line.
Therefore HAS_SEPARATE_PREEMPT_RESCHED_BITS is introduced to allow
architectures to select this. Note that under this Kconfig, preempt
count is maintained in a 64-bit word however preempt_count() still
remains as an int because all the effective bits still fit in
(previously we mask out NEED_RESCHED bit in preempt_count()). This
should make no functional changes for existing preempt_count() users.
Enable this for x86_64 along with the introduction of the Kconfig.
[boqun: Undo the __preempt_count_{add,sub}() optimization in 32-bit
preempt count since it may introduce {over,under}flow]
Originally-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-11-boqun@kernel.org
---
arch/x86/Kconfig | 1 +-
arch/x86/include/asm/preempt.h | 55 ++++++++++++++++++++++-----------
arch/x86/kernel/cpu/common.c | 2 +-
include/linux/hardirq.h | 47 ++++++++++++++++++++--------
include/linux/preempt.h | 23 ++++++++++++--
kernel/Kconfig.preempt | 4 ++-
kernel/sched/core.c | 12 +++++--
kernel/softirq.c | 6 ++++-
lib/locking-selftest.c | 2 +-
9 files changed, 115 insertions(+), 37 deletions(-)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index bdad90f..6a7067d 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -326,6 +326,7 @@ config X86
select USER_STACKTRACE_SUPPORT
select HAVE_ARCH_KCSAN if X86_64
select PROC_PID_ARCH_STATUS if PROC_FS
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS if X86_64 && PREEMPT_COUNT
select HAVE_ARCH_NODE_DEV_GROUP if X86_SGX
select FUNCTION_ALIGNMENT_16B if X86_64 || X86_ALIGNMENT_16
select FUNCTION_ALIGNMENT_4B
diff --git a/arch/x86/include/asm/preempt.h b/arch/x86/include/asm/preempt.h
index 1220656..fafb6f8 100644
--- a/arch/x86/include/asm/preempt.h
+++ b/arch/x86/include/asm/preempt.h
@@ -7,10 +7,20 @@
#include <linux/static_call_types.h>
-DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);
+DECLARE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count);
-/* We use the MSB mostly because its available */
-#define PREEMPT_NEED_RESCHED 0x80000000
+/*
+ * We use the MSB for PREEMPT_NEED_RESCHED mostly because it is available.
+ */
+#define PREEMPT_NEED_RESCHED (~(((unsigned long)-1L) >> 1))
+
+#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+#define __pc_dec "decq"
+#define __pc_op(op, ...) raw_cpu_##op##_8(__VA_ARGS__)
+#else
+#define __pc_dec "decl"
+#define __pc_op(op, ...) raw_cpu_##op##_4(__VA_ARGS__)
+#endif
/*
* We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
@@ -24,18 +34,26 @@ DECLARE_PER_CPU_CACHE_HOT(int, __preempt_count);
*/
static __always_inline int preempt_count(void)
{
- return raw_cpu_read_4(__preempt_count) & ~PREEMPT_NEED_RESCHED;
+ return __pc_op(read, __preempt_count) & ~PREEMPT_NEED_RESCHED;
}
-static __always_inline void preempt_count_set(int pc)
+/*
+ * unsigned long preempt count parameter works for both 32bit and 64bit cases:
+ *
+ * - For 32bit, "int" (the return of preempt_count()) and "unsigned long" have
+ * the same size.
+ * - For 64bit, the effective bits of a preempt count sit in 32bit, and we
+ * preserve the NEED_RESCHED bit from the old count.
+ */
+static __always_inline void preempt_count_set(unsigned long pc)
{
- int old, new;
+ unsigned long old, new;
- old = raw_cpu_read_4(__preempt_count);
+ old = __pc_op(read, __preempt_count);
do {
new = (old & PREEMPT_NEED_RESCHED) |
(pc & ~PREEMPT_NEED_RESCHED);
- } while (!raw_cpu_try_cmpxchg_4(__preempt_count, &old, new));
+ } while (!__pc_op(try_cmpxchg, __preempt_count, &old, new));
}
/*
@@ -58,17 +76,17 @@ static __always_inline void preempt_count_set(int pc)
static __always_inline void set_preempt_need_resched(void)
{
- raw_cpu_and_4(__preempt_count, ~PREEMPT_NEED_RESCHED);
+ __pc_op(and, __preempt_count, ~PREEMPT_NEED_RESCHED);
}
static __always_inline void clear_preempt_need_resched(void)
{
- raw_cpu_or_4(__preempt_count, PREEMPT_NEED_RESCHED);
+ __pc_op(or, __preempt_count, PREEMPT_NEED_RESCHED);
}
static __always_inline bool test_preempt_need_resched(void)
{
- return !(raw_cpu_read_4(__preempt_count) & PREEMPT_NEED_RESCHED);
+ return !(__pc_op(read, __preempt_count) & PREEMPT_NEED_RESCHED);
}
/*
@@ -77,22 +95,22 @@ static __always_inline bool test_preempt_need_resched(void)
static __always_inline void __preempt_count_add(int val)
{
- raw_cpu_add_4(__preempt_count, val);
+ __pc_op(add, __preempt_count, val);
}
static __always_inline void __preempt_count_sub(int val)
{
- raw_cpu_add_4(__preempt_count, -val);
+ __pc_op(add, __preempt_count, -val);
}
static __always_inline int __preempt_count_add_return(int val)
{
- return raw_cpu_add_return_4(__preempt_count, val);
+ return __pc_op(add_return, __preempt_count, val);
}
static __always_inline int __preempt_count_sub_return(int val)
{
- return raw_cpu_add_return_4(__preempt_count, -val);
+ return __pc_op(add_return, __preempt_count, -val);
}
/*
@@ -102,7 +120,7 @@ static __always_inline int __preempt_count_sub_return(int val)
*/
static __always_inline bool __preempt_count_dec_and_test(void)
{
- return GEN_UNARY_RMWcc("decl", __my_cpu_var(__preempt_count), e,
+ return GEN_UNARY_RMWcc(__pc_dec, __my_cpu_var(__preempt_count), e,
__percpu_arg([var]));
}
@@ -111,7 +129,7 @@ static __always_inline bool __preempt_count_dec_and_test(void)
*/
static __always_inline bool should_resched(int preempt_offset)
{
- return unlikely(raw_cpu_read_4(__preempt_count) == preempt_offset);
+ return unlikely(__pc_op(read, __preempt_count) == preempt_offset);
}
#ifdef CONFIG_PREEMPTION
@@ -158,4 +176,7 @@ do { \
#endif /* PREEMPTION */
+#undef __pc_op
+#undef __pc_dec
+
#endif /* __ASM_PREEMPT_H */
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index a3df21d..73a6d9f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -2236,7 +2236,7 @@ DEFINE_PER_CPU_CACHE_HOT(struct task_struct *, current_task) = &init_task;
EXPORT_PER_CPU_SYMBOL(current_task);
EXPORT_PER_CPU_SYMBOL(const_current_task);
-DEFINE_PER_CPU_CACHE_HOT(int, __preempt_count) = INIT_PREEMPT_COUNT;
+DEFINE_PER_CPU_CACHE_HOT(unsigned long, __preempt_count) = INIT_PREEMPT_COUNT;
EXPORT_PER_CPU_SYMBOL(__preempt_count);
DEFINE_PER_CPU_CACHE_HOT(unsigned long, cpu_current_top_of_stack) = TOP_OF_INIT_STACK;
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 8d48955..73b48dd 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -10,8 +10,6 @@
#include <linux/vtime.h>
#include <asm/hardirq.h>
-DECLARE_PER_CPU(unsigned int, nmi_nesting);
-
extern void synchronize_irq(unsigned int irq);
extern bool synchronize_hardirq(unsigned int irq);
@@ -94,6 +92,37 @@ void irq_exit_rcu(void);
#define arch_nmi_exit() do { } while (0)
#endif
+#ifdef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+static __always_inline void __preempt_count_nmi_enter(void)
+{
+ __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET);
+}
+
+static __always_inline void __preempt_count_nmi_exit(void)
+{
+ __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET);
+}
+#else
+DECLARE_PER_CPU(unsigned int, nmi_nesting);
+
+#define __preempt_count_nmi_enter() \
+ do { \
+ __preempt_count_add(HARDIRQ_OFFSET); \
+ /* Maximum NMI nesting is 15. */ \
+ BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
+ __this_cpu_inc(nmi_nesting); \
+ preempt_count_set(preempt_count() | NMI_MASK); \
+ } while (0)
+
+#define __preempt_count_nmi_exit() \
+ do { \
+ __preempt_count_sub(HARDIRQ_OFFSET); \
+ if (!__this_cpu_dec_return(nmi_nesting)) \
+ preempt_count_set(preempt_count() & ~NMI_MASK); \
+ } while (0)
+
+#endif
+
/*
* NMI vs Tracing
* --------------
@@ -110,18 +139,14 @@ void irq_exit_rcu(void);
do { \
lockdep_off(); \
arch_nmi_enter(); \
- /* Maximum NMI nesting is 15. */ \
- BUG_ON(__this_cpu_read(nmi_nesting) >= 15); \
- __this_cpu_inc(nmi_nesting); \
- __preempt_count_add(HARDIRQ_OFFSET); \
- preempt_count_set(preempt_count() | NMI_MASK); \
+ __preempt_count_nmi_enter(); \
} while (0)
#define nmi_enter() \
do { \
__nmi_enter(); \
lockdep_hardirq_enter(); \
- ct_nmi_enter(); \
+ ct_nmi_enter(); \
instrumentation_begin(); \
ftrace_nmi_enter(); \
instrumentation_end(); \
@@ -129,12 +154,8 @@ void irq_exit_rcu(void);
#define __nmi_exit() \
do { \
- unsigned int nesting; \
BUG_ON(!in_nmi()); \
- __preempt_count_sub(HARDIRQ_OFFSET); \
- nesting = __this_cpu_dec_return(nmi_nesting); \
- if (!nesting) \
- preempt_count_set(preempt_count() & ~NMI_MASK); \
+ __preempt_count_nmi_exit(); \
arch_nmi_exit(); \
lockdep_on(); \
} while (0)
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 33fc4c8..8299657 100644
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -34,14 +34,31 @@
* SOFTIRQ_MASK: 0x0000ff00
* HARDIRQ_DISABLE_MASK: 0x00ff0000
* HARDIRQ_MASK: 0x0f000000
+ *
+ * When HAS_SEPARATE_PREEMPT_RESCHED_BITS=y, PREEMPT_NEED_RESCHED is put in a
+ * separate word and that allows 64bit load-store architectures to 'set'
+ * PREEMPT_NEED_RESCHED without messing up the otherwise symmetric
+ * modifications used on preempt_count and still load the whole thing
+ * (single-copy) atomically, without having to resort to full atomic
+ * operations.
+ *
+ * Because of the above, NMI_MASK bits are different depending on
+ * HAS_SEPARATE_PREEMPT_RESCHED_BITS:
+ *
+ * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=n:
+ *
* NMI_MASK: 0x10000000
* PREEMPT_NEED_RESCHED: 0x80000000
+ *
+ * - HAS_SEPARATE_PREEMPT_RESCHED_BITS=y:
+ * NMI_MASK: 0xf0000000
+ * (PREEMPT_NEED_RESCHED is in a different word)
*/
#define PREEMPT_BITS 8
#define SOFTIRQ_BITS 8
#define HARDIRQ_DISABLE_BITS 8
#define HARDIRQ_BITS 4
-#define NMI_BITS 1
+#define NMI_BITS (1 + 3*IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS))
#define PREEMPT_SHIFT 0
#define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS)
@@ -116,8 +133,8 @@ static __always_inline unsigned char interrupt_context_level(void)
* preempt_count() is commonly implemented with READ_ONCE().
*/
-#define nmi_count() (preempt_count() & NMI_MASK)
-#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
+#define nmi_count() (preempt_count() & NMI_MASK)
+#define hardirq_count() (preempt_count() & HARDIRQ_MASK)
#ifdef CONFIG_PREEMPT_RT
# define softirq_count() (current->softirq_disable_cnt & SOFTIRQ_MASK)
# define irq_count() ((preempt_count() & (NMI_MASK | HARDIRQ_MASK)) | softirq_count())
diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
index 88c594c..35f546a 100644
--- a/kernel/Kconfig.preempt
+++ b/kernel/Kconfig.preempt
@@ -122,6 +122,10 @@ config PREEMPT_RT_NEEDS_BH_LOCK
config PREEMPT_COUNT
bool
+config HAS_SEPARATE_PREEMPT_RESCHED_BITS
+ bool
+ depends on PREEMPT_COUNT && 64BIT
+
config PREEMPTION
bool
select PREEMPT_COUNT
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 9b3f176..6d88343 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -5973,8 +5973,13 @@ void preempt_count_add(int val)
#ifdef CONFIG_DEBUG_PREEMPT
/*
* Underflow?
+ *
+ * Cannot detect underflow based on the current preempt_count() value
+ * if using HAS_SEPARATE_PREEMPT_RESCHED_BITS because preempt count takes all 32
+ * bits.
*/
- if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
+ if (!IS_ENABLED(CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS) &&
+ DEBUG_LOCKS_WARN_ON((preempt_count() < 0)))
return;
#endif
__preempt_count_add(val);
@@ -6006,7 +6011,10 @@ void preempt_count_sub(int val)
/*
* Underflow?
*/
- if (DEBUG_LOCKS_WARN_ON(val > preempt_count()))
+ unsigned int uval = val;
+ unsigned int pc = preempt_count();
+
+ if (DEBUG_LOCKS_WARN_ON(pc - uval > pc))
return;
/*
* Is the spinlock portion underflowing?
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 0c9b226..7980a4a 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -103,7 +103,13 @@ void _local_interrupt_enable(void)
}
EXPORT_SYMBOL(_local_interrupt_enable);
+#ifndef CONFIG_HAS_SEPARATE_PREEMPT_RESCHED_BITS
+/*
+ * Any 32bit architecture that still cares about performance should
+ * probably ensure this is near preempt_count.
+ */
DEFINE_PER_CPU(unsigned int, nmi_nesting);
+#endif
/*
* SOFTIRQ_OFFSET usage:
diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c
index bfafe12..c3d976c 100644
--- a/lib/locking-selftest.c
+++ b/lib/locking-selftest.c
@@ -1429,7 +1429,7 @@ static int unexpected_testcase_failures;
static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask)
{
- int saved_preempt_count = preempt_count();
+ long saved_preempt_count = preempt_count();
#ifdef CONFIG_PREEMPT_RT
int saved_mgd_count = current->migration_disabled;
int saved_rcu_count = current->rcu_read_lock_nesting;
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 11/17] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (9 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 10/17] preempt: Introduce HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
2026-08-04 16:14 ` [PATCH v4 12/17] s390/preempt: " Boqun Feng
` (5 subsequent siblings)
16 siblings, 2 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
Arm64 already uses 64-bit preempt count and the need reschedule bit is
maintained in a separate 32-bit word from the preempt count. Therefore
preempt count has enough bits to represent 16 levels of NMI nesting,
hence enable it for arm64. This saves a per-CPU variable and additional
instructions in the NMI path.
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
arch/arm64/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b3afe0688919..349c3533cd1e 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -247,6 +247,7 @@ config ARM64
select PCI_SYSCALL if PCI
select POWER_RESET
select POWER_SUPPLY
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS
select SPARSE_IRQ
select SWIOTLB
select SYSCTL_EXCEPTION_TRACE
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 11/17] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
@ 2026-08-08 20:48 ` tip-bot2 for Boqun Feng
2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: c9bae17d96b93ce05ff2eeef08165c24c7f5e7e3
Gitweb: https://git.kernel.org/tip/c9bae17d96b93ce05ff2eeef08165c24c7f5e7e3
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:33 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:08 +02:00
arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
Arm64 already uses 64-bit preempt count and the need reschedule bit is
maintained in a separate 32-bit word from the preempt count. Therefore
preempt count has enough bits to represent 16 levels of NMI nesting,
hence enable it for arm64. This saves a per-CPU variable and additional
instructions in the NMI path.
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-12-boqun@kernel.org
---
arch/arm64/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b3afe06..349c353 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -247,6 +247,7 @@ config ARM64
select PCI_SYSCALL if PCI
select POWER_RESET
select POWER_SUPPLY
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS
select SPARSE_IRQ
select SWIOTLB
select SYSCTL_EXCEPTION_TRACE
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [tip: locking/core] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 11/17] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Boqun Feng
@ 2026-08-10 8:57 ` tip-bot2 for Boqun Feng
1 sibling, 0 replies; 66+ messages in thread
From: tip-bot2 for Boqun Feng @ 2026-08-10 8:57 UTC (permalink / raw)
To: linux-tip-commits; +Cc: Boqun Feng, Peter Zijlstra (Intel), x86, linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 264bbd32a31ba80e634bec5c1df8514c0e97fe7e
Gitweb: https://git.kernel.org/tip/264bbd32a31ba80e634bec5c1df8514c0e97fe7e
Author: Boqun Feng <boqun@kernel.org>
AuthorDate: Tue, 04 Aug 2026 09:14:33 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Mon, 10 Aug 2026 10:50:20 +02:00
arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
Arm64 already uses 64-bit preempt count and the need reschedule bit is
maintained in a separate 32-bit word from the preempt count. Therefore
preempt count has enough bits to represent 16 levels of NMI nesting,
hence enable it for arm64. This saves a per-CPU variable and additional
instructions in the NMI path.
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-12-boqun@kernel.org
---
arch/arm64/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index b3afe06..349c353 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -247,6 +247,7 @@ config ARM64
select PCI_SYSCALL if PCI
select POWER_RESET
select POWER_SUPPLY
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS
select SPARSE_IRQ
select SWIOTLB
select SYSCTL_EXCEPTION_TRACE
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 12/17] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (10 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 11/17] arm64: sched/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-04 20:27 ` Shrikanth Hegde
` (2 more replies)
2026-08-04 16:14 ` [PATCH v4 13/17] rust: Introduce interrupt module Boqun Feng
` (4 subsequent siblings)
16 siblings, 3 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux,
Heiko Carstens
From: Heiko Carstens <hca@linux.ibm.com>
Convert s390's preempt_count to 64 bit, and change the preempt
primitives accordingly.
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
[boqun: Apply the corrected comment for asm block]
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
arch/s390/Kconfig | 1 +
arch/s390/include/asm/lowcore.h | 13 +++++++---
arch/s390/include/asm/preempt.h | 43 +++++++++++++++------------------
3 files changed, 30 insertions(+), 27 deletions(-)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 84404e6778d5..378fcd2b6181 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -273,6 +273,7 @@ config S390
select PCI_MSI if PCI
select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
select PCI_QUIRKS if PCI
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS
select SPARSE_IRQ
select SWIOTLB
select SYSCTL_EXCEPTION_TRACE
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index 3b3ecc647993..5cef215d30e7 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -160,10 +160,15 @@ struct lowcore {
/* SMP info area */
__u32 cpu_nr; /* 0x03a0 */
__u32 softirq_pending; /* 0x03a4 */
- __s32 preempt_count; /* 0x03a8 */
- __u32 spinlock_lockval; /* 0x03ac */
- __u32 spinlock_index; /* 0x03b0 */
- __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */
+ union {
+ struct {
+ __u32 need_resched; /* 0x03a8 */
+ __u32 count; /* 0x03ac */
+ } preempt;
+ __u64 preempt_count; /* 0x03a8 */
+ };
+ __u32 spinlock_lockval; /* 0x03b0 */
+ __u32 spinlock_index; /* 0x03b4 */
__u64 percpu_offset; /* 0x03b8 */
__u8 percpu_register; /* 0x03c0 */
__u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */
diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h
index 0a25d4648b4c..5560d5fca2a3 100644
--- a/arch/s390/include/asm/preempt.h
+++ b/arch/s390/include/asm/preempt.h
@@ -8,11 +8,8 @@
#include <asm/cmpxchg.h>
#include <asm/march.h>
-/*
- * Use MSB so it is possible to read preempt_count with LLGT which
- * reads the least significant 31 bits with a single instruction.
- */
-#define PREEMPT_NEED_RESCHED 0x80000000
+/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */
+#define PREEMPT_NEED_RESCHED 0x8000000000000000UL
/*
* We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
@@ -26,25 +23,25 @@
*/
static __always_inline int preempt_count(void)
{
- unsigned long lc_preempt, count;
+ unsigned long lc_preempt;
+ int count;
- BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) != sizeof(int));
- lc_preempt = offsetof(struct lowcore, preempt_count);
- /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */
+ lc_preempt = offsetof(struct lowcore, preempt.count);
+ /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED) */
asm_inline(
- ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n",
- "llgt %[count],%[offalt](%%r0)\n",
+ ALTERNATIVE("ly %[count],%[offzero](%%r0)\n",
+ "ly %[count],%[offalt](%%r0)\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: [count] "=d" (count)
: [offzero] "i" (lc_preempt),
[offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS),
- "m" (((struct lowcore *)0)->preempt_count));
+ "m" (((struct lowcore *)0)->preempt.count));
return count;
}
-static __always_inline void preempt_count_set(int pc)
+static __always_inline void preempt_count_set(unsigned long pc)
{
- int old, new;
+ unsigned long old, new;
old = READ_ONCE(get_lowcore()->preempt_count);
do {
@@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc)
static __always_inline void set_preempt_need_resched(void)
{
- __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
+ __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count);
}
static __always_inline void clear_preempt_need_resched(void)
{
- __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
+ __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count);
}
static __always_inline bool test_preempt_need_resched(void)
@@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val)
lc_preempt = offsetof(struct lowcore, preempt_count);
asm_inline(
- ALTERNATIVE("asi %[offzero](%%r0),%[val]\n",
- "asi %[offalt](%%r0),%[val]\n",
+ ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n",
+ "agsi %[offalt](%%r0),%[val]\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: "+m" (((struct lowcore *)0)->preempt_count)
: [offzero] "i" (lc_preempt), [val] "i" (val),
@@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val)
return;
}
}
- __atomic_add(val, &get_lowcore()->preempt_count);
+ __atomic64_add(val, (long *)&get_lowcore()->preempt_count);
}
static __always_inline void __preempt_count_sub(int val)
@@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_test(void)
lc_preempt = offsetof(struct lowcore, preempt_count);
asm_inline(
- ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n",
- "alsi %[offalt](%%r0),%[val]\n",
+ ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n",
+ "algsi %[offalt](%%r0),%[val]\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: "=@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count)
: [offzero] "i" (lc_preempt), [val] "i" (-1),
[offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS));
return (cc == 0) || (cc == 2);
#else
- return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count);
+ return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_count);
#endif
}
@@ -141,7 +138,7 @@ static __always_inline bool should_resched(int preempt_offset)
static __always_inline int __preempt_count_add_return(int val)
{
- return val + __atomic_add(val, &get_lowcore()->preempt_count);
+ return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count);
}
static __always_inline int __preempt_count_sub_return(int val)
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* Re: [PATCH v4 12/17] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 12/17] s390/preempt: " Boqun Feng
@ 2026-08-04 20:27 ` Shrikanth Hegde
2026-08-05 9:42 ` Peter Zijlstra
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Heiko Carstens
2026-08-10 8:57 ` tip-bot2 for Heiko Carstens
2 siblings, 1 reply; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-04 20:27 UTC (permalink / raw)
To: Boqun Feng, Madhavan Srinivasan, Christophe Leroy (CS GROUP)
Cc: Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux, Heiko Carstens,
Peter Zijlstra
+maddy, christophe,
On 8/4/26 9:44 PM, Boqun Feng wrote:
> From: Heiko Carstens <hca@linux.ibm.com>
>
> Convert s390's preempt_count to 64 bit, and change the preempt
> primitives accordingly.
>
> + union {
> + struct {
> + __u32 need_resched; /* 0x03a8 */
> + __u32 count; /* 0x03ac */
> + } preempt;
> + __u64 preempt_count; /* 0x03a8 */
> + };
This is interesting. So arm64, s390 will have similar implementation
w.r.t to preempt_count.
I guess PoC that i was trying for ppc64 is worth pursuing even more,
since ppc64 is one missing major arch which still uses old method
of querying tif_need_resched.
With similar change ppc64 can set HAS_SEPARATE_PREEMPT_RESCHED_BITS too.
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 12/17] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 20:27 ` Shrikanth Hegde
@ 2026-08-05 9:42 ` Peter Zijlstra
2026-08-05 12:37 ` Shrikanth Hegde
0 siblings, 1 reply; 66+ messages in thread
From: Peter Zijlstra @ 2026-08-05 9:42 UTC (permalink / raw)
To: Shrikanth Hegde
Cc: Boqun Feng, Madhavan Srinivasan, Christophe Leroy (CS GROUP),
Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux, Heiko Carstens
On Wed, Aug 05, 2026 at 01:57:04AM +0530, Shrikanth Hegde wrote:
> +maddy, christophe,
>
> On 8/4/26 9:44 PM, Boqun Feng wrote:
> > From: Heiko Carstens <hca@linux.ibm.com>
> >
> > Convert s390's preempt_count to 64 bit, and change the preempt
> > primitives accordingly.
> >
>
> > + union {
> > + struct {
> > + __u32 need_resched; /* 0x03a8 */
> > + __u32 count; /* 0x03ac */
> > + } preempt;
> > + __u64 preempt_count; /* 0x03a8 */
> > + };
>
>
> This is interesting. So arm64, s390 will have similar implementation
> w.r.t to preempt_count.
>
> I guess PoC that i was trying for ppc64 is worth pursuing even more,
> since ppc64 is one missing major arch which still uses old method
> of querying tif_need_resched.
> With similar change ppc64 can set HAS_SEPARATE_PREEMPT_RESCHED_BITS too.
Yes. PPC should be able to the same thing ARM64 does.
^ permalink raw reply [flat|nested] 66+ messages in thread* Re: [PATCH v4 12/17] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-05 9:42 ` Peter Zijlstra
@ 2026-08-05 12:37 ` Shrikanth Hegde
0 siblings, 0 replies; 66+ messages in thread
From: Shrikanth Hegde @ 2026-08-05 12:37 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Boqun Feng, Madhavan Srinivasan, Christophe Leroy (CS GROUP),
Ingo Molnar, Will Deacon, Waiman Long, Gary Guo, Alice Ryhl,
Lyude Paul, Daniel Almeida, Onur Özkan, Miguel Ojeda,
Danilo Krummrich, linux-kernel, rust-for-linux, Heiko Carstens
Hi Peter.
On 8/5/26 3:12 PM, Peter Zijlstra wrote:
> On Wed, Aug 05, 2026 at 01:57:04AM +0530, Shrikanth Hegde wrote:
>> +maddy, christophe,
>>
>> On 8/4/26 9:44 PM, Boqun Feng wrote:
>>> From: Heiko Carstens <hca@linux.ibm.com>
>>>
>>> Convert s390's preempt_count to 64 bit, and change the preempt
>>> primitives accordingly.
>>>
>>
>>> + union {
>>> + struct {
>>> + __u32 need_resched; /* 0x03a8 */
>>> + __u32 count; /* 0x03ac */
>>> + } preempt;
>>> + __u64 preempt_count; /* 0x03a8 */
>>> + };
>>
>>
>> This is interesting. So arm64, s390 will have similar implementation
>> w.r.t to preempt_count.
>>
>> I guess PoC that i was trying for ppc64 is worth pursuing even more,
>> since ppc64 is one missing major arch which still uses old method
>> of querying tif_need_resched.
>> With similar change ppc64 can set HAS_SEPARATE_PREEMPT_RESCHED_BITS too.
>
> Yes. PPC should be able to the same thing ARM64 does.
Yes. I am thinking of moving it to PACA instead of keeping it in thread_info
as PACA is a natural per-CPU storage.
If i hit any major hurdle there, will use thread info similar to arm64.
^ permalink raw reply [flat|nested] 66+ messages in thread
* [tip: locking/core] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 12/17] s390/preempt: " Boqun Feng
2026-08-04 20:27 ` Shrikanth Hegde
@ 2026-08-08 20:48 ` tip-bot2 for Heiko Carstens
2026-08-10 8:57 ` tip-bot2 for Heiko Carstens
2 siblings, 0 replies; 66+ messages in thread
From: tip-bot2 for Heiko Carstens @ 2026-08-08 20:48 UTC (permalink / raw)
To: linux-tip-commits
Cc: Heiko Carstens, Boqun Feng, Peter Zijlstra (Intel), x86,
linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: c915500b3a095ee7647337f0ed13efe8b1e6e99e
Gitweb: https://git.kernel.org/tip/c915500b3a095ee7647337f0ed13efe8b1e6e99e
Author: Heiko Carstens <hca@linux.ibm.com>
AuthorDate: Tue, 04 Aug 2026 09:14:34 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Sat, 08 Aug 2026 22:44:09 +02:00
s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
Convert s390's preempt_count to 64 bit, and change the preempt
primitives accordingly.
[boqun: Apply the corrected comment for asm block]
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-13-boqun@kernel.org
---
arch/s390/Kconfig | 1 +-
arch/s390/include/asm/lowcore.h | 13 ++++++----
arch/s390/include/asm/preempt.h | 43 ++++++++++++++------------------
3 files changed, 30 insertions(+), 27 deletions(-)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 84404e6..378fcd2 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -273,6 +273,7 @@ config S390
select PCI_MSI if PCI
select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
select PCI_QUIRKS if PCI
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS
select SPARSE_IRQ
select SWIOTLB
select SYSCTL_EXCEPTION_TRACE
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index 3b3ecc6..5cef215 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -160,10 +160,15 @@ struct lowcore {
/* SMP info area */
__u32 cpu_nr; /* 0x03a0 */
__u32 softirq_pending; /* 0x03a4 */
- __s32 preempt_count; /* 0x03a8 */
- __u32 spinlock_lockval; /* 0x03ac */
- __u32 spinlock_index; /* 0x03b0 */
- __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */
+ union {
+ struct {
+ __u32 need_resched; /* 0x03a8 */
+ __u32 count; /* 0x03ac */
+ } preempt;
+ __u64 preempt_count; /* 0x03a8 */
+ };
+ __u32 spinlock_lockval; /* 0x03b0 */
+ __u32 spinlock_index; /* 0x03b4 */
__u64 percpu_offset; /* 0x03b8 */
__u8 percpu_register; /* 0x03c0 */
__u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */
diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h
index 0a25d46..5560d5f 100644
--- a/arch/s390/include/asm/preempt.h
+++ b/arch/s390/include/asm/preempt.h
@@ -8,11 +8,8 @@
#include <asm/cmpxchg.h>
#include <asm/march.h>
-/*
- * Use MSB so it is possible to read preempt_count with LLGT which
- * reads the least significant 31 bits with a single instruction.
- */
-#define PREEMPT_NEED_RESCHED 0x80000000
+/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */
+#define PREEMPT_NEED_RESCHED 0x8000000000000000UL
/*
* We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
@@ -26,25 +23,25 @@
*/
static __always_inline int preempt_count(void)
{
- unsigned long lc_preempt, count;
+ unsigned long lc_preempt;
+ int count;
- BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) != sizeof(int));
- lc_preempt = offsetof(struct lowcore, preempt_count);
- /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */
+ lc_preempt = offsetof(struct lowcore, preempt.count);
+ /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED) */
asm_inline(
- ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n",
- "llgt %[count],%[offalt](%%r0)\n",
+ ALTERNATIVE("ly %[count],%[offzero](%%r0)\n",
+ "ly %[count],%[offalt](%%r0)\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: [count] "=d" (count)
: [offzero] "i" (lc_preempt),
[offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS),
- "m" (((struct lowcore *)0)->preempt_count));
+ "m" (((struct lowcore *)0)->preempt.count));
return count;
}
-static __always_inline void preempt_count_set(int pc)
+static __always_inline void preempt_count_set(unsigned long pc)
{
- int old, new;
+ unsigned long old, new;
old = READ_ONCE(get_lowcore()->preempt_count);
do {
@@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc)
static __always_inline void set_preempt_need_resched(void)
{
- __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
+ __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count);
}
static __always_inline void clear_preempt_need_resched(void)
{
- __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
+ __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count);
}
static __always_inline bool test_preempt_need_resched(void)
@@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val)
lc_preempt = offsetof(struct lowcore, preempt_count);
asm_inline(
- ALTERNATIVE("asi %[offzero](%%r0),%[val]\n",
- "asi %[offalt](%%r0),%[val]\n",
+ ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n",
+ "agsi %[offalt](%%r0),%[val]\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: "+m" (((struct lowcore *)0)->preempt_count)
: [offzero] "i" (lc_preempt), [val] "i" (val),
@@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val)
return;
}
}
- __atomic_add(val, &get_lowcore()->preempt_count);
+ __atomic64_add(val, (long *)&get_lowcore()->preempt_count);
}
static __always_inline void __preempt_count_sub(int val)
@@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_test(void)
lc_preempt = offsetof(struct lowcore, preempt_count);
asm_inline(
- ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n",
- "alsi %[offalt](%%r0),%[val]\n",
+ ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n",
+ "algsi %[offalt](%%r0),%[val]\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: "=@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count)
: [offzero] "i" (lc_preempt), [val] "i" (-1),
[offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS));
return (cc == 0) || (cc == 2);
#else
- return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count);
+ return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_count);
#endif
}
@@ -141,7 +138,7 @@ static __always_inline bool should_resched(int preempt_offset)
static __always_inline int __preempt_count_add_return(int val)
{
- return val + __atomic_add(val, &get_lowcore()->preempt_count);
+ return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count);
}
static __always_inline int __preempt_count_sub_return(int val)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [tip: locking/core] s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
2026-08-04 16:14 ` [PATCH v4 12/17] s390/preempt: " Boqun Feng
2026-08-04 20:27 ` Shrikanth Hegde
2026-08-08 20:48 ` [tip: locking/core] " tip-bot2 for Heiko Carstens
@ 2026-08-10 8:57 ` tip-bot2 for Heiko Carstens
2 siblings, 0 replies; 66+ messages in thread
From: tip-bot2 for Heiko Carstens @ 2026-08-10 8:57 UTC (permalink / raw)
To: linux-tip-commits
Cc: Heiko Carstens, Boqun Feng, Peter Zijlstra (Intel), x86,
linux-kernel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: 3484440a7b194ed80fa4ae2d25fe1a3fcade98f3
Gitweb: https://git.kernel.org/tip/3484440a7b194ed80fa4ae2d25fe1a3fcade98f3
Author: Heiko Carstens <hca@linux.ibm.com>
AuthorDate: Tue, 04 Aug 2026 09:14:34 -07:00
Committer: Peter Zijlstra <peterz@infradead.org>
CommitterDate: Mon, 10 Aug 2026 10:50:20 +02:00
s390/preempt: Enable HAS_SEPARATE_PREEMPT_RESCHED_BITS
Convert s390's preempt_count to 64 bit, and change the preempt
primitives accordingly.
[boqun: Apply the corrected comment for asm block]
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://patch.msgid.link/20260804161447.84806-13-boqun@kernel.org
---
arch/s390/Kconfig | 1 +-
arch/s390/include/asm/lowcore.h | 13 ++++++----
arch/s390/include/asm/preempt.h | 43 ++++++++++++++------------------
3 files changed, 30 insertions(+), 27 deletions(-)
diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig
index 84404e6..378fcd2 100644
--- a/arch/s390/Kconfig
+++ b/arch/s390/Kconfig
@@ -273,6 +273,7 @@ config S390
select PCI_MSI if PCI
select PCI_MSI_ARCH_FALLBACKS if PCI_MSI
select PCI_QUIRKS if PCI
+ select HAS_SEPARATE_PREEMPT_RESCHED_BITS
select SPARSE_IRQ
select SWIOTLB
select SYSCTL_EXCEPTION_TRACE
diff --git a/arch/s390/include/asm/lowcore.h b/arch/s390/include/asm/lowcore.h
index 3b3ecc6..5cef215 100644
--- a/arch/s390/include/asm/lowcore.h
+++ b/arch/s390/include/asm/lowcore.h
@@ -160,10 +160,15 @@ struct lowcore {
/* SMP info area */
__u32 cpu_nr; /* 0x03a0 */
__u32 softirq_pending; /* 0x03a4 */
- __s32 preempt_count; /* 0x03a8 */
- __u32 spinlock_lockval; /* 0x03ac */
- __u32 spinlock_index; /* 0x03b0 */
- __u8 pad_0x03b4[0x03b8-0x03b4]; /* 0x03b4 */
+ union {
+ struct {
+ __u32 need_resched; /* 0x03a8 */
+ __u32 count; /* 0x03ac */
+ } preempt;
+ __u64 preempt_count; /* 0x03a8 */
+ };
+ __u32 spinlock_lockval; /* 0x03b0 */
+ __u32 spinlock_index; /* 0x03b4 */
__u64 percpu_offset; /* 0x03b8 */
__u8 percpu_register; /* 0x03c0 */
__u8 pad_0x03c1[0x0400-0x03c1]; /* 0x03c1 */
diff --git a/arch/s390/include/asm/preempt.h b/arch/s390/include/asm/preempt.h
index 0a25d46..5560d5f 100644
--- a/arch/s390/include/asm/preempt.h
+++ b/arch/s390/include/asm/preempt.h
@@ -8,11 +8,8 @@
#include <asm/cmpxchg.h>
#include <asm/march.h>
-/*
- * Use MSB so it is possible to read preempt_count with LLGT which
- * reads the least significant 31 bits with a single instruction.
- */
-#define PREEMPT_NEED_RESCHED 0x80000000
+/* Use MSB for PREEMPT_NEED_RESCHED mostly because it is available. */
+#define PREEMPT_NEED_RESCHED 0x8000000000000000UL
/*
* We use the PREEMPT_NEED_RESCHED bit as an inverted NEED_RESCHED such
@@ -26,25 +23,25 @@
*/
static __always_inline int preempt_count(void)
{
- unsigned long lc_preempt, count;
+ unsigned long lc_preempt;
+ int count;
- BUILD_BUG_ON(sizeof_field(struct lowcore, preempt_count) != sizeof(int));
- lc_preempt = offsetof(struct lowcore, preempt_count);
- /* READ_ONCE(get_lowcore()->preempt_count) & ~PREEMPT_NEED_RESCHED */
+ lc_preempt = offsetof(struct lowcore, preempt.count);
+ /* READ_ONCE(get_lowcore()->preempt.count) (without PREEMPT_NEED_RESCHED) */
asm_inline(
- ALTERNATIVE("llgt %[count],%[offzero](%%r0)\n",
- "llgt %[count],%[offalt](%%r0)\n",
+ ALTERNATIVE("ly %[count],%[offzero](%%r0)\n",
+ "ly %[count],%[offalt](%%r0)\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: [count] "=d" (count)
: [offzero] "i" (lc_preempt),
[offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS),
- "m" (((struct lowcore *)0)->preempt_count));
+ "m" (((struct lowcore *)0)->preempt.count));
return count;
}
-static __always_inline void preempt_count_set(int pc)
+static __always_inline void preempt_count_set(unsigned long pc)
{
- int old, new;
+ unsigned long old, new;
old = READ_ONCE(get_lowcore()->preempt_count);
do {
@@ -63,12 +60,12 @@ static __always_inline void preempt_count_set(int pc)
static __always_inline void set_preempt_need_resched(void)
{
- __atomic_and(~PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
+ __atomic64_and(~PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count);
}
static __always_inline void clear_preempt_need_resched(void)
{
- __atomic_or(PREEMPT_NEED_RESCHED, &get_lowcore()->preempt_count);
+ __atomic64_or(PREEMPT_NEED_RESCHED, (long *)&get_lowcore()->preempt_count);
}
static __always_inline bool test_preempt_need_resched(void)
@@ -88,8 +85,8 @@ static __always_inline void __preempt_count_add(int val)
lc_preempt = offsetof(struct lowcore, preempt_count);
asm_inline(
- ALTERNATIVE("asi %[offzero](%%r0),%[val]\n",
- "asi %[offalt](%%r0),%[val]\n",
+ ALTERNATIVE("agsi %[offzero](%%r0),%[val]\n",
+ "agsi %[offalt](%%r0),%[val]\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: "+m" (((struct lowcore *)0)->preempt_count)
: [offzero] "i" (lc_preempt), [val] "i" (val),
@@ -98,7 +95,7 @@ static __always_inline void __preempt_count_add(int val)
return;
}
}
- __atomic_add(val, &get_lowcore()->preempt_count);
+ __atomic64_add(val, (long *)&get_lowcore()->preempt_count);
}
static __always_inline void __preempt_count_sub(int val)
@@ -119,15 +116,15 @@ static __always_inline bool __preempt_count_dec_and_test(void)
lc_preempt = offsetof(struct lowcore, preempt_count);
asm_inline(
- ALTERNATIVE("alsi %[offzero](%%r0),%[val]\n",
- "alsi %[offalt](%%r0),%[val]\n",
+ ALTERNATIVE("algsi %[offzero](%%r0),%[val]\n",
+ "algsi %[offalt](%%r0),%[val]\n",
ALT_FEATURE(MFEATURE_LOWCORE))
: "=@cc" (cc), "+m" (((struct lowcore *)0)->preempt_count)
: [offzero] "i" (lc_preempt), [val] "i" (-1),
[offalt] "i" (lc_preempt + LOWCORE_ALT_ADDRESS));
return (cc == 0) || (cc == 2);
#else
- return __atomic_add_const_and_test(-1, &get_lowcore()->preempt_count);
+ return __atomic64_add_const_and_test(-1, (long *)&get_lowcore()->preempt_count);
#endif
}
@@ -141,7 +138,7 @@ static __always_inline bool should_resched(int preempt_offset)
static __always_inline int __preempt_count_add_return(int val)
{
- return val + __atomic_add(val, &get_lowcore()->preempt_count);
+ return val + __atomic64_add(val, (long *)&get_lowcore()->preempt_count);
}
static __always_inline int __preempt_count_sub_return(int val)
^ permalink raw reply related [flat|nested] 66+ messages in thread
* [PATCH v4 13/17] rust: Introduce interrupt module
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (11 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 12/17] s390/preempt: " Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-04 16:14 ` [PATCH v4 14/17] rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers Boqun Feng
` (3 subsequent siblings)
16 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux,
Benno Lossin, Andreas Hindborg
From: Lyude Paul <lyude@redhat.com>
This introduces a module for dealing with interrupt-disabled contexts,
including the ability to enable and disable interrupts along with the
ability to annotate functions as expecting that IRQs are already
disabled on the local CPU.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Reviewed-by: Benno Lossin <lossin@kernel.org>
Reviewed-by: Andreas Hindborg <a.hindborg@kernel.org>
Reviewed-by: Gary Guo <gary@garyguo.net>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
rust/helpers/helpers.c | 1 +
rust/helpers/interrupt.c | 18 ++++++++
rust/helpers/sync.c | 5 +++
rust/kernel/interrupt.rs | 89 ++++++++++++++++++++++++++++++++++++++++
rust/kernel/lib.rs | 1 +
5 files changed, 114 insertions(+)
create mode 100644 rust/helpers/interrupt.c
create mode 100644 rust/kernel/interrupt.rs
diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c
index 998e31052e66..0d85b5e68ec2 100644
--- a/rust/helpers/helpers.c
+++ b/rust/helpers/helpers.c
@@ -65,6 +65,7 @@
#include "irq.c"
#include "fs.c"
#include "gpu.c"
+#include "interrupt.c"
#include "io.c"
#include "jump_label.c"
#include "kunit.c"
diff --git a/rust/helpers/interrupt.c b/rust/helpers/interrupt.c
new file mode 100644
index 000000000000..51b319bd4c00
--- /dev/null
+++ b/rust/helpers/interrupt.c
@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <linux/spinlock.h>
+
+__rust_helper void rust_helper_local_interrupt_disable(void)
+{
+ local_interrupt_disable();
+}
+
+__rust_helper void rust_helper_local_interrupt_enable(void)
+{
+ local_interrupt_enable();
+}
+
+__rust_helper bool rust_helper_irqs_disabled(void)
+{
+ return irqs_disabled();
+}
diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c
index 82d6aff73b04..4f474fe847c4 100644
--- a/rust/helpers/sync.c
+++ b/rust/helpers/sync.c
@@ -11,3 +11,8 @@ __rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k)
{
lockdep_unregister_key(k);
}
+
+__rust_helper void rust_helper_lockdep_assert_irqs_disabled(void)
+{
+ lockdep_assert_irqs_disabled();
+}
diff --git a/rust/kernel/interrupt.rs b/rust/kernel/interrupt.rs
new file mode 100644
index 000000000000..a880ec3b8538
--- /dev/null
+++ b/rust/kernel/interrupt.rs
@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: GPL-2.0
+
+//! Interrupt controls
+//!
+//! This module allows Rust code to annotate areas of code where local processor interrupts should
+//! be disabled, along with actually disabling local processor interrupts.
+//!
+//! # ⚠️ Warning! ⚠️
+//!
+//! The usage of this module can be more complicated than meets the eye, especially surrounding
+//! [preemptible kernels]. It's recommended to take care when using the functions and types defined
+//! here and familiarize yourself with the various documentation we have before using them, along
+//! with the various documents we link to here.
+//!
+//! # Reading material
+//!
+//! - [Software interrupts and realtime (LWN)](https://lwn.net/Articles/520076)
+//!
+//! [preemptible kernels]: https://www.kernel.org/doc/html/latest/locking/preempt-locking.html
+
+use crate::types::NotThreadSafe;
+
+/// A guard that represents local processor interrupt disablement on preemptible kernels.
+///
+/// [`LocalInterruptDisabled`] is a guard type that represents that local processor interrupts have
+/// been disabled on a preemptible kernel.
+///
+/// Certain functions take an immutable reference of [`LocalInterruptDisabled`] in order to require
+/// that they may only be run in local-interrupt-disabled contexts on preemptible kernels.
+///
+/// This is a marker type; it has no size, and is simply used as a compile-time guarantee that local
+/// processor interrupts are disabled on preemptible kernels. Note that no guarantees about the
+/// state of interrupts are made by this type on non-preemptible kernels.
+///
+/// # Invariants
+///
+/// Local processor interrupts are disabled on preemptible kernels for as long as an object of this
+/// type exists.
+pub struct LocalInterruptDisabled(NotThreadSafe);
+
+/// Disable local processor interrupts on a preemptible kernel.
+///
+/// This function disables local processor interrupts on a preemptible kernel, and returns a
+/// [`LocalInterruptDisabled`] token as proof of this. On non-preemptible kernels, this function is
+/// a no-op.
+///
+/// **Usage of this function is discouraged** unless you are absolutely sure you know what you are
+/// doing, as kernel interfaces for Rust that deal with interrupt state will typically handle local
+/// processor interrupt state management on their own and managing this by hand is quite error
+/// prone.
+#[inline]
+pub fn local_interrupt_disable() -> LocalInterruptDisabled {
+ // SAFETY: It's always safe to call `local_interrupt_disable()`.
+ unsafe { bindings::local_interrupt_disable() };
+
+ LocalInterruptDisabled(NotThreadSafe)
+}
+
+impl Drop for LocalInterruptDisabled {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: Per type invariants, a `local_interrupt_disable()` must be called to create this
+ // object, hence calling the corresponding `local_interrupt_enable()` is safe.
+ unsafe { bindings::local_interrupt_enable() };
+ }
+}
+
+impl LocalInterruptDisabled {
+ /// Assume that local processor interrupts are disabled on preemptible kernels.
+ ///
+ /// This can be used for annotating code that is known to be run in contexts where local
+ /// processor interrupts are disabled on preemptible kernels. It makes no changes to the local
+ /// interrupt state on its own.
+ ///
+ /// # Safety
+ ///
+ /// For the whole life `'a`, local interrupts must be disabled on preemptible kernels. This
+ /// could be a context like, for example, an interrupt handler.
+ #[inline]
+ pub unsafe fn assume_disabled<'a>() -> &'a LocalInterruptDisabled {
+ const ASSUME_DISABLED: &LocalInterruptDisabled = &LocalInterruptDisabled(NotThreadSafe);
+
+ // Confirm they're actually disabled if lockdep is available
+ // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`.
+ unsafe { bindings::lockdep_assert_irqs_disabled() };
+
+ ASSUME_DISABLED
+ }
+}
diff --git a/rust/kernel/lib.rs b/rust/kernel/lib.rs
index 9512af7156df..2ee6c24d39c2 100644
--- a/rust/kernel/lib.rs
+++ b/rust/kernel/lib.rs
@@ -82,6 +82,7 @@
pub mod impl_flags;
pub mod init;
pub mod interop;
+pub mod interrupt;
pub mod io;
pub mod ioctl;
pub mod iommu;
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [PATCH v4 14/17] rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (12 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 13/17] rust: Introduce interrupt module Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-04 16:14 ` [PATCH v4 15/17] rust: sync: Use super::* in spinlock.rs Boqun Feng
` (2 subsequent siblings)
16 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux,
Andreas Hindborg
spin_lock_irq_disable() and spin_unlock_irq_enable() are inline
functions, to use them in Rust helpers are introduced. This is for
interrupt disabling lock abstraction in Rust.
Reviewed-by: Andreas Hindborg <a.hindborg@kernel.org>
Reviewed-by: Gary Guo <gary@garyguo.net>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
rust/helpers/spinlock.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c
index 4d13062cf253..d53400c15022 100644
--- a/rust/helpers/spinlock.c
+++ b/rust/helpers/spinlock.c
@@ -36,3 +36,18 @@ __rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock)
{
lockdep_assert_held(lock);
}
+
+__rust_helper void rust_helper_spin_lock_irq_disable(spinlock_t *lock)
+{
+ spin_lock_irq_disable(lock);
+}
+
+__rust_helper void rust_helper_spin_unlock_irq_enable(spinlock_t *lock)
+{
+ spin_unlock_irq_enable(lock);
+}
+
+__rust_helper int rust_helper_spin_trylock_irq_disable(spinlock_t *lock)
+{
+ return spin_trylock_irq_disable(lock);
+}
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [PATCH v4 15/17] rust: sync: Use super::* in spinlock.rs
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (13 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 14/17] rust: helper: Add spin_{un,}lock_irq_{enable,disable}() helpers Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-04 16:14 ` [PATCH v4 16/17] rust: sync: Add SpinLockIrq Boqun Feng
2026-08-04 16:14 ` [PATCH v4 17/17] rust: sync: Introduce SpinLockIrq::lock_with() and friends Boqun Feng
16 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
From: Lyude Paul <lyude@redhat.com>
No functional changes.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
rust/kernel/sync/lock/spinlock.rs | 9 ++++-----
1 file changed, 4 insertions(+), 5 deletions(-)
diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs
index ef76fa07ca3a..d75af32218ba 100644
--- a/rust/kernel/sync/lock/spinlock.rs
+++ b/rust/kernel/sync/lock/spinlock.rs
@@ -3,6 +3,7 @@
//! A kernel spinlock.
//!
//! This module allows Rust code to use the kernel's `spinlock_t`.
+use super::*;
/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
///
@@ -82,7 +83,7 @@ macro_rules! new_spinlock {
/// ```
///
/// [`spinlock_t`]: srctree/include/linux/spinlock.h
-pub type SpinLock<T> = super::Lock<T, SpinLockBackend>;
+pub type SpinLock<T> = Lock<T, SpinLockBackend>;
/// A kernel `spinlock_t` lock backend.
pub struct SpinLockBackend;
@@ -91,13 +92,11 @@ macro_rules! new_spinlock {
///
/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLock`]. It will unlock
/// the [`SpinLock`] upon being dropped.
-///
-/// [`Guard`]: super::Guard
-pub type SpinLockGuard<'a, T> = super::Guard<'a, T, SpinLockBackend>;
+pub type SpinLockGuard<'a, T> = Guard<'a, T, SpinLockBackend>;
// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
// default implementation that always calls the same locking method.
-unsafe impl super::Backend for SpinLockBackend {
+unsafe impl Backend for SpinLockBackend {
type State = bindings::spinlock_t;
type GuardState = ();
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [PATCH v4 16/17] rust: sync: Add SpinLockIrq
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (14 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 15/17] rust: sync: Use super::* in spinlock.rs Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
2026-08-04 16:14 ` [PATCH v4 17/17] rust: sync: Introduce SpinLockIrq::lock_with() and friends Boqun Feng
16 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
From: Lyude Paul <lyude@redhat.com>
A variant of `SpinLock` that ensures interrupts are disabled in the
critical section. `lock()` will ensure that either interrupts are
already disabled or disable them. `unlock()` will reverse the respective
operation.
[Boqun: Port to use spin_lock_irq_disable() and
spin_unlock_irq_enable()]
Signed-off-by: Lyude Paul <lyude@redhat.com>
Reviewed-by: Gary Guo <gary@garyguo.net>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
rust/kernel/sync.rs | 9 +-
rust/kernel/sync/lock/global.rs | 3 +
rust/kernel/sync/lock/spinlock.rs | 230 ++++++++++++++++++++++++++++++
3 files changed, 241 insertions(+), 1 deletion(-)
diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs
index 993dbf2caa0e..df4f2604ff9b 100644
--- a/rust/kernel/sync.rs
+++ b/rust/kernel/sync.rs
@@ -27,7 +27,14 @@
pub use condvar::{new_condvar, CondVar, CondVarTimeoutResult};
pub use lock::global::{global_lock, GlobalGuard, GlobalLock, GlobalLockBackend, GlobalLockedBy};
pub use lock::mutex::{new_mutex, Mutex, MutexGuard};
-pub use lock::spinlock::{new_spinlock, SpinLock, SpinLockGuard};
+pub use lock::spinlock::{
+ new_spinlock,
+ new_spinlock_irq,
+ SpinLock,
+ SpinLockGuard,
+ SpinLockIrq,
+ SpinLockIrqGuard, //
+};
pub use locked_by::LockedBy;
pub use refcount::Refcount;
pub use set_once::SetOnce;
diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs
index ec2dd84316fc..ebb10521d8bd 100644
--- a/rust/kernel/sync/lock/global.rs
+++ b/rust/kernel/sync/lock/global.rs
@@ -306,4 +306,7 @@ macro_rules! global_lock_inner {
(backend SpinLock) => {
$crate::sync::lock::spinlock::SpinLockBackend
};
+ (backend SpinLockIrq) => {
+ $crate::sync::lock::spinlock::SpinLockIrqBackend
+ };
}
diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs
index d75af32218ba..872544948e5d 100644
--- a/rust/kernel/sync/lock/spinlock.rs
+++ b/rust/kernel/sync/lock/spinlock.rs
@@ -4,6 +4,7 @@
//!
//! This module allows Rust code to use the kernel's `spinlock_t`.
use super::*;
+use crate::prelude::*;
/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
///
@@ -143,3 +144,232 @@ unsafe fn assert_is_held(ptr: *mut Self::State) {
unsafe { bindings::spin_assert_is_held(ptr) }
}
}
+
+/// Creates a [`SpinLockIrq`] initialiser with the given name and a newly-created lock class.
+///
+/// It uses the name if one is given, otherwise it generates one based on the file name and line
+/// number.
+#[macro_export]
+macro_rules! new_spinlock_irq {
+ ($inner:expr $(, $name:literal)? $(,)?) => {
+ $crate::sync::SpinLockIrq::new(
+ $inner, $crate::optional_name!($($name)?), $crate::static_lock_class!())
+ };
+}
+pub use new_spinlock_irq;
+
+/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section.
+///
+/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts,
+/// [see the interrupt module](kernel::interrupt).
+///
+/// # Examples
+///
+/// The following example shows how to declare, allocate initialise and access a struct (`Example`)
+/// that contains an inner struct (`Inner`) that is protected by a spinlock that requires local
+/// processor interrupts to be disabled.
+///
+/// ```
+/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
+///
+/// struct Inner {
+/// a: u32,
+/// b: u32,
+/// }
+///
+/// #[pin_data]
+/// struct Example {
+/// #[pin]
+/// c: SpinLockIrq<Inner>,
+/// #[pin]
+/// d: SpinLockIrq<Inner>,
+/// }
+///
+/// impl Example {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {
+/// c <- new_spinlock_irq!(Inner { a: 0, b: 10 }),
+/// d <- new_spinlock_irq!(Inner { a: 20, b: 30 }),
+/// })
+/// }
+/// }
+///
+/// // Allocate a boxed `Example`
+/// let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
+///
+/// // Accessing an `Example` from a context where interrupts may not be disabled already.
+/// let c_guard = e.c.lock(); // interrupts are disabled now, +1 interrupt disable refcount
+/// let d_guard = e.d.lock(); // no interrupt state change, +1 interrupt disable refcount
+///
+/// assert_eq!(c_guard.a, 0);
+/// assert_eq!(c_guard.b, 10);
+/// assert_eq!(d_guard.a, 20);
+/// assert_eq!(d_guard.b, 30);
+///
+/// drop(c_guard); // Dropping c_guard will not re-enable interrupts just yet, since d_guard is
+/// // still in scope.
+/// drop(d_guard); // Last interrupt disable reference dropped here, so interrupts are re-enabled
+/// // now
+/// # Ok::<(), Error>(())
+/// ```
+///
+/// [`lock()`]: SpinLockIrq::lock
+pub type SpinLockIrq<T> = super::Lock<T, SpinLockIrqBackend>;
+
+/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts.
+pub struct SpinLockIrqBackend;
+
+/// A [`Guard`] acquired from locking a [`SpinLockIrq`] using [`lock()`].
+///
+/// This is simply a type alias for a [`Guard`] returned from locking a [`SpinLockIrq`] using
+/// [`lock()`]. It will unlock the [`SpinLockIrq`] and decrement the local processor's interrupt
+/// disablement refcount upon being dropped.
+///
+/// [`lock()`]: SpinLockIrq::lock
+pub type SpinLockIrqGuard<'a, T> = Guard<'a, T, SpinLockIrqBackend>;
+
+// SAFETY: The underlying kernel `spinlock_t` object ensures mutual exclusion. `relock` uses the
+// default implementation that always calls the same locking method.
+unsafe impl Backend for SpinLockIrqBackend {
+ type State = bindings::spinlock_t;
+ type GuardState = ();
+
+ #[inline]
+ unsafe fn init(
+ ptr: *mut Self::State,
+ name: *const crate::ffi::c_char,
+ key: *mut bindings::lock_class_key,
+ ) {
+ // SAFETY: The safety requirements ensure that `ptr` is valid for writes, and `name` and
+ // `key` are valid for read indefinitely.
+ unsafe { bindings::__spin_lock_init(ptr, name, key) }
+ }
+
+ #[inline]
+ unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState {
+ // SAFETY: The safety requirements of this function ensure that `ptr` points to valid
+ // memory, and that it has been initialised before.
+ unsafe { bindings::spin_lock_irq_disable(ptr) }
+ }
+
+ #[inline]
+ unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) {
+ // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the
+ // caller is the owner of the spinlock.
+ unsafe { bindings::spin_unlock_irq_enable(ptr) }
+ }
+
+ #[inline]
+ unsafe fn try_lock(ptr: *mut Self::State) -> Option<Self::GuardState> {
+ // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
+ let result = unsafe { bindings::spin_trylock_irq_disable(ptr) };
+
+ if result != 0 {
+ Some(())
+ } else {
+ None
+ }
+ }
+
+ #[inline]
+ unsafe fn assert_is_held(ptr: *mut Self::State) {
+ // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use.
+ unsafe { bindings::spin_assert_is_held(ptr) }
+ }
+}
+
+#[kunit_tests(rust_spinlock_irq_condvar)]
+mod tests {
+ use super::*;
+ use crate::{
+ sync::*,
+ workqueue::{
+ self,
+ impl_has_work,
+ new_work,
+ Work,
+ WorkItem, //
+ },
+ };
+
+ struct TestState {
+ value: u32,
+ waiter_ready: bool,
+ }
+
+ #[pin_data]
+ struct Test {
+ #[pin]
+ state: SpinLockIrq<TestState>,
+
+ #[pin]
+ state_changed: CondVar,
+
+ #[pin]
+ waiter_state_changed: CondVar,
+
+ #[pin]
+ wait_work: Work<Self>,
+ }
+
+ impl_has_work! {
+ impl HasWork<Self> for Test { self.wait_work }
+ }
+
+ impl Test {
+ pub(crate) fn new() -> Result<Arc<Self>> {
+ Arc::try_pin_init(
+ try_pin_init!(
+ Self {
+ state <- new_spinlock_irq!(TestState {
+ value: 1,
+ waiter_ready: false
+ }),
+ state_changed <- new_condvar!(),
+ waiter_state_changed <- new_condvar!(),
+ wait_work <- new_work!("IrqCondvarTest::wait_work")
+ }
+ ),
+ GFP_KERNEL,
+ )
+ }
+ }
+
+ impl WorkItem for Test {
+ type Pointer = Arc<Self>;
+
+ fn run(this: Arc<Self>) {
+ // Wait for the test to be ready to wait for us
+ let mut state = this.state.lock();
+
+ // Make sure the interrupts actually turned off
+ // SAFETY: It's always safe to call `lockdep_assert_irqs_disabled()`
+ unsafe { bindings::lockdep_assert_irqs_disabled() };
+
+ while !state.waiter_ready {
+ this.waiter_state_changed.wait(&mut state);
+ }
+
+ // Deliver the exciting value update our test has been waiting for
+ state.value += 1;
+ this.state_changed.notify_sync();
+ }
+ }
+
+ #[test]
+ fn spinlock_irq_condvar() -> Result {
+ let testdata = Test::new()?;
+
+ let _ = workqueue::system().enqueue(testdata.clone());
+
+ // Let the updater know when we're ready to wait
+ let mut state = testdata.state.lock();
+ state.waiter_ready = true;
+ testdata.waiter_state_changed.notify_sync();
+
+ // Wait for the exciting value update
+ testdata.state_changed.wait(&mut state);
+ assert_eq!(state.value, 2);
+ Ok(())
+ }
+}
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread* [PATCH v4 17/17] rust: sync: Introduce SpinLockIrq::lock_with() and friends
2026-08-04 16:14 [PATCH v4 00/17] Refcounted interrupt disable and SpinLockIrq for Rust Boqun Feng
` (15 preceding siblings ...)
2026-08-04 16:14 ` [PATCH v4 16/17] rust: sync: Add SpinLockIrq Boqun Feng
@ 2026-08-04 16:14 ` Boqun Feng
16 siblings, 0 replies; 66+ messages in thread
From: Boqun Feng @ 2026-08-04 16:14 UTC (permalink / raw)
To: Peter Zijlstra
Cc: Ingo Molnar, Will Deacon, Boqun Feng, Waiman Long, Gary Guo,
Alice Ryhl, Lyude Paul, Daniel Almeida, Onur Özkan,
Miguel Ojeda, Danilo Krummrich, linux-kernel, rust-for-linux
From: Lyude Paul <lyude@redhat.com>
`SpinLockIrq` and `SpinLock` use the exact same underlying C structure,
with the only real difference being that the former uses the
irq_disable() and irq_enable() variants for locking/unlocking. These
variants can introduce some minor overhead in contexts where we already
know that local processor interrupts are disabled, and as such we want a
way to be able to skip modifying processor interrupt state in said
contexts in order to avoid some overhead - just like the current C API
allows us to do.
In order to do this, we add some special functions for SpinLockIrq:
lock_with() and try_lock_with(), which allow acquiring the lock without
changing the interrupt state - as long as the caller can provide a
LocalInterruptDisabled reference to prove that local processor
interrupts have been disabled.
In some hacked-together benchmarks we ran, most of the time this did
actually seem to lead to a noticeable difference in overhead:
From an aarch64 VM running on a MacBook M4:
lock() when irq is disabled, 100 times cost Delta { nanos: 500 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 292 }
lock() when irq is enabled, 100 times cost Delta { nanos: 834 }
lock() when irq is disabled, 100 times cost Delta { nanos: 459 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 291 }
lock() when irq is enabled, 100 times cost Delta { nanos: 709 }
From an x86_64 VM (qemu/kvm) running on a i7-13700H
lock() when irq is disabled, 100 times cost Delta { nanos: 1002 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 729 }
lock() when irq is enabled, 100 times cost Delta { nanos: 1516 }
lock() when irq is disabled, 100 times cost Delta { nanos: 754 }
lock_with() when irq is disabled, 100 times cost Delta { nanos: 966 }
lock() when irq is enabled, 100 times cost Delta { nanos: 1227 }
(note that there were some runs on x86_64 where lock() on irq
disabled vs. lock_with() on irq disabled had equivalent benchmarks,
but it very much appeared to be a minority of test runs.)
While it's not clear how this affects real-world workloads yet, let's
add this for the time being so we can find out.
This makes it so that a `SpinLockIrq` will work like a `SpinLock` if
interrupts are disabled. So a function:
(&'a SpinLockIrq, &'a LocalInterruptDisabled) -> Guard<'a, .., SpinLockBackend>
makes sense. Note that due to `Guard` and `LocalInterruptDisabled`
having the same lifetime, interrupts cannot be enabled while the Guard
exists.
Signed-off-by: Lyude Paul <lyude@redhat.com>
Reviewed-by: Gary Guo <gary@garyguo.net>
Signed-off-by: Boqun Feng <boqun@kernel.org>
---
rust/kernel/sync/lock/spinlock.rs | 92 ++++++++++++++++++++++++++++++-
1 file changed, 91 insertions(+), 1 deletion(-)
diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs
index 872544948e5d..aafc80125f59 100644
--- a/rust/kernel/sync/lock/spinlock.rs
+++ b/rust/kernel/sync/lock/spinlock.rs
@@ -4,7 +4,10 @@
//!
//! This module allows Rust code to use the kernel's `spinlock_t`.
use super::*;
-use crate::prelude::*;
+use crate::{
+ interrupt::LocalInterruptDisabled,
+ prelude::*, //
+};
/// Creates a [`SpinLock`] initialiser with the given name and a newly-created lock class.
///
@@ -160,6 +163,16 @@ macro_rules! new_spinlock_irq {
/// A variant of `SpinLock` that ensures interrupts are disabled in the critical section.
///
+/// This lock can be acquired in two ways:
+///
+/// - Using [`lock()`] like any other type of lock, in which case the bindings will modify the
+/// interrupt state to ensure that local processor interrupts remain disabled for at least as
+/// long as the [`SpinLockIrqGuard`] exists.
+/// - Using [`lock_with()`] in contexts where a [`LocalInterruptDisabled`] token is present and
+/// local processor interrupts are already known to be disabled, in which case the local
+/// interrupt state will not be touched. This method should be preferred if a
+/// [`LocalInterruptDisabled`] token is present in the scope.
+///
/// For more info on spinlocks, see [`SpinLock`]. For more information on interrupts,
/// [see the interrupt module](kernel::interrupt).
///
@@ -213,7 +226,47 @@ macro_rules! new_spinlock_irq {
/// # Ok::<(), Error>(())
/// ```
///
+/// The next example demonstrates locking a [`SpinLockIrq`] using [`lock_with()`] in a function
+/// which can only be called when local processor interrupts are already disabled.
+///
+/// ```
+/// use kernel::sync::{new_spinlock_irq, SpinLockIrq};
+/// use kernel::interrupt::*;
+///
+/// struct Inner {
+/// a: u32,
+/// }
+///
+/// #[pin_data]
+/// struct Example {
+/// #[pin]
+/// inner: SpinLockIrq<Inner>,
+/// }
+///
+/// impl Example {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {
+/// inner <- new_spinlock_irq!(Inner { a: 20 }),
+/// })
+/// }
+/// }
+///
+/// // Accessing an `Example` from a function that can only be called in no-interrupt contexts.
+/// fn noirq_work(e: &Example, interrupt_disabled: &LocalInterruptDisabled) {
+/// // Because we know interrupts are disabled from interrupt_disable, we can skip toggling
+/// // interrupt state using lock_with() and the provided token
+/// assert_eq!(e.inner.lock_with(interrupt_disabled).a, 20);
+/// }
+///
+/// # let e = KBox::pin_init(Example::new(), GFP_KERNEL)?;
+/// # let interrupt_guard = local_interrupt_disable();
+/// # noirq_work(&e, &interrupt_guard);
+/// #
+/// # Ok::<(), Error>(())
+/// ```
+///
/// [`lock()`]: SpinLockIrq::lock
+/// [`lock_with()`]: SpinLockIrq::lock_with
pub type SpinLockIrq<T> = super::Lock<T, SpinLockIrqBackend>;
/// A kernel `spinlock_t` lock backend that can only be acquired in interrupt disabled contexts.
@@ -278,6 +331,43 @@ unsafe fn assert_is_held(ptr: *mut Self::State) {
}
}
+impl<T: ?Sized> Lock<T, SpinLockIrqBackend> {
+ /// Casts the lock as a `Lock<T, SpinLockBackend>`.
+ #[inline]
+ fn as_lock_in_interrupt<'a>(&'a self, _context: &'a LocalInterruptDisabled) -> &'a SpinLock<T> {
+ // SAFETY:
+ // - `Lock<T, SpinLockBackend>` and `Lock<T, SpinLockIrqBackend>` both have identical data
+ // layouts.
+ // - As long as local interrupts are disabled (which is proven to be true by _context), it
+ // is safe to treat a lock with SpinLockIrqBackend as a SpinLockBackend lock.
+ unsafe { core::mem::transmute(self) }
+ }
+
+ /// Acquires the lock without modifying local interrupt state.
+ ///
+ /// This function should be used in place of the more expensive [`Lock::lock()`] function when
+ /// possible for [`SpinLockIrq`] locks.
+ #[inline]
+ pub fn lock_with<'a>(&'a self, context: &'a LocalInterruptDisabled) -> SpinLockGuard<'a, T> {
+ self.as_lock_in_interrupt(context).lock()
+ }
+
+ /// Tries to acquire the lock without modifying local interrupt state.
+ ///
+ /// This function should be used in place of the more expensive [`Lock::try_lock()`] function
+ /// when possible for [`SpinLockIrq`] locks.
+ ///
+ /// Returns a guard that can be used to access the data protected by the lock if successful.
+ #[must_use = "if unused, the lock will be immediately unlocked"]
+ #[inline]
+ pub fn try_lock_with<'a>(
+ &'a self,
+ context: &'a LocalInterruptDisabled,
+ ) -> Option<SpinLockGuard<'a, T>> {
+ self.as_lock_in_interrupt(context).try_lock()
+ }
+}
+
#[kunit_tests(rust_spinlock_irq_condvar)]
mod tests {
use super::*;
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 66+ messages in thread