[PATCH v13 01/17] preempt: Track NMI nesting to separate per-CPU counter

Lyude Paul
2025-10-13T15:53:56Z

From: Joel Fernandes <joelagnelf@nvidia.com>

Move NMI nesting tracking from the preempt_count bits to a separate per-CPU
counter (nmi_nesting). This is to free up the NMI bits in the preempt_count,
allowing those bits to be repurposed for other uses. This also has the benefit
of tracking more than 16-levels deep if there is ever a need.

Suggested-by: Boqun Feng <boqun.feng@gmail.com>
Signed-off-by: Joel Fernandes <joelaf@google.com>
Signed-off-by: Lyude Paul <lyude@redhat.com>
---
include/linux/hardirq.h | 17 +++++++++++++----
kernel/softirq.c | 2 ++
rust/kernel/alloc/kvec.rs | 5 +----
rust/kernel/cpufreq.rs | 3 +--
4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index d57cab4d4c06f..177eed1de35cc 100644
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -10,6 +10,8 @@
#include <linux/vtime.h>
#include <asm/hardirq.h>

+DECLARE_PER_CPU(unsigned int, nmi_nesting);
+
extern void synchronize_irq(unsigned int irq);
extern bool synchronize_hardirq(unsigned int irq);

@@ -102,14 +104,17 @@ void irq_exit_rcu(void);
*/

/*
- * nmi_enter() can nest up to 15 times; see NMI_BITS.
+ * nmi_enter() can nest - nesting is tracked in a per-CPU counter.
*/
#define __nmi_enter() \
do { \
lockdep_off(); \
arch_nmi_enter(); \
- BUG_ON(in_nmi() == NMI_MASK); \
- __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \
+ BUG_ON(__this_cpu_read(nmi_nesting) == UINT_MAX); \
+ __this_cpu_inc(nmi_nesting); \
+ __preempt_count_add(HARDIRQ_OFFSET); \
+ if (__this_cpu_read(nmi_nesting) == 1) \
+ __preempt_count_add(NMI_OFFSET); \
} while (0)

#define nmi_enter() \
@@ -124,8 +129,12 @@ void irq_exit_rcu(void);

#define __nmi_exit() \
do { \
+ unsigned int nesting; \
BUG_ON(!in_nmi()); \
- __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \
+ __preempt_count_sub(HARDIRQ_OFFSET); \
+ nesting = __this_cpu_dec_return(nmi_nesting); \
+ if (!nesting) \
+ __preempt_count_sub(NMI_OFFSET); \
arch_nmi_exit(); \
lockdep_on(); \
} while (0)
diff --git a/kernel/softirq.c b/kernel/softirq.c
index 77198911b8dd4..af47ea23aba3b 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -88,6 +88,8 @@ EXPORT_PER_CPU_SYMBOL_GPL(hardirqs_enabled);
EXPORT_PER_CPU_SYMBOL_GPL(hardirq_context);
#endif

+DEFINE_PER_CPU(unsigned int, nmi_nesting);
+
/*
* SOFTIRQ_OFFSET usage:
*
diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs
index e94aebd084c83..1d6cc81bdeef5 100644
--- a/rust/kernel/alloc/kvec.rs
+++ b/rust/kernel/alloc/kvec.rs
@@ -7,10 +7,7 @@
layout::ArrayLayout,
AllocError, Allocator, Box, Flags, NumaNode,
};
-use crate::{
- fmt,
- page::AsPageIter,
-};
+use crate::{fmt, page::AsPageIter};
use core::{
borrow::{Borrow, BorrowMut},
marker::PhantomData,
diff --git a/rust/kernel/cpufreq.rs b/rust/kernel/cpufreq.rs
index 21b5b9b8acc10..1a555fcb120a9 100644
--- a/rust/kernel/cpufreq.rs
+++ b/rust/kernel/cpufreq.rs
@@ -38,8 +38,7 @@
const CPUFREQ_NAME_LEN: usize = bindings::CPUFREQ_NAME_LEN as usize;

/// Default transition latency value in nanoseconds.
-pub const DEFAULT_TRANSITION_LATENCY_NS: u32 =
- bindings::CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS;
+pub const DEFAULT_TRANSITION_LATENCY_NS: u32 = bindings::CPUFREQ_DEFAULT_TRANSITION_LATENCY_NS;

/// CPU frequency driver flags.
pub mod flags {
--
2.51.0