From: "Paul E. McKenney" <paulmck@kernel.org>
To: rcu@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com,
rostedt@goodmis.org, Kunwu Chan <kunwu.chan@gmail.com>,
Zqiang <qiang.zhang@linux.dev>, Boqun Feng <boqun@kernel.org>,
"Paul E . McKenney" <paulmck@kernel.org>
Subject: [PATCH v2 21/21] srcutree: Preserve IRQ state in synchronize_srcu_atomic() callchain
Date: Wed, 7 Oct 2026 13:59:24 -0700 [thread overview]
Message-ID: <20261007205924.1983367-21-paulmck@kernel.org> (raw)
In-Reply-To: <546c81a7-f677-4d8e-a304-746b227058b0@paulmck-laptop>
From: Kunwu Chan <kunwu.chan@gmail.com>
synchronize_srcu_atomic() may be called with interrupts disabled, but
its callchain uses raw_spin_lock_irq_rcu_node() /
raw_spin_unlock_irq_rcu_node(), which unconditionally enables
interrupts on unlock. This can incorrectly change the caller's IRQ
state.
Use the irqsave/irqrestore variants for the locks in this callchain
so that the caller's IRQ state is preserved.
Co-developed-by: Zqiang <qiang.zhang@linux.dev>
Signed-off-by: Zqiang <qiang.zhang@linux.dev>
Signed-off-by: Kunwu Chan <kunwu.chan@gmail.com>
Acked-by: Boqun Feng <boqun@kernel.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
kernel/rcu/srcutree.c | 42 +++++++++++++++++++++++-------------------
1 file changed, 23 insertions(+), 19 deletions(-)
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index 7890f7c7c2ee7..c8209aa6552a1 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -985,6 +985,7 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic)
bool cbs;
bool last_lvl;
int cpu;
+ unsigned long flags;
unsigned long gpseq;
int idx;
unsigned long mask;
@@ -999,7 +1000,7 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic)
mutex_lock(&sup->srcu_cb_mutex);
/* End the current grace period. */
- raw_spin_lock_irq_rcu_node(sup);
+ raw_spin_lock_irqsave_rcu_node(sup, flags);
idx = rcu_seq_state(sup->srcu_gp_seq);
WARN_ON_ONCE(idx != SRCU_STATE_SCAN2);
if (srcu_gp_is_expedited(ssp))
@@ -1010,7 +1011,7 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic)
gpseq = rcu_seq_current(&sup->srcu_gp_seq);
if (ULONG_CMP_LT(sup->srcu_gp_seq_needed_exp, gpseq))
WRITE_ONCE(sup->srcu_gp_seq_needed_exp, gpseq);
- raw_spin_unlock_irq_rcu_node(sup);
+ raw_spin_unlock_irqrestore_rcu_node(sup, flags);
if (!is_atomic)
mutex_unlock(&sup->srcu_gp_mutex);
/* A new grace period can start at this point. But only one. */
@@ -1048,12 +1049,12 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic)
if (!(gpseq & counter_wrap_check))
for_each_possible_cpu(cpu) {
sdp = per_cpu_ptr(ssp->sda, cpu);
- raw_spin_lock_irq_rcu_node(sdp);
+ raw_spin_lock_irqsave_rcu_node(sdp, flags);
if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed + 100))
sdp->srcu_gp_seq_needed = gpseq;
if (ULONG_CMP_GE(gpseq, sdp->srcu_gp_seq_needed_exp + 100))
sdp->srcu_gp_seq_needed_exp = gpseq;
- raw_spin_unlock_irq_rcu_node(sdp);
+ raw_spin_unlock_irqrestore_rcu_node(sdp, flags);
}
/* Callback initiation done, allow grace periods after next. */
@@ -1061,16 +1062,16 @@ static void srcu_gp_end(struct srcu_struct *ssp, bool is_atomic)
mutex_unlock(&sup->srcu_cb_mutex);
/* Start a new grace period if needed. */
- raw_spin_lock_irq_rcu_node(sup);
+ raw_spin_lock_irqsave_rcu_node(sup, flags);
gpseq = rcu_seq_current(&sup->srcu_gp_seq);
if (!rcu_seq_state(gpseq) &&
ULONG_CMP_LT(gpseq, sup->srcu_gp_seq_needed)) {
WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC);
srcu_gp_start(ssp);
- raw_spin_unlock_irq_rcu_node(sup);
+ raw_spin_unlock_irqrestore_rcu_node(sup, flags);
srcu_reschedule(ssp, 0);
} else {
- raw_spin_unlock_irq_rcu_node(sup);
+ raw_spin_unlock_irqrestore_rcu_node(sup, flags);
}
/* Transition to big if needed, but never for atomic SRCU. */
@@ -1220,10 +1221,11 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
static bool try_check_zero(struct srcu_struct *ssp, int idx, int trycount)
{
unsigned long curdelay;
+ unsigned long flags;
- raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
+ raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);
curdelay = !srcu_get_delay(ssp);
- raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
+ raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
for (;;) {
if (srcu_readers_active_idx_check(ssp, idx))
@@ -2023,6 +2025,7 @@ EXPORT_SYMBOL_GPL(srcu_batches_completed);
*/
static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic)
{
+ unsigned long flags;
int idx;
if (!is_atomic)
@@ -2040,10 +2043,10 @@ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic)
*/
idx = rcu_seq_state(smp_load_acquire(&ssp->srcu_sup->srcu_gp_seq)); /* ^^^ */
if (idx == SRCU_STATE_IDLE) {
- raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
+ raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);
if (ULONG_CMP_GE(ssp->srcu_sup->srcu_gp_seq, ssp->srcu_sup->srcu_gp_seq_needed)) {
WARN_ON_ONCE(rcu_seq_state(ssp->srcu_sup->srcu_gp_seq));
- raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
+ raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
if (!is_atomic)
mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
return;
@@ -2053,7 +2056,7 @@ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic)
WARN_ON_ONCE(ssp->srcu_reader_flavor & SRCU_READ_FLAVOR_ATOMIC);
srcu_gp_start(ssp);
}
- raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
+ raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
if (idx != SRCU_STATE_IDLE) {
if (!is_atomic)
mutex_unlock(&ssp->srcu_sup->srcu_gp_mutex);
@@ -2069,10 +2072,10 @@ static void srcu_advance_state(struct srcu_struct *ssp, bool is_atomic)
return; /* readers present, retry later. */
}
srcu_flip(ssp);
- raw_spin_lock_irq_rcu_node(ssp->srcu_sup);
+ raw_spin_lock_irqsave_rcu_node(ssp->srcu_sup, flags);
rcu_seq_set_state(&ssp->srcu_sup->srcu_gp_seq, SRCU_STATE_SCAN2);
ssp->srcu_sup->srcu_n_exp_nodelay = 0;
- raw_spin_unlock_irq_rcu_node(ssp->srcu_sup);
+ raw_spin_unlock_irqrestore_rcu_node(ssp->srcu_sup, flags);
}
if (rcu_seq_state(READ_ONCE(ssp->srcu_sup->srcu_gp_seq)) == SRCU_STATE_SCAN2) {
@@ -2119,6 +2122,7 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp)
{
unsigned long srcu_state;
struct srcu_usage *sup = ssp->srcu_sup;
+ unsigned long flags;
unsigned long rdm0, rdm1;
unsigned long unlocks0, unlocks1;
@@ -2149,9 +2153,9 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp)
}
// One last check for others doing our work for us under the lock.
- raw_spin_lock_irq_rcu_node(sup);
+ raw_spin_lock_irqsave_rcu_node(sup, flags);
if (poll_state_synchronize_srcu(ssp, srcu_state)) {
- raw_spin_unlock_irq_rcu_node(sup);
+ raw_spin_unlock_irqrestore_rcu_node(sup, flags);
atomic_set(&sup->srcu_atomic_gp_flag, 0);
preempt_enable();
return;
@@ -2162,7 +2166,7 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp)
smp_store_release(&sup->srcu_gp_seq_needed, srcu_state); // See srcu_funnel_gp_start().
ASSERT_EXCLUSIVE_WRITER(ssp->srcu_sup->srcu_gp_seq);
srcu_gp_start(ssp);
- raw_spin_unlock_irq_rcu_node(sup);
+ raw_spin_unlock_irqrestore_rcu_node(sup, flags);
//
// Fastpath: If there are no readers at all, neither grace-period
@@ -2200,9 +2204,9 @@ void synchronize_srcu_atomic(struct srcu_struct *ssp)
// poll_state_synchronize_srcu() working, all under ->lock
// and ->srcu_atomic_gp_flag, which excludes concurrent
// sequence-number updates.
- raw_spin_lock_irq_rcu_node(sup);
+ raw_spin_lock_irqsave_rcu_node(sup, flags);
rcu_seq_end(&sup->srcu_gp_seq);
- raw_spin_unlock_irq_rcu_node(sup);
+ raw_spin_unlock_irqrestore_rcu_node(sup, flags);
WARN_ON_ONCE(!poll_state_synchronize_srcu(ssp, srcu_state));
atomic_set_release(&sup->srcu_atomic_gp_flag, 0);
preempt_enable();
--
2.40.1
prev parent reply other threads:[~2026-10-07 20:59 UTC|newest]
Thread overview: 30+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-10-07 20:59 [PATCH v2 0/21] Add atomic SRCU &c for v7.4 Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 01/21] srcutiny: Make a Tiny SRCU grace period imply an RCU grace period Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 02/21] srcutree: Suppress srcu_advance_state() mutex_lock in atomic Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 03/21] srcutree: Suppress to-big transition for atomic SRCU Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 04/21] srcutree: Add an atomic Tree SRCU Paul E. McKenney
2026-10-08 7:39 ` David Woodhouse
2026-10-08 15:51 ` Paul E. McKenney
2026-10-12 12:49 ` David Woodhouse
2026-10-14 21:14 ` Paul E. McKenney
2026-10-14 21:31 ` David Woodhouse
2026-10-14 22:02 ` Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 05/21] srcutiny: Add an atomic Tiny SRCU Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 06/21] rcutorture: Add support for testing synchronize_srcu_atomic() Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 07/21] srcutree: Disable preemption across synchronize_srcu_atomic() Paul E. McKenney
2026-10-23 0:19 ` Joel Fernandes
2026-10-23 2:22 ` Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 08/21] srcu: Use IRQ_WORK_INIT_HARD for srcu's irq_work Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 09/21] srcutree: Warn if Tiny SRCU readers are preempted Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 10/21] srcutree: Explicitly note DEFINE_SRCU() needs for srcu_barrier() Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 11/21] rcutorture: Add atomic-SRCU support to torture.sh Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 12/21] srcutree: Add reader-free fastpath to synchronize_srcu_atomic() Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 13/21] srcutree: Skip callback scheduling for atomic SRCU grace periods Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 14/21] srcutree: Remove srcu_barrier() sleep for atomic SRCU Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 15/21] srcu: Restrict atomic-SRCU non_block annotation to task context Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 16/21] srcutree: Make init_srcu_struct_atomic() prevent transition to big Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 17/21] srcutree: Don't transition atomic SRCU to big in srcu_gp_end() Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 18/21] srcutree: Skip torture to-big transition for atomic SRCU Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 19/21] srcutree: Add lockdep and early-boot checks to synchronize_srcu_atomic() Paul E. McKenney
2026-10-07 20:59 ` [PATCH v2 20/21] srcutiny: " Paul E. McKenney
2026-10-07 20:59 ` Paul E. McKenney [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261007205924.1983367-21-paulmck@kernel.org \
--to=paulmck@kernel.org \
--cc=boqun@kernel.org \
--cc=kernel-team@meta.com \
--cc=kunwu.chan@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=qiang.zhang@linux.dev \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.