From: "Paul E. McKenney" <paulmck@kernel.org>
To: rcu@vger.kernel.org
Cc: linux-kernel@vger.kernel.org, kernel-team@meta.com,
rostedt@goodmis.org, hch@lst.de,
"Paul E. McKenney" <paulmck@kernel.org>,
Sachin Sant <sachinp@linux.ibm.com>,
"Zhang, Qiang1" <qiang1.zhang@intel.com>
Subject: [PATCH rcu 06/20] srcu: Move ->srcu_size_state from srcu_struct to srcu_usage
Date: Thu, 30 Mar 2023 15:47:12 -0700 [thread overview]
Message-ID: <20230330224726.662344-6-paulmck@kernel.org> (raw)
In-Reply-To: <f1b6cd5f-f0b7-4748-abd5-0dcfef0ce126@paulmck-laptop>
This commit moves the ->srcu_size_state field from the srcu_struct
structure to the srcu_usage structure to reduce the size of the former
in order to improve cache locality.
Suggested-by: Christoph Hellwig <hch@lst.de>
Tested-by: Sachin Sant <sachinp@linux.ibm.com>
Tested-by: "Zhang, Qiang1" <qiang1.zhang@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
---
include/linux/srcutree.h | 2 +-
kernel/rcu/srcutree.c | 37 +++++++++++++++++++------------------
2 files changed, 20 insertions(+), 19 deletions(-)
diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h
index c7373fe5c14b..443d27a214ef 100644
--- a/include/linux/srcutree.h
+++ b/include/linux/srcutree.h
@@ -64,13 +64,13 @@ struct srcu_usage {
struct srcu_node *node; /* Combining tree. */
struct srcu_node *level[RCU_NUM_LVLS + 1];
/* First node at each level. */
+ int srcu_size_state; /* Small-to-big transition state. */
};
/*
* Per-SRCU-domain structure, similar in function to rcu_state.
*/
struct srcu_struct {
- int srcu_size_state; /* Small-to-big transition state. */
struct mutex srcu_cb_mutex; /* Serialize CB preparation. */
spinlock_t __private lock; /* Protect counters and size state. */
struct mutex srcu_gp_mutex; /* Serialize GP work. */
diff --git a/kernel/rcu/srcutree.c b/kernel/rcu/srcutree.c
index acb0862faafa..8428a184d506 100644
--- a/kernel/rcu/srcutree.c
+++ b/kernel/rcu/srcutree.c
@@ -225,7 +225,7 @@ static bool init_srcu_struct_nodes(struct srcu_struct *ssp, gfp_t gfp_flags)
}
sdp->grpmask = 1 << (cpu - sdp->mynode->grplo);
}
- smp_store_release(&ssp->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);
+ smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_WAIT_BARRIER);
return true;
}
@@ -240,7 +240,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
ssp->srcu_sup = kzalloc(sizeof(*ssp->srcu_sup), GFP_KERNEL);
if (!ssp->srcu_sup)
return -ENOMEM;
- ssp->srcu_size_state = SRCU_SIZE_SMALL;
+ ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;
ssp->srcu_sup->node = NULL;
mutex_init(&ssp->srcu_cb_mutex);
mutex_init(&ssp->srcu_gp_mutex);
@@ -261,7 +261,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
init_srcu_struct_data(ssp);
ssp->srcu_gp_seq_needed_exp = 0;
ssp->srcu_last_gp_end = ktime_get_mono_fast_ns();
- if (READ_ONCE(ssp->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {
+ if (READ_ONCE(ssp->srcu_sup->srcu_size_state) == SRCU_SIZE_SMALL && SRCU_SIZING_IS_INIT()) {
if (!init_srcu_struct_nodes(ssp, GFP_ATOMIC)) {
if (!ssp->sda_is_static) {
free_percpu(ssp->sda);
@@ -270,7 +270,7 @@ static int init_srcu_struct_fields(struct srcu_struct *ssp, bool is_static)
return -ENOMEM;
}
} else {
- WRITE_ONCE(ssp->srcu_size_state, SRCU_SIZE_BIG);
+ WRITE_ONCE(ssp->srcu_sup->srcu_size_state, SRCU_SIZE_BIG);
}
}
smp_store_release(&ssp->srcu_gp_seq_needed, 0); /* Init done. */
@@ -315,7 +315,7 @@ EXPORT_SYMBOL_GPL(init_srcu_struct);
static void __srcu_transition_to_big(struct srcu_struct *ssp)
{
lockdep_assert_held(&ACCESS_PRIVATE(ssp, lock));
- smp_store_release(&ssp->srcu_size_state, SRCU_SIZE_ALLOC);
+ smp_store_release(&ssp->srcu_sup->srcu_size_state, SRCU_SIZE_ALLOC);
}
/*
@@ -326,10 +326,10 @@ static void srcu_transition_to_big(struct srcu_struct *ssp)
unsigned long flags;
/* Double-checked locking on ->srcu_size-state. */
- if (smp_load_acquire(&ssp->srcu_size_state) != SRCU_SIZE_SMALL)
+ if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL)
return;
spin_lock_irqsave_rcu_node(ssp, flags);
- if (smp_load_acquire(&ssp->srcu_size_state) != SRCU_SIZE_SMALL) {
+ if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) != SRCU_SIZE_SMALL) {
spin_unlock_irqrestore_rcu_node(ssp, flags);
return;
}
@@ -345,7 +345,7 @@ static void spin_lock_irqsave_check_contention(struct srcu_struct *ssp)
{
unsigned long j;
- if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_size_state)
+ if (!SRCU_SIZING_IS_CONTEND() || ssp->srcu_sup->srcu_size_state)
return;
j = jiffies;
if (ssp->srcu_size_jiffies != j) {
@@ -666,7 +666,7 @@ void cleanup_srcu_struct(struct srcu_struct *ssp)
}
kfree(ssp->srcu_sup->node);
ssp->srcu_sup->node = NULL;
- ssp->srcu_size_state = SRCU_SIZE_SMALL;
+ ssp->srcu_sup->srcu_size_state = SRCU_SIZE_SMALL;
if (!ssp->sda_is_static) {
free_percpu(ssp->sda);
ssp->sda = NULL;
@@ -770,7 +770,7 @@ static void srcu_gp_start(struct srcu_struct *ssp)
struct srcu_data *sdp;
int state;
- if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)
+ if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)
sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());
else
sdp = this_cpu_ptr(ssp->sda);
@@ -880,7 +880,7 @@ static void srcu_gp_end(struct srcu_struct *ssp)
/* A new grace period can start at this point. But only one. */
/* Initiate callback invocation as needed. */
- ss_state = smp_load_acquire(&ssp->srcu_size_state);
+ ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);
if (ss_state < SRCU_SIZE_WAIT_BARRIER) {
srcu_schedule_cbs_sdp(per_cpu_ptr(ssp->sda, get_boot_cpu_id()),
cbdelay);
@@ -940,7 +940,7 @@ static void srcu_gp_end(struct srcu_struct *ssp)
if (ss_state == SRCU_SIZE_ALLOC)
init_srcu_struct_nodes(ssp, GFP_KERNEL);
else
- smp_store_release(&ssp->srcu_size_state, ss_state + 1);
+ smp_store_release(&ssp->srcu_sup->srcu_size_state, ss_state + 1);
}
}
@@ -1002,7 +1002,7 @@ static void srcu_funnel_gp_start(struct srcu_struct *ssp, struct srcu_data *sdp,
unsigned long snp_seq;
/* Ensure that snp node tree is fully initialized before traversing it */
- if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)
+ if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)
snp_leaf = NULL;
else
snp_leaf = sdp->mynode;
@@ -1209,7 +1209,7 @@ static unsigned long srcu_gp_start_if_needed(struct srcu_struct *ssp,
* sequence number cannot wrap around in the meantime.
*/
idx = __srcu_read_lock_nmisafe(ssp);
- ss_state = smp_load_acquire(&ssp->srcu_size_state);
+ ss_state = smp_load_acquire(&ssp->srcu_sup->srcu_size_state);
if (ss_state < SRCU_SIZE_WAIT_CALL)
sdp = per_cpu_ptr(ssp->sda, get_boot_cpu_id());
else
@@ -1546,7 +1546,7 @@ void srcu_barrier(struct srcu_struct *ssp)
atomic_set(&ssp->srcu_barrier_cpu_cnt, 1);
idx = __srcu_read_lock_nmisafe(ssp);
- if (smp_load_acquire(&ssp->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)
+ if (smp_load_acquire(&ssp->srcu_sup->srcu_size_state) < SRCU_SIZE_WAIT_BARRIER)
srcu_barrier_one_cpu(ssp, per_cpu_ptr(ssp->sda, get_boot_cpu_id()));
else
for_each_possible_cpu(cpu)
@@ -1784,7 +1784,7 @@ void srcu_torture_stats_print(struct srcu_struct *ssp, char *tt, char *tf)
int cpu;
int idx;
unsigned long s0 = 0, s1 = 0;
- int ss_state = READ_ONCE(ssp->srcu_size_state);
+ int ss_state = READ_ONCE(ssp->srcu_sup->srcu_size_state);
int ss_state_idx = ss_state;
idx = ssp->srcu_idx & 0x1;
@@ -1871,8 +1871,9 @@ void __init srcu_init(void)
ssp = list_first_entry(&srcu_boot_list, struct srcu_struct,
work.work.entry);
list_del_init(&ssp->work.work.entry);
- if (SRCU_SIZING_IS(SRCU_SIZING_INIT) && ssp->srcu_size_state == SRCU_SIZE_SMALL)
- ssp->srcu_size_state = SRCU_SIZE_ALLOC;
+ if (SRCU_SIZING_IS(SRCU_SIZING_INIT) &&
+ ssp->srcu_sup->srcu_size_state == SRCU_SIZE_SMALL)
+ ssp->srcu_sup->srcu_size_state = SRCU_SIZE_ALLOC;
queue_work(rcu_gp_wq, &ssp->work.work);
}
}
--
2.40.0.rc2
next prev parent reply other threads:[~2023-03-30 22:48 UTC|newest]
Thread overview: 37+ messages / expand[flat|nested] mbox.gz Atom feed top
2023-03-30 22:47 [PATCH rcu 0/20] Further shrink srcu_struct to promote cache locality Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 01/20] rcu-tasks: Fix warning for unused tasks_rcu_exit_srcu Paul E. McKenney
2023-03-31 11:58 ` Frederic Weisbecker
2023-03-31 18:35 ` Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 02/20] srcu: Add whitespace to __SRCU_STRUCT_INIT() & __DEFINE_SRCU() Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 03/20] srcu: Use static init for statically allocated in-module srcu_struct Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 04/20] srcu: Begin offloading srcu_struct fields to srcu_update Paul E. McKenney
2023-04-04 0:35 ` Joel Fernandes
2023-04-04 1:06 ` Paul E. McKenney
2023-04-04 1:16 ` Joel Fernandes
2023-03-30 22:47 ` [PATCH rcu 05/20] srcu: Move ->level from srcu_struct to srcu_usage Paul E. McKenney
2023-03-30 22:47 ` Paul E. McKenney [this message]
2023-03-30 22:47 ` [PATCH rcu 07/20] srcu: Move ->srcu_cb_mutex " Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 08/20] srcu: Move ->lock initialization after srcu_usage allocation Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 09/20] srcu: Move ->lock from srcu_struct to srcu_usage Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 10/20] srcu: Move ->srcu_gp_mutex " Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 11/20] srcu: Move grace-period fields " Paul E. McKenney
2023-06-01 11:14 ` Jon Hunter
[not found] ` <CALm+0cVXGdLNQpfJxnAnq2j2Ybs_rVAEqNzxgLSq7bDJp1KnfA@mail.gmail.com>
2023-06-01 13:46 ` Paul E. McKenney
2023-06-01 17:14 ` Jon Hunter
2023-06-01 19:21 ` Paul E. McKenney
2023-06-02 2:52 ` Z qiang
2023-06-02 3:09 ` Paul E. McKenney
2023-06-04 9:53 ` Linux regression tracking #adding (Thorsten Leemhuis)
2023-03-30 22:47 ` [PATCH rcu 12/20] srcu: Move heuristics " Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 13/20] srcu: Move ->sda_is_static " Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 14/20] srcu: Move srcu_barrier() fields " Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 15/20] srcu: Move work-scheduling " Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 16/20] srcu: Check for readers at module-exit time Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 17/20] srcu: Fix long lines in srcu_get_delay() Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 18/20] srcu: Fix long lines in cleanup_srcu_struct() Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 19/20] srcu: Fix long lines in srcu_gp_end() Paul E. McKenney
2023-03-30 22:47 ` [PATCH rcu 20/20] srcu: Fix long lines in srcu_funnel_gp_start() Paul E. McKenney
2023-04-04 13:57 ` [PATCH rcu 0/20] Further shrink srcu_struct to promote cache locality Joel Fernandes
2023-04-04 14:09 ` Paul E. McKenney
2023-04-04 17:01 ` Joel Fernandes
2023-04-04 17:17 ` Paul E. McKenney
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20230330224726.662344-6-paulmck@kernel.org \
--to=paulmck@kernel.org \
--cc=hch@lst.de \
--cc=kernel-team@meta.com \
--cc=linux-kernel@vger.kernel.org \
--cc=qiang1.zhang@intel.com \
--cc=rcu@vger.kernel.org \
--cc=rostedt@goodmis.org \
--cc=sachinp@linux.ibm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.