From: Miroslav Cimerman <mc@raidlab.org>
To: Song Liu <song@kernel.org>, Yu Kuai <yukuai@fygo.io>
Cc: Li Nan <magiclinan@didiglobal.com>, Xiao Ni <xiao@kernel.org>,
linux-raid@vger.kernel.org, linux-kernel@vger.kernel.org,
Anton Gavriliuk <antosha20xx@gmail.com>
Subject: [PATCH] md/raid5: grow stripe hash table with the size of stripe cache
Date: Mon, 16 Nov 2026 13:39:54 +0000 [thread overview]
Message-ID: <20261116133954.5111-1-mc@raidlab.org> (raw)
The stripe hash table has a fixed size of one page, 512 buckets with
4 KiB pages. With stripe_cache_size at its current maximum (32768), a
bucket contains 64 stripes on average. Every stripe lookup walks one
chain with interrupts off and the hash lock held, and with random I/O
most lookups miss and walk the whole chain. This costs CPU time on
every request and makes the hash locks more contended.
Grow the table when stripe_cache_size is written, to about
STRIPES_PER_HASH_BUCKET (two) stripes per bucket, rounded up to a
power of two. The new table is allocated before any hash lock is taken,
and the stripes are moved to it with all hash locks held. The table
size stays a power of two and a multiple of NR_STRIPE_HASH_LOCKS, so
every stripe keeps its hash lock. If the allocation fails, the old
table stays. Moving about 24k stripes (only stripes that have been
used are in the table) took about 2 ms with interrupts off on the
Xeon machine below. This happens only when stripe_cache_size is set.
The table starts at one page as before. It needs at most 8 bytes per
stripe: 128 KiB at stripe_cache_size=32768, where the stripe pages
alone take 768 MiB on RAID-6 4+2. The table is not shrunk when
stripe_cache_size is lowered.
2x Xeon E5-2699 v3 (one NUMA node used), RAM disks, chunk 512K,
stripe_cache_size=32768, group_thread_cnt=8, io_uring, without -> with
this patch:
RAID-6 4+2 4k random write, 16 jobs QD64 320k -> 355k IOPS (+11%)
RAID-6 6+2 4k random write, 16 jobs QD64 345k -> 381k IOPS (+10%)
RAID-5 5+1 4k random write, 16 jobs QD64 437k -> 514k IOPS (+18%)
RAID-6 4+2 1M seq write, 4 jobs QD16 4410 -> 4971 MiB/s (+13%)
RAID-6 4+2 4k random read, degraded 543k -> 705k IOPS (+30%)
HPE DL345 Gen11, AMD EPYC 9455, 6x PCIe 5.0 NVMe (KIOXIA CM7-V 3.2 TB),
the same md settings and chunk size:
RAID-6 4+2 4k random write, 16 jobs QD64 501k -> 536k IOPS (+7%)
RAID-6 4+2 2M seq write, 1 job QD64 10342 -> 13523 MiB/s (+31%)
The CPU time per I/O drops by 21-34% on the Xeon, and by 12% (random)
and 49% (sequential) on the EPYC. In the 4k random write tests,
find_get_stripe() drops from 9.2% to 0.6% of all cycles on the Xeon,
and from 3.4% to 0.2% on the EPYC. With md's default stripe_cache_size
nothing changes.
Signed-off-by: Miroslav Cimerman <mc@raidlab.org>
---
drivers/md/raid5.c | 60 +++++++++++++++++++++++++++++++++++++++++++---
drivers/md/raid5.h | 14 ++++++++---
2 files changed, 68 insertions(+), 6 deletions(-)
diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index efb4b0d37cb0..5e28ab750398 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -77,7 +77,7 @@ static void raid5_quiesce(struct mddev *mddev, int quiesce);
static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
{
- int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
+ int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & conf->hash_mask;
return &conf->stripe_hashtbl[hash];
}
@@ -7061,6 +7061,57 @@ raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
return ret;
}
+static unsigned int nr_hash_buckets(int nr_stripes)
+{
+ unsigned int buckets = max(nr_stripes / STRIPES_PER_HASH_BUCKET, 1);
+
+ /*
+ * The table size must be a multiple of NR_STRIPE_HASH_LOCKS for a
+ * stripe to keep its lock when the table grows.
+ */
+ BUILD_BUG_ON(NR_HASH_MIN % NR_STRIPE_HASH_LOCKS);
+ return max_t(unsigned int, roundup_pow_of_two(buckets), NR_HASH_MIN);
+}
+
+/*
+ * Grow the stripe hash table so that there are fewer stripes hashed to
+ * a single bucket when the stripe cache is enlarged.
+ *
+ * This is only an optimization. If the allocation of the new table fails,
+ * the old table stays.
+ */
+static void raid5_grow_stripe_hashtbl(struct r5conf *conf, int nr_stripes)
+{
+ unsigned int new_size = nr_hash_buckets(nr_stripes);
+ unsigned int i, old_size;
+ struct hlist_head *new, *old;
+ struct stripe_head *sh;
+ struct hlist_node *tmp;
+
+ lockdep_assert_held(&conf->cache_size_mutex);
+ old = conf->stripe_hashtbl;
+ old_size = conf->hash_mask + 1;
+ if (new_size <= old_size)
+ return;
+ new = kvzalloc_objs(*new, new_size, GFP_KERNEL | __GFP_NOWARN);
+ if (!new)
+ return;
+
+ lock_all_device_hash_locks_irq(conf);
+ conf->stripe_hashtbl = new;
+ conf->hash_mask = new_size - 1;
+ for (i = 0; i < old_size; i++) {
+ hlist_for_each_entry_safe(sh, tmp, &old[i], hash) {
+ hlist_del(&sh->hash);
+ hlist_add_head(&sh->hash,
+ stripe_hash(conf, sh->sector));
+ }
+ }
+ unlock_all_device_hash_locks_irq(conf);
+
+ kvfree(old);
+}
+
int
raid5_set_cache_size(struct mddev *mddev, int size)
{
@@ -7086,6 +7137,7 @@ raid5_set_cache_size(struct mddev *mddev, int size)
result = -ENOMEM;
break;
}
+ raid5_grow_stripe_hashtbl(conf, conf->max_nr_stripes);
mutex_unlock(&conf->cache_size_mutex);
return result;
@@ -7589,7 +7641,7 @@ static void free_conf(struct r5conf *conf)
put_page(conf->disks[i].extra_page);
kfree(conf->disks);
bioset_exit(&conf->bio_split);
- kfree(conf->stripe_hashtbl);
+ kvfree(conf->stripe_hashtbl);
kfree(conf->pending_data);
mempool_destroy(conf->ctx_pool);
@@ -7785,9 +7837,11 @@ static struct r5conf *setup_conf(struct mddev *mddev)
goto abort;
ret = -ENOMEM;
- conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL);
+ conf->stripe_hashtbl = kvzalloc_objs(*conf->stripe_hashtbl,
+ NR_HASH_MIN);
if (!conf->stripe_hashtbl)
goto abort;
+ conf->hash_mask = NR_HASH_MIN - 1;
/* We init hash_locks[0] separately to that it can be used
* as the reference lock in the spin_lock_nest_lock() call
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index 1a61b474fc22..6d1d4aa125f2 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -489,8 +489,12 @@ struct disk_info {
#define IO_THRESHOLD 1
#define BYPASS_THRESHOLD 1
-#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head))
-#define HASH_MASK (NR_HASH - 1)
+/*
+ * The stripe hash table starts at one page and is enlarged with
+ * stripe_cache_size, to about STRIPES_PER_HASH_BUCKET stripes per bucket.
+ */
+#define NR_HASH_MIN (PAGE_SIZE / sizeof(struct hlist_head))
+#define STRIPES_PER_HASH_BUCKET 2
#define MAX_STRIPE_BATCH 8
/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
@@ -570,7 +574,11 @@ struct raid5_percpu {
struct r5conf {
struct hlist_head *stripe_hashtbl;
- /* only protect corresponding hash list and inactive_list */
+ unsigned int hash_mask; /* stripe_hashtbl size - 1 */
+ /*
+ * Protect corresponding hash list and inactive_list. All of them are
+ * held to replace stripe_hashtbl and hash_mask during the table growth.
+ */
spinlock_t hash_locks[NR_STRIPE_HASH_LOCKS];
struct mddev *mddev;
int chunk_sectors;
base-commit: 47e1b18f38d74015cb9fd30b5ff2a1eb88e77d31
--
2.53.0
reply other threads:[~2026-11-16 13:40 UTC|newest]
Thread overview: [no followups] expand[flat|nested] mbox.gz Atom feed
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261116133954.5111-1-mc@raidlab.org \
--to=mc@raidlab.org \
--cc=antosha20xx@gmail.com \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-raid@vger.kernel.org \
--cc=magiclinan@didiglobal.com \
--cc=song@kernel.org \
--cc=xiao@kernel.org \
--cc=yukuai@fygo.io \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.