Linux RAID subsystem development
 help / color / mirror / Atom feed
From: Miroslav Cimerman <mc@raidlab.org>
To: Song Liu <song@kernel.org>, Yu Kuai <yukuai@fygo.io>
Cc: Li Nan <magiclinan@didiglobal.com>, Xiao Ni <xiao@kernel.org>,
	linux-raid@vger.kernel.org, linux-kernel@vger.kernel.org,
	Anton Gavriliuk <antosha20xx@gmail.com>
Subject: [PATCH] md/raid5: grow stripe hash table with the size of stripe cache
Date: Mon, 16 Nov 2026 13:39:54 +0000	[thread overview]
Message-ID: <20261116133954.5111-1-mc@raidlab.org> (raw)

The stripe hash table has a fixed size of one page, 512 buckets with
4 KiB pages. With stripe_cache_size at its current maximum (32768), a
bucket contains 64 stripes on average. Every stripe lookup walks one
chain with interrupts off and the hash lock held, and with random I/O
most lookups miss and walk the whole chain. This costs CPU time on
every request and makes the hash locks more contended.

Grow the table when stripe_cache_size is written, to about
STRIPES_PER_HASH_BUCKET (two) stripes per bucket, rounded up to a
power of two. The new table is allocated before any hash lock is taken,
and the stripes are moved to it with all hash locks held. The table
size stays a power of two and a multiple of NR_STRIPE_HASH_LOCKS, so
every stripe keeps its hash lock. If the allocation fails, the old
table stays. Moving about 24k stripes (only stripes that have been
used are in the table) took about 2 ms with interrupts off on the
Xeon machine below. This happens only when stripe_cache_size is set.

The table starts at one page as before. It needs at most 8 bytes per
stripe: 128 KiB at stripe_cache_size=32768, where the stripe pages
alone take 768 MiB on RAID-6 4+2. The table is not shrunk when
stripe_cache_size is lowered.

2x Xeon E5-2699 v3 (one NUMA node used), RAM disks, chunk 512K,
stripe_cache_size=32768, group_thread_cnt=8, io_uring, without -> with
this patch:

  RAID-6 4+2  4k random write, 16 jobs QD64  320k -> 355k IOPS  (+11%)
  RAID-6 6+2  4k random write, 16 jobs QD64  345k -> 381k IOPS  (+10%)
  RAID-5 5+1  4k random write, 16 jobs QD64  437k -> 514k IOPS  (+18%)
  RAID-6 4+2  1M seq write, 4 jobs QD16      4410 -> 4971 MiB/s (+13%)
  RAID-6 4+2  4k random read, degraded       543k -> 705k IOPS  (+30%)

HPE DL345 Gen11, AMD EPYC 9455, 6x PCIe 5.0 NVMe (KIOXIA CM7-V 3.2 TB),
the same md settings and chunk size:

  RAID-6 4+2  4k random write, 16 jobs QD64  501k -> 536k IOPS   (+7%)
  RAID-6 4+2  2M seq write, 1 job QD64      10342 -> 13523 MiB/s (+31%)

The CPU time per I/O drops by 21-34% on the Xeon, and by 12% (random)
and 49% (sequential) on the EPYC. In the 4k random write tests,
find_get_stripe() drops from 9.2% to 0.6% of all cycles on the Xeon,
and from 3.4% to 0.2% on the EPYC. With md's default stripe_cache_size
nothing changes.

Signed-off-by: Miroslav Cimerman <mc@raidlab.org>
---
 drivers/md/raid5.c | 60 +++++++++++++++++++++++++++++++++++++++++++---
 drivers/md/raid5.h | 14 ++++++++---
 2 files changed, 68 insertions(+), 6 deletions(-)

diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c
index efb4b0d37cb0..5e28ab750398 100644
--- a/drivers/md/raid5.c
+++ b/drivers/md/raid5.c
@@ -77,7 +77,7 @@ static void raid5_quiesce(struct mddev *mddev, int quiesce);
 
 static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
 {
-	int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
+	int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & conf->hash_mask;
 	return &conf->stripe_hashtbl[hash];
 }
 
@@ -7061,6 +7061,57 @@ raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
 	return ret;
 }
 
+static unsigned int nr_hash_buckets(int nr_stripes)
+{
+	unsigned int buckets = max(nr_stripes / STRIPES_PER_HASH_BUCKET, 1);
+
+	/*
+	 * The table size must be a multiple of NR_STRIPE_HASH_LOCKS for a
+	 * stripe to keep its lock when the table grows.
+	 */
+	BUILD_BUG_ON(NR_HASH_MIN % NR_STRIPE_HASH_LOCKS);
+	return max_t(unsigned int, roundup_pow_of_two(buckets), NR_HASH_MIN);
+}
+
+/*
+ * Grow the stripe hash table so that there are fewer stripes hashed to
+ * a single bucket when the stripe cache is enlarged.
+ *
+ * This is only an optimization. If the allocation of the new table fails,
+ * the old table stays.
+ */
+static void raid5_grow_stripe_hashtbl(struct r5conf *conf, int nr_stripes)
+{
+	unsigned int new_size = nr_hash_buckets(nr_stripes);
+	unsigned int i, old_size;
+	struct hlist_head *new, *old;
+	struct stripe_head *sh;
+	struct hlist_node *tmp;
+
+	lockdep_assert_held(&conf->cache_size_mutex);
+	old = conf->stripe_hashtbl;
+	old_size = conf->hash_mask + 1;
+	if (new_size <= old_size)
+		return;
+	new = kvzalloc_objs(*new, new_size, GFP_KERNEL | __GFP_NOWARN);
+	if (!new)
+		return;
+
+	lock_all_device_hash_locks_irq(conf);
+	conf->stripe_hashtbl = new;
+	conf->hash_mask = new_size - 1;
+	for (i = 0; i < old_size; i++) {
+		hlist_for_each_entry_safe(sh, tmp, &old[i], hash) {
+			hlist_del(&sh->hash);
+			hlist_add_head(&sh->hash,
+				       stripe_hash(conf, sh->sector));
+		}
+	}
+	unlock_all_device_hash_locks_irq(conf);
+
+	kvfree(old);
+}
+
 int
 raid5_set_cache_size(struct mddev *mddev, int size)
 {
@@ -7086,6 +7137,7 @@ raid5_set_cache_size(struct mddev *mddev, int size)
 			result = -ENOMEM;
 			break;
 		}
+	raid5_grow_stripe_hashtbl(conf, conf->max_nr_stripes);
 	mutex_unlock(&conf->cache_size_mutex);
 
 	return result;
@@ -7589,7 +7641,7 @@ static void free_conf(struct r5conf *conf)
 			put_page(conf->disks[i].extra_page);
 	kfree(conf->disks);
 	bioset_exit(&conf->bio_split);
-	kfree(conf->stripe_hashtbl);
+	kvfree(conf->stripe_hashtbl);
 	kfree(conf->pending_data);
 
 	mempool_destroy(conf->ctx_pool);
@@ -7785,9 +7837,11 @@ static struct r5conf *setup_conf(struct mddev *mddev)
 		goto abort;
 
 	ret = -ENOMEM;
-	conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL);
+	conf->stripe_hashtbl = kvzalloc_objs(*conf->stripe_hashtbl,
+					     NR_HASH_MIN);
 	if (!conf->stripe_hashtbl)
 		goto abort;
+	conf->hash_mask = NR_HASH_MIN - 1;
 
 	/* We init hash_locks[0] separately to that it can be used
 	 * as the reference lock in the spin_lock_nest_lock() call
diff --git a/drivers/md/raid5.h b/drivers/md/raid5.h
index 1a61b474fc22..6d1d4aa125f2 100644
--- a/drivers/md/raid5.h
+++ b/drivers/md/raid5.h
@@ -489,8 +489,12 @@ struct disk_info {
 
 #define	IO_THRESHOLD		1
 #define BYPASS_THRESHOLD	1
-#define NR_HASH			(PAGE_SIZE / sizeof(struct hlist_head))
-#define HASH_MASK		(NR_HASH - 1)
+/*
+ * The stripe hash table starts at one page and is enlarged with
+ * stripe_cache_size, to about STRIPES_PER_HASH_BUCKET stripes per bucket.
+ */
+#define NR_HASH_MIN		(PAGE_SIZE / sizeof(struct hlist_head))
+#define STRIPES_PER_HASH_BUCKET	2
 #define MAX_STRIPE_BATCH	8
 
 /* NOTE NR_STRIPE_HASH_LOCKS must remain below 64.
@@ -570,7 +574,11 @@ struct raid5_percpu {
 
 struct r5conf {
 	struct hlist_head	*stripe_hashtbl;
-	/* only protect corresponding hash list and inactive_list */
+	unsigned int		hash_mask; /* stripe_hashtbl size - 1 */
+	/*
+	 * Protect corresponding hash list and inactive_list. All of them are
+	 * held to replace stripe_hashtbl and hash_mask during the table growth.
+	 */
 	spinlock_t		hash_locks[NR_STRIPE_HASH_LOCKS];
 	struct mddev		*mddev;
 	int			chunk_sectors;

base-commit: 47e1b18f38d74015cb9fd30b5ff2a1eb88e77d31
-- 
2.53.0


                 reply	other threads:[~2026-11-16 13:40 UTC|newest]

Thread overview: [no followups] expand[flat|nested]  mbox.gz  Atom feed

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261116133954.5111-1-mc@raidlab.org \
    --to=mc@raidlab.org \
    --cc=antosha20xx@gmail.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-raid@vger.kernel.org \
    --cc=magiclinan@didiglobal.com \
    --cc=song@kernel.org \
    --cc=xiao@kernel.org \
    --cc=yukuai@fygo.io \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox