[RFC PATCH 4/6] blk-mq: add a shared zone to tag fairness

[Keith Busch <kbusch@meta.com>]

From: Keith Busch <kbusch@kernel.org>

Private slices of the sbitmap can strand an under utilized queue's tags.
Allow a driver to request some percentage of tags to be reserved into a
common shared pool that anyone may allocate from. The default 0 means
the entire tag set is divided up among its users; 100 means no one has
an exclusive window and can therefore allocate from the entire tag
space.

Shared tags being outside the private zone makes it so we can't do a
sbitmap range weight to find its active commands, so the debugfs count
may no longer accurate. Count the hctx's actual tags via
__blk_mq_hctx_tag_busy_iter() to provide accuracy over performance for
debugfs.

Signed-off-by: Keith Busch <kbusch@kernel.org>
---
 block/blk-mq-tag.c     | 94 +++++++++++++++++++++++++++++++-----------
 block/blk-mq.c         |  3 ++
 include/linux/blk-mq.h |  5 +++
 3 files changed, 79 insertions(+), 23 deletions(-)

diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
index 58cf480df9c69..aa7dbceb60dcd 100644
--- a/block/blk-mq-tag.c
+++ b/block/blk-mq-tag.c
@@ -144,54 +144,79 @@ static inline bool blk_mq_tag_is_windowed(struct blk_mq_alloc_data *data)
 	       (data->hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
 }
 
+struct blk_mq_tag_win {
+	unsigned int priv_min;
+	unsigned int priv_max;
+	unsigned int shared_min;
+	unsigned int shared_max;
+};
+
 static bool blk_mq_tag_active_window(struct blk_mq_hw_ctx *hctx,
-				     unsigned int depth, unsigned int *min,
-				     unsigned int *max)
+				     unsigned int depth,
+				     struct blk_mq_tag_win *w)
 {
 	unsigned int users = READ_ONCE(hctx->tags->active_queues);
 	int slot = READ_ONCE(*blk_mq_tag_win_slot(hctx));
-	unsigned int p, rem, pos;
+	unsigned int fair, priv, pct, pos;
+
+	if (users <= 1 || depth <= 1)
+		return false;
 
-	if (users <= 1 || slot < 0 || depth <= 1)
+	fair = depth / users;
+	if (!fair)
 		return false;
 
-	p = depth / users;
-	if (!p)
+	pct = 100 - min(hctx->queue->tag_set->shared_pct, 100u);
+	priv = fair * pct / 100;
+	if (!priv)
 		return false;
 
 	/*
 	 * Slots are allocated from a bitmap and freed out of order, so a slot
 	 * index can exceed the active-user count once lower slots are freed.
 	 * Convert it to a position based on how many active slots precede it.
+	 * If a slot could not be assigned, the window is confined to the
+	 * shared range.
 	 */
-	pos = bitmap_weight(hctx->tags->active_slots, slot);
-	rem = depth - p * users;
-	*min = pos * p;
-	*max = pos * p + p + rem;
-	if (*max > depth)
-		*max = depth;
+	pos = slot >= 0 ? bitmap_weight(hctx->tags->active_slots, slot) : users;
+	if (pos < users) {
+		w->priv_min = pos * priv;
+		w->priv_max = w->priv_min + priv;
+	} else {
+		w->priv_min = w->priv_max = 0;
+	}
 
+	w->shared_min = min(users * priv, depth);
+	w->shared_max = depth;
+
+	if (w->priv_min == w->priv_max && w->shared_min == w->shared_max)
+		return false;
 	return true;
 }
 
-int blk_mq_get_tag_window(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt)
+static int __blk_mq_get_tag_window(struct sbitmap_queue *bt,
+				   struct blk_mq_tag_win *w)
 {
-	unsigned int min, max;
+	if (w->priv_max > w->priv_min) {
+		int tag = sbitmap_queue_get_range(bt, w->priv_min, w->priv_max);
 
-	if (blk_mq_tag_active_window(hctx, bt->sb.depth, &min, &max))
-		return sbitmap_queue_get_range(bt, min, max);
-	return __sbitmap_queue_get(bt);
+		if (tag >= 0)
+			return tag;
+	}
+
+	if (w->shared_max > w->shared_min)
+		return sbitmap_queue_get_range(bt, w->shared_min, w->shared_max);
 
+	return BLK_MQ_NO_TAG;
 }
 
-unsigned int blk_mq_hctx_active(struct blk_mq_hw_ctx *hctx)
+int blk_mq_get_tag_window(struct blk_mq_hw_ctx *hctx, struct sbitmap_queue *bt)
 {
-	struct sbitmap *sb = &hctx->tags->bitmap_tags.sb;
-	unsigned int min, max;
+	struct blk_mq_tag_win w;
 
-	if (!blk_mq_tag_active_window(hctx, sb->depth, &min, &max))
-		return sbitmap_weight(sb);
-	return sbitmap_weight_range(sb, min, max - min);
+	if (blk_mq_tag_active_window(hctx, bt->sb.depth, &w))
+		return __blk_mq_get_tag_window(bt, &w);
+	return __sbitmap_queue_get(bt);
 }
 
 static int __blk_mq_get_tag(struct blk_mq_alloc_data *data,
@@ -604,6 +629,29 @@ static void __blk_mq_hctx_tag_busy_iter(struct blk_mq_hw_ctx *hctx,
 	bt_for_each(hctx, hctx->queue, &tags->bitmap_tags, fn, priv, false);
 }
 
+static bool blk_mq_count_active(struct request *rq, void *priv)
+{
+	(*(unsigned int *)priv)++;
+	return true;
+}
+
+unsigned int blk_mq_hctx_active(struct blk_mq_hw_ctx *hctx)
+{
+	struct request_queue *q = hctx->queue;
+	unsigned int count = 0;
+	int srcu_idx;
+
+	if (!percpu_ref_tryget(&q->q_usage_counter))
+	        return 0;
+
+	srcu_idx = srcu_read_lock(&q->tag_set->tags_srcu);
+	__blk_mq_hctx_tag_busy_iter(hctx, blk_mq_count_active, &count);
+	srcu_read_unlock(&q->tag_set->tags_srcu, srcu_idx);
+
+	blk_queue_exit(q);
+	return count;
+}
+
 /**
  * blk_mq_queue_tag_busy_iter - iterate over all requests with a driver tag
  * @q:		Request queue to examine.
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 56ab6ac5ec696..cc56baee74fe8 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -4845,6 +4845,9 @@ int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
 		set->queue_depth = BLK_MQ_MAX_DEPTH;
 	}
 
+	if (set->shared_pct > 100)
+		set->shared_pct = 100;
+
 	if (!set->nr_maps)
 		set->nr_maps = 1;
 	else if (set->nr_maps > HCTX_MAX_TYPES)
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 22cc09d5ef320..59847ac7d319c 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -515,6 +515,10 @@ enum hctx_type {
  * @numa_node:	   NUMA node the storage adapter has been connected to.
  * @timeout:	   Request processing timeout in jiffies.
  * @flags:	   Zero or more BLK_MQ_F_* flags.
+ * @shared_pct:	   Percent of tags dedicated to non-exclusive use. 0 means
+ *		   the entire tagset is divided among the users for exclusive
+ *		   per-user access, and 100 means the entire tag space is
+ *		   available to all queue contexts that share it.
  * @driver_data:   Pointer to data owned by the block driver that created this
  *		   tag set.
  * @tags:	   Tag sets. One tag set per hardware queue. Has @nr_hw_queues
@@ -544,6 +548,7 @@ struct blk_mq_tag_set {
 	int			numa_node;
 	unsigned int		timeout;
 	unsigned int		flags;
+	unsigned int		shared_pct;
 	void			*driver_data;
 
 	struct blk_mq_tags	**tags;
-- 
2.52.0