[PATCH V6 10/12] block: limit hw queues to be polled in each blk_poll()

Ming Lei
2021-04-22T12:22:49Z

Limit at most 8 queues are polled in each blk_pull(), avoid to
add extra latency when queue depth is high.

Reviewed-by: Jeffle Xu <jefflexu@linux.alibaba.com>
Signed-off-by: Ming Lei <ming.lei@redhat.com>
---
block/blk-poll.c | 78 ++++++++++++++++++++++++++++++++++--------------
1 file changed, 55 insertions(+), 23 deletions(-)

diff --git a/block/blk-poll.c b/block/blk-poll.c
index 249d73ff6f81..20e7c47cc984 100644
--- a/block/blk-poll.c
+++ b/block/blk-poll.c
@@ -288,36 +288,32 @@ static void bio_grp_list_move(struct bio_grp_list *dst,
src->nr_grps -= cnt;
}

-static int blk_mq_poll_io(struct bio *bio)
+#define POLL_HCTX_MAX_CNT 8
+
+static bool blk_add_unique_hctx(struct blk_mq_hw_ctx **data, int *cnt,
+ struct blk_mq_hw_ctx *hctx)
{
- struct request_queue *q = bio->bi_bdev->bd_disk->queue;
- blk_qc_t cookie = bio_get_poll_data(bio);
- int ret = 0;
+ int i;

- /* wait until the bio is submitted really */
- if (!blk_qc_t_ready(cookie))
- return 0;

- if (!bio_flagged(bio, BIO_DONE) && blk_qc_t_valid(cookie)) {
- struct blk_mq_hw_ctx *hctx =
- q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
+ for (i = 0; i < *cnt; i++) {
+ if (data[i] == hctx)
+ goto exit;
+ }

- ret += blk_mq_poll_hctx(q, hctx);
+ if (i < POLL_HCTX_MAX_CNT) {
+ data[i] = hctx;
+ (*cnt)++;
}
- return ret;
+ exit:
+ return *cnt == POLL_HCTX_MAX_CNT;
}

-static int blk_bio_poll_and_end_io(struct bio_grp_list *grps)
+static void blk_build_poll_queues(struct bio_grp_list *grps,
+ struct blk_mq_hw_ctx **data, int *cnt)
{
- int ret = 0;
int i;

- /*
- * Poll hw queue first.
- *
- * TODO: limit max poll times and make sure to not poll same
- * hw queue one more time.
- */
for (i = 0; i < grps->nr_grps; i++) {
struct bio_grp_list_data *grp = &grps->head[i];
struct bio *bio;
@@ -325,11 +321,31 @@ static int blk_bio_poll_and_end_io(struct bio_grp_list *grps)
if (bio_grp_list_grp_empty(grp))
continue;

- for (bio = grp->list.head; bio; bio = bio->bi_poll)
- ret += blk_mq_poll_io(bio);
+ for (bio = grp->list.head; bio; bio = bio->bi_poll) {
+ blk_qc_t cookie;
+ struct blk_mq_hw_ctx *hctx;
+ struct request_queue *q;
+
+ if (bio_flagged(bio, BIO_DONE))
+ continue;
+
+ /* wait until the bio is submitted really */
+ cookie = bio_get_poll_data(bio);
+ if (!blk_qc_t_ready(cookie) || !blk_qc_t_valid(cookie))
+ continue;
+
+ q = bio->bi_bdev->bd_disk->queue;
+ hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
+ if (blk_add_unique_hctx(data, cnt, hctx))
+ return;
+ }
}
+}
+
+static void blk_bio_poll_reap_ios(struct bio_grp_list *grps)
+{
+ int i;

- /* reap bios */
for (i = 0; i < grps->nr_grps; i++) {
struct bio_grp_list_data *grp = &grps->head[i];
struct bio *bio;
@@ -354,6 +370,22 @@ static int blk_bio_poll_and_end_io(struct bio_grp_list *grps)
}
__bio_grp_list_merge(&grp->list, &bl);
}
+}
+
+static int blk_bio_poll_and_end_io(struct bio_grp_list *grps)
+{
+ int ret = 0;
+ int i;
+ struct blk_mq_hw_ctx *hctx[POLL_HCTX_MAX_CNT];
+ int cnt = 0;
+
+ blk_build_poll_queues(grps, hctx, &cnt);
+
+ for (i = 0; i < cnt; i++)
+ ret += blk_mq_poll_hctx(hctx[i]->queue, hctx[i]);
+
+ blk_bio_poll_reap_ios(grps);
+
return ret;
}

--
2.29.2