[PATCH 6/8] io_uring: replace list with array for compl batch

Pavel Begunkov
2021-01-25T12:12:05Z

Reincarnation of an old patch that replaces a list in struct
io_compl_batch with an array. It's needed to avoid hooking requests via
their compl.list, because it won't be always available in the future.

It's also nice to split io_submit_flush_completions() to avoid free
under locks and remove unlock/lock with a long comment describing when
it can be done.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
---
fs/io_uring.c | 35 +++++++++++------------------------
1 file changed, 11 insertions(+), 24 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index 08d0c8b60c2a..fcd8df43b6bf 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -260,10 +260,11 @@ struct io_sq_data {
};

#define IO_IOPOLL_BATCH 8
+#define IO_COMPL_BATCH 32

struct io_comp_state {
unsigned int nr;
- struct list_head list;
+ struct io_kiocb *reqs[IO_COMPL_BATCH];
};

struct io_submit_state {
@@ -1338,7 +1339,6 @@ static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
init_llist_head(&ctx->rsrc_put_llist);

submit_state = &ctx->submit_state;
- INIT_LIST_HEAD(&submit_state->comp.list);
submit_state->comp.nr = 0;
submit_state->file_refs = 0;
submit_state->free_reqs = 0;
@@ -1901,33 +1901,20 @@ static void io_req_complete_nostate(struct io_kiocb *req, long res,
static void io_submit_flush_completions(struct io_comp_state *cs,
struct io_ring_ctx *ctx)
{
+ int i, nr = cs->nr;
+
spin_lock_irq(&ctx->completion_lock);
- while (!list_empty(&cs->list)) {
- struct io_kiocb *req;
+ for (i = 0; i < nr; i++) {
+ struct io_kiocb *req = cs->reqs[i];

- req = list_first_entry(&cs->list, struct io_kiocb, compl.list);
- list_del(&req->compl.list);
__io_cqring_fill_event(req, req->result, req->compl.cflags);
-
- /*
- * io_free_req() doesn't care about completion_lock unless one
- * of these flags is set. REQ_F_WORK_INITIALIZED is in the list
- * because of a potential deadlock with req->work.fs->lock
- * We defer both, completion and submission refs.
- */
- if (req->flags & (REQ_F_FAIL_LINK|REQ_F_LINK_TIMEOUT
- |REQ_F_WORK_INITIALIZED)) {
- spin_unlock_irq(&ctx->completion_lock);
- io_double_put_req(req);
- spin_lock_irq(&ctx->completion_lock);
- } else {
- io_double_put_req(req);
- }
}
io_commit_cqring(ctx);
spin_unlock_irq(&ctx->completion_lock);

io_cqring_ev_posted(ctx);
+ for (i = 0; i < nr; i++)
+ io_double_put_req(cs->reqs[i]);
cs->nr = 0;
}

@@ -6565,8 +6552,8 @@ static void __io_queue_sqe(struct io_kiocb *req, struct io_comp_state *cs)
} else if (likely(!ret)) {
/* drop submission reference */
if (req->flags & REQ_F_COMPLETE_INLINE) {
- list_add_tail(&req->compl.list, &cs->list);
- if (++cs->nr >= 32)
+ cs->reqs[cs->nr++] = req;
+ if (cs->nr == IO_COMPL_BATCH)
io_submit_flush_completions(cs, req->ctx);
req = NULL;
} else {
@@ -6705,7 +6692,7 @@ static int io_submit_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe,
static void io_submit_state_end(struct io_submit_state *state,
struct io_ring_ctx *ctx)
{
- if (!list_empty(&state->comp.list))
+ if (state->comp.nr)
io_submit_flush_completions(&state->comp, ctx);
if (state->plug_started)
blk_finish_plug(&state->plug);
--
2.24.0