[RFC 2/3] io_uring: don't get completion_lock in io_poll_rewait()

Xiaoguang Wang
2021-09-22T12:34:28Z

In current implementation, if there are not available events,
io_poll_rewait() just gets completion_lock, and unlocks it in
io_poll_task_func() or io_async_task_func(), which isn't necessary.

Change this logic to let io_poll_task_func() or io_async_task_func()
get the completion_lock lock, this is also a preparation for
later patch, which will batch poll request completion.

Signed-off-by: Xiaoguang Wang <xiaoguang.wang@linux.alibaba.com>
---
fs/io_uring.c | 50 ++++++++++++++++++++++----------------------------
1 file changed, 22 insertions(+), 28 deletions(-)

diff --git a/fs/io_uring.c b/fs/io_uring.c
index ca4464a75c7b..6fdfb688cf91 100644
--- a/fs/io_uring.c
+++ b/fs/io_uring.c
@@ -5040,10 +5040,7 @@ static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
}

static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
- __acquires(&req->ctx->completion_lock)
{
- struct io_ring_ctx *ctx = req->ctx;
-
/* req->task == current here, checking PF_EXITING is safe */
if (unlikely(req->task->flags & PF_EXITING))
WRITE_ONCE(poll->canceled, true);
@@ -5054,7 +5051,6 @@ static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
req->result = vfs_poll(req->file, &pt) & poll->events;
}

- spin_lock(&ctx->completion_lock);
if (!req->result && !READ_ONCE(poll->canceled)) {
if (req->opcode == IORING_OP_POLL_ADD)
WRITE_ONCE(req->poll.active, true);
@@ -5142,30 +5138,29 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked)
{
struct io_ring_ctx *ctx = req->ctx;
struct io_kiocb *nxt;
+ bool done;

- if (io_poll_rewait(req, &req->poll)) {
- spin_unlock(&ctx->completion_lock);
- } else {
- bool done;
+ if (io_poll_rewait(req, &req->poll))
+ return;

- done = __io_poll_complete(req, req->result);
- if (done) {
- io_poll_remove_double(req);
- __io_poll_remove_one(req, io_poll_get_single(req), true);
- hash_del(&req->hash_node);
- } else {
- req->result = 0;
- WRITE_ONCE(req->poll.active, true);
- }
- io_commit_cqring(ctx);
- spin_unlock(&ctx->completion_lock);
- io_cqring_ev_posted(ctx);
+ spin_lock(&ctx->completion_lock);
+ done = __io_poll_complete(req, req->result);
+ if (done) {
+ io_poll_remove_double(req);
+ __io_poll_remove_one(req, io_poll_get_single(req), true);
+ hash_del(&req->hash_node);
+ } else {
+ req->result = 0;
+ WRITE_ONCE(req->poll.active, true);
+ }
+ io_commit_cqring(ctx);
+ spin_unlock(&ctx->completion_lock);
+ io_cqring_ev_posted(ctx);

- if (done) {
- nxt = io_put_req_find_next(req);
- if (nxt)
- io_req_task_submit(nxt, locked);
- }
+ if (done) {
+ nxt = io_put_req_find_next(req);
+ if (nxt)
+ io_req_task_submit(nxt, locked);
}
}

@@ -5284,11 +5279,10 @@ static void io_async_task_func(struct io_kiocb *req, bool *locked)

trace_io_uring_task_run(req->ctx, req, req->opcode, req->user_data);

- if (io_poll_rewait(req, &apoll->poll)) {
- spin_unlock(&ctx->completion_lock);
+ if (io_poll_rewait(req, &apoll->poll))
return;
- }

+ spin_lock(&ctx->completion_lock);
hash_del(&req->hash_node);
io_poll_remove_double(req);
spin_unlock(&ctx->completion_lock);
--
2.14.4.44.g2045bb6