[PATCH 6.1.y] io_uring/rw: end write accounting from ->ki_complete

[Sasha Levin <sashal@kernel.org>]

From: Jens Axboe <axboe@kernel.dk>

[ Upstream commit 796aa0547557e63338657ed1c487906f9fac4c73 ]

Commit b000145e9907 moved both the fsnotify calls and the write
accounting out of the kiocb completion handler and into the
io_req_rw_complete() task_work. However, only the fsnotify part actually
needed to move as it may sleep. Ending the write accounting is just a
percpu_up_read() on the superblock writers sem.

Deferring it is a problem, because it makes dropping SB_FREEZE_WRITE
protection depend on the ring owner getting to running task_work. But
the task may be blocked in freeze_super(), causing it to never get to
that:

  task                             io-wq worker
  --------------------------------------------------------------
  io_write()
    io_kiocb_start_write()         (takes sb_writers, hidden from
                                    lockdep by __sb_writers_release)
    write_iter() -> -EIOCBQUEUED
  ioctl(FS_IOC_SHUTDOWN)
    bdev_freeze()
      freeze_super()
        percpu_down_write()        <- waits for the reader above
                                   io_write()
                                     kiocb_start_write()
                                       percpu_down_read()  <- queued
                                                              behind the
                                                              writer
  <bio completes>
    io_complete_rw()
      queues io_req_rw_complete()  <- never runs, task is in D state

End the write from io_complete_rw() instead, and leave only the fsnotify
calls in task_work.

Reported-by: syzbot+2eb3d983669d3e49d4fa@syzkaller.appspotmail.com
Cc: stable@vger.kernel.org
Fixes: b000145e9907 ("io_uring/rw: defer fsnotify calls to task context")
Signed-off-by: Jens Axboe <axboe@kernel.dk>
[ replaced upstream’s separate notification helper with an end_write Boolean parameter in io_req_io_end() ]
Signed-off-by: Sasha Levin <sashal@kernel.org>
---
 io_uring/rw.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/io_uring/rw.c b/io_uring/rw.c
index cbd3ee9a9373a..bf66ffef7e434 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -235,14 +235,15 @@ static void io_req_end_write(struct io_kiocb *req)
 
 /*
  * Trigger the notifications after having done some IO, and finish the write
- * accounting, if any.
+ * accounting for inline completions.
  */
-static void io_req_io_end(struct io_kiocb *req)
+static void io_req_io_end(struct io_kiocb *req, bool end_write)
 {
 	struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
 
 	if (rw->kiocb.ki_flags & IOCB_WRITE) {
-		io_req_end_write(req);
+		if (end_write)
+			io_req_end_write(req);
 		fsnotify_modify(req->file);
 	} else {
 		fsnotify_access(req->file);
@@ -255,10 +256,10 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res)
 		if ((res == -EAGAIN || res == -EOPNOTSUPP) &&
 		    io_rw_should_reissue(req)) {
 			/*
-			 * Reissue will start accounting again, finish the
-			 * current cycle.
+			 * Write accounting is already finished by
+			 * io_complete_rw().
 			 */
-			io_req_io_end(req);
+			io_req_io_end(req, false);
 			req->flags |= REQ_F_REISSUE | REQ_F_PARTIAL_IO;
 			return true;
 		}
@@ -284,7 +285,7 @@ static inline int io_fixup_rw_res(struct io_kiocb *req, long res)
 
 static void io_req_rw_complete(struct io_kiocb *req, bool *locked)
 {
-	io_req_io_end(req);
+	io_req_io_end(req, false);
 	io_req_task_complete(req, locked);
 }
 
@@ -293,6 +294,10 @@ static void io_complete_rw(struct kiocb *kiocb, long res)
 	struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb);
 	struct io_kiocb *req = cmd_to_io_kiocb(rw);
 
+	/* ring owner may block in freeze_super() before task_work runs */
+	if (kiocb->ki_flags & IOCB_WRITE)
+		io_req_end_write(req);
+
 	if (__io_complete_rw_common(req, res))
 		return;
 	io_req_set_res(req, io_fixup_rw_res(req, res), 0);
@@ -333,7 +338,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret,
 			 * Safe to call io_end from here as we're inline
 			 * from the submission path.
 			 */
-			io_req_io_end(req);
+			io_req_io_end(req, true);
 			io_req_set_res(req, final_ret,
 				       io_put_kbuf(req, issue_flags));
 			return IOU_OK;
-- 
2.53.0