[PATCH 0/2] io_uring passthru: set result on blk-mq request completion

[PATCH 0/2] io_uring passthru: set result on blk-mq request completion

[Caleb Sander Mateos]

io_uring NVMe passthru currently receives the NVMe status and result in
the blk-mq request completion callback nvme_uring_cmd_end_io() but
doesn't post the io_uring CQE until the io_uring task work callback
nvme_uring_task_cb(). The status and result must be plumbed through
struct nvme_uring_cmd_pdu, taking up 16 bytes of the 32 available.

Store the status and result on the io_uring request in
nvme_uring_cmd_end_io() instead of nvme_uring_task_cb() so it doesn't
need to be passed through struct nvme_uring_cmd_pdu.

Caleb Sander Mateos (2):
  io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()
  nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()

 block/ioctl.c                | 10 ++++++----
 drivers/block/ublk_drv.c     | 26 ++++++++++++++++++--------
 drivers/nvme/host/ioctl.c    | 19 +++++++++----------
 drivers/scsi/scsi_bsg.c      |  4 ++--
 fs/btrfs/ioctl.c             |  3 ++-
 fs/fuse/dev_uring.c          | 15 ++++++++++-----
 include/linux/io_uring/cmd.h | 29 +++++++++++++----------------
 io_uring/uring_cmd.c         | 34 ++++++++++++++++++++++------------
 8 files changed, 82 insertions(+), 58 deletions(-)

-- 
2.55.0

[PATCH 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

[Caleb Sander Mateos]

In preparation for setting the io_uring NVMe passthru CQE results from
the blk-mq request completion rather than the task work callback, split
out functions io_uring_cmd_set_res{,32}() from io_uring_cmd_done{,32}().
This allows io_uring_cmd_done{,32}() and __io_uring_cmd_done() to be
consolidated into a single CQE-size-agnostic function with 3 fewer
arguments than __io_uring_cmd_done().

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 block/ioctl.c                |  7 ++++---
 drivers/block/ublk_drv.c     | 24 ++++++++++++++++--------
 drivers/nvme/host/ioctl.c    |  7 ++++---
 drivers/scsi/scsi_bsg.c      |  4 ++--
 fs/btrfs/ioctl.c             |  3 ++-
 fs/fuse/dev_uring.c          | 15 ++++++++++-----
 include/linux/io_uring/cmd.h | 30 ++++++++++++++----------------
 io_uring/uring_cmd.c         | 35 +++++++++++++++++++----------------
 8 files changed, 71 insertions(+), 54 deletions(-)

diff --git a/block/ioctl.c b/block/ioctl.c
index 64b4e6c0f696..946cab74f253 100644
--- a/block/ioctl.c
+++ b/block/ioctl.c
@@ -869,13 +869,14 @@ static void blk_cmd_complete(struct io_tw_req tw_req, io_tw_token_t tw)
 	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
 	struct blk_iou_cmd *bic = io_uring_cmd_to_pdu(cmd, struct blk_iou_cmd);
 
 	if (bic->res == -EAGAIN && bic->nowait)
 		io_uring_cmd_issue_blocking(cmd);
-	else
-		io_uring_cmd_done(cmd, bic->res,
-				  IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+	else {
+		io_uring_cmd_set_res(cmd, bic->res);
+		io_uring_cmd_done(cmd, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+	}
 }
 
 static void bio_cmd_bio_end_io(struct bio *bio)
 {
 	struct io_uring_cmd *cmd = bio->bi_private;
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 4d17ed264da1..51f15e5b3c01 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -837,11 +837,12 @@ static void ublk_batch_deinit_fetch_buf(struct ublk_queue *ubq,
 	list_del_init(&fcmd->node);
 	WARN_ON_ONCE(fcmd != ubq->active_fcmd);
 	__ublk_release_fcmd(ubq);
 	spin_unlock(&ubq->evts_lock);
 
-	io_uring_cmd_done(fcmd->cmd, res, data->issue_flags);
+	io_uring_cmd_set_res(fcmd->cmd, res);
+	io_uring_cmd_done(fcmd->cmd, data->issue_flags);
 	ublk_batch_free_fcmd(fcmd);
 }
 
 static int ublk_batch_fetch_post_cqe(struct ublk_batch_fetch_cmd *fcmd,
 				     struct io_br_sel *sel,
@@ -1640,11 +1641,12 @@ static void ublk_complete_io_cmd(struct ublk_io *io, struct request *req,
 				 int res, unsigned issue_flags)
 {
 	struct io_uring_cmd *cmd = __ublk_prep_compl_io_cmd(io, req);
 
 	/* tell ublksrv one io request is coming */
-	io_uring_cmd_done(cmd, res, issue_flags);
+	io_uring_cmd_set_res(cmd, res);
+	io_uring_cmd_done(cmd, issue_flags);
 }
 
 #define UBLK_REQUEUE_DELAY_MS	3
 
 static inline void __ublk_abort_rq(struct ublk_queue *ubq,
@@ -1726,11 +1728,12 @@ static void ublk_auto_buf_dispatch(const struct ublk_queue *ubq,
 	enum auto_buf_reg_res res = ublk_auto_buf_register(ubq, req, io, cmd,
 			issue_flags);
 
 	if (res != AUTO_BUF_REG_FAIL) {
 		ublk_auto_buf_io_setup(ubq, req, io, cmd, res);
-		io_uring_cmd_done(cmd, UBLK_IO_RES_OK, issue_flags);
+		io_uring_cmd_set_res(cmd, UBLK_IO_RES_OK);
+		io_uring_cmd_done(cmd, issue_flags);
 	}
 }
 
 static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req,
 			  struct ublk_io *io)
@@ -2787,12 +2790,14 @@ static void ublk_cancel_cmd(struct ublk_queue *ubq, u16 tag,
 		cmd = io->cmd;
 		io->cmd = NULL;
 	}
 	spin_unlock(&ubq->cancel_lock);
 
-	if (!done && cmd)
-		io_uring_cmd_done(cmd, UBLK_IO_RES_ABORT, issue_flags);
+	if (!done && cmd) {
+		io_uring_cmd_set_res(cmd, UBLK_IO_RES_ABORT);
+		io_uring_cmd_done(cmd, issue_flags);
+	}
 }
 
 /*
  * Cancel a batch fetch command if it hasn't been claimed by another path.
  *
@@ -2814,11 +2819,12 @@ static void ublk_batch_cancel_cmd(struct ublk_queue *ubq,
 	if (done)
 		list_del_init(&fcmd->node);
 	spin_unlock(&ubq->evts_lock);
 
 	if (done) {
-		io_uring_cmd_done(fcmd->cmd, UBLK_IO_RES_ABORT, issue_flags);
+		io_uring_cmd_set_res(fcmd->cmd, UBLK_IO_RES_ABORT);
+		io_uring_cmd_done(fcmd->cmd, issue_flags);
 		ublk_batch_free_fcmd(fcmd);
 	}
 }
 
 static void ublk_batch_cancel_queue(struct ublk_queue *ubq)
@@ -3524,12 +3530,14 @@ static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 	struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req);
 	int ret = -ECANCELED;
 
 	if (!tw.cancel)
 		ret = ublk_ch_uring_cmd_local(cmd, issue_flags);
-	if (ret != -EIOCBQUEUED)
-		io_uring_cmd_done(cmd, ret, issue_flags);
+	if (ret != -EIOCBQUEUED) {
+		io_uring_cmd_set_res(cmd, ret);
+		io_uring_cmd_done(cmd, issue_flags);
+	}
 }
 
 static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags)
 {
 	if (unlikely(issue_flags & IO_URING_F_CANCEL)) {
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 6539d4750098..131959c00287 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -438,12 +438,12 @@ static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 	struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
 
 	if (pdu->bio)
 		blk_rq_unmap_user(pdu->bio);
-	io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,
-			    IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+	io_uring_cmd_set_res32(ioucmd, pdu->status, pdu->result);
+	io_uring_cmd_done(ioucmd, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
 }
 
 static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
 						blk_status_t err,
 						const struct io_comp_batch *iob)
@@ -469,11 +469,12 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
 	 */
 	if (blk_rq_is_poll(req) && iob &&
 	    iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
 		if (pdu->bio)
 			blk_rq_unmap_user(pdu->bio);
-		io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, 0);
+		io_uring_cmd_set_res32(ioucmd, pdu->status, pdu->result);
+		io_uring_cmd_done(ioucmd, 0);
 	} else {
 		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
 	}
 	return RQ_END_IO_FREE;
 }
diff --git a/drivers/scsi/scsi_bsg.c b/drivers/scsi/scsi_bsg.c
index e80dec53174e..8b32d8e8f29f 100644
--- a/drivers/scsi/scsi_bsg.c
+++ b/drivers/scsi/scsi_bsg.c
@@ -58,12 +58,12 @@ static void scsi_bsg_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 	res2 = bsg_scsi_res2_build(status_byte(scmd->result), driver_status,
 				  host_byte(scmd->result), sense_len_wr,
 				  scmd->resid_len);
 
 	blk_mq_free_request(rq);
-	io_uring_cmd_done32(ioucmd, ret, res2,
-			    IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+	io_uring_cmd_set_res32(ioucmd, ret, res2);
+	io_uring_cmd_done(ioucmd, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
 }
 
 static enum rq_end_io_ret scsi_bsg_uring_cmd_done(struct request *req,
 						  blk_status_t status,
 						  const struct io_comp_batch *iocb)
diff --git a/fs/btrfs/ioctl.c b/fs/btrfs/ioctl.c
index 72bc9d4f7708..07057a708452 100644
--- a/fs/btrfs/ioctl.c
+++ b/fs/btrfs/ioctl.c
@@ -4619,11 +4619,12 @@ static void btrfs_uring_read_finished(struct io_tw_req tw_req, io_tw_token_t tw)
 
 out:
 	btrfs_unlock_extent(io_tree, priv->start, priv->lockend, &priv->cached_state);
 	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
 
-	io_uring_cmd_done(cmd, ret, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+	io_uring_cmd_set_res(cmd, ret);
+	io_uring_cmd_done(cmd, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
 	add_rchar(current, ret);
 
 	for (index = 0; index < priv->nr_pages; index++)
 		__free_page(priv->pages[index]);
 
diff --git a/fs/fuse/dev_uring.c b/fs/fuse/dev_uring.c
index 77c8cec43d9c..c5bb50183e1c 100644
--- a/fs/fuse/dev_uring.c
+++ b/fs/fuse/dev_uring.c
@@ -364,12 +364,14 @@ static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent)
 	 */
 	list_move(&ent->list, &queue->ent_released);
 	ent->state = FRRS_RELEASED;
 	spin_unlock(&queue->lock);
 
-	if (cmd)
-		io_uring_cmd_done(cmd, -ENOTCONN, IO_URING_F_UNLOCKED);
+	if (cmd) {
+		io_uring_cmd_set_res(cmd, -ENOTCONN);
+		io_uring_cmd_done(cmd, IO_URING_F_UNLOCKED);
+	}
 
 	if (req)
 		fuse_uring_stop_fuse_req_end(req);
 }
 
@@ -529,11 +531,12 @@ static void fuse_uring_cancel(struct io_uring_cmd *cmd,
 	}
 	spin_unlock(&queue->lock);
 
 	if (need_cmd_done) {
 		/* no queue lock to avoid lock order issues */
-		io_uring_cmd_done(cmd, -ENOTCONN, issue_flags);
+		io_uring_cmd_set_res(cmd, -ENOTCONN);
+		io_uring_cmd_done(cmd, issue_flags);
 		kfree(ent);
 		if (atomic_dec_and_test(&queue->ring->queue_refs))
 			wake_up_all(&queue->ring->stop_waitq);
 	}
 }
@@ -940,11 +943,12 @@ static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd,
 	list_move_tail(&ent->list, &queue->ent_in_userspace);
 	ent->cmd = NULL;
 	fuse_uring_add_to_pq(ent);
 	spin_unlock(&queue->lock);
 
-	io_uring_cmd_done(cmd, ret, issue_flags);
+	io_uring_cmd_set_res(cmd, ret);
+	io_uring_cmd_done(cmd, issue_flags);
 }
 
 /* FUSE_URING_CMD_COMMIT_AND_FETCH handler */
 static int fuse_uring_commit_fetch(struct io_uring_cmd *cmd, int issue_flags,
 				   struct fuse_chan *fch)
@@ -1307,11 +1311,12 @@ static void fuse_uring_send_in_task(struct io_tw_req tw_req, io_tw_token_t tw)
 
 		spin_lock(&queue->lock);
 		list_del_init(&ent->list);
 		spin_unlock(&queue->lock);
 
-		io_uring_cmd_done(cmd, err, issue_flags);
+		io_uring_cmd_set_res(cmd, err);
+		io_uring_cmd_done(cmd, issue_flags);
 
 		fuse_uring_req_end(ent, ent->fuse_req, err);
 		kfree(ent);
 		if (atomic_dec_and_test(&queue->ring->queue_refs))
 			wake_up_all(&queue->ring->stop_waitq);
diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h
index 331dcbefe72f..96fcc06097c4 100644
--- a/include/linux/io_uring/cmd.h
+++ b/include/linux/io_uring/cmd.h
@@ -50,19 +50,22 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
 				  const struct iovec __user *uvec,
 				  size_t uvec_segs,
 				  int ddir, struct iov_iter *iter,
 				  unsigned issue_flags);
 
+/* One of these must be called prior to io_uring_cmd_done() */
+void io_uring_cmd_set_res(struct io_uring_cmd *, s32 ret);
+void io_uring_cmd_set_res32(struct io_uring_cmd *, s32 ret, u64 res2);
+
 /*
  * Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
  * and the corresponding io_uring request.
  *
  * Note: the caller should never hard code @issue_flags and is only allowed
  * to pass the mask provided by the core io_uring code.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
-			 unsigned issue_flags, bool is_cqe32);
+void io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);
 
 void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
 			    io_req_tw_func_t task_work_cb,
 			    unsigned flags);
 
@@ -105,12 +108,19 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
 						int ddir, struct iov_iter *iter,
 						unsigned issue_flags)
 {
 	return -EOPNOTSUPP;
 }
-static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
-		u64 ret2, unsigned issue_flags, bool is_cqe32)
+static inline void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
+{
+}
+static inline void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret,
+					  u64 res2)
+{
+}
+static inline void io_uring_cmd_done(struct io_uring_cmd *cmd,
+				     unsigned issue_flags)
 {
 }
 static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
 			    io_req_tw_func_t task_work_cb, unsigned flags)
 {
@@ -168,22 +178,10 @@ static inline struct task_struct *io_uring_cmd_get_task(struct io_uring_cmd *cmd
 static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd)
 {
 	return cmd_to_io_kiocb(cmd)->ctx;
 }
 
-static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret,
-				     unsigned issue_flags)
-{
-	return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false);
-}
-
-static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret,
-				       u64 res2, unsigned issue_flags)
-{
-	return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true);
-}
-
 int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
 			    void (*release)(void *), unsigned int index,
 			    unsigned int issue_flags);
 int io_buffer_unregister_bvec(struct io_uring_cmd *cmd, unsigned int index,
 			      unsigned int issue_flags);
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 726a659f38c3..ef3e7e352c6a 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -136,40 +136,43 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
 	req->io_task_work.func = task_work_cb;
 	__io_req_task_work_add(req, flags);
 }
 EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
 
-static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
-					  u64 extra1, u64 extra2)
+void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
 {
-	req->big_cqe.extra1 = extra1;
-	req->big_cqe.extra2 = extra2;
+	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+	if (ret < 0)
+		req_set_fail(req);
+	io_req_set_res(req, ret, 0);
 }
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res);
+
+void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2)
+{
+	struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+	if (ret < 0)
+		req_set_fail(req);
+	io_req_set_res32(req, ret, 0, res2, 0);
+}
+EXPORT_SYMBOL_GPL(io_uring_cmd_set_res32);
 
 /*
  * Called by consumers of io_uring_cmd, if they originally returned
  * -EIOCBQUEUED upon receiving the command.
  */
-void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
-		       unsigned issue_flags, bool is_cqe32)
+void io_uring_cmd_done(struct io_uring_cmd *ioucmd, unsigned issue_flags)
 {
 	struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
 
 	if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
 		return;
 
 	io_uring_cmd_del_cancelable(ioucmd, issue_flags);
 
-	if (ret < 0)
-		req_set_fail(req);
-
-	io_req_set_res(req, ret, 0);
-	if (is_cqe32) {
-		if (req->ctx->flags & IORING_SETUP_CQE_MIXED)
-			req->cqe.flags |= IORING_CQE_F_32;
-		io_req_set_cqe32_extra(req, res2, 0);
-	}
 	io_req_uring_cleanup(req, issue_flags);
 	if (req->flags & REQ_F_IOPOLL) {
 		/* order with io_do_iopoll() checking ->iopoll_completed */
 		smp_store_release(&req->iopoll_completed, 1);
 	} else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
@@ -179,11 +182,11 @@ void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
 	} else {
 		req->io_task_work.func = io_req_task_complete;
 		io_req_task_work_add(req);
 	}
 }
-EXPORT_SYMBOL_GPL(__io_uring_cmd_done);
+EXPORT_SYMBOL_GPL(io_uring_cmd_done);
 
 int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
 {
 	struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
 	struct io_async_cmd *ac;
-- 
2.55.0

Re: [PATCH 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

[Joanne Koong]

Hi Caleb,

On Thu, Aug 27, 2026 at 11:57 AM Caleb Sander Mateos
<csander@purestorage.com> wrote:
>
> In preparation for setting the io_uring NVMe passthru CQE results from
> the blk-mq request completion rather than the task work callback, split
> out functions io_uring_cmd_set_res{,32}() from io_uring_cmd_done{,32}().
> This allows io_uring_cmd_done{,32}() and __io_uring_cmd_done() to be
> consolidated into a single CQE-size-agnostic function with 3 fewer
> arguments than __io_uring_cmd_done().
>

The conversion looks logically correct to me but the interface feels a
bit annoying / fragile on the caller side.

Is there a way nvme can get what it needs without changing the other
callers? Would it work to just keep the existing
io_uring_cmd_done{,32}() as is and add a separate API for special
callers like nvme, eg:

  void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2);
  void __io_uring_cmd_done(struct io_uring_cmd *cmd, unsigned issue_flags);

where io_uring_cmd_done{,32} would then just be inline wrappers over
those? afaict, then only the nvme callsite would have to change and
fuse, btrfs, block, ublk, and scsi_bsg could just stay as is. Do you
think something like that makes sense to do?

Thanks,
Joanne

Re: [PATCH 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

[Caleb Sander Mateos]

On Thu, Aug 27, 2026 at 1:05 PM Joanne Koong <joannelkoong@gmail.com> wrote:
>
> Hi Caleb,
>
> On Thu, Aug 27, 2026 at 11:57 AM Caleb Sander Mateos
> <csander@purestorage.com> wrote:
> >
> > In preparation for setting the io_uring NVMe passthru CQE results from
> > the blk-mq request completion rather than the task work callback, split
> > out functions io_uring_cmd_set_res{,32}() from io_uring_cmd_done{,32}().
> > This allows io_uring_cmd_done{,32}() and __io_uring_cmd_done() to be
> > consolidated into a single CQE-size-agnostic function with 3 fewer
> > arguments than __io_uring_cmd_done().
> >
>
> The conversion looks logically correct to me but the interface feels a
> bit annoying / fragile on the caller side.
>
> Is there a way nvme can get what it needs without changing the other
> callers? Would it work to just keep the existing
> io_uring_cmd_done{,32}() as is and add a separate API for special
> callers like nvme, eg:
>
>   void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2);
>   void __io_uring_cmd_done(struct io_uring_cmd *cmd, unsigned issue_flags);
>
> where io_uring_cmd_done{,32} would then just be inline wrappers over
> those? afaict, then only the nvme callsite would have to change and
> fuse, btrfs, block, ublk, and scsi_bsg could just stay as is. Do you
> think something like that makes sense to do?

Sure, sounds fine to me. Thanks for taking a look.

Best,
Caleb

Re: [PATCH 1/2] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()

[Ming Lei]

On Thu, Aug 27, 2026 at 4:44 PM Caleb Sander Mateos
<csander@purestorage.com> wrote:
>
> On Thu, Aug 27, 2026 at 1:05 PM Joanne Koong <joannelkoong@gmail.com> wrote:
> >
> > Hi Caleb,
> >
> > On Thu, Aug 27, 2026 at 11:57 AM Caleb Sander Mateos
> > <csander@purestorage.com> wrote:
> > >
> > > In preparation for setting the io_uring NVMe passthru CQE results from
> > > the blk-mq request completion rather than the task work callback, split
> > > out functions io_uring_cmd_set_res{,32}() from io_uring_cmd_done{,32}().
> > > This allows io_uring_cmd_done{,32}() and __io_uring_cmd_done() to be
> > > consolidated into a single CQE-size-agnostic function with 3 fewer
> > > arguments than __io_uring_cmd_done().
> > >
> >
> > The conversion looks logically correct to me but the interface feels a
> > bit annoying / fragile on the caller side.
> >
> > Is there a way nvme can get what it needs without changing the other
> > callers? Would it work to just keep the existing
> > io_uring_cmd_done{,32}() as is and add a separate API for special
> > callers like nvme, eg:
> >
> >   void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2);
> >   void __io_uring_cmd_done(struct io_uring_cmd *cmd, unsigned issue_flags);
> >
> > where io_uring_cmd_done{,32} would then just be inline wrappers over
> > those? afaict, then only the nvme callsite would have to change and
> > fuse, btrfs, block, ublk, and scsi_bsg could just stay as is. Do you
> > think something like that makes sense to do?

Yeah, this way is definitely much better from a driver viewpoint.



Thanks,
Ming Lei

[PATCH 2/2] nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()

[Caleb Sander Mateos]

io_uring_cmd_set_res32() only performs loads and stores to the io_uring
request state, so it's safe to call in interrupt context. Move the call
from the nvme_uring_task_cb() task work to nvme_uring_cmd_end_io(). This
unifies the 2 io_uring_cmd_set_res32() callers and removes the need to
pass result and status through struct nvme_uring_cmd_pdu, saving 16
bytes and a couple memory accesses.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
 drivers/nvme/host/ioctl.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 131959c00287..b50bba207818 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -421,12 +421,10 @@ struct nvme_uring_data {
  * Expect build errors if this grows larger than that.
  */
 struct nvme_uring_cmd_pdu {
 	struct request *req;
 	struct bio *bio;
-	u64 result;
-	int status;
 };
 
 static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
 		struct io_uring_cmd *ioucmd)
 {
@@ -438,29 +436,30 @@ static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
 	struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
 
 	if (pdu->bio)
 		blk_rq_unmap_user(pdu->bio);
-	io_uring_cmd_set_res32(ioucmd, pdu->status, pdu->result);
 	io_uring_cmd_done(ioucmd, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
 }
 
 static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
 						blk_status_t err,
 						const struct io_comp_batch *iob)
 {
 	struct io_uring_cmd *ioucmd = req->end_io_data;
 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
+	u64 result = le64_to_cpu(nvme_req(req)->result.u64);
+	int status;
 
 	if (nvme_req(req)->flags & NVME_REQ_CANCELLED) {
-		pdu->status = -EINTR;
+		status = -EINTR;
 	} else {
-		pdu->status = nvme_req(req)->status;
-		if (!pdu->status)
-			pdu->status = blk_status_to_errno(err);
+		status = nvme_req(req)->status;
+		if (!status)
+			status = blk_status_to_errno(err);
 	}
-	pdu->result = le64_to_cpu(nvme_req(req)->result.u64);
+	io_uring_cmd_set_res32(ioucmd, status, result);
 
 	/*
 	 * For IOPOLL, check if this completion is happening in the context
 	 * of the same io_ring that owns the request (local context). If so,
 	 * we can complete inline without task_work overhead. Otherwise, we
@@ -469,11 +468,10 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
 	 */
 	if (blk_rq_is_poll(req) && iob &&
 	    iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
 		if (pdu->bio)
 			blk_rq_unmap_user(pdu->bio);
-		io_uring_cmd_set_res32(ioucmd, pdu->status, pdu->result);
 		io_uring_cmd_done(ioucmd, 0);
 	} else {
 		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
 	}
 	return RQ_END_IO_FREE;
-- 
2.55.0

Re: [PATCH 2/2] nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()

[sashiko-bot]

Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:

Pre-existing issues:
- [Critical] Polled NVMe uring_cmd requests completing via IRQ cause a Use-After-Free in nvme_ns_chr_uring_cmd_iopoll.
- [Critical] io_do_iopoll unconditionally overwrites req->cqe.flags, destroying IORING_CQE_F_32 for polled uring_cmd requests.
--

commit 67604b5180877187b81d959a1a1acc3b5bc52ca4
Author: Caleb Sander Mateos <csander@purestorage.com>

nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()

This commit moves io_uring_cmd_set_res32() from the task work callback
into the end_io handler for nvme uring commands. This eliminates the need
to store the result and status in the PDU, saving memory and unifying
the completion path.

> diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
> index 269afd5bc02e3..8d357c47111f8 100644
> --- a/drivers/nvme/host/ioctl.c
> +++ b/drivers/nvme/host/ioctl.c

[ ... ]

> @@ -414,15 +411,17 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
>  {
>  	struct io_uring_cmd *ioucmd = req->end_io_data;
>  	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
> +	u64 result = le64_to_cpu(nvme_req(req)->result.u64);
> +	int status;
>  
>  	if (nvme_req(req)->flags & NVME_REQ_CANCELLED) {
> -		pdu->status = -EINTR;
> +		status = -EINTR;
>  	} else {
> -		pdu->status = nvme_req(req)->status;
> -		if (!pdu->status)
> -			pdu->status = blk_status_to_errno(err);
> +		status = nvme_req(req)->status;
> +		if (!status)
> +			status = blk_status_to_errno(err);
>  	}
> -	pdu->result = le64_to_cpu(nvme_req(req)->result.u64);
> +	io_uring_cmd_set_res32(ioucmd, status, result);

[Severity: Critical]
This is a pre-existing issue, but does moving this call before polling expose
a regression with IORING_CQE_F_32?

When io_uring_cmd_set_res32() posts 32-byte CQE results, it sets:
    req->cqe.flags |= IORING_CQE_F_32;

However, for polled requests, io_do_iopoll() later executes:
    req->cqe.flags = io_put_kbuf(req, max(req->cqe.res, 0), NULL);

This unconditional assignment completely overwrites the flags, erasing
IORING_CQE_F_32. As a result, when io_fill_cqe_req() processes the
completion, it allocates a 16-byte slot instead of 32 bytes and silently
drops extra1 and extra2 (the NVMe-specific result values).

>  
>  	/*
>  	 * For IOPOLL, check if this completion is happening in the context
> @@ -435,7 +434,6 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
>  	    iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
>  		if (pdu->bio)
>  			blk_rq_unmap_user(pdu->bio);
> -		io_uring_cmd_set_res32(ioucmd, pdu->status, pdu->result);
>  		io_uring_cmd_done(ioucmd, 0);
>  	} else {
>  		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
>  	}
>  	return RQ_END_IO_FREE;

[Severity: Critical]
This isn't a bug introduced by this patch, but is there a use-after-free
regression when a polled NVMe uring_cmd completes via an interrupt?

If the completion is not in the expected poll context (e.g. hybrid polling
or a spurious interrupt), this path queues nvme_uring_task_cb() via
io_uring_cmd_do_in_task_lazy() and returns RQ_END_IO_FREE, which frees the
block request immediately.

However, the io_uring state flag req->iopoll_completed is not set to 1 until
the task_work runs. Concurrently, the io_uring polling thread in
io_do_iopoll() sees req->iopoll_completed == 0 and invokes
nvme_ns_chr_uring_cmd_iopoll().

This function then dereferences pdu->req (which points to the just-freed
block request) to check blk_rq_is_poll(req):

drivers/nvme/host/ioctl.c:nvme_ns_chr_uring_cmd_iopoll() {
    ...
    struct request *req = pdu->req;

    if (req && blk_rq_is_poll(req))
    ...
}

Can this lead to memory corruption if the unprivileged user issuing the
command continues polling while the interrupt path frees the request?

-- 
Sashiko AI review · https://sashiko.dev/#/patchset/20260827185722.3234622-1-csander@purestorage.com?part=2

Re: [PATCH 2/2] nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()

[Caleb Sander Mateos]

On Thu, Aug 27, 2026 at 12:17 PM <sashiko-bot@kernel.org> wrote:
>
> Thank you for your contribution! Sashiko AI review found 2 potential issue(s) to consider:
>
> Pre-existing issues:
> - [Critical] Polled NVMe uring_cmd requests completing via IRQ cause a Use-After-Free in nvme_ns_chr_uring_cmd_iopoll.
> - [Critical] io_do_iopoll unconditionally overwrites req->cqe.flags, destroying IORING_CQE_F_32 for polled uring_cmd requests.
> --
>
> commit 67604b5180877187b81d959a1a1acc3b5bc52ca4
> Author: Caleb Sander Mateos <csander@purestorage.com>
>
> nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()
>
> This commit moves io_uring_cmd_set_res32() from the task work callback
> into the end_io handler for nvme uring commands. This eliminates the need
> to store the result and status in the PDU, saving memory and unifying
> the completion path.
>
> > diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
> > index 269afd5bc02e3..8d357c47111f8 100644
> > --- a/drivers/nvme/host/ioctl.c
> > +++ b/drivers/nvme/host/ioctl.c
>
> [ ... ]
>
> > @@ -414,15 +411,17 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
> >  {
> >       struct io_uring_cmd *ioucmd = req->end_io_data;
> >       struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
> > +     u64 result = le64_to_cpu(nvme_req(req)->result.u64);
> > +     int status;
> >
> >       if (nvme_req(req)->flags & NVME_REQ_CANCELLED) {
> > -             pdu->status = -EINTR;
> > +             status = -EINTR;
> >       } else {
> > -             pdu->status = nvme_req(req)->status;
> > -             if (!pdu->status)
> > -                     pdu->status = blk_status_to_errno(err);
> > +             status = nvme_req(req)->status;
> > +             if (!status)
> > +                     status = blk_status_to_errno(err);
> >       }
> > -     pdu->result = le64_to_cpu(nvme_req(req)->result.u64);
> > +     io_uring_cmd_set_res32(ioucmd, status, result);
>
> [Severity: Critical]
> This is a pre-existing issue, but does moving this call before polling expose
> a regression with IORING_CQE_F_32?
>
> When io_uring_cmd_set_res32() posts 32-byte CQE results, it sets:
>     req->cqe.flags |= IORING_CQE_F_32;
>
> However, for polled requests, io_do_iopoll() later executes:
>     req->cqe.flags = io_put_kbuf(req, max(req->cqe.res, 0), NULL);
>
> This unconditional assignment completely overwrites the flags, erasing
> IORING_CQE_F_32. As a result, when io_fill_cqe_req() processes the
> completion, it allocates a 16-byte slot instead of 32 bytes and silently
> drops extra1 and extra2 (the NVMe-specific result values).

Yes, this does look like an existing issue. Perhaps the req->cqe.flags
= io_put_kbuf(...) should be using |= instead? That would match how
io_req_rw_complete() uses it.
>
> >
> >       /*
> >        * For IOPOLL, check if this completion is happening in the context
> > @@ -435,7 +434,6 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
> >           iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
> >               if (pdu->bio)
> >                       blk_rq_unmap_user(pdu->bio);
> > -             io_uring_cmd_set_res32(ioucmd, pdu->status, pdu->result);
> >               io_uring_cmd_done(ioucmd, 0);
> >       } else {
> >               io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
> >       }
> >       return RQ_END_IO_FREE;
>
> [Severity: Critical]
> This isn't a bug introduced by this patch, but is there a use-after-free
> regression when a polled NVMe uring_cmd completes via an interrupt?

Aren't the blk-mq poll queues mutually exclusive with the
interrupt-driven queues? I'm not sure this is possible.

Best,
Caleb

>
> If the completion is not in the expected poll context (e.g. hybrid polling
> or a spurious interrupt), this path queues nvme_uring_task_cb() via
> io_uring_cmd_do_in_task_lazy() and returns RQ_END_IO_FREE, which frees the
> block request immediately.
>
> However, the io_uring state flag req->iopoll_completed is not set to 1 until
> the task_work runs. Concurrently, the io_uring polling thread in
> io_do_iopoll() sees req->iopoll_completed == 0 and invokes
> nvme_ns_chr_uring_cmd_iopoll().
>
> This function then dereferences pdu->req (which points to the just-freed
> block request) to check blk_rq_is_poll(req):
>
> drivers/nvme/host/ioctl.c:nvme_ns_chr_uring_cmd_iopoll() {
>     ...
>     struct request *req = pdu->req;
>
>     if (req && blk_rq_is_poll(req))
>     ...
> }
>
> Can this lead to memory corruption if the unprivileged user issuing the
> command continues polling while the interrupt path frees the request?
>
> --
> Sashiko AI review · https://sashiko.dev/#/patchset/20260827185722.3234622-1-csander@purestorage.com?part=2