* [PATCH v3 1/3] io_uring: move req_set_*() to public header
2026-09-09 15:58 [PATCH v3 0/3] io_uring passthru: set result on blk-mq request completion Caleb Sander Mateos
@ 2026-09-09 15:58 ` Caleb Sander Mateos
2026-09-09 15:58 ` [PATCH v3 2/3] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done() Caleb Sander Mateos
` (2 subsequent siblings)
3 siblings, 0 replies; 5+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 15:58 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: Joanne Koong, Ming Lei, io-uring, linux-nvme, linux-kernel,
Caleb Sander Mateos
In preparation for using req_set_fail(), req_set_res(), and
req_set_res32() in include/linux/io_uring/cmd.h, move them from the
private header io_uring/io_uring.h to include/linux/io_uring.h.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
include/linux/io_uring.h | 32 ++++++++++++++++++++++++++++++++
io_uring/io_uring.h | 31 -------------------------------
2 files changed, 32 insertions(+), 31 deletions(-)
diff --git a/include/linux/io_uring.h b/include/linux/io_uring.h
index d1aa4edfc2a5..505caa99c621 100644
--- a/include/linux/io_uring.h
+++ b/include/linux/io_uring.h
@@ -1,13 +1,45 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#ifndef _LINUX_IO_URING_H
#define _LINUX_IO_URING_H
+#include <linux/io_uring_types.h>
#include <linux/sched.h>
#include <linux/xarray.h>
#include <uapi/linux/io_uring.h>
+static inline void req_set_fail(struct io_kiocb *req)
+{
+ req->flags |= REQ_F_FAIL;
+ if (req->flags & REQ_F_CQE_SKIP) {
+ req->flags &= ~REQ_F_CQE_SKIP;
+ req->flags |= REQ_F_SKIP_LINK_CQES;
+ }
+}
+
+static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags)
+{
+ req->cqe.res = res;
+ req->cqe.flags = cflags;
+}
+
+static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx)
+{
+ if (ctx->flags & IORING_SETUP_CQE_MIXED)
+ return IORING_CQE_F_32;
+ return 0;
+}
+
+static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags,
+ __u64 extra1, __u64 extra2)
+{
+ req->cqe.res = res;
+ req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx);
+ req->big_cqe.extra1 = extra1;
+ req->big_cqe.extra2 = extra2;
+}
+
#if defined(CONFIG_IO_URING)
void __io_uring_cancel(bool cancel_all);
void __io_uring_free(struct task_struct *tsk);
void io_uring_unreg_ringfd(void);
const char *io_uring_get_opcode(u8 opcode);
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 896aab1ed026..109261722ca7 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -316,41 +316,10 @@ static __always_inline bool io_fill_cqe_req(struct io_ring_ctx *ctx,
if (trace_io_uring_complete_enabled())
trace_call__io_uring_complete(req->ctx, req, cqe);
return true;
}
-static inline void req_set_fail(struct io_kiocb *req)
-{
- req->flags |= REQ_F_FAIL;
- if (req->flags & REQ_F_CQE_SKIP) {
- req->flags &= ~REQ_F_CQE_SKIP;
- req->flags |= REQ_F_SKIP_LINK_CQES;
- }
-}
-
-static inline void io_req_set_res(struct io_kiocb *req, s32 res, u32 cflags)
-{
- req->cqe.res = res;
- req->cqe.flags = cflags;
-}
-
-static inline u32 ctx_cqe32_flags(struct io_ring_ctx *ctx)
-{
- if (ctx->flags & IORING_SETUP_CQE_MIXED)
- return IORING_CQE_F_32;
- return 0;
-}
-
-static inline void io_req_set_res32(struct io_kiocb *req, s32 res, u32 cflags,
- __u64 extra1, __u64 extra2)
-{
- req->cqe.res = res;
- req->cqe.flags = cflags | ctx_cqe32_flags(req->ctx);
- req->big_cqe.extra1 = extra1;
- req->big_cqe.extra2 = extra2;
-}
-
static inline void *io_uring_alloc_async_data(struct io_alloc_cache *cache,
struct io_kiocb *req)
{
if (cache) {
req->async_data = io_cache_alloc(cache, GFP_KERNEL);
--
2.55.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH v3 2/3] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done()
2026-09-09 15:58 [PATCH v3 0/3] io_uring passthru: set result on blk-mq request completion Caleb Sander Mateos
2026-09-09 15:58 ` [PATCH v3 1/3] io_uring: move req_set_*() to public header Caleb Sander Mateos
@ 2026-09-09 15:58 ` Caleb Sander Mateos
2026-09-09 15:58 ` [PATCH v3 3/3] nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io() Caleb Sander Mateos
2026-09-10 20:55 ` [PATCH v3 0/3] io_uring passthru: set result on blk-mq request completion Jens Axboe
3 siblings, 0 replies; 5+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 15:58 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: Joanne Koong, Ming Lei, io-uring, linux-nvme, linux-kernel,
Caleb Sander Mateos
In preparation for setting the io_uring NVMe passthru CQE results from
the blk-mq request completion rather than the task work callback, split
out functions io_uring_cmd_set_res{,32}() from __io_uring_cmd_done().
io_uring_cmd_done{,32}() now call io_uring_cmd_set_res{,32}() and then
__io_uring_cmd_done(). This allows __io_uring_cmd_done() to be made
CQE-size-agnostic, with 3 fewer arguments.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Reviewed-by: Joanne Koong <joannelkoong@gmail.com>
---
include/linux/io_uring/cmd.h | 34 ++++++++++++++++++++++++++++------
io_uring/uring_cmd.c | 19 +------------------
2 files changed, 29 insertions(+), 24 deletions(-)
diff --git a/include/linux/io_uring/cmd.h b/include/linux/io_uring/cmd.h
index 331dcbefe72f..2e4368d611ee 100644
--- a/include/linux/io_uring/cmd.h
+++ b/include/linux/io_uring/cmd.h
@@ -1,10 +1,11 @@
/* SPDX-License-Identifier: GPL-2.0-or-later */
#ifndef _LINUX_IO_URING_CMD_H
#define _LINUX_IO_URING_CMD_H
#include <uapi/linux/io_uring.h>
+#include <linux/io_uring.h>
#include <linux/io_uring_types.h>
#include <linux/blk-mq.h>
/* only top 8 bits of sqe->uring_cmd_flags for kernel internal use */
#define IORING_URING_CMD_CANCELABLE (1U << 30)
@@ -39,10 +40,28 @@ static inline void io_uring_cmd_private_sz_check(size_t cmd_sz)
#define io_uring_cmd_to_pdu(cmd, pdu_type) ( \
io_uring_cmd_private_sz_check(sizeof(pdu_type)), \
((pdu_type *)&(cmd)->pdu) \
)
+static inline void io_uring_cmd_set_res(struct io_uring_cmd *cmd, s32 ret)
+{
+ struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+ if (ret < 0)
+ req_set_fail(req);
+ io_req_set_res(req, ret, 0);
+}
+
+static inline void io_uring_cmd_set_res32(struct io_uring_cmd *cmd, s32 ret, u64 res2)
+{
+ struct io_kiocb *req = cmd_to_io_kiocb(cmd);
+
+ if (ret < 0)
+ req_set_fail(req);
+ io_req_set_res32(req, ret, 0, res2, 0);
+}
+
#if defined(CONFIG_IO_URING)
int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter,
struct io_uring_cmd *ioucmd,
unsigned int issue_flags);
@@ -54,15 +73,16 @@ int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
/*
* Completes the request, i.e. posts an io_uring CQE and deallocates @ioucmd
* and the corresponding io_uring request.
*
+ * io_uring_cmd_set_res()/io_uring_cmd_set_res32() must be called first.
+ *
* Note: the caller should never hard code @issue_flags and is only allowed
* to pass the mask provided by the core io_uring code.
*/
-void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret, u64 res2,
- unsigned issue_flags, bool is_cqe32);
+void __io_uring_cmd_done(struct io_uring_cmd *, unsigned issue_flags);
void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
io_req_tw_func_t task_work_cb,
unsigned flags);
@@ -105,12 +125,12 @@ static inline int io_uring_cmd_import_fixed_vec(struct io_uring_cmd *ioucmd,
int ddir, struct iov_iter *iter,
unsigned issue_flags)
{
return -EOPNOTSUPP;
}
-static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd, s32 ret,
- u64 ret2, unsigned issue_flags, bool is_cqe32)
+static inline void __io_uring_cmd_done(struct io_uring_cmd *cmd,
+ unsigned issue_flags)
{
}
static inline void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
io_req_tw_func_t task_work_cb, unsigned flags)
{
@@ -171,17 +191,19 @@ static inline void *io_uring_cmd_ctx_handle(struct io_uring_cmd *cmd)
}
static inline void io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret,
unsigned issue_flags)
{
- return __io_uring_cmd_done(ioucmd, ret, 0, issue_flags, false);
+ io_uring_cmd_set_res(ioucmd, ret);
+ __io_uring_cmd_done(ioucmd, issue_flags);
}
static inline void io_uring_cmd_done32(struct io_uring_cmd *ioucmd, s32 ret,
u64 res2, unsigned issue_flags)
{
- return __io_uring_cmd_done(ioucmd, ret, res2, issue_flags, true);
+ io_uring_cmd_set_res32(ioucmd, ret, res2);
+ __io_uring_cmd_done(ioucmd, issue_flags);
}
int io_buffer_register_bvec(struct io_uring_cmd *cmd, struct request *rq,
void (*release)(void *), unsigned int index,
unsigned int issue_flags);
diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c
index 726a659f38c3..3d5d8b5f4ebb 100644
--- a/io_uring/uring_cmd.c
+++ b/io_uring/uring_cmd.c
@@ -136,40 +136,23 @@ void __io_uring_cmd_do_in_task(struct io_uring_cmd *ioucmd,
req->io_task_work.func = task_work_cb;
__io_req_task_work_add(req, flags);
}
EXPORT_SYMBOL_GPL(__io_uring_cmd_do_in_task);
-static inline void io_req_set_cqe32_extra(struct io_kiocb *req,
- u64 extra1, u64 extra2)
-{
- req->big_cqe.extra1 = extra1;
- req->big_cqe.extra2 = extra2;
-}
-
/*
* Called by consumers of io_uring_cmd, if they originally returned
* -EIOCBQUEUED upon receiving the command.
*/
-void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, s32 ret, u64 res2,
- unsigned issue_flags, bool is_cqe32)
+void __io_uring_cmd_done(struct io_uring_cmd *ioucmd, unsigned issue_flags)
{
struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
if (WARN_ON_ONCE(req->flags & REQ_F_APOLL_MULTISHOT))
return;
io_uring_cmd_del_cancelable(ioucmd, issue_flags);
- if (ret < 0)
- req_set_fail(req);
-
- io_req_set_res(req, ret, 0);
- if (is_cqe32) {
- if (req->ctx->flags & IORING_SETUP_CQE_MIXED)
- req->cqe.flags |= IORING_CQE_F_32;
- io_req_set_cqe32_extra(req, res2, 0);
- }
io_req_uring_cleanup(req, issue_flags);
if (req->flags & REQ_F_IOPOLL) {
/* order with io_do_iopoll() checking ->iopoll_completed */
smp_store_release(&req->iopoll_completed, 1);
} else if (issue_flags & IO_URING_F_COMPLETE_DEFER) {
--
2.55.0
^ permalink raw reply related [flat|nested] 5+ messages in thread* [PATCH v3 3/3] nvme/ioctl: call io_uring_cmd_set_res32() in ->end_io()
2026-09-09 15:58 [PATCH v3 0/3] io_uring passthru: set result on blk-mq request completion Caleb Sander Mateos
2026-09-09 15:58 ` [PATCH v3 1/3] io_uring: move req_set_*() to public header Caleb Sander Mateos
2026-09-09 15:58 ` [PATCH v3 2/3] io_uring/cmd: split io_uring_cmd_set_res() from io_uring_cmd_done() Caleb Sander Mateos
@ 2026-09-09 15:58 ` Caleb Sander Mateos
2026-09-10 20:55 ` [PATCH v3 0/3] io_uring passthru: set result on blk-mq request completion Jens Axboe
3 siblings, 0 replies; 5+ messages in thread
From: Caleb Sander Mateos @ 2026-09-09 15:58 UTC (permalink / raw)
To: Jens Axboe, Keith Busch, Christoph Hellwig, Sagi Grimberg
Cc: Joanne Koong, Ming Lei, io-uring, linux-nvme, linux-kernel,
Caleb Sander Mateos
io_uring_cmd_set_res32() only performs loads and stores to the io_uring
request state, so it's safe to call in interrupt context. Move the call
from the nvme_uring_task_cb() task work to nvme_uring_cmd_end_io(). This
unifies the 2 places setting the NVMe status and result on the uring_cmd
and removes the need to pass them through struct nvme_uring_cmd_pdu,
saving 16 bytes and a couple memory accesses.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/nvme/host/ioctl.c | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c
index 6539d4750098..3aa5d2f2dfbb 100644
--- a/drivers/nvme/host/ioctl.c
+++ b/drivers/nvme/host/ioctl.c
@@ -421,12 +421,10 @@ struct nvme_uring_data {
* Expect build errors if this grows larger than that.
*/
struct nvme_uring_cmd_pdu {
struct request *req;
struct bio *bio;
- u64 result;
- int status;
};
static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu(
struct io_uring_cmd *ioucmd)
{
@@ -438,29 +436,30 @@ static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw)
struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req);
struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
if (pdu->bio)
blk_rq_unmap_user(pdu->bio);
- io_uring_cmd_done32(ioucmd, pdu->status, pdu->result,
- IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
+ __io_uring_cmd_done(ioucmd, IO_URING_CMD_TASK_WORK_ISSUE_FLAGS);
}
static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
blk_status_t err,
const struct io_comp_batch *iob)
{
struct io_uring_cmd *ioucmd = req->end_io_data;
struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);
+ u64 result = le64_to_cpu(nvme_req(req)->result.u64);
+ int status;
if (nvme_req(req)->flags & NVME_REQ_CANCELLED) {
- pdu->status = -EINTR;
+ status = -EINTR;
} else {
- pdu->status = nvme_req(req)->status;
- if (!pdu->status)
- pdu->status = blk_status_to_errno(err);
+ status = nvme_req(req)->status;
+ if (!status)
+ status = blk_status_to_errno(err);
}
- pdu->result = le64_to_cpu(nvme_req(req)->result.u64);
+ io_uring_cmd_set_res32(ioucmd, status, result);
/*
* For IOPOLL, check if this completion is happening in the context
* of the same io_ring that owns the request (local context). If so,
* we can complete inline without task_work overhead. Otherwise, we
@@ -469,11 +468,11 @@ static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req,
*/
if (blk_rq_is_poll(req) && iob &&
iob->poll_ctx == io_uring_cmd_ctx_handle(ioucmd)) {
if (pdu->bio)
blk_rq_unmap_user(pdu->bio);
- io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, 0);
+ __io_uring_cmd_done(ioucmd, 0);
} else {
io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);
}
return RQ_END_IO_FREE;
}
--
2.55.0
^ permalink raw reply related [flat|nested] 5+ messages in thread