From: Jens Axboe <axboe@kernel.dk>
To: io-uring@vger.kernel.org
Cc: linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org, tglx@kernel.org, mingo@redhat.com,
peterz@infradead.org, Jens Axboe <axboe@kernel.dk>
Subject: [PATCH 07/15] io_uring: add uring_lock section depth tracking and blockable opdef flag
Date: Fri, 11 Sep 2026 09:40:57 -0600 [thread overview]
Message-ID: <20260911154148.644489-8-axboe@kernel.dk> (raw)
In-Reply-To: <20260911154148.644489-1-axboe@kernel.dk>
Prep patch for issuing requests inline in blocking mode and catching the
sleep when it happens, rather than punting to io-wq upfront because an
operation may block.
If a request blocks inline, uring_lock must be dropped on behalf of the
sleeping task, which is only safe outside the sections that rely on the
lock being held. Track those with a depth counter in
io_ring_submit_lock() and io_ring_submit_unlock().
Add a "blockable" flag to io_issue_def for opcodes whose issue path can
cope with blocking inline: read/write, the forced async fs ops, open,
close and splice/tee. uring_cmd is excluded for now, drivers may bind
state to the submitting task.
No functional changes in this patch.
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
include/linux/io_uring_types.h | 5 +++++
io_uring/io_uring.h | 3 +++
io_uring/opdef.c | 29 +++++++++++++++++++++++++++++
io_uring/opdef.h | 2 ++
4 files changed, 39 insertions(+)
diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 4af3d579ead6..50a4a0ad222f 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -352,6 +352,11 @@ struct io_ring_ctx {
/* submission data */
struct {
struct mutex uring_lock;
+ /*
+ * io_ring_submit_lock() nesting depth, non-zero means the
+ * issue path relies on the lock being held.
+ */
+ unsigned int submit_lock_depth;
/*
* Ring buffer of indices into array of io_uring_sqe, which is
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 896aab1ed026..870bb4dcc415 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -393,6 +393,8 @@ static inline void io_ring_submit_unlock(struct io_ring_ctx *ctx,
unsigned issue_flags)
{
lockdep_assert_held(&ctx->uring_lock);
+ lockdep_assert(ctx->submit_lock_depth > 0);
+ ctx->submit_lock_depth--;
if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
mutex_unlock(&ctx->uring_lock);
}
@@ -409,6 +411,7 @@ static inline void io_ring_submit_lock(struct io_ring_ctx *ctx,
if (unlikely(issue_flags & IO_URING_F_UNLOCKED))
mutex_lock(&ctx->uring_lock);
lockdep_assert_held(&ctx->uring_lock);
+ ctx->submit_lock_depth++;
}
static inline void io_commit_cqring(struct io_ring_ctx *ctx)
diff --git a/io_uring/opdef.c b/io_uring/opdef.c
index cf3aa2242cd7..fa07a2b94536 100644
--- a/io_uring/opdef.c
+++ b/io_uring/opdef.c
@@ -69,6 +69,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_readv,
.issue = io_read,
},
@@ -83,12 +84,14 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_writev,
.issue = io_write,
},
[IORING_OP_FSYNC] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_fsync_prep,
.issue = io_fsync,
},
@@ -101,6 +104,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_read_fixed,
.issue = io_read_fixed,
},
@@ -114,6 +118,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_write_fixed,
.issue = io_write_fixed,
},
@@ -132,6 +137,7 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_SYNC_FILE_RANGE] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_sfr_prep,
.issue = io_sync_file_range,
},
@@ -215,16 +221,19 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_FALLOCATE] = {
.needs_file = 1,
.hash_reg_file = 1,
+ .blockable = 1,
.prep = io_fallocate_prep,
.issue = io_fallocate,
},
[IORING_OP_OPENAT] = {
.filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
+ .blockable = 1,
.prep = io_openat_prep,
.issue = io_openat,
.filter_populate = io_openat_bpf_populate,
},
[IORING_OP_CLOSE] = {
+ .blockable = 1,
.prep = io_close_prep,
.issue = io_close,
},
@@ -236,6 +245,7 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_STATX] = {
.audit_skip = 1,
+ .blockable = 1,
.prep = io_statx_prep,
.issue = io_statx,
},
@@ -249,6 +259,7 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_read,
.issue = io_read,
},
@@ -262,17 +273,20 @@ const struct io_issue_def io_issue_defs[] = {
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_write,
.issue = io_write,
},
[IORING_OP_FADVISE] = {
.needs_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_fadvise_prep,
.issue = io_fadvise,
},
[IORING_OP_MADVISE] = {
.audit_skip = 1,
+ .blockable = 1,
.prep = io_madvise_prep,
.issue = io_madvise,
},
@@ -308,6 +322,7 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_OPENAT2] = {
.filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
+ .blockable = 1,
.prep = io_openat2_prep,
.issue = io_openat2,
.filter_populate = io_openat_bpf_populate,
@@ -327,6 +342,7 @@ const struct io_issue_def io_issue_defs[] = {
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_splice_prep,
.issue = io_splice,
},
@@ -347,6 +363,7 @@ const struct io_issue_def io_issue_defs[] = {
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
+ .blockable = 1,
.prep = io_tee_prep,
.issue = io_tee,
},
@@ -360,22 +377,27 @@ const struct io_issue_def io_issue_defs[] = {
#endif
},
[IORING_OP_RENAMEAT] = {
+ .blockable = 1,
.prep = io_renameat_prep,
.issue = io_renameat,
},
[IORING_OP_UNLINKAT] = {
+ .blockable = 1,
.prep = io_unlinkat_prep,
.issue = io_unlinkat,
},
[IORING_OP_MKDIRAT] = {
+ .blockable = 1,
.prep = io_mkdirat_prep,
.issue = io_mkdirat,
},
[IORING_OP_SYMLINKAT] = {
+ .blockable = 1,
.prep = io_symlinkat_prep,
.issue = io_symlinkat,
},
[IORING_OP_LINKAT] = {
+ .blockable = 1,
.prep = io_linkat_prep,
.issue = io_linkat,
},
@@ -387,19 +409,23 @@ const struct io_issue_def io_issue_defs[] = {
},
[IORING_OP_FSETXATTR] = {
.needs_file = 1,
+ .blockable = 1,
.prep = io_fsetxattr_prep,
.issue = io_fsetxattr,
},
[IORING_OP_SETXATTR] = {
+ .blockable = 1,
.prep = io_setxattr_prep,
.issue = io_setxattr,
},
[IORING_OP_FGETXATTR] = {
.needs_file = 1,
+ .blockable = 1,
.prep = io_fgetxattr_prep,
.issue = io_fgetxattr,
},
[IORING_OP_GETXATTR] = {
+ .blockable = 1,
.prep = io_getxattr_prep,
.issue = io_getxattr,
},
@@ -497,6 +523,7 @@ const struct io_issue_def io_issue_defs[] = {
[IORING_OP_FTRUNCATE] = {
.needs_file = 1,
.hash_reg_file = 1,
+ .blockable = 1,
.prep = io_ftruncate_prep,
.issue = io_ftruncate,
},
@@ -553,6 +580,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_readv_fixed,
.issue = io_read,
},
@@ -567,6 +595,7 @@ const struct io_issue_def io_issue_defs[] = {
.iopoll = 1,
.vectored = 1,
.async_size = sizeof(struct io_async_rw),
+ .blockable = 1,
.prep = io_prep_writev_fixed,
.issue = io_write,
},
diff --git a/io_uring/opdef.h b/io_uring/opdef.h
index 667f981e63b0..45c2f77cf782 100644
--- a/io_uring/opdef.h
+++ b/io_uring/opdef.h
@@ -29,6 +29,8 @@ struct io_issue_def {
unsigned vectored : 1;
/* set to 1 if this opcode uses 128b sqes in a mixed sq */
unsigned is_128 : 1;
+ /* issue path is safe to run inline in blocking mode */
+ unsigned blockable : 1;
/* size of async data needed, if any */
unsigned short async_size;
--
2.55.0
next prev parent reply other threads:[~2026-09-11 15:42 UTC|newest]
Thread overview: 18+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-09-11 15:40 [RFC PATCH 00/15] io_uring: thread identity handoff for blocking inline issue Jens Axboe
2026-09-11 15:40 ` [PATCH 01/15] kernel: add thread identity handoff Jens Axboe
2026-09-11 15:40 ` [PATCH 02/15] sched: call into io_uring when a PF_IO_HANDOFF task blocks Jens Axboe
2026-09-11 15:40 ` [PATCH 03/15] arm64: implement thread identity handoff Jens Axboe
2026-09-11 15:40 ` [PATCH 04/15] x86: " Jens Axboe
2026-09-11 15:40 ` [PATCH 05/15] io_uring/kbuf: use io_ring_submit_unlock() helper Jens Axboe
2026-09-11 15:40 ` [PATCH 06/15] io_uring: keep the tctx nodes on a list Jens Axboe
2026-09-11 15:40 ` Jens Axboe [this message]
2026-09-11 15:40 ` [PATCH 08/15] io_uring: split io_uring_enter() and io_submit_sqes() into helpers Jens Axboe
2026-09-11 15:40 ` [PATCH 09/15] io_uring: keep the submission plug on the io_submit_sqes() stack Jens Axboe
2026-09-11 15:41 ` [PATCH 10/15] io-wq: support handing a task identity to an idle worker Jens Axboe
2026-09-11 15:41 ` [PATCH 11/15] io_uring: enable handing submitter identity to an io-wq worker Jens Axboe
2026-09-11 15:41 ` [PATCH 12/15] io_uring: defer the identity migration to the end of the submission Jens Axboe
2026-09-11 15:41 ` [PATCH 13/15] io_uring: issue blockable requests inline in blocking mode Jens Axboe
2026-09-11 15:41 ` [PATCH 14/15] io_uring: add tracepoints for the handoff operation Jens Axboe
2026-09-11 15:41 ` [PATCH 15/15] io_uring: issue IOSQE_ASYNC requests inline when a handoff is possible Jens Axboe
2026-09-11 17:33 ` [RFC PATCH 00/15] io_uring: thread identity handoff for blocking inline issue Gabriel Krisman Bertazi
2026-09-11 17:51 ` Jens Axboe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260911154148.644489-8-axboe@kernel.dk \
--to=axboe@kernel.dk \
--cc=io-uring@vger.kernel.org \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=mingo@redhat.com \
--cc=peterz@infradead.org \
--cc=tglx@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.