* [PATCH 6.18.y 1/8] io_uring/napi: don't end the single NAPI busy poll early on need_resched()
2026-11-21 21:31 [PATCH 6.18.y 0/8] io_uring fixes for 6.18-stable Jens Axboe
@ 2026-11-21 21:31 ` Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 2/8] io_uring: fail soft links on negative IOU_COMPLETE results Jens Axboe
` (6 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2026-11-21 21:31 UTC (permalink / raw)
To: stable
From: Cinq Prairie <notprairie@gmail.com>
Commit 39c7210e970200f8b17de3767c73efe64173a95f upstream.
With a single registered NAPI, io_napi_blocking_busy_loop() hands
io_napi_busy_loop_should_end() to napi_busy_loop_rcu() as loop_end and
runs its own loop once, relying on __napi_busy_loop() to spin until
that predicate fires. napi_busy_loop_rcu() sets NAPI_F_END_ON_RESCHED,
so __napi_busy_loop() also returns on the first need_resched(). io_uring
treats the return as the end of the busy poll period and sleeps in
schedule_hrtimeout() for the rest of the wait timeout.
With two or more NAPIs the inner loop is single-shot and io_uring's own
loop runs until its predicate is met, so busy_poll_to is honoured. The
same registration therefore behaves differently for one NAPI than for
several.
Make the busy poll deadline absolute, stored in io_wait_queue, so that
io_napi_busy_loop_should_end() no longer depends on the start_time the
napi loop snapshots on every entry. The single NAPI loop can then be
re-entered after an early need_resched() exit if there is budget left,
and the loop condition drops the "run once" special case. The fast path
that holds the NAPI across polls is kept.
Measured on io_uring-7.4 at 9c207aa58e54, x86-64, HZ=1000,
PREEMPT_DYNAMIC (lazy), no CPU isolation, low RX traffic, r8169 with one
RX queue, prefer_busy_poll, busy_poll_to = wait timeout = 10 ms.
"waker" is a SCHED_FIFO task pinned to the polling CPU waking every 1 ms:
1 NAPI, waker, before: busy 0.5-1.5 ms per 10 ms wait, ~0.8M polls/s
1 NAPI, waker, after: busy 10 ms per 10 ms wait, ~16.2M polls/s
1 NAPI, idle, before: busy 10 ms per 10 ms wait, ~15.3M polls/s
1 NAPI, idle, after: busy 10 ms per 10 ms wait, ~16.2M polls/s
2 NAPIs, waker, before: busy 10 ms per 10 ms wait, ~7.2M polls/s
2 NAPIs, waker, after: busy 10 ms per 10 ms wait, ~7.1M polls/s
The held-NAPI fast path polls about twice as often as the single-shot
path used for several NAPIs, which is why it is kept.
Fixes: 8d0c12a80cde ("io-uring: add napi busy poll support")
Cc: stable@vger.kernel.org
Signed-off-by: Cinq Prairie <notprairie@gmail.com>
Link: https://patch.msgid.link/20261027204840.7823-1-notprairie@gmail.com
[axboe: trim comments, drop __always_unused]
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
io_uring/io_uring.h | 1 +
io_uring/napi.c | 31 ++++++++++++++-----------------
2 files changed, 15 insertions(+), 17 deletions(-)
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 922bdf68aec8..4f1d20d28f72 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -113,6 +113,7 @@ struct io_wait_queue {
#ifdef CONFIG_NET_RX_BUSY_POLL
ktime_t napi_busy_poll_dt;
+ ktime_t napi_busy_poll_end;
bool napi_prefer_busy_poll;
#endif
};
diff --git a/io_uring/napi.c b/io_uring/napi.c
index 45180b172bec..46f5854f05ef 100644
--- a/io_uring/napi.c
+++ b/io_uring/napi.c
@@ -128,19 +128,15 @@ static inline void io_napi_remove_stale(struct io_ring_ctx *ctx, bool is_stale)
__io_napi_remove_stale(ctx);
}
-static inline bool io_napi_busy_loop_timeout(ktime_t start_time,
- ktime_t bp)
+static inline bool io_napi_busy_loop_timeout(struct io_wait_queue *iowq)
{
- if (bp) {
- ktime_t end_time = ktime_add(start_time, bp);
- ktime_t now = net_to_ktime(busy_loop_current_time());
-
- return ktime_after(now, end_time);
- }
-
- return true;
+ if (!iowq->napi_busy_poll_dt)
+ return true;
+ return ktime_after(net_to_ktime(busy_loop_current_time()),
+ iowq->napi_busy_poll_end);
}
+/* Uses an absolute deadline, so the napi loop can be re-entered */
static bool io_napi_busy_loop_should_end(void *data,
unsigned long start_time)
{
@@ -150,8 +146,7 @@ static bool io_napi_busy_loop_should_end(void *data,
return true;
if (io_should_wake(iowq) || io_has_work(iowq->ctx))
return true;
- if (io_napi_busy_loop_timeout(net_to_ktime(start_time),
- iowq->napi_busy_poll_dt))
+ if (io_napi_busy_loop_timeout(iowq))
return true;
return false;
@@ -204,13 +199,16 @@ __io_napi_do_busy_loop(struct io_ring_ctx *ctx,
static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx,
struct io_wait_queue *iowq)
{
- unsigned long start_time = busy_loop_current_time();
+ ktime_t now = net_to_ktime(busy_loop_current_time());
bool (*loop_end)(void *, unsigned long) = NULL;
void *loop_end_arg = NULL;
bool is_stale = false;
- /* Singular lists use a different napi loop end check function and are
- * only executed once.
+ iowq->napi_busy_poll_end = ktime_add(now, iowq->napi_busy_poll_dt);
+
+ /*
+ * A single NAPI is held by __napi_busy_loop() for the whole spin. It
+ * may return early on need_resched(), loop until our own deadline.
*/
if (list_is_singular(&ctx->napi_list)) {
loop_end = io_napi_busy_loop_should_end;
@@ -221,8 +219,7 @@ static void io_napi_blocking_busy_loop(struct io_ring_ctx *ctx,
do {
is_stale = __io_napi_do_busy_loop(ctx, loop_end,
loop_end_arg);
- } while (!io_napi_busy_loop_should_end(iowq, start_time) &&
- !loop_end_arg);
+ } while (!io_napi_busy_loop_should_end(iowq, 0));
}
io_napi_remove_stale(ctx, is_stale);
--
2.55.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6.18.y 2/8] io_uring: fail soft links on negative IOU_COMPLETE results
2026-11-21 21:31 [PATCH 6.18.y 0/8] io_uring fixes for 6.18-stable Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 1/8] io_uring/napi: don't end the single NAPI busy poll early on need_resched() Jens Axboe
@ 2026-11-21 21:31 ` Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 3/8] io_uring/futex: fail soft links when cancelling a wait Jens Axboe
` (5 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2026-11-21 21:31 UTC (permalink / raw)
To: stable
From: Yang Xiuwei <yangxiuwei@kylinos.cn>
Commit 9ed19503a7b34e1ffb2b81236843204c473b0ae0 upstream.
Handlers such as fsync return IOU_COMPLETE with a negative cqe.res.
The old io_req_complete() path called req_set_fail() for res < 0.
After completions moved into the core, that check was dropped,
so IOSQE_IO_LINK dependents kept running after the head failed.
Fixes: 97b388d70b53 ("io_uring: handle completions in the core")
Signed-off-by: Yang Xiuwei <yangxiuwei@kylinos.cn>
Link: https://patch.msgid.link/20261114041123.109163-1-yangxiuwei@kylinos.cn
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
io_uring/io_uring.c | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 4a9f3a0fe506..df5f33a0f937 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -1971,9 +1971,11 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
ret = __io_issue_sqe(req, issue_flags, def);
if (ret == IOU_COMPLETE) {
- /* not all handlers take care of the internal restart codes */
- if (unlikely(req->cqe.res <= -ERESTARTSYS))
- req->cqe.res = io_fixup_restart_res(req->cqe.res);
+ if (unlikely(req->cqe.res < 0)) {
+ if (req->cqe.res <= -ERESTARTSYS)
+ req->cqe.res = io_fixup_restart_res(req->cqe.res);
+ req_set_fail(req);
+ }
if (issue_flags & IO_URING_F_COMPLETE_DEFER)
io_req_complete_defer(req);
else
--
2.55.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6.18.y 3/8] io_uring/futex: fail soft links when cancelling a wait
2026-11-21 21:31 [PATCH 6.18.y 0/8] io_uring fixes for 6.18-stable Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 1/8] io_uring/napi: don't end the single NAPI busy poll early on need_resched() Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 2/8] io_uring: fail soft links on negative IOU_COMPLETE results Jens Axboe
@ 2026-11-21 21:31 ` Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 4/8] io_uring/rw: fail soft links on IOPOLL short or error IO Jens Axboe
` (4 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2026-11-21 21:31 UTC (permalink / raw)
To: stable
From: Yang Xiuwei <yangxiuwei@kylinos.cn>
Commit 9a7cbe50dd95594d6fb3c57b8454ace58d8d8d0a upstream.
__io_futex_cancel() completes the request with -ECANCELED but does
not call req_set_fail(). IOSQE_IO_LINK dependents then still run.
Fixes: 194bb58c6090 ("io_uring: add support for futex wake and wait")
Signed-off-by: Yang Xiuwei <yangxiuwei@kylinos.cn>
Link: https://patch.msgid.link/20261114072246.741950-2-yangxiuwei@kylinos.cn
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
io_uring/futex.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/io_uring/futex.c b/io_uring/futex.c
index 6f756de08705..5986ce2a121e 100644
--- a/io_uring/futex.c
+++ b/io_uring/futex.c
@@ -104,6 +104,7 @@ static bool __io_futex_cancel(struct io_kiocb *req)
}
hlist_del_init(&req->hash_node);
+ req_set_fail(req);
io_req_set_res(req, -ECANCELED, 0);
io_req_task_work_add(req);
return true;
--
2.55.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6.18.y 4/8] io_uring/rw: fail soft links on IOPOLL short or error IO
2026-11-21 21:31 [PATCH 6.18.y 0/8] io_uring fixes for 6.18-stable Jens Axboe
` (2 preceding siblings ...)
2026-11-21 21:31 ` [PATCH 6.18.y 3/8] io_uring/futex: fail soft links when cancelling a wait Jens Axboe
@ 2026-11-21 21:31 ` Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 5/8] io_uring/timeout: don't rearm a multishot timeout on cancel Jens Axboe
` (3 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2026-11-21 21:31 UTC (permalink / raw)
To: stable
From: Yang Xiuwei <yangxiuwei@kylinos.cn>
Commit 4d3db66c5363dcf42c3d43ebaea330ac2d1888bb upstream.
io_complete_rw_iopoll() updates cqe.res when the transfer does not
match the requested length, but it does not call req_set_fail().
Soft linked requests then still run after a short or failed IOPOLL
read or write.
Fixes: b66ceaf324b3 ("io_uring: move iopoll reissue into regular IO path")
Signed-off-by: Yang Xiuwei <yangxiuwei@kylinos.cn>
Link: https://patch.msgid.link/20261114072246.741950-3-yangxiuwei@kylinos.cn
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
io_uring/rw.c | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/io_uring/rw.c b/io_uring/rw.c
index f46d08bbe9d8..48ef87b4697b 100644
--- a/io_uring/rw.c
+++ b/io_uring/rw.c
@@ -618,8 +618,10 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res)
if (res == -EAGAIN && io_rw_should_reissue(req))
req->flags |= REQ_F_REISSUE | REQ_F_BL_NO_RECYCLE;
- else if (unlikely(final_res != req->cqe.res))
+ else if (unlikely(final_res != req->cqe.res)) {
+ req_set_fail(req);
req->cqe.res = final_res;
+ }
/* order with io_iopoll_complete() checking ->iopoll_completed */
smp_store_release(&req->iopoll_completed, 1);
--
2.55.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6.18.y 5/8] io_uring/timeout: don't rearm a multishot timeout on cancel
2026-11-21 21:31 [PATCH 6.18.y 0/8] io_uring fixes for 6.18-stable Jens Axboe
` (3 preceding siblings ...)
2026-11-21 21:31 ` [PATCH 6.18.y 4/8] io_uring/rw: fail soft links on IOPOLL short or error IO Jens Axboe
@ 2026-11-21 21:31 ` Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 6/8] io_uring/rsrc: keep the old file when a file update fails Jens Axboe
` (2 subsequent siblings)
7 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2026-11-21 21:31 UTC (permalink / raw)
To: stable
From: Cheng Yongkang <teel4res@gmail.com>
Commit 97d4a585db2b91d855f957be0a9f126068c5be93 upstream.
io_timeout_complete() does not check tw.cancel. When it runs from
io_cancel_local_task_work(), a multishot timeout that has not reached
its count still posts an IORING_CQE_F_MORE completion and rearms its
timer. With a zero interval the timer expires again right away and
queues new task work, so the drain keeps running it until posting a
CQE fails.
Normally the CQ fills up quickly and the request then completes. But
if userspace moves the CQ head far ahead of the tail, for example to
cq.tail + 0x7fffffff, posting keeps succeeding and ring teardown is
delayed for a long time.
Don't post another CQE or rearm the timer when tw.cancel is set, so
the request completes through io_req_task_complete(), like the poll and
linked timeout task work does when cancelled.
Fixes: ea97f6c8558e ("io_uring: add support for multishot timeouts")
Signed-off-by: Cheng Yongkang <teel4res@gmail.com>
Link: https://patch.msgid.link/20261116044519.281756-1-teel4res@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
io_uring/timeout.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/io_uring/timeout.c b/io_uring/timeout.c
index 2637c8bb61a0..36c8f3dae27a 100644
--- a/io_uring/timeout.c
+++ b/io_uring/timeout.c
@@ -76,7 +76,8 @@ static void io_timeout_complete(struct io_tw_req tw_req, io_tw_token_t tw)
struct io_ring_ctx *ctx = req->ctx;
if (!io_timeout_finish(timeout, data)) {
- if (io_req_post_cqe(req, -ETIME, IORING_CQE_F_MORE)) {
+ if (!tw.cancel &&
+ io_req_post_cqe(req, -ETIME, IORING_CQE_F_MORE)) {
/* re-arm timer */
raw_spin_lock_irq(&ctx->timeout_lock);
list_add(&timeout->list, ctx->timeout_list.prev);
--
2.55.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6.18.y 6/8] io_uring/rsrc: keep the old file when a file update fails
2026-11-21 21:31 [PATCH 6.18.y 0/8] io_uring fixes for 6.18-stable Jens Axboe
` (4 preceding siblings ...)
2026-11-21 21:31 ` [PATCH 6.18.y 5/8] io_uring/timeout: don't rearm a multishot timeout on cancel Jens Axboe
@ 2026-11-21 21:31 ` Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 7/8] io_uring/kbuf: initialize arg->out_len from arg->max_len in io_buffers_select() Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 8/8] io_uring/io-wq: apply the current cpuset to new workers Jens Axboe
7 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2026-11-21 21:31 UTC (permalink / raw)
To: stable
From: Cheng Yongkang <teel4res@gmail.com>
Commit e5a346e4309d720c47fe8331a00068608fad47dd upstream.
__io_sqe_files_update() resets the slot before it calls fget() on the
new fd. If fget() fails, the fd is an io_uring file, or the node
allocation fails, the update stops and reports that entry as not done,
but the old file has already been removed. Later requests that use the
slot fail with -EBADF, and if the old file was registered with a tag,
its tag CQE is posted as if it had been removed on purpose.
Reset the slot only after the new file has been checked and its node
allocated, as io_install_fixed_file() does. Removal with fd == -1 still
resets the slot right away.
Fixes: c3a31e605620 ("io_uring: add support for IORING_REGISTER_FILES_UPDATE")
Cc: stable@vger.kernel.org
Signed-off-by: Cheng Yongkang <teel4res@gmail.com>
Link: https://patch.msgid.link/20261116044531.281978-1-teel4res@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
[axboe: 6.18 doesn't clamp the slot index with array_index_nospec() in
this function]
---
io_uring/rsrc.c | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 7170b3f001d4..36a9bc279e74 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -237,9 +237,6 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
continue;
i = up->offset + done;
- if (io_reset_rsrc_node(ctx, &ctx->file_table.data, i))
- io_file_bitmap_clear(&ctx->file_table, i);
-
if (fd != -1) {
struct file *file = fget(fd);
struct io_rsrc_node *node;
@@ -262,11 +259,15 @@ static int __io_sqe_files_update(struct io_ring_ctx *ctx,
fput(file);
break;
}
+ if (io_reset_rsrc_node(ctx, &ctx->file_table.data, i))
+ io_file_bitmap_clear(&ctx->file_table, i);
ctx->file_table.data.nodes[i] = node;
if (tag)
node->tag = tag;
io_fixed_file_set(node, file);
io_file_bitmap_set(&ctx->file_table, i);
+ } else if (io_reset_rsrc_node(ctx, &ctx->file_table.data, i)) {
+ io_file_bitmap_clear(&ctx->file_table, i);
}
}
return done ? done : err;
--
2.55.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6.18.y 7/8] io_uring/kbuf: initialize arg->out_len from arg->max_len in io_buffers_select()
2026-11-21 21:31 [PATCH 6.18.y 0/8] io_uring fixes for 6.18-stable Jens Axboe
` (5 preceding siblings ...)
2026-11-21 21:31 ` [PATCH 6.18.y 6/8] io_uring/rsrc: keep the old file when a file update fails Jens Axboe
@ 2026-11-21 21:31 ` Jens Axboe
2026-11-21 21:31 ` [PATCH 6.18.y 8/8] io_uring/io-wq: apply the current cpuset to new workers Jens Axboe
7 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2026-11-21 21:31 UTC (permalink / raw)
To: stable
From: Siddharth Chintamaneni <sidchintamaneni@gmail.com>
Commit b7322810d1870ccc0d293ad383d94ef583d5057b upstream.
io_buffers_select() passes &arg->out_len to io_provided_buffers_select()
on the legacy provided-buffer path. Because callers initialize
arg->max_len (from sqe->len) while leaving arg->out_len as 0,
io_provided_buffers_select() sees *len == 0 and always selects the full
buffer size buf->len, ignoring the user-specified sqe->len limit.
Initialize arg->out_len = arg->max_len before calling
io_provided_buffers_select().
Fixes: 35c8711c8fc4 ("io_uring/kbuf: add helpers for getting/peeking multiple buffers")
Signed-off-by: Siddharth Chintamaneni <sidchintamaneni@gmail.com>
Link: https://patch.msgid.link/20261116193146.65315-1-sidchintamaneni@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
io_uring/kbuf.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c
index c091c3c462e1..6659f56e91fc 100644
--- a/io_uring/kbuf.c
+++ b/io_uring/kbuf.c
@@ -355,6 +355,7 @@ int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
req->flags |= REQ_F_BUF_MORE;
}
} else {
+ arg->out_len = arg->max_len;
ret = io_provided_buffers_select(req, &arg->out_len, sel->buf_list, arg->iovs);
}
out_unlock:
--
2.55.0
^ permalink raw reply related [flat|nested] 9+ messages in thread* [PATCH 6.18.y 8/8] io_uring/io-wq: apply the current cpuset to new workers
2026-11-21 21:31 [PATCH 6.18.y 0/8] io_uring fixes for 6.18-stable Jens Axboe
` (6 preceding siblings ...)
2026-11-21 21:31 ` [PATCH 6.18.y 7/8] io_uring/kbuf: initialize arg->out_len from arg->max_len in io_buffers_select() Jens Axboe
@ 2026-11-21 21:31 ` Jens Axboe
7 siblings, 0 replies; 9+ messages in thread
From: Jens Axboe @ 2026-11-21 21:31 UTC (permalink / raw)
To: stable
From: Mikhail Malyshev <mike.malyshev@gmail.com>
Commit 0659e44ce739ecd856164c75a5325b7959a2921d upstream.
Since commit 84eacf177faa ("io_uring/io-wq: inherit cpuset of cgroup in
io worker"), io_wq_create() copies the cpuset of the creating task into
wq->cpu_mask, and io_init_new_worker() gives each new worker that copy.
A cpuset change, or a move of the task to another cgroup, updates the
workers that exist but not the copy, so workers started later run on
the CPUs the cgroup had when the io-wq was created, which may since
have been given to other tasks. Workers come and go for the life of the
io-wq, and a worker that cpuset moved is also retired at its next idle
timeout and replaced on the old CPUs.
Keep wq->cpu_mask for the affinity of the io-wq itself, all CPUs or the
mask set with IORING_REGISTER_IOWQ_AFF, and intersect it with the cpuset
of the worker when the worker starts, falling back to the cpuset if the
two do not intersect, as __set_cpus_allowed_ptr() does. Use the same
mask in the idle exit test, and go back to all CPUs when the registered
affinity is removed.
Fixes: 84eacf177faa ("io_uring/io-wq: inherit cpuset of cgroup in io worker")
Cc: stable@vger.kernel.org
Signed-off-by: Mikhail Malyshev <mike.malyshev@gmail.com>
Link: https://patch.msgid.link/20261111162705.1150983-1-mike.malyshev@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
---
io_uring/io-wq.c | 44 +++++++++++++++++++++++++++++++++++++++-----
1 file changed, 39 insertions(+), 5 deletions(-)
diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c
index fd53297af26e..0d74e365d54a 100644
--- a/io_uring/io-wq.c
+++ b/io_uring/io-wq.c
@@ -731,6 +731,34 @@ static bool io_worker_retire(struct io_wq_acct *acct, struct io_worker *worker)
return false;
}
+/*
+ * The CPUs a worker of @wq runs on: those of the io-wq affinity that the
+ * worker's cpuset allows, or the cpuset if they have none in common, as the
+ * scheduler does with a task's requested affinity when its cpuset changes.
+ * Read the cpuset each time: it can change while the io-wq exists.
+ */
+static void io_wq_worker_cpus(struct io_wq *wq, struct task_struct *tsk,
+ struct cpumask *cpus)
+{
+ cpuset_cpus_allowed(tsk, cpus);
+ if (cpumask_intersects(cpus, wq->cpu_mask))
+ cpumask_and(cpus, cpus, wq->cpu_mask);
+}
+
+/* Is the current worker on a CPU that a new worker would not get? */
+static bool io_wq_worker_off_cpus(struct io_wq *wq)
+{
+ cpumask_var_t cpus;
+ bool off;
+
+ if (!alloc_cpumask_var(&cpus, GFP_KERNEL))
+ return false;
+ io_wq_worker_cpus(wq, current, cpus);
+ off = !cpumask_test_cpu(raw_smp_processor_id(), cpus);
+ free_cpumask_var(cpus);
+ return off;
+}
+
static int io_wq_worker(void *data)
{
struct io_worker *worker = data;
@@ -809,8 +837,7 @@ static int io_wq_worker(void *data)
}
if (!ret) {
last_timeout = true;
- exit_mask = !cpumask_test_cpu(raw_smp_processor_id(),
- wq->cpu_mask);
+ exit_mask = io_wq_worker_off_cpus(wq);
}
}
@@ -860,9 +887,16 @@ void io_wq_worker_sleeping(struct task_struct *tsk)
static void io_init_new_worker(struct io_wq *wq, struct io_wq_acct *acct, struct io_worker *worker,
struct task_struct *tsk)
{
+ cpumask_var_t cpus;
+
tsk->worker_private = worker;
worker->task = tsk;
- set_cpus_allowed_ptr(tsk, wq->cpu_mask);
+ /* Without a mask, the worker keeps its creator's CPUs. */
+ if (alloc_cpumask_var(&cpus, GFP_KERNEL)) {
+ io_wq_worker_cpus(wq, tsk, cpus);
+ set_cpus_allowed_ptr(tsk, cpus);
+ free_cpumask_var(cpus);
+ }
raw_spin_lock(&acct->workers_lock);
hlist_nulls_add_head_rcu(&worker->nulls_node, &acct->free_list);
@@ -1361,7 +1395,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
if (!alloc_cpumask_var(&wq->cpu_mask, GFP_KERNEL))
goto err;
- cpuset_cpus_allowed(data->task, wq->cpu_mask);
+ cpumask_copy(wq->cpu_mask, cpu_possible_mask);
wq->acct[IO_WQ_ACCT_BOUND].max_workers = bounded;
wq->acct[IO_WQ_ACCT_UNBOUND].max_workers =
task_rlimit(current, RLIMIT_NPROC);
@@ -1545,7 +1579,7 @@ int io_wq_cpu_affinity(struct io_uring_task *tctx, cpumask_var_t mask)
else
ret = -EINVAL;
} else {
- cpumask_copy(tctx->io_wq->cpu_mask, allowed_mask);
+ cpumask_copy(tctx->io_wq->cpu_mask, cpu_possible_mask);
}
rcu_read_unlock();
--
2.55.0
^ permalink raw reply related [flat|nested] 9+ messages in thread