[PATCH 13/20] ublk: optimize ublk_user_copy() on daemon task

Caleb Sander Mateos
2025-12-17T05:35:16Z

ublk user copy syscalls may be issued from any task, so they take a
reference count on the struct ublk_io to check whether it is owned by
the ublk server and prevent a concurrent UBLK_IO_COMMIT_AND_FETCH_REQ
from completing the request. However, if the user copy syscall is issued
on the io's daemon task, a concurrent UBLK_IO_COMMIT_AND_FETCH_REQ isn't
possible, so the atomic reference count dance is unnecessary. Check for
UBLK_IO_FLAG_OWNED_BY_SRV to ensure the request is dispatched to the
sever and obtain the request from ublk_io's req field instead of looking
it up on the tagset. Skip the reference count increment and decrement.
Commit 8a8fe42d765b ("ublk: optimize UBLK_IO_REGISTER_IO_BUF on daemon
task") made an analogous optimization for ublk zero copy buffer
registration.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/block/ublk_drv.c | 23 ++++++++++++++++++-----
1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 042df4de9253..a0fbabd49feb 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -180,11 +180,11 @@ struct ublk_io {
/*
* The number of uses of this I/O by the ublk server
* if user copy or zero copy are enabled:
* - UBLK_REFCOUNT_INIT from dispatch to the server
* until UBLK_IO_COMMIT_AND_FETCH_REQ
- * - 1 for each inflight ublk_ch_{read,write}_iter() call
+ * - 1 for each inflight ublk_ch_{read,write}_iter() call not on task
* - 1 for each io_uring registered buffer not registered on task
* The I/O can only be completed once all references are dropped.
* User copy and buffer registration operations are only permitted
* if the reference count is nonzero.
*/
@@ -2644,10 +2644,11 @@ ublk_user_copy(struct kiocb *iocb, struct iov_iter *iter, int dir)
struct ublk_queue *ubq;
struct request *req;
struct ublk_io *io;
unsigned data_len;
bool is_integrity;
+ bool on_daemon;
size_t buf_off;
u16 tag, q_id;
ssize_t ret;

if (!user_backed_iter(iter))
@@ -2670,13 +2671,24 @@ ublk_user_copy(struct kiocb *iocb, struct iov_iter *iter, int dir)

if (tag >= ub->dev_info.queue_depth)
return -EINVAL;

io = &ubq->ios[tag];
- req = __ublk_check_and_get_req(ub, q_id, tag, io);
- if (!req)
- return -EINVAL;
+ on_daemon = current == READ_ONCE(io->task);
+ if (on_daemon) {
+ /* On daemon, io can't be completed concurrently, so skip ref */
+ if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV))
+ return -EINVAL;
+
+ req = io->req;
+ if (!ublk_rq_has_data(req))
+ return -EINVAL;
+ } else {
+ req = __ublk_check_and_get_req(ub, q_id, tag, io);
+ if (!req)
+ return -EINVAL;
+ }

if (is_integrity) {
struct blk_integrity *bi = &req->q->limits.integrity;

data_len = bio_integrity_bytes(bi, blk_rq_sectors(req));
@@ -2697,11 +2709,12 @@ ublk_user_copy(struct kiocb *iocb, struct iov_iter *iter, int dir)
ret = ublk_copy_user_integrity(req, buf_off, iter, dir);
else
ret = ublk_copy_user_pages(req, buf_off, iter, dir);

out:
- ublk_put_req_ref(io, req);
+ if (!on_daemon)
+ ublk_put_req_ref(io, req);
return ret;
}

static ssize_t ublk_ch_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
--
2.45.2