* [PATCH v3 1/2] ublk: honor rq_affinity on request completion
2026-10-26 12:06 [PATCH v3 0/2] ublk: honor rq_affinity on request completion Yoav Cohen
@ 2026-10-26 12:06 ` Yoav Cohen
2026-10-26 12:06 ` [PATCH v3 2/2] selftests: ublk: add rq_affinity test Yoav Cohen
1 sibling, 0 replies; 4+ messages in thread
From: Yoav Cohen @ 2026-10-26 12:06 UTC (permalink / raw)
To: Ming Lei, Jens Axboe, linux-block, csander
Cc: jholzman, omril, ofer, Yoav Cohen
ublk ends successfully completed requests inline on the server thread
that issued UBLK_IO_COMMIT_AND_FETCH_REQ, so rq_affinity never
redirects completion back to the submitting CPU, unlike NVMe, SCSI,
virtio-blk, loop, nbd and rnbd.
Add UBLK_F_SUPPORT_RQ_AFFINITY. When a server requests it at ADD_DEV
time, a request that completed in full (io->res == blk_rq_bytes()) is
routed through blk_mq_complete_request_remote() with a new ->complete()
callback, ublk_end_rq(), unless rq_affinity is 0. Partial completion,
errors and copy-back failures keep completing inline.
ublk_end_rq() may run on a CPU other than the server's, so it does not
touch ublk_io or ublk_queue. Whether the device opted in is taken from
the ublk_device the callers already have, avoiding extra dereferences of
the request's hctx and queue on the completion fast path.
Servers that don't request the feature keep today's behavior.
Signed-off-by: Yoav Cohen <yoav@nvidia.com>
---
drivers/block/ublk_drv.c | 68 ++++++++++++++++++++++++++++-------
include/uapi/linux/ublk_cmd.h | 8 +++++
2 files changed, 64 insertions(+), 12 deletions(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 66eb55e7162e..b626993936fc 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -90,7 +90,8 @@
| UBLK_F_BATCH_IO \
| UBLK_F_NO_AUTO_PART_SCAN \
| UBLK_F_SHMEM_ZC \
- | UBLK_F_IO_DESC_SIZE)
+ | UBLK_F_IO_DESC_SIZE \
+ | UBLK_F_SUPPORT_RQ_AFFINITY)
#define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \
| UBLK_F_USER_RECOVERY_REISSUE \
@@ -457,6 +458,11 @@ static inline bool ublk_dev_support_user_copy(const struct ublk_device *ub)
return ub->dev_info.flags & UBLK_F_USER_COPY;
}
+static inline bool ublk_dev_support_rq_affinity(const struct ublk_device *ub)
+{
+ return ub->dev_info.flags & UBLK_F_SUPPORT_RQ_AFFINITY;
+}
+
static inline bool ublk_dev_is_zoned(const struct ublk_device *ub)
{
return ub->dev_info.flags & UBLK_F_ZONED;
@@ -789,7 +795,8 @@ static void ublk_setup_iod_zoned(struct ublk_queue *ubq, struct request *req)
#endif
static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
- bool need_map, struct io_comp_batch *iob);
+ bool need_map, bool rq_affinity,
+ struct io_comp_batch *iob);
static dev_t ublk_chr_devt;
static const struct class ublk_chr_class = {
@@ -1199,13 +1206,15 @@ static inline bool ublk_get_req_ref(struct ublk_io *io)
return refcount_inc_not_zero(&io->ref);
}
-static inline void ublk_put_req_ref(struct ublk_io *io, struct request *req)
+static inline void ublk_put_req_ref(const struct ublk_device *ub,
+ struct ublk_io *io, struct request *req)
{
if (!refcount_dec_and_test(&io->ref))
return;
/* ublk_need_map_io() and ublk_need_req_ref() are mutually exclusive */
- __ublk_complete_rq(req, io, false, NULL);
+ __ublk_complete_rq(req, io, false, ublk_dev_support_rq_affinity(ub),
+ NULL);
}
static inline bool ublk_sub_req_ref(struct ublk_io *io)
@@ -1549,9 +1558,33 @@ static void ublk_end_request(struct request *req, blk_status_t error)
local_bh_enable();
}
+/*
+ * Runs after blk_mq_complete_request_remote() redirected a fully successful
+ * @req to the submitting CPU. May run on a CPU other than the ublk server's,
+ * so don't touch ublk_io/ublk_queue here.
+ */
+static void ublk_end_rq(struct request *req)
+{
+ blk_mq_end_request(req, BLK_STS_OK);
+}
+
+/*
+ * Only a fully completed request is redirected: partial completion,
+ * errors and the copy-back failure cases keep completing inline.
+ * rq_affinity=0 also completes inline, even on a single hw queue.
+ */
+static inline bool ublk_complete_rq_remote(struct request *req,
+ struct ublk_io *io, bool rq_affinity)
+{
+ return rq_affinity && io->res == blk_rq_bytes(req) &&
+ test_bit(QUEUE_FLAG_SAME_COMP, &req->q->queue_flags) &&
+ blk_mq_complete_request_remote(req);
+}
+
/* todo: handle partial completion */
static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
- bool need_map, struct io_comp_batch *iob)
+ bool need_map, bool rq_affinity,
+ struct io_comp_batch *iob)
{
unsigned int unmapped_bytes;
blk_status_t res = BLK_STS_OK;
@@ -1588,6 +1621,9 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
io->res = unmapped_bytes;
}
+ if (ublk_complete_rq_remote(req, io, rq_affinity))
+ return;
+
/*
* Run bio->bi_end_io() with softirqs disabled. If the final fput
* happens off this path, then that will prevent ublk's blkdev_release()
@@ -1614,6 +1650,8 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
return;
exit:
+ if (res == BLK_STS_OK && ublk_complete_rq_remote(req, io, rq_affinity))
+ return;
ublk_end_request(req, res);
}
@@ -2347,6 +2385,7 @@ static const struct blk_mq_ops ublk_mq_ops = {
.queue_rqs = ublk_queue_rqs,
.init_hctx = ublk_init_hctx,
.timeout = ublk_timeout,
+ .complete = ublk_end_rq,
};
static const struct blk_mq_ops ublk_batch_mq_ops = {
@@ -2355,6 +2394,7 @@ static const struct blk_mq_ops ublk_batch_mq_ops = {
.queue_rqs = ublk_batch_queue_rqs,
.init_hctx = ublk_init_hctx,
.timeout = ublk_timeout,
+ .complete = ublk_end_rq,
};
static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq)
@@ -2685,7 +2725,8 @@ static void __ublk_fail_req(struct ublk_device *ub, struct ublk_io *io,
blk_mq_requeue_request(req, false);
else {
io->res = -EIO;
- __ublk_complete_rq(req, io, ublk_dev_need_map_io(ub), NULL);
+ __ublk_complete_rq(req, io, ublk_dev_need_map_io(ub),
+ ublk_dev_support_rq_affinity(ub), NULL);
}
}
@@ -3180,7 +3221,7 @@ static void ublk_io_release(void *priv)
if (current == io->task && io->task_registered_buffers)
io->task_registered_buffers--;
else
- ublk_put_req_ref(io, rq);
+ ublk_put_req_ref(ubq->dev, io, rq);
}
static int ublk_register_io_buf(struct io_uring_cmd *cmd,
@@ -3202,7 +3243,7 @@ static int ublk_register_io_buf(struct io_uring_cmd *cmd,
ret = io_buffer_register_request(cmd, req, ublk_io_release, index,
issue_flags);
if (ret) {
- ublk_put_req_ref(io, req);
+ ublk_put_req_ref(ub, io, req);
return ret;
}
@@ -3469,7 +3510,8 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd,
if (req_op(req) == REQ_OP_ZONE_APPEND)
req->__sector = addr;
if (compl)
- __ublk_complete_rq(req, io, ublk_dev_need_map_io(ub), NULL);
+ __ublk_complete_rq(req, io, ublk_dev_need_map_io(ub),
+ ublk_dev_support_rq_affinity(ub), NULL);
break;
}
case UBLK_IO_NEED_GET_DATA:
@@ -3521,7 +3563,7 @@ static inline struct request *__ublk_check_and_get_req(struct ublk_device *ub,
return req;
fail_put:
- ublk_put_req_ref(io, req);
+ ublk_put_req_ref(ub, io, req);
return NULL;
}
@@ -3812,7 +3854,9 @@ static int ublk_batch_commit_io(struct ublk_queue *ubq,
if (req_op(req) == REQ_OP_ZONE_APPEND)
req->__sector = ublk_batch_zone_lba(uc, elem);
if (compl)
- __ublk_complete_rq(req, io, ublk_dev_need_map_io(data->ub), data->iob);
+ __ublk_complete_rq(req, io, ublk_dev_need_map_io(data->ub),
+ ublk_dev_support_rq_affinity(data->ub),
+ data->iob);
return 0;
}
@@ -4134,7 +4178,7 @@ ublk_user_copy(struct kiocb *iocb, struct iov_iter *iter, int dir)
out:
if (!on_daemon)
- ublk_put_req_ref(io, req);
+ ublk_put_req_ref(ub, io, req);
return ret;
}
diff --git a/include/uapi/linux/ublk_cmd.h b/include/uapi/linux/ublk_cmd.h
index 33b25dd13965..47b5fba9519c 100644
--- a/include/uapi/linux/ublk_cmd.h
+++ b/include/uapi/linux/ublk_cmd.h
@@ -420,6 +420,14 @@ struct ublk_shmem_buf_reg {
/* ublksrv_io_desc size is specified by ublksrv_ctrl_dev_info's io_desc_size */
#define UBLK_F_IO_DESC_SIZE (1ULL << 20)
+/*
+ * Request completion honors the block device's rq_affinity setting
+ * (/sys/block/ublkbN/queue/rq_affinity): the submitting CPU's completion
+ * work can run there instead of always on the ublk server's CPU. Without
+ * this feature, rq_affinity has no effect on ublk devices.
+ */
+#define UBLK_F_SUPPORT_RQ_AFFINITY (1ULL << 21)
+
/* device state */
#define UBLK_S_DEV_DEAD 0
#define UBLK_S_DEV_LIVE 1
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH v3 2/2] selftests: ublk: add rq_affinity test
2026-10-26 12:06 [PATCH v3 0/2] ublk: honor rq_affinity on request completion Yoav Cohen
2026-10-26 12:06 ` [PATCH v3 1/2] " Yoav Cohen
@ 2026-10-26 12:06 ` Yoav Cohen
2026-10-26 14:45 ` Ming Lei
1 sibling, 1 reply; 4+ messages in thread
From: Yoav Cohen @ 2026-10-26 12:06 UTC (permalink / raw)
To: Ming Lei, Jens Axboe, linux-block, csander
Cc: jholzman, omril, ofer, Yoav Cohen
Add UBLK_F_SUPPORT_RQ_AFFINITY to kublk's feature table and a
--rq_affinity flag to request it at ADD_DEV time, and add
test_generic_18.sh: writes and verifies data on a loop device with
I/O pinned to one CPU while cycling rq_affinity through 0, 1 and 2.
Skipped on kernels without UBLK_F_SUPPORT_RQ_AFFINITY.
Signed-off-by: Yoav Cohen <yoav@nvidia.com>
---
tools/testing/selftests/ublk/Makefile | 1 +
tools/testing/selftests/ublk/kublk.c | 6 ++-
.../testing/selftests/ublk/test_generic_18.sh | 42 +++++++++++++++++++
3 files changed, 48 insertions(+), 1 deletion(-)
create mode 100755 tools/testing/selftests/ublk/test_generic_18.sh
diff --git a/tools/testing/selftests/ublk/Makefile b/tools/testing/selftests/ublk/Makefile
index 37883e9d50ec..d12e9879b89a 100644
--- a/tools/testing/selftests/ublk/Makefile
+++ b/tools/testing/selftests/ublk/Makefile
@@ -19,6 +19,7 @@ TEST_PROGS += test_generic_12.sh
TEST_PROGS += test_generic_13.sh
TEST_PROGS += test_generic_16.sh
TEST_PROGS += test_generic_17.sh
+TEST_PROGS += test_generic_18.sh
TEST_PROGS += test_batch_01.sh
TEST_PROGS += test_batch_02.sh
diff --git a/tools/testing/selftests/ublk/kublk.c b/tools/testing/selftests/ublk/kublk.c
index 2400b4615766..e3afef0e7628 100644
--- a/tools/testing/selftests/ublk/kublk.c
+++ b/tools/testing/selftests/ublk/kublk.c
@@ -2083,6 +2083,7 @@ static int cmd_dev_get_features(void)
FEAT_NAME(UBLK_F_NO_AUTO_PART_SCAN),
FEAT_NAME(UBLK_F_SHMEM_ZC),
FEAT_NAME(UBLK_F_IO_DESC_SIZE),
+ FEAT_NAME(UBLK_F_SUPPORT_RQ_AFFINITY),
};
struct ublk_dev *dev;
__u64 features = 0;
@@ -2180,7 +2181,7 @@ static void __cmd_create_help(char *exe, bool recovery)
printf("\t[--nthreads threads] [--per_io_tasks]\n");
printf("\t[--integrity_capable] [--integrity_reftag] [--metadata_size SIZE] "
"[--pi_offset OFFSET] [--csum_type ip|t10dif|nvme] [--tag_size SIZE]\n");
- printf("\t[--batch|-b] [--rotate_auto_buf] [--no_auto_part_scan]\n");
+ printf("\t[--batch|-b] [--rotate_auto_buf] [--no_auto_part_scan] [--rq_affinity]\n");
printf("\t[--io_desc_size SIZE]\n");
printf("\t[target options] [backfile1] [backfile2] ...\n");
printf("\tdefault: nr_queues=2(max 32), depth=128(max 1024), dev_id=-1(auto allocation)\n");
@@ -2260,6 +2261,7 @@ int main(int argc, char *argv[])
{ "batch", 0, NULL, 'b'},
{ "rotate_auto_buf", 0, NULL, 0 },
{ "no_auto_part_scan", 0, NULL, 0 },
+ { "rq_affinity", 0, NULL, 0 },
{ "shmem_zc", 0, NULL, 0 },
{ "htlb", 1, NULL, 0 },
{ "rdonly_shmem_buf", 0, NULL, 0 },
@@ -2407,6 +2409,8 @@ int main(int argc, char *argv[])
ctx.safe_stop = 1;
if (!strcmp(longopts[option_idx].name, "no_auto_part_scan"))
ctx.flags |= UBLK_F_NO_AUTO_PART_SCAN;
+ if (!strcmp(longopts[option_idx].name, "rq_affinity"))
+ ctx.flags |= UBLK_F_SUPPORT_RQ_AFFINITY;
if (!strcmp(longopts[option_idx].name, "shmem_zc"))
ctx.flags |= UBLK_F_SHMEM_ZC;
if (!strcmp(longopts[option_idx].name, "htlb"))
diff --git a/tools/testing/selftests/ublk/test_generic_18.sh b/tools/testing/selftests/ublk/test_generic_18.sh
new file mode 100755
index 000000000000..c899a5bf0364
--- /dev/null
+++ b/tools/testing/selftests/ublk/test_generic_18.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# SPDX-License-Identifier: GPL-2.0
+
+. "$(cd "$(dirname "$0")" && pwd)"/test_common.sh
+
+ERR_CODE=0
+
+if ! _have_program fio; then
+ exit "$UBLK_SKIP_CODE"
+fi
+
+if ! _have_feature "SUPPORT_RQ_AFFINITY"; then
+ exit "$UBLK_SKIP_CODE"
+fi
+
+_prep_test "loop" "write and verify I/O across rq_affinity settings"
+
+_create_backfile 0 256M
+
+dev_id=$(_add_ublk_dev -t loop --rq_affinity "${UBLK_BACKFILES[0]}")
+_check_add_dev $TID $?
+
+rq_affinity_path=/sys/block/ublkb"${dev_id}"/queue/rq_affinity
+
+for affinity in 0 1 2; do
+ echo "${affinity}" > "${rq_affinity_path}"
+
+ # Pin submission to a single CPU so rq_affinity has somewhere to
+ # redirect completion to.
+ taskset -c 0 fio --name=verify --rw=randwrite --direct=1 \
+ --ioengine=libaio --bs=8k --iodepth=32 --verify=crc32c \
+ --do_verify=1 --verify_state_save=0 \
+ --filename=/dev/ublkb"${dev_id}" --size=256M > /dev/null
+ if [ $? -ne 0 ]; then
+ echo "fio verify failed with rq_affinity=${affinity}"
+ ERR_CODE=255
+ break
+ fi
+done
+
+_cleanup_test
+_show_result $TID $ERR_CODE
--
2.50.1 (Apple Git-155)
^ permalink raw reply related [flat|nested] 4+ messages in thread