* [PATCH 1/6] ublk: consistently use u16 for queue and tag numbers
2026-07-29 1:29 [PATCH 0/6] ublk: io_desc optimizations Caleb Sander Mateos
@ 2026-07-29 1:29 ` Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 2/6] ublk: remove struct ublk_zoned_report_desc's operation field Caleb Sander Mateos
` (4 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-07-29 1:29 UTC (permalink / raw)
To: Ming Lei, Jens Axboe; +Cc: linux-block, linux-kernel, Caleb Sander Mateos
The u16 nr_hw_queues and queue_depth fields of the ublk UAPI struct
ublksrv_ctrl_dev_info constrain the number of queues and queue depth of
each ublk device. However, the ublk driver is a bit inconsistent with
the type it uses to represent these values, mixing u16 with int and
unsigned int. Change all queue number, queue depth, q_id, and tag
variables/fields to u16 to save some space.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/block/ublk_drv.c | 80 ++++++++++++++++++++--------------------
1 file changed, 41 insertions(+), 39 deletions(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 4f30644756df..257c61bc8222 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -235,22 +235,22 @@ struct ublk_io {
void *buf_ctx_handle;
spinlock_t lock;
} ____cacheline_aligned_in_smp;
struct ublk_queue {
- int q_id;
- int q_depth;
+ u16 q_id;
+ u16 q_depth;
unsigned long flags;
struct ublksrv_io_desc *io_cmd_buf;
bool force_abort;
bool canceling;
bool fail_io; /* copy of dev->state == UBLK_S_DEV_FAIL_IO */
spinlock_t cancel_lock;
struct ublk_device *dev;
- u32 nr_io_ready;
+ u16 nr_io_ready;
/*
* For supporting UBLK_F_BATCH_IO only.
*
* Inflight ublk request tag is saved in this fifo
@@ -325,11 +325,11 @@ struct ublk_device {
spinlock_t lock;
struct mm_struct *mm;
struct ublk_params params;
- u32 nr_queue_ready;
+ u16 nr_queue_ready;
bool unprivileged_daemons;
struct mutex cancel_mutex;
bool canceling;
pid_t ublksrv_tgid;
struct delayed_work exit_work;
@@ -401,11 +401,11 @@ static inline void ublk_io_evts_deinit(struct ublk_queue *q)
WARN_ON_ONCE(!kfifo_is_empty(&q->evts_fifo));
kfifo_free(&q->evts_fifo);
}
static inline struct ublksrv_io_desc *
-ublk_get_iod(const struct ublk_queue *ubq, unsigned tag)
+ublk_get_iod(const struct ublk_queue *ubq, u16 tag)
{
return &ubq->io_cmd_buf[tag];
}
static inline bool ublk_support_zero_copy(const struct ublk_queue *ubq)
@@ -421,12 +421,11 @@ static inline bool ublk_dev_support_zero_copy(const struct ublk_device *ub)
static inline bool ublk_support_shmem_zc(const struct ublk_queue *ubq)
{
return ubq->flags & UBLK_F_SHMEM_ZC;
}
-static inline bool ublk_iod_is_shmem_zc(const struct ublk_queue *ubq,
- unsigned int tag)
+static inline bool ublk_iod_is_shmem_zc(const struct ublk_queue *ubq, u16 tag)
{
return ublk_get_iod(ubq, tag)->op_flags & UBLK_IO_F_SHMEM_ZC;
}
static inline bool ublk_dev_support_shmem_zc(const struct ublk_device *ub)
@@ -862,22 +861,22 @@ static ssize_t ublk_batch_copy_io_tags(struct ublk_batch_fetch_cmd *fcmd,
static unsigned int unprivileged_ublks_max = 64;
static unsigned int unprivileged_ublks_added; /* protected by ublk_ctl_mutex */
static struct miscdevice ublk_misc;
-static inline unsigned ublk_pos_to_hwq(loff_t pos)
+static inline u16 ublk_pos_to_hwq(loff_t pos)
{
return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_QID_OFF) &
UBLK_QID_BITS_MASK;
}
static inline unsigned ublk_pos_to_buf_off(loff_t pos)
{
return (pos - UBLKSRV_IO_BUF_OFFSET) & UBLK_IO_BUF_BITS_MASK;
}
-static inline unsigned ublk_pos_to_tag(loff_t pos)
+static inline u16 ublk_pos_to_tag(loff_t pos)
{
return ((pos - UBLKSRV_IO_BUF_OFFSET) >> UBLK_TAG_OFF) &
UBLK_TAG_BITS_MASK;
}
@@ -1229,22 +1228,22 @@ static noinline void ublk_put_device(struct ublk_device *ub)
{
put_device(&ub->cdev_dev);
}
static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev,
- int qid)
+ u16 qid)
{
return dev->queues[qid];
}
static inline struct ublksrv_io_desc *
-ublk_queue_cmd_buf(struct ublk_device *ub, int q_id)
+ublk_queue_cmd_buf(struct ublk_device *ub, u16 q_id)
{
return ublk_get_queue(ub, q_id)->io_cmd_buf;
}
-static inline int __ublk_queue_cmd_buf_size(int depth)
+static inline int __ublk_queue_cmd_buf_size(u16 depth)
{
return round_up(depth * sizeof(struct ublksrv_io_desc), PAGE_SIZE);
}
static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub)
@@ -1653,11 +1652,11 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq,
else
ublk_end_request(rq, BLK_STS_IOERR);
}
static void
-ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, unsigned tag)
+ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, u16 tag)
{
struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag);
iod->op_flags |= UBLK_IO_F_NEED_REG_BUF;
}
@@ -1764,11 +1763,11 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req,
}
static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)
{
unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
- int tag = req->tag;
+ u16 tag = req->tag;
struct ublk_io *io = &ubq->ios[tag];
pr_devel("%s: complete: qid %d tag %d io_flags %x addr %llx\n",
__func__, ubq->q_id, req->tag, io->flags,
ublk_get_iod(ubq, req->tag)->addr);
@@ -2354,11 +2353,11 @@ static const struct blk_mq_ops ublk_batch_mq_ops = {
.timeout = ublk_timeout,
};
static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq)
{
- int i;
+ u16 i;
ubq->nr_io_ready = 0;
for (i = 0; i < ubq->q_depth; i++) {
struct ublk_io *io = &ubq->ios[i];
@@ -2399,11 +2398,11 @@ static int ublk_ch_open(struct inode *inode, struct file *filp)
return 0;
}
static void ublk_reset_ch_dev(struct ublk_device *ub)
{
- int i;
+ u16 i;
for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
struct ublk_queue *ubq = ublk_get_queue(ub, i);
/* Sync with ublk_cancel_cmd() */
@@ -2471,20 +2470,20 @@ static void ublk_partition_scan_work(struct work_struct *work)
* means.
*/
static void ublk_set_canceling(struct ublk_device *ub, bool canceling)
__must_hold(&ub->cancel_mutex)
{
- int i;
+ u16 i;
ub->canceling = canceling;
for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
ublk_get_queue(ub, i)->canceling = canceling;
}
static bool ublk_check_and_reset_active_ref(struct ublk_device *ub)
{
- int i, j;
+ u16 i, j;
if (!ublk_dev_need_req_ref(ub))
return false;
for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
@@ -2513,11 +2512,11 @@ static bool ublk_check_and_reset_active_ref(struct ublk_device *ub)
static void ublk_ch_release_work_fn(struct work_struct *work)
{
struct ublk_device *ub =
container_of(work, struct ublk_device, exit_work.work);
struct gendisk *disk;
- int i;
+ u16 i;
/*
* For zero-copy and auto buffer register modes, I/O references
* might not be dropped naturally when the daemon is killed, but
* io_uring guarantees that registered bvec kernel buffers are
@@ -2632,11 +2631,12 @@ static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct ublk_device *ub = filp->private_data;
size_t sz = vma->vm_end - vma->vm_start;
unsigned max_sz = ublk_max_cmd_buf_size();
unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT;
- int q_id, ret = 0;
+ int ret = 0;
+ u16 q_id;
spin_lock(&ub->lock);
if (!ub->mm)
ub->mm = current->mm;
if (current->mm != ub->mm)
@@ -2705,11 +2705,11 @@ static void ublk_abort_batch_queue(struct ublk_device *ub,
* So no one can hold our request IO reference any more, simply ignore the
* reference, and complete the request immediately
*/
static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq)
{
- int i;
+ u16 i;
for (i = 0; i < ubq->q_depth; i++) {
struct ublk_io *io = &ubq->ios[i];
if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)
@@ -2748,11 +2748,11 @@ static void ublk_start_cancel(struct ublk_device *ub)
out:
mutex_unlock(&ub->cancel_mutex);
ublk_put_disk(disk);
}
-static void ublk_cancel_cmd(struct ublk_queue *ubq, unsigned tag,
+static void ublk_cancel_cmd(struct ublk_queue *ubq, u16 tag,
unsigned int issue_flags)
{
struct ublk_io *io = &ubq->ios[tag];
struct ublk_device *ub = ubq->dev;
struct io_uring_cmd *cmd = NULL;
@@ -2899,11 +2899,11 @@ static inline bool ublk_dev_ready(const struct ublk_device *ub)
return ub->nr_queue_ready == ub->dev_info.nr_hw_queues;
}
static void ublk_cancel_queue(struct ublk_queue *ubq)
{
- int i;
+ u16 i;
if (ublk_support_batch_io(ubq)) {
ublk_batch_cancel_queue(ubq);
return;
}
@@ -2913,11 +2913,11 @@ static void ublk_cancel_queue(struct ublk_queue *ubq)
}
/* Cancel all pending commands, must be called after del_gendisk() returns */
static void ublk_cancel_dev(struct ublk_device *ub)
{
- int i;
+ u16 i;
for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
ublk_cancel_queue(ublk_get_queue(ub, i));
}
@@ -2947,11 +2947,11 @@ static void ublk_wait_tagset_rqs_idle(struct ublk_device *ub)
}
}
static void ublk_force_abort_dev(struct ublk_device *ub)
{
- int i;
+ u16 i;
pr_devel("%s: force abort ub: dev_id %d state %s\n",
__func__, ub->dev_info.dev_id,
ub->dev_info.state == UBLK_S_DEV_LIVE ?
"LIVE" : "QUIESCED");
@@ -3144,11 +3144,11 @@ ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io,
return 0;
}
static inline void ublk_prep_cancel(struct io_uring_cmd *cmd,
unsigned int issue_flags,
- struct ublk_queue *ubq, unsigned int tag)
+ struct ublk_queue *ubq, u16 tag)
{
struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd);
/*
* Safe to refer to @ubq since ublk_queue won't be died until its
@@ -3948,12 +3948,12 @@ static int ublk_handle_non_batch_cmd(struct io_uring_cmd *cmd,
unsigned int issue_flags)
{
const struct ublksrv_io_cmd *ub_cmd = io_uring_sqe_cmd(cmd->sqe,
struct ublksrv_io_cmd);
struct ublk_device *ub = cmd->file->private_data;
- unsigned tag = READ_ONCE(ub_cmd->tag);
- unsigned q_id = READ_ONCE(ub_cmd->q_id);
+ u16 tag = READ_ONCE(ub_cmd->tag);
+ u16 q_id = READ_ONCE(ub_cmd->q_id);
unsigned index = READ_ONCE(ub_cmd->addr);
struct ublk_queue *ubq;
struct ublk_io *io;
if (cmd->cmd_op == UBLK_U_IO_UNREGISTER_IO_BUF)
@@ -4155,11 +4155,12 @@ static const struct file_operations ublk_ch_batch_io_fops = {
.mmap = ublk_ch_mmap,
};
static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq)
{
- int size, i;
+ int size;
+ u16 i;
size = ublk_queue_cmd_buf_size(ub);
for (i = 0; i < ubq->q_depth; i++) {
struct ublk_io *io = &ubq->ios[i];
@@ -4176,22 +4177,22 @@ static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq)
ublk_io_evts_deinit(ubq);
kvfree(ubq);
}
-static void ublk_deinit_queue(struct ublk_device *ub, int q_id)
+static void ublk_deinit_queue(struct ublk_device *ub, u16 q_id)
{
struct ublk_queue *ubq = ub->queues[q_id];
if (!ubq)
return;
__ublk_deinit_queue(ub, ubq);
ub->queues[q_id] = NULL;
}
-static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id)
+static int ublk_get_queue_numa_node(struct ublk_device *ub, u16 q_id)
{
unsigned int cpu;
/* Find first CPU mapped to this queue */
for_each_possible_cpu(cpu) {
@@ -4200,18 +4201,19 @@ static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id)
}
return NUMA_NO_NODE;
}
-static int ublk_init_queue(struct ublk_device *ub, int q_id)
+static int ublk_init_queue(struct ublk_device *ub, u16 q_id)
{
- int depth = ub->dev_info.queue_depth;
+ u16 depth = ub->dev_info.queue_depth;
gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
struct ublk_queue *ubq;
struct page *page;
int numa_node;
- int size, i, ret;
+ int size, ret;
+ u16 i;
/* Determine NUMA node based on queue's CPU affinity */
numa_node = ublk_get_queue_numa_node(ub, q_id);
/* Allocate queue structure on local NUMA node */
@@ -4252,19 +4254,20 @@ static int ublk_init_queue(struct ublk_device *ub, int q_id)
return ret;
}
static void ublk_deinit_queues(struct ublk_device *ub)
{
- int i;
+ u16 i;
for (i = 0; i < ub->dev_info.nr_hw_queues; i++)
ublk_deinit_queue(ub, i);
}
static int ublk_init_queues(struct ublk_device *ub)
{
- int i, ret;
+ int ret;
+ u16 i;
for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
ret = ublk_init_queue(ub, i);
if (ret)
goto fail;
@@ -5158,11 +5161,11 @@ static int ublk_ctrl_set_size(struct ublk_device *ub, const struct ublksrv_ctrl_
return ret;
}
struct count_busy {
const struct ublk_queue *ubq;
- unsigned int nr_busy;
+ u16 nr_busy;
};
static bool ublk_count_busy_req(struct request *rq, void *data)
{
struct count_busy *idle = data;
@@ -5196,12 +5199,11 @@ static int ublk_wait_for_idle_io(struct ublk_device *ub,
*/
if (ublk_dev_support_batch_io(ub))
return 0;
while (elapsed < timeout_ms && !signal_pending(current)) {
- unsigned int queues_cancelable = 0;
- int i;
+ u16 i, queues_cancelable = 0;
for (i = 0; i < ub->dev_info.nr_hw_queues; i++) {
struct ublk_queue *ubq = ublk_get_queue(ub, i);
queues_cancelable += !!ubq_has_idle_io(ubq);
--
2.54.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 2/6] ublk: remove struct ublk_zoned_report_desc's operation field
2026-07-29 1:29 [PATCH 0/6] ublk: io_desc optimizations Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 1/6] ublk: consistently use u16 for queue and tag numbers Caleb Sander Mateos
@ 2026-07-29 1:29 ` Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 3/6] ublk: split request validation from io_desc init Caleb Sander Mateos
` (3 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-07-29 1:29 UTC (permalink / raw)
To: Ming Lei, Jens Axboe; +Cc: linux-block, linux-kernel, Caleb Sander Mateos
struct ublk_zoned_report_desc's operation field is only ever set to
UBLK_IO_OP_REPORT_ZONES, so remove it. Replace its one load with the
constant.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/block/ublk_drv.c | 14 +++-----------
1 file changed, 3 insertions(+), 11 deletions(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 257c61bc8222..fada30d8d350 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -526,11 +526,10 @@ static void ublk_init_iod(struct ublk_queue *ubq, struct request *req,
#ifdef CONFIG_BLK_DEV_ZONED
struct ublk_zoned_report_desc {
__u64 sector;
- __u32 operation;
__u32 nr_zones;
};
static DEFINE_XARRAY(ublk_zoned_report_descs);
@@ -656,11 +655,10 @@ static int ublk_report_zones(struct gendisk *disk, sector_t sector,
if (IS_ERR(req)) {
ret = PTR_ERR(req);
goto out;
}
- desc.operation = UBLK_IO_OP_REPORT_ZONES;
desc.sector = sector;
desc.nr_zones = zones_in_request;
ret = ublk_zoned_insert_report_desc(req, &desc);
if (ret)
goto free_req;
@@ -729,19 +727,13 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
break;
case REQ_OP_DRV_IN:
desc = ublk_zoned_get_report_desc(req);
if (!desc)
return BLK_STS_IOERR;
- ublk_op = desc->operation;
- switch (ublk_op) {
- case UBLK_IO_OP_REPORT_ZONES:
- ublk_init_iod(ubq, req, ublk_op, desc->nr_zones,
- desc->sector);
- return BLK_STS_OK;
- default:
- return BLK_STS_IOERR;
- }
+ ublk_init_iod(ubq, req, UBLK_IO_OP_REPORT_ZONES, desc->nr_zones,
+ desc->sector);
+ return BLK_STS_OK;
case REQ_OP_DRV_OUT:
/* We do not support drv_out */
return BLK_STS_NOTSUPP;
default:
return BLK_STS_IOERR;
--
2.54.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 3/6] ublk: split request validation from io_desc init
2026-07-29 1:29 [PATCH 0/6] ublk: io_desc optimizations Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 1/6] ublk: consistently use u16 for queue and tag numbers Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 2/6] ublk: remove struct ublk_zoned_report_desc's operation field Caleb Sander Mateos
@ 2026-07-29 1:29 ` Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 4/6] ublk: initialize io_desc on daemon task Caleb Sander Mateos
` (2 subsequent siblings)
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-07-29 1:29 UTC (permalink / raw)
To: Ming Lei, Jens Axboe; +Cc: linux-block, linux-kernel, Caleb Sander Mateos
In preparation for moving the struct ublksrv_io_desc initialization from
the thread submitting ublk requests to the daemon thread receiving them,
split the fallible part of ublk_setup_iod{,_zoned}() into new helper
ublk_validate_req{,_zoned}(). Only ublk_setup_iod{,_zoned}() accesses
the io_desc and cannot error out.
Return a bool value from ublk_validate_req{,_zoned}() as the existing
error code ublk_setup_iod{,_zoned}() returns is only checked against
BLK_STS_OK.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/block/ublk_drv.c | 70 +++++++++++++++++++++++++++-------------
1 file changed, 48 insertions(+), 22 deletions(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index fada30d8d350..efbb0d6af128 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -698,12 +698,28 @@ static int ublk_report_zones(struct gendisk *disk, sector_t sector,
out:
kvfree(buffer);
return ret;
}
-static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
- struct request *req)
+static bool ublk_validate_req_zoned(const struct request *req)
+{
+ switch (req_op(req)) {
+ case REQ_OP_ZONE_OPEN:
+ case REQ_OP_ZONE_CLOSE:
+ case REQ_OP_ZONE_FINISH:
+ case REQ_OP_ZONE_RESET:
+ case REQ_OP_ZONE_APPEND:
+ case REQ_OP_ZONE_RESET_ALL:
+ return true;
+ case REQ_OP_DRV_IN:
+ return !!ublk_zoned_get_report_desc(req);
+ default:
+ return false;
+ }
+}
+
+static void ublk_setup_iod_zoned(struct ublk_queue *ubq, struct request *req)
{
struct ublk_zoned_report_desc *desc;
u32 ublk_op;
switch (req_op(req)) {
@@ -725,24 +741,19 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
case REQ_OP_ZONE_RESET_ALL:
ublk_op = UBLK_IO_OP_ZONE_RESET_ALL;
break;
case REQ_OP_DRV_IN:
desc = ublk_zoned_get_report_desc(req);
- if (!desc)
- return BLK_STS_IOERR;
ublk_init_iod(ubq, req, UBLK_IO_OP_REPORT_ZONES, desc->nr_zones,
desc->sector);
- return BLK_STS_OK;
- case REQ_OP_DRV_OUT:
- /* We do not support drv_out */
- return BLK_STS_NOTSUPP;
+ return;
default:
- return BLK_STS_IOERR;
+ WARN_ON_ONCE(1);
+ return;
}
ublk_init_iod(ubq, req, ublk_op, blk_rq_sectors(req), blk_rq_pos(req));
- return BLK_STS_OK;
}
#else
#define ublk_report_zones (NULL)
@@ -759,14 +770,18 @@ static void ublk_dev_param_zoned_apply(struct ublk_device *ub)
static int ublk_revalidate_disk_zones(struct ublk_device *ub)
{
return 0;
}
-static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq,
- struct request *req)
+static bool ublk_validate_req_zoned(const struct request *req)
{
- return BLK_STS_NOTSUPP;
+ return false;
+}
+
+static void ublk_setup_iod_zoned(struct ublk_queue *ubq, struct request *req)
+{
+ WARN_ON_ONCE(1);
}
#endif
static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
@@ -1489,11 +1504,26 @@ static unsigned int ublk_unmap_io(bool need_map,
return ublk_copy_user_pages(req, 0, &iter, dir);
}
return rq_bytes;
}
-static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req)
+static bool ublk_validate_req(const struct ublk_queue *ubq,
+ const struct request *req)
+{
+ switch (req_op(req)) {
+ case REQ_OP_READ:
+ case REQ_OP_WRITE:
+ case REQ_OP_FLUSH:
+ case REQ_OP_DISCARD:
+ case REQ_OP_WRITE_ZEROES:
+ return true;
+ default:
+ return ublk_queue_is_zoned(ubq) && ublk_validate_req_zoned(req);
+ }
+}
+
+static void ublk_setup_iod(struct ublk_queue *ubq, struct request *req)
{
u32 ublk_op;
switch (req_op(req)) {
case REQ_OP_READ:
@@ -1510,17 +1540,15 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req)
break;
case REQ_OP_WRITE_ZEROES:
ublk_op = UBLK_IO_OP_WRITE_ZEROES;
break;
default:
- if (ublk_queue_is_zoned(ubq))
- return ublk_setup_iod_zoned(ubq, req);
- return BLK_STS_IOERR;
+ ublk_setup_iod_zoned(ubq, req);
+ return;
}
ublk_init_iod(ubq, req, ublk_op, blk_rq_sectors(req), blk_rq_pos(req));
- return BLK_STS_OK;
}
static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu(
struct io_uring_cmd *ioucmd)
{
@@ -2124,12 +2152,10 @@ static enum blk_eh_timer_return ublk_timeout(struct request *rq)
}
static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq,
bool check_cancel)
{
- blk_status_t res;
-
if (unlikely(READ_ONCE(ubq->fail_io)))
return BLK_STS_TARGET;
/* With recovery feature enabled, force_abort is set in
* ublk_stop_dev() before calling del_gendisk(). We have to
@@ -2146,14 +2172,14 @@ static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq,
if (check_cancel && unlikely(ubq->canceling))
return BLK_STS_IOERR;
/* fill iod to slot in io cmd buffer */
- res = ublk_setup_iod(ubq, rq);
- if (unlikely(res != BLK_STS_OK))
+ if (unlikely(!ublk_validate_req(ubq, rq)))
return BLK_STS_IOERR;
+ ublk_setup_iod(ubq, rq);
blk_mq_start_request(rq);
return BLK_STS_OK;
}
/*
--
2.54.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 4/6] ublk: initialize io_desc on daemon task
2026-07-29 1:29 [PATCH 0/6] ublk: io_desc optimizations Caleb Sander Mateos
` (2 preceding siblings ...)
2026-07-29 1:29 ` [PATCH 3/6] ublk: split request validation from io_desc init Caleb Sander Mateos
@ 2026-07-29 1:29 ` Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 5/6] ublk: add UBLK_F_IO_DESC_SIZE Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 6/6] ublk: lift need_map check out of ublk_{,un}map_io() Caleb Sander Mateos
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-07-29 1:29 UTC (permalink / raw)
To: Ming Lei, Jens Axboe; +Cc: linux-block, linux-kernel, Caleb Sander Mateos
ublk_setup_iod() is currently called to populate struct ublksrv_io_desc
on the thread submitting I/O to a ublk device. However, only the ublk
server threads read the io_descs. This basically guarantees a cache miss
on both threads for each ublk I/O. There's really no need to initialize
the io_descs on the submitting thread. Move the ublk_setup_iod() call to
ublk_dispatch_req() (for non-UBLK_F_BATCH_IO) and
__ublk_batch_prep_dispatch() (for UBLK_F_BATCH_IO), which runs on the
ublk server daemon thread before dispatching the I/O to userspace.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/block/ublk_drv.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index efbb0d6af128..46f5ab13f87e 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -1786,10 +1786,11 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req)
{
unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS;
u16 tag = req->tag;
struct ublk_io *io = &ubq->ios[tag];
+ ublk_setup_iod(ubq, req);
pr_devel("%s: complete: qid %d tag %d io_flags %x addr %llx\n",
__func__, ubq->q_id, req->tag, io->flags,
ublk_get_iod(ubq, req->tag)->addr);
/*
@@ -1839,10 +1840,11 @@ static bool __ublk_batch_prep_dispatch(struct ublk_queue *ubq,
struct ublk_io *io = &ubq->ios[tag];
struct request *req = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], tag);
enum auto_buf_reg_res res = AUTO_BUF_REG_FALLBACK;
struct io_uring_cmd *cmd = data->cmd;
+ ublk_setup_iod(ubq, req);
if (!ublk_start_io(ubq, req, io))
return false;
if (ublk_support_auto_buf_reg(ubq) && blk_rq_has_data(req)) {
res = ublk_auto_buf_register(ubq, req, io, cmd,
@@ -2175,11 +2177,10 @@ static blk_status_t ublk_prep_req(struct ublk_queue *ubq, struct request *rq,
/* fill iod to slot in io cmd buffer */
if (unlikely(!ublk_validate_req(ubq, rq)))
return BLK_STS_IOERR;
- ublk_setup_iod(ubq, rq);
blk_mq_start_request(rq);
return BLK_STS_OK;
}
/*
--
2.54.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 5/6] ublk: add UBLK_F_IO_DESC_SIZE
2026-07-29 1:29 [PATCH 0/6] ublk: io_desc optimizations Caleb Sander Mateos
` (3 preceding siblings ...)
2026-07-29 1:29 ` [PATCH 4/6] ublk: initialize io_desc on daemon task Caleb Sander Mateos
@ 2026-07-29 1:29 ` Caleb Sander Mateos
2026-07-29 1:29 ` [PATCH 6/6] ublk: lift need_map check out of ublk_{,un}map_io() Caleb Sander Mateos
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-07-29 1:29 UTC (permalink / raw)
To: Ming Lei, Jens Axboe; +Cc: linux-block, linux-kernel, Caleb Sander Mateos
ublk passes the parameters of incoming I/O in memory shared between the
kernel ublk driver and userspace ublk server in struct ublksrv_io_desc.
The size of this struct is currently fixed to 24 bytes, which has been
an obstacle to extending it with additional fields [1]. Additionally,
with multiple ublk server threads handling I/Os from the same ublk queue
(possible with UBLK_F_PER_IO_DAEMON or UBLK_F_BATCH_IO), false sharing
results from adjacent io_descs sharing the same cache line.
Add a ublk feature UBLK_F_IO_DESC_SIZE to allow a ublk server to
override the size of each io_desc. The size must be at least 24 and a
multiple of 8 to store a properly-aligned struct ublksrv_io_desc. The
struct ublksrv_io_desc is located at the beginning of each io_desc and
the remainder is padding. The mmap() performed for each queue must have
a length of queue_depth * io_desc_size, rounded up to the page size. The
mmap() offset must be q_id * UBLK_MAX_QUEUE_DEPTH * io_desc_size, also
rounded up to the page size.
[1]: https://lore.kernel.org/linux-block/aV8QfvaNO5P6vOs6@fedora/
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/block/ublk_drv.c | 35 ++++++++++++++++++++++++-----------
include/uapi/linux/ublk_cmd.h | 5 ++++-
2 files changed, 28 insertions(+), 12 deletions(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 46f5ab13f87e..1da45e382253 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -87,11 +87,12 @@
| UBLK_F_BUF_REG_OFF_DAEMON \
| (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) ? UBLK_F_INTEGRITY : 0) \
| UBLK_F_SAFE_STOP_DEV \
| UBLK_F_BATCH_IO \
| UBLK_F_NO_AUTO_PART_SCAN \
- | UBLK_F_SHMEM_ZC)
+ | UBLK_F_SHMEM_ZC \
+ | UBLK_F_IO_DESC_SIZE)
#define UBLK_F_ALL_RECOVERY_FLAGS (UBLK_F_USER_RECOVERY \
| UBLK_F_USER_RECOVERY_REISSUE \
| UBLK_F_USER_RECOVERY_FAIL_IO)
@@ -237,10 +238,11 @@ struct ublk_io {
} ____cacheline_aligned_in_smp;
struct ublk_queue {
u16 q_id;
u16 q_depth;
+ u16 io_desc_size;
unsigned long flags;
struct ublksrv_io_desc *io_cmd_buf;
bool force_abort;
@@ -403,11 +405,11 @@ static inline void ublk_io_evts_deinit(struct ublk_queue *q)
}
static inline struct ublksrv_io_desc *
ublk_get_iod(const struct ublk_queue *ubq, u16 tag)
{
- return &ubq->io_cmd_buf[tag];
+ return (void *)ubq->io_cmd_buf + tag * (size_t)ubq->io_desc_size;
}
static inline bool ublk_support_zero_copy(const struct ublk_queue *ubq)
{
return ubq->flags & UBLK_F_SUPPORT_ZERO_COPY;
@@ -1246,23 +1248,24 @@ static inline struct ublksrv_io_desc *
ublk_queue_cmd_buf(struct ublk_device *ub, u16 q_id)
{
return ublk_get_queue(ub, q_id)->io_cmd_buf;
}
-static inline int __ublk_queue_cmd_buf_size(u16 depth)
+static inline size_t __ublk_queue_cmd_buf_size(const struct ublk_device *ub,
+ u16 depth)
{
- return round_up(depth * sizeof(struct ublksrv_io_desc), PAGE_SIZE);
+ return round_up(depth * (size_t)ub->dev_info.io_desc_size, PAGE_SIZE);
}
-static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub)
+static inline size_t ublk_queue_cmd_buf_size(const struct ublk_device *ub)
{
- return __ublk_queue_cmd_buf_size(ub->dev_info.queue_depth);
+ return __ublk_queue_cmd_buf_size(ub, ub->dev_info.queue_depth);
}
-static int ublk_max_cmd_buf_size(void)
+static size_t ublk_max_cmd_buf_size(const struct ublk_device *ub)
{
- return __ublk_queue_cmd_buf_size(UBLK_MAX_QUEUE_DEPTH);
+ return __ublk_queue_cmd_buf_size(ub, UBLK_MAX_QUEUE_DEPTH);
}
/*
* Should I/O outstanding to the ublk server when it exits be reissued?
* If not, outstanding I/O will get errors.
@@ -2648,11 +2651,11 @@ static int ublk_ch_release(struct inode *inode, struct file *filp)
/* map pre-allocated per-queue cmd buffer to ublksrv daemon */
static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma)
{
struct ublk_device *ub = filp->private_data;
size_t sz = vma->vm_end - vma->vm_start;
- unsigned max_sz = ublk_max_cmd_buf_size();
+ size_t max_sz = ublk_max_cmd_buf_size(ub);
unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT;
int ret = 0;
u16 q_id;
spin_lock(&ub->lock);
@@ -4174,11 +4177,11 @@ static const struct file_operations ublk_ch_batch_io_fops = {
.mmap = ublk_ch_mmap,
};
static void __ublk_deinit_queue(struct ublk_device *ub, struct ublk_queue *ubq)
{
- int size;
+ size_t size;
u16 i;
size = ublk_queue_cmd_buf_size(ub);
for (i = 0; i < ubq->q_depth; i++) {
@@ -4227,11 +4230,12 @@ static int ublk_init_queue(struct ublk_device *ub, u16 q_id)
u16 depth = ub->dev_info.queue_depth;
gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO;
struct ublk_queue *ubq;
struct page *page;
int numa_node;
- int size, ret;
+ size_t size;
+ int ret;
u16 i;
/* Determine NUMA node based on queue's CPU affinity */
numa_node = ublk_get_queue_numa_node(ub, q_id);
@@ -4252,10 +4256,11 @@ static int ublk_init_queue(struct ublk_device *ub, u16 q_id)
if (!page) {
kvfree(ubq);
return -ENOMEM;
}
ubq->io_cmd_buf = page_address(page);
+ ubq->io_desc_size = ub->dev_info.io_desc_size;
for (i = 0; i < ubq->q_depth; i++)
spin_lock_init(&ubq->ios[i].lock);
if (ublk_dev_support_batch_io(ub)) {
@@ -4736,10 +4741,18 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header)
/* User copy is required to access integrity buffer */
if (info.flags & UBLK_F_INTEGRITY && !(info.flags & UBLK_F_USER_COPY))
return -EINVAL;
+ if (info.flags & UBLK_F_IO_DESC_SIZE) {
+ if (info.io_desc_size < sizeof(struct ublksrv_io_desc) ||
+ info.io_desc_size % _Alignof(struct ublksrv_io_desc))
+ return -EINVAL;
+ } else {
+ info.io_desc_size = sizeof(struct ublksrv_io_desc);
+ }
+
/* the created device is always owned by current user */
ublk_store_owner_uid_gid(&info.owner_uid, &info.owner_gid);
if (header->dev_id != info.dev_id) {
pr_warn("%s: dev id not match %u %u\n",
diff --git a/include/uapi/linux/ublk_cmd.h b/include/uapi/linux/ublk_cmd.h
index 6991370a72ce..33b25dd13965 100644
--- a/include/uapi/linux/ublk_cmd.h
+++ b/include/uapi/linux/ublk_cmd.h
@@ -415,10 +415,13 @@ struct ublk_shmem_buf_reg {
* pages match a registered buffer, UBLK_IO_F_SHMEM_ZC is set and addr
* encodes the buffer index + offset instead of a userspace buffer address.
*/
#define UBLK_F_SHMEM_ZC (1ULL << 19)
+/* ublksrv_io_desc size is specified by ublksrv_ctrl_dev_info's io_desc_size */
+#define UBLK_F_IO_DESC_SIZE (1ULL << 20)
+
/* device state */
#define UBLK_S_DEV_DEAD 0
#define UBLK_S_DEV_LIVE 1
#define UBLK_S_DEV_QUIESCED 2
#define UBLK_S_DEV_FAIL_IO 3
@@ -450,11 +453,11 @@ struct ublksrv_ctrl_cmd {
struct ublksrv_ctrl_dev_info {
__u16 nr_hw_queues;
__u16 queue_depth;
__u16 state;
- __u16 pad0;
+ __u16 io_desc_size;
__u32 max_io_buf_bytes;
__u32 dev_id;
__s32 ublksrv_pid;
--
2.54.0
^ permalink raw reply related [flat|nested] 7+ messages in thread* [PATCH 6/6] ublk: lift need_map check out of ublk_{,un}map_io()
2026-07-29 1:29 [PATCH 0/6] ublk: io_desc optimizations Caleb Sander Mateos
` (4 preceding siblings ...)
2026-07-29 1:29 ` [PATCH 5/6] ublk: add UBLK_F_IO_DESC_SIZE Caleb Sander Mateos
@ 2026-07-29 1:29 ` Caleb Sander Mateos
5 siblings, 0 replies; 7+ messages in thread
From: Caleb Sander Mateos @ 2026-07-29 1:29 UTC (permalink / raw)
To: Ming Lei, Jens Axboe; +Cc: linux-block, linux-kernel, Caleb Sander Mateos
ublk_map_io() and ublk_unmap_io() are no-ops for ublk devices that
enable user copy or zero copy. However, the implementation is a bit
convoluted, returning the full request data length and relying on the
caller to check the return value against the request length.
UBLK_F_SHMEM_ZC recently added branches to skip the ublk_{,un}map_io()
call for I/Os using a shared-memory buffer. This is a more logical place
for the need_map check, so move it there from ublk_{,un}map_io().
Checking need_map early also avoids the expensive pointer-chasing for
the ublk_iod_is_shmem_zc() check in __ublk_complete_rq() in the common
case of a ublk device using user copy or zero copy.
Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
---
drivers/block/ublk_drv.c | 21 +++++++--------------
1 file changed, 7 insertions(+), 14 deletions(-)
diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c
index 1da45e382253..d4443a43229e 100644
--- a/drivers/block/ublk_drv.c
+++ b/drivers/block/ublk_drv.c
@@ -1464,19 +1464,15 @@ static inline bool ublk_need_unmap_req(const struct request *req)
{
return blk_rq_has_data(req) &&
(req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN);
}
-static unsigned int ublk_map_io(const struct ublk_queue *ubq,
- const struct request *req,
+static unsigned int ublk_map_io(const struct request *req,
const struct ublk_io *io)
{
const unsigned int rq_bytes = blk_rq_bytes(req);
- if (!ublk_need_map_io(ubq))
- return rq_bytes;
-
/*
* no zero copy, we delay copy WRITE request data into ublksrv
* context and the big benefit is that pinning pages in current
* context is pretty fast, see ublk_pin_user_pages
*/
@@ -1488,19 +1484,15 @@ static unsigned int ublk_map_io(const struct ublk_queue *ubq,
return ublk_copy_user_pages(req, 0, &iter, dir);
}
return rq_bytes;
}
-static unsigned int ublk_unmap_io(bool need_map,
- const struct request *req,
+static unsigned int ublk_unmap_io(const struct request *req,
const struct ublk_io *io)
{
const unsigned int rq_bytes = blk_rq_bytes(req);
- if (!need_map)
- return rq_bytes;
-
if (ublk_need_unmap_req(req)) {
struct iov_iter iter;
const int dir = ITER_SOURCE;
import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter);
@@ -1591,15 +1583,16 @@ static inline void __ublk_complete_rq(struct request *req, struct ublk_io *io,
if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE &&
req_op(req) != REQ_OP_DRV_IN)
goto exit;
/* shmem zero copy: no data to unmap, pages already shared */
- if (ublk_iod_is_shmem_zc(req->mq_hctx->driver_data, req->tag))
+ if (!need_map ||
+ ublk_iod_is_shmem_zc(req->mq_hctx->driver_data, req->tag))
goto exit;
/* for READ request, writing data in iod->addr to rq buffers */
- unmapped_bytes = ublk_unmap_io(need_map, req, io);
+ unmapped_bytes = ublk_unmap_io(req, io);
/*
* Extremely impossible since we got data filled in just before
*
* Re-read simply for this unlikely case.
@@ -1755,14 +1748,14 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req,
struct ublk_io *io)
{
unsigned mapped_bytes;
/* shmem zero copy: skip data copy, pages already shared */
- if (ublk_iod_is_shmem_zc(ubq, req->tag))
+ if (!ublk_need_map_io(ubq) || ublk_iod_is_shmem_zc(ubq, req->tag))
return true;
- mapped_bytes = ublk_map_io(ubq, req, io);
+ mapped_bytes = ublk_map_io(req, io);
/* partially mapped, update io descriptor */
if (unlikely(mapped_bytes != blk_rq_bytes(req))) {
/*
* Nothing mapped, retry until we succeed.
--
2.54.0
^ permalink raw reply related [flat|nested] 7+ messages in thread