* [PATCH V10 19/19] virtio_ring: add in order support
2025-12-24 0:32 [PATCH V10 00/19] virtio_ring " Jason Wang
@ 2025-12-24 0:33 ` Jason Wang
2025-12-24 8:05 ` Eugenio Perez Martin
0 siblings, 1 reply; 4+ messages in thread
From: Jason Wang @ 2025-12-24 0:33 UTC (permalink / raw)
To: mst, jasowang, xuanzhuo, eperezma, virtualization, linux-kernel
This patch implements in order support for both split virtqueue and
packed virtqueue. Performance could be gained for the device where the
memory access could be expensive (e.g vhost-net or a real PCI device):
Benchmark with KVM guest:
Vhost-net on the host: (pktgen + XDP_DROP):
in_order=off | in_order=on | +%
TX: 4.51Mpps | 5.30Mpps | +17%
RX: 3.47Mpps | 3.61Mpps | + 4%
Vhost-user(testpmd) on the host: (pktgen/XDP_DROP):
For split virtqueue:
in_order=off | in_order=on | +%
TX: 5.60Mpps | 5.60Mpps | +0.0%
RX: 9.16Mpps | 9.61Mpps | +4.9%
For packed virtqueue:
in_order=off | in_order=on | +%
TX: 5.60Mpps | 5.70Mpps | +1.7%
RX: 10.6Mpps | 10.8Mpps | +1.8%
Benchmark also shows no performance impact for in_order=off for queue
size with 256 and 1024.
Signed-off-by: Jason Wang <jasowang@redhat.com>
fix
---
drivers/virtio/virtio_ring.c | 459 +++++++++++++++++++++++++++++++++--
1 file changed, 436 insertions(+), 23 deletions(-)
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 61884e031b94..a059b77dc021 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -70,6 +70,8 @@
enum vq_layout {
SPLIT = 0,
PACKED,
+ SPLIT_IN_ORDER,
+ PACKED_IN_ORDER,
VQ_TYPE_MAX,
};
@@ -80,6 +82,7 @@ struct vring_desc_state_split {
* allocated together. So we won't stress more to the memory allocator.
*/
struct vring_desc *indir_desc;
+ u32 total_in_len;
};
struct vring_desc_state_packed {
@@ -91,6 +94,7 @@ struct vring_desc_state_packed {
struct vring_packed_desc *indir_desc;
u16 num; /* Descriptor list length. */
u16 last; /* The last desc state in a list. */
+ u32 total_in_len;
};
struct vring_desc_extra {
@@ -205,8 +209,24 @@ struct vring_virtqueue {
enum vq_layout layout;
- /* Head of free buffer list. */
+ /*
+ * Without IN_ORDER it's the head of free buffer list. With
+ * IN_ORDER and SPLIT, it's the next available buffer
+ * index. With IN_ORDER and PACKED, it's unused.
+ */
unsigned int free_head;
+
+ /*
+ * With IN_ORDER, once we see an in-order batch, this stores
+ * this last entry, and until we return the last buffer.
+ * After this, id is set to UINT_MAX to mark it invalid.
+ * Unused without IN_ORDER.
+ */
+ struct used_entry {
+ u32 id;
+ u32 len;
+ } batch_last;
+
/* Number we've added since last sync. */
unsigned int num_added;
@@ -218,6 +238,11 @@ struct vring_virtqueue {
*/
u16 last_used_idx;
+ /* With IN_ORDER and SPLIT, last descriptor id we used to
+ * detach buffer.
+ */
+ u16 last_used;
+
/* Hint for event idx: already triggered no need to disable. */
bool event_triggered;
@@ -259,7 +284,12 @@ static void vring_free(struct virtqueue *_vq);
static inline bool virtqueue_is_packed(const struct vring_virtqueue *vq)
{
- return vq->layout == PACKED;
+ return vq->layout == PACKED || vq->layout == PACKED_IN_ORDER;
+}
+
+static inline bool virtqueue_is_in_order(const struct vring_virtqueue *vq)
+{
+ return vq->layout == SPLIT_IN_ORDER || vq->layout == PACKED_IN_ORDER;
}
static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
@@ -469,6 +499,8 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
else
vq->last_used_idx = 0;
+ vq->last_used = 0;
+
vq->event_triggered = false;
vq->num_added = 0;
@@ -576,6 +608,8 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
struct scatterlist *sg;
struct vring_desc *desc;
unsigned int i, n, avail, descs_used, err_idx, sg_count = 0;
+ /* Total length for in-order */
+ unsigned int total_in_len = 0;
int head;
bool indirect;
@@ -667,6 +701,7 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
*/
i = virtqueue_add_desc_split(vq, desc, extra, i, addr,
len, flags, premapped);
+ total_in_len += len;
}
}
@@ -689,7 +724,12 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
vq->vq.num_free -= descs_used;
/* Update free pointer */
- if (indirect)
+ if (virtqueue_is_in_order(vq)) {
+ vq->free_head += descs_used;
+ if (vq->free_head >= vq->split.vring.num)
+ vq->free_head -= vq->split.vring.num;
+ vq->split.desc_state[head].total_in_len = total_in_len;
+ } else if (indirect)
vq->free_head = vq->split.desc_extra[head].next;
else
vq->free_head = i;
@@ -862,6 +902,14 @@ static bool more_used_split(const struct vring_virtqueue *vq)
return virtqueue_poll_split(vq, vq->last_used_idx);
}
+static bool more_used_split_in_order(const struct vring_virtqueue *vq)
+{
+ if (vq->batch_last.id != UINT_MAX)
+ return true;
+
+ return virtqueue_poll_split(vq, vq->last_used_idx);
+}
+
static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
unsigned int *len,
void **ctx)
@@ -919,6 +967,80 @@ static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
return ret;
}
+static void *virtqueue_get_buf_ctx_split_in_order(struct vring_virtqueue *vq,
+ unsigned int *len,
+ void **ctx)
+{
+ void *ret;
+ unsigned int num = vq->split.vring.num;
+ unsigned int num_free = vq->vq.num_free;
+ u16 last_used, last_used_idx;
+
+ START_USE(vq);
+
+ if (unlikely(vq->broken)) {
+ END_USE(vq);
+ return NULL;
+ }
+
+ last_used = vq->last_used & (num - 1);
+ last_used_idx = vq->last_used_idx & (num - 1);
+
+ if (vq->batch_last.id == UINT_MAX) {
+ if (!more_used_split_in_order(vq)) {
+ pr_debug("No more buffers in queue\n");
+ END_USE(vq);
+ return NULL;
+ }
+
+ /*
+ * Only get used array entries after they have been
+ * exposed by host.
+ */
+ virtio_rmb(vq->weak_barriers);
+
+ vq->batch_last.id = virtio32_to_cpu(vq->vq.vdev,
+ vq->split.vring.used->ring[last_used_idx].id);
+ vq->batch_last.len = virtio32_to_cpu(vq->vq.vdev,
+ vq->split.vring.used->ring[last_used_idx].len);
+ }
+
+ if (vq->batch_last.id == last_used) {
+ vq->batch_last.id = UINT_MAX;
+ *len = vq->batch_last.len;
+ } else {
+ *len = vq->split.desc_state[last_used].total_in_len;
+ }
+
+ if (unlikely(last_used >= num)) {
+ BAD_RING(vq, "id %u out of range\n", last_used);
+ return NULL;
+ }
+ if (unlikely(!vq->split.desc_state[last_used].data)) {
+ BAD_RING(vq, "id %u is not a head!\n", last_used);
+ return NULL;
+ }
+
+ /* detach_buf_split clears data, so grab it now. */
+ ret = vq->split.desc_state[last_used].data;
+ detach_buf_split_in_order(vq, last_used, ctx);
+
+ vq->last_used_idx++;
+ vq->last_used += (vq->vq.num_free - num_free);
+ /* If we expect an interrupt for the next entry, tell host
+ * by writing event index and flush out the write before
+ * the read in the next get_buf call. */
+ if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
+ virtio_store_mb(vq->weak_barriers,
+ &vring_used_event(&vq->split.vring),
+ cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx));
+
+ LAST_ADD_TIME_INVALID(vq);
+
+ END_USE(vq);
+ return ret;
+}
+
static void virtqueue_disable_cb_split(struct vring_virtqueue *vq)
{
if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
@@ -1012,7 +1134,10 @@ static void *virtqueue_detach_unused_buf_split(struct vring_virtqueue *vq)
continue;
/* detach_buf_split clears data, so grab it now. */
buf = vq->split.desc_state[i].data;
- detach_buf_split(vq, i, NULL);
+ if (virtqueue_is_in_order(vq))
+ detach_buf_split_in_order(vq, i, NULL);
+ else
+ detach_buf_split(vq, i, NULL);
vq->split.avail_idx_shadow--;
vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
vq->split.avail_idx_shadow);
@@ -1075,6 +1200,7 @@ static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
/* Put everything in free lists. */
vq->free_head = 0;
+ vq->batch_last.id = UINT_MAX;
}
static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
@@ -1186,7 +1312,6 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
if (!vq)
return NULL;
- vq->layout = SPLIT;
vq->vq.callback = callback;
vq->vq.vdev = vdev;
vq->vq.name = name;
@@ -1206,6 +1331,8 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
!context;
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
+ vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ?
+ SPLIT_IN_ORDER : SPLIT;
if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
vq->weak_barriers = false;
@@ -1363,13 +1490,14 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
unsigned int in_sgs,
void *data,
bool premapped,
- gfp_t gfp)
+ gfp_t gfp,
+ u16 id)
{
struct vring_desc_extra *extra;
struct vring_packed_desc *desc;
struct scatterlist *sg;
- unsigned int i, n, err_idx, len;
- u16 head, id;
+ unsigned int i, n, err_idx, len, total_in_len = 0;
+ u16 head;
dma_addr_t addr;
head = vq->packed.next_avail_idx;
@@ -1387,8 +1515,6 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
}
i = 0;
- id = vq->free_head;
- BUG_ON(id == vq->packed.vring.num);
for (n = 0; n < out_sgs + in_sgs; n++) {
for (sg = sgs[n]; sg; sg = sg_next(sg)) {
@@ -1408,6 +1534,8 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
extra[i].flags = n < out_sgs ? 0 : VRING_DESC_F_WRITE;
}
+ if (n >= out_sgs)
+ total_in_len += len;
i++;
}
}
@@ -1454,13 +1582,15 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
1 << VRING_PACKED_DESC_F_USED;
}
vq->packed.next_avail_idx = n;
- vq->free_head = vq->packed.desc_extra[id].next;
+ if (!virtqueue_is_in_order(vq))
+ vq->free_head = vq->packed.desc_extra[id].next;
/* Store token and indirect buffer state. */
vq->packed.desc_state[id].num = 1;
vq->packed.desc_state[id].data = data;
vq->packed.desc_state[id].indir_desc = desc;
vq->packed.desc_state[id].last = id;
+ vq->packed.desc_state[id].total_in_len = total_in_len;
vq->num_added += 1;
@@ -1513,8 +1643,11 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
BUG_ON(total_sg == 0);
if (virtqueue_use_indirect(vq, total_sg)) {
+ id = vq->free_head;
+ BUG_ON(id == vq->packed.vring.num);
err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
- in_sgs, data, premapped, gfp);
+ in_sgs, data, premapped,
+ gfp, id);
if (err != -ENOMEM) {
END_USE(vq);
return err;
@@ -1635,6 +1768,157 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
return -EIO;
}
+static inline int virtqueue_add_packed_in_order(struct vring_virtqueue *vq,
+ struct scatterlist *sgs[],
+ unsigned int total_sg,
+ unsigned int out_sgs,
+ unsigned int in_sgs,
+ void *data,
+ void *ctx,
+ bool premapped,
+ gfp_t gfp)
+{
+ struct vring_packed_desc *desc;
+ struct scatterlist *sg;
+ unsigned int i, n, sg_count, err_idx, total_in_len = 0;
+ __le16 head_flags, flags;
+ u16 head, avail_used_flags;
+ int err;
+
+ START_USE(vq);
+
+ BUG_ON(data == NULL);
+ BUG_ON(ctx && vq->indirect);
+
+ if (unlikely(vq->broken)) {
+ END_USE(vq);
+ return -EIO;
+ }
+
+ LAST_ADD_TIME_UPDATE(vq);
+
+ BUG_ON(total_sg == 0);
+
+ if (virtqueue_use_indirect(vq, total_sg)) {
+ err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
+ in_sgs, data, premapped, gfp,
+ vq->packed.next_avail_idx);
+ if (err != -ENOMEM) {
+ END_USE(vq);
+ return err;
+ }
+
+ /* fall back on direct */
+ }
+
+ head = vq->packed.next_avail_idx;
+ avail_used_flags = vq->packed.avail_used_flags;
+
+ WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
+
+ desc = vq->packed.vring.desc;
+ i = head;
+
+ if (unlikely(vq->vq.num_free < total_sg)) {
+ pr_debug("Can't add buf len %i - avail = %i\n",
+ total_sg, vq->vq.num_free);
+ END_USE(vq);
+ return -ENOSPC;
+ }
+
+ sg_count = 0;
+ for (n = 0; n < out_sgs + in_sgs; n++) {
+ for (sg = sgs[n]; sg; sg = sg_next(sg)) {
+ dma_addr_t addr;
+ u32 len;
+
+ flags = 0;
+ if (++sg_count != total_sg)
+ flags |= cpu_to_le16(VRING_DESC_F_NEXT);
+ if (n >= out_sgs)
+ flags |= cpu_to_le16(VRING_DESC_F_WRITE);
+
+ if (vring_map_one_sg(vq, sg, n < out_sgs ?
+ DMA_TO_DEVICE : DMA_FROM_DEVICE,
+ &addr, &len, premapped))
+ goto unmap_release;
+
+ flags |= cpu_to_le16(vq->packed.avail_used_flags);
+
+ if (i == head)
+ head_flags = flags;
+ else
+ desc[i].flags = flags;
+
+ desc[i].addr = cpu_to_le64(addr);
+ desc[i].len = cpu_to_le32(len);
+ desc[i].id = cpu_to_le16(head);
+
+ if (unlikely(vq->use_map_api)) {
+ vq->packed.desc_extra[i].addr = premapped ?
+ DMA_MAPPING_ERROR: addr;
+ vq->packed.desc_extra[i].len = len;
+ vq->packed.desc_extra[i].flags =
+ le16_to_cpu(flags);
+ }
+
+ if ((unlikely(++i >= vq->packed.vring.num))) {
+ i = 0;
+ vq->packed.avail_used_flags ^=
+ 1 << VRING_PACKED_DESC_F_AVAIL |
+ 1 << VRING_PACKED_DESC_F_USED;
+ vq->packed.avail_wrap_counter ^= 1;
+ }
+
+ if (n >= out_sgs)
+ total_in_len += len;
+ }
+ }
+
+ /* We're using some buffers from the free list. */
+ vq->vq.num_free -= total_sg;
+
+ /* Update free pointer */
+ vq->packed.next_avail_idx = i;
+
+ /* Store token. */
+ vq->packed.desc_state[head].num = total_sg;
+ vq->packed.desc_state[head].data = data;
+ vq->packed.desc_state[head].indir_desc = ctx;
+ vq->packed.desc_state[head].total_in_len = total_in_len;
+
+ /*
+ * A driver MUST NOT make the first descriptor in the list
+ * available before all subsequent descriptors comprising
+ * the list are made available.
+ */
+ virtio_wmb(vq->weak_barriers);
+ vq->packed.vring.desc[head].flags = head_flags;
+ vq->num_added += total_sg;
+
+ pr_debug("Added buffer head %i to %p\n", head, vq);
+ END_USE(vq);
+
+ return 0;
+
+unmap_release:
+ err_idx = i;
+ i = head;
+ vq->packed.avail_used_flags = avail_used_flags;
+
+ for (n = 0; n < total_sg; n++) {
+ if (i == err_idx)
+ break;
+ vring_unmap_extra_packed(vq, &vq->packed.desc_extra[i]);
+ i++;
+ if (i >= vq->packed.vring.num)
+ i = 0;
+ }
+
+ END_USE(vq);
+ return -EIO;
+}
+
static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
{
u16 new, old, off_wrap, flags, wrap_counter, event_idx;
@@ -1796,10 +2080,82 @@ static void update_last_used_idx_packed(struct vring_virtqueue *vq,
cpu_to_le16(vq->last_used_idx));
}
+static bool more_used_packed_in_order(const struct vring_virtqueue *vq)
+{
+ if (vq->batch_last.id != UINT_MAX)
+ return true;
+
+ return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
+}
+
+static void *virtqueue_get_buf_ctx_packed_in_order(struct vring_virtqueue *vq,
+ unsigned int *len,
+ void **ctx)
+{
+ unsigned int num = vq->packed.vring.num;
+ u16 last_used, last_used_idx;
+ bool used_wrap_counter;
+ void *ret;
+
+ START_USE(vq);
+
+ if (unlikely(vq->broken)) {
+ END_USE(vq);
+ return NULL;
+ }
+
+ last_used_idx = vq->last_used_idx;
+ used_wrap_counter = packed_used_wrap_counter(last_used_idx);
+ last_used = packed_last_used(last_used_idx);
+
+ if (vq->batch_last.id == UINT_MAX) {
+ if (!more_used_packed_in_order(vq)) {
+ pr_debug("No more buffers in queue\n");
+ END_USE(vq);
+ return NULL;
+ }
+ /* Only get used elements after they have been exposed by host. */
+ virtio_rmb(vq->weak_barriers);
+ vq->batch_last.id =
+ le16_to_cpu(vq->packed.vring.desc[last_used].id);
+ vq->batch_last.len =
+ le32_to_cpu(vq->packed.vring.desc[last_used].len);
+ }
+
+ if (vq->batch_last.id == last_used) {
+ vq->batch_last.id = UINT_MAX;
+ *len = vq->batch_last.len;
+ } else {
+ *len = vq->packed.desc_state[last_used].total_in_len;
+ }
+
+ if (unlikely(last_used >= num)) {
+ BAD_RING(vq, "id %u out of range\n", last_used);
+ return NULL;
+ }
+ if (unlikely(!vq->packed.desc_state[last_used].data)) {
+ BAD_RING(vq, "id %u is not a head!\n", last_used);
+ return NULL;
+ }
+
+ /* detach_buf_packed clears data, so grab it now. */
+ ret = vq->packed.desc_state[last_used].data;
+ detach_buf_packed_in_order(vq, last_used, ctx);
+
+ update_last_used_idx_packed(vq, last_used, last_used,
+ used_wrap_counter);
+
+ LAST_ADD_TIME_INVALID(vq);
+
+ END_USE(vq);
+ return ret;
+}
+
static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
unsigned int *len,
void **ctx)
{
+ unsigned int num = vq->packed.vring.num;
u16 last_used, id, last_used_idx;
bool used_wrap_counter;
void *ret;
@@ -1826,7 +2182,7 @@ static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
- if (unlikely(id >= vq->packed.vring.num)) {
+ if (unlikely(id >= num)) {
BAD_RING(vq, "id %u out of range\n", id);
return NULL;
}
@@ -1967,7 +2323,10 @@ static void *virtqueue_detach_unused_buf_packed(struct vring_virtqueue *vq)
continue;
/* detach_buf clears data, so grab it now. */
buf = vq->packed.desc_state[i].data;
- detach_buf_packed(vq, i, NULL);
+ if (virtqueue_is_in_order(vq))
+ detach_buf_packed_in_order(vq, i, NULL);
+ else
+ detach_buf_packed(vq, i, NULL);
END_USE(vq);
return buf;
}
@@ -1993,6 +2352,8 @@ static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
for (i = 0; i < num - 1; i++)
desc_extra[i].next = i + 1;
+ desc_extra[num - 1].next = 0;
+
return desc_extra;
}
@@ -2124,10 +2485,17 @@ static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
{
vq->packed = *vring_packed;
- /* Put everything in free lists. */
- vq->free_head = 0;
+ if (virtqueue_is_in_order(vq)) {
+ vq->batch_last.id = UINT_MAX;
+ } else {
+ /*
+ * Put everything in free lists. Note that
+ * next_avail_idx is sufficient with IN_ORDER so
+ * free_head is unused.
+ */
+ vq->free_head = 0;
+ }
}
-
static void virtqueue_reset_packed(struct vring_virtqueue *vq)
{
memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
@@ -2172,13 +2540,14 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
#else
vq->broken = false;
#endif
- vq->layout = PACKED;
vq->map = map;
vq->use_map_api = vring_use_map_api(vdev);
vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
!context;
vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
+ vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ?
+ PACKED_IN_ORDER : PACKED;
if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
vq->weak_barriers = false;
@@ -2288,9 +2657,39 @@ static const struct virtqueue_ops packed_ops = {
.reset = virtqueue_reset_packed,
};
+static const struct virtqueue_ops split_in_order_ops = {
+ .add = virtqueue_add_split,
+ .get = virtqueue_get_buf_ctx_split_in_order,
+ .kick_prepare = virtqueue_kick_prepare_split,
+ .disable_cb = virtqueue_disable_cb_split,
+ .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
+ .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
+ .poll = virtqueue_poll_split,
+ .detach_unused_buf = virtqueue_detach_unused_buf_split,
+ .more_used = more_used_split_in_order,
+ .resize = virtqueue_resize_split,
+ .reset = virtqueue_reset_split,
+};
+
+static const struct virtqueue_ops packed_in_order_ops = {
+ .add = virtqueue_add_packed_in_order,
+ .get = virtqueue_get_buf_ctx_packed_in_order,
+ .kick_prepare = virtqueue_kick_prepare_packed,
+ .disable_cb = virtqueue_disable_cb_packed,
+ .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
+ .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
+ .poll = virtqueue_poll_packed,
+ .detach_unused_buf = virtqueue_detach_unused_buf_packed,
+ .more_used = more_used_packed_in_order,
+ .resize = virtqueue_resize_packed,
+ .reset = virtqueue_reset_packed,
+};
+
static const struct virtqueue_ops *const all_ops[VQ_TYPE_MAX] = {
[SPLIT] = &split_ops,
- [PACKED] = &packed_ops
+ [PACKED] = &packed_ops,
+ [SPLIT_IN_ORDER] = &split_in_order_ops,
+ [PACKED_IN_ORDER] = &packed_in_order_ops,
};
static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
@@ -2346,6 +2745,12 @@ static int virtqueue_enable_after_reset(struct virtqueue *_vq)
case PACKED: \
ret = all_ops[PACKED]->op(vq, ##__VA_ARGS__); \
break; \
+ case SPLIT_IN_ORDER: \
+ ret = all_ops[SPLIT_IN_ORDER]->op(vq, ##__VA_ARGS__); \
+ break; \
+ case PACKED_IN_ORDER: \
+ ret = all_ops[PACKED_IN_ORDER]->op(vq, ##__VA_ARGS__); \
+ break; \
default: \
BUG(); \
break; \
@@ -2362,10 +2767,16 @@ static int virtqueue_enable_after_reset(struct virtqueue *_vq)
case PACKED: \
all_ops[PACKED]->op(vq, ##__VA_ARGS__); \
break; \
- default: \
- BUG(); \
- break; \
- } \
+ case SPLIT_IN_ORDER: \
+ all_ops[SPLIT_IN_ORDER]->op(vq, ##__VA_ARGS__); \
+ break; \
+ case PACKED_IN_ORDER: \
+ all_ops[PACKED_IN_ORDER]->op(vq, ##__VA_ARGS__); \
+ break; \
+ default: \
+ BUG(); \
+ break; \
+ } \
})
static inline int virtqueue_add(struct virtqueue *_vq,
@@ -3081,6 +3492,8 @@ void vring_transport_features(struct virtio_device *vdev)
break;
case VIRTIO_F_NOTIFICATION_DATA:
break;
+ case VIRTIO_F_IN_ORDER:
+ break;
default:
/* We don't understand this bit. */
__virtio_clear_bit(vdev, i);
--
2.31.1
^ permalink raw reply related [flat|nested] 4+ messages in thread
* Re: [PATCH V10 19/19] virtio_ring: add in order support
2025-12-24 0:33 ` [PATCH V10 19/19] virtio_ring: add " Jason Wang
@ 2025-12-24 8:05 ` Eugenio Perez Martin
2025-12-25 4:20 ` Jason Wang
0 siblings, 1 reply; 4+ messages in thread
From: Eugenio Perez Martin @ 2025-12-24 8:05 UTC (permalink / raw)
To: Jason Wang; +Cc: mst, xuanzhuo, virtualization, linux-kernel
On Wed, Dec 24, 2025 at 1:34 AM Jason Wang <jasowang@redhat.com> wrote:
>
> This patch implements in order support for both split virtqueue and
> packed virtqueue. Performance could be gained for the device where the
> memory access could be expensive (e.g vhost-net or a real PCI device):
>
> Benchmark with KVM guest:
>
> Vhost-net on the host: (pktgen + XDP_DROP):
>
> in_order=off | in_order=on | +%
> TX: 4.51Mpps | 5.30Mpps | +17%
> RX: 3.47Mpps | 3.61Mpps | + 4%
>
> Vhost-user(testpmd) on the host: (pktgen/XDP_DROP):
>
> For split virtqueue:
>
> in_order=off | in_order=on | +%
> TX: 5.60Mpps | 5.60Mpps | +0.0%
> RX: 9.16Mpps | 9.61Mpps | +4.9%
>
> For packed virtqueue:
>
> in_order=off | in_order=on | +%
> TX: 5.60Mpps | 5.70Mpps | +1.7%
> RX: 10.6Mpps | 10.8Mpps | +1.8%
>
> Benchmark also shows no performance impact for in_order=off for queue
> size with 256 and 1024.
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>
>
> fix
Extra "fix" here?
Reviewed-by: Eugenio Pérez <eperezma@redhat.com>
Thanks!
> ---
> drivers/virtio/virtio_ring.c | 459 +++++++++++++++++++++++++++++++++--
> 1 file changed, 436 insertions(+), 23 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 61884e031b94..a059b77dc021 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -70,6 +70,8 @@
> enum vq_layout {
> SPLIT = 0,
> PACKED,
> + SPLIT_IN_ORDER,
> + PACKED_IN_ORDER,
> VQ_TYPE_MAX,
> };
>
> @@ -80,6 +82,7 @@ struct vring_desc_state_split {
> * allocated together. So we won't stress more to the memory allocator.
> */
> struct vring_desc *indir_desc;
> + u32 total_in_len;
> };
>
> struct vring_desc_state_packed {
> @@ -91,6 +94,7 @@ struct vring_desc_state_packed {
> struct vring_packed_desc *indir_desc;
> u16 num; /* Descriptor list length. */
> u16 last; /* The last desc state in a list. */
> + u32 total_in_len;
> };
>
> struct vring_desc_extra {
> @@ -205,8 +209,24 @@ struct vring_virtqueue {
>
> enum vq_layout layout;
>
> - /* Head of free buffer list. */
> + /*
> + * Without IN_ORDER it's the head of free buffer list. With
> + * IN_ORDER and SPLIT, it's the next available buffer
> + * index. With IN_ORDER and PACKED, it's unused.
> + */
> unsigned int free_head;
> +
> + /*
> + * With IN_ORDER, once we see an in-order batch, this stores
> + * this last entry, and until we return the last buffer.
> + * After this, id is set to UINT_MAX to mark it invalid.
> + * Unused without IN_ORDER.
> + */
> + struct used_entry {
> + u32 id;
> + u32 len;
> + } batch_last;
> +
> /* Number we've added since last sync. */
> unsigned int num_added;
>
> @@ -218,6 +238,11 @@ struct vring_virtqueue {
> */
> u16 last_used_idx;
>
> + /* With IN_ORDER and SPLIT, last descriptor id we used to
> + * detach buffer.
> + */
> + u16 last_used;
> +
> /* Hint for event idx: already triggered no need to disable. */
> bool event_triggered;
>
> @@ -259,7 +284,12 @@ static void vring_free(struct virtqueue *_vq);
>
> static inline bool virtqueue_is_packed(const struct vring_virtqueue *vq)
> {
> - return vq->layout == PACKED;
> + return vq->layout == PACKED || vq->layout == PACKED_IN_ORDER;
> +}
> +
> +static inline bool virtqueue_is_in_order(const struct vring_virtqueue *vq)
> +{
> + return vq->layout == SPLIT_IN_ORDER || vq->layout == PACKED_IN_ORDER;
> }
>
> static bool virtqueue_use_indirect(const struct vring_virtqueue *vq,
> @@ -469,6 +499,8 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
> else
> vq->last_used_idx = 0;
>
> + vq->last_used = 0;
> +
> vq->event_triggered = false;
> vq->num_added = 0;
>
> @@ -576,6 +608,8 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
> struct scatterlist *sg;
> struct vring_desc *desc;
> unsigned int i, n, avail, descs_used, err_idx, sg_count = 0;
> + /* Total length for in-order */
> + unsigned int total_in_len = 0;
> int head;
> bool indirect;
>
> @@ -667,6 +701,7 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
> */
> i = virtqueue_add_desc_split(vq, desc, extra, i, addr,
> len, flags, premapped);
> + total_in_len += len;
> }
> }
>
> @@ -689,7 +724,12 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
> vq->vq.num_free -= descs_used;
>
> /* Update free pointer */
> - if (indirect)
> + if (virtqueue_is_in_order(vq)) {
> + vq->free_head += descs_used;
> + if (vq->free_head >= vq->split.vring.num)
> + vq->free_head -= vq->split.vring.num;
> + vq->split.desc_state[head].total_in_len = total_in_len;
> + } else if (indirect)
> vq->free_head = vq->split.desc_extra[head].next;
> else
> vq->free_head = i;
> @@ -862,6 +902,14 @@ static bool more_used_split(const struct vring_virtqueue *vq)
> return virtqueue_poll_split(vq, vq->last_used_idx);
> }
>
> +static bool more_used_split_in_order(const struct vring_virtqueue *vq)
> +{
> + if (vq->batch_last.id != UINT_MAX)
> + return true;
> +
> + return virtqueue_poll_split(vq, vq->last_used_idx);
> +}
> +
> static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
> unsigned int *len,
> void **ctx)
> @@ -919,6 +967,80 @@ static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
> return ret;
> }
>
> +static void *virtqueue_get_buf_ctx_split_in_order(struct vring_virtqueue *vq,
> + unsigned int *len,
> + void **ctx)
> +{
> + void *ret;
> + unsigned int num = vq->split.vring.num;
> + unsigned int num_free = vq->vq.num_free;
> + u16 last_used, last_used_idx;
> +
> + START_USE(vq);
> +
> + if (unlikely(vq->broken)) {
> + END_USE(vq);
> + return NULL;
> + }
> +
> + last_used = vq->last_used & (num - 1);
> + last_used_idx = vq->last_used_idx & (num - 1);
> +
> + if (vq->batch_last.id == UINT_MAX) {
> + if (!more_used_split_in_order(vq)) {
> + pr_debug("No more buffers in queue\n");
> + END_USE(vq);
> + return NULL;
> + }
> +
> + /*
> + * Only get used array entries after they have been
> + * exposed by host.
> + */
> + virtio_rmb(vq->weak_barriers);
> +
> + vq->batch_last.id = virtio32_to_cpu(vq->vq.vdev,
> + vq->split.vring.used->ring[last_used_idx].id);
> + vq->batch_last.len = virtio32_to_cpu(vq->vq.vdev,
> + vq->split.vring.used->ring[last_used_idx].len);
> + }
> +
> + if (vq->batch_last.id == last_used) {
> + vq->batch_last.id = UINT_MAX;
> + *len = vq->batch_last.len;
> + } else {
> + *len = vq->split.desc_state[last_used].total_in_len;
> + }
> +
> + if (unlikely(last_used >= num)) {
> + BAD_RING(vq, "id %u out of range\n", last_used);
> + return NULL;
> + }
This check cannot be true, but it does not harm so it is totally ok to
fix on top later.
> + if (unlikely(!vq->split.desc_state[last_used].data)) {
> + BAD_RING(vq, "id %u is not a head!\n", last_used);
> + return NULL;
> + }
> +
> + /* detach_buf_split clears data, so grab it now. */
> + ret = vq->split.desc_state[last_used].data;
> + detach_buf_split_in_order(vq, last_used, ctx);
> +
> + vq->last_used_idx++;
> + vq->last_used += (vq->vq.num_free - num_free);
> + /* If we expect an interrupt for the next entry, tell host
> + * by writing event index and flush out the write before
> + * the read in the next get_buf call. */
> + if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
> + virtio_store_mb(vq->weak_barriers,
> + &vring_used_event(&vq->split.vring),
> + cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx));
> +
> + LAST_ADD_TIME_INVALID(vq);
> +
> + END_USE(vq);
> + return ret;
> +}
> +
> static void virtqueue_disable_cb_split(struct vring_virtqueue *vq)
> {
> if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
> @@ -1012,7 +1134,10 @@ static void *virtqueue_detach_unused_buf_split(struct vring_virtqueue *vq)
> continue;
> /* detach_buf_split clears data, so grab it now. */
> buf = vq->split.desc_state[i].data;
> - detach_buf_split(vq, i, NULL);
> + if (virtqueue_is_in_order(vq))
> + detach_buf_split_in_order(vq, i, NULL);
> + else
> + detach_buf_split(vq, i, NULL);
> vq->split.avail_idx_shadow--;
> vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
> vq->split.avail_idx_shadow);
> @@ -1075,6 +1200,7 @@ static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
>
> /* Put everything in free lists. */
> vq->free_head = 0;
> + vq->batch_last.id = UINT_MAX;
> }
>
> static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
> @@ -1186,7 +1312,6 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> if (!vq)
> return NULL;
>
> - vq->layout = SPLIT;
> vq->vq.callback = callback;
> vq->vq.vdev = vdev;
> vq->vq.name = name;
> @@ -1206,6 +1331,8 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
> !context;
> vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
> + vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ?
> + SPLIT_IN_ORDER : SPLIT;
>
> if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
> vq->weak_barriers = false;
> @@ -1363,13 +1490,14 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
> unsigned int in_sgs,
> void *data,
> bool premapped,
> - gfp_t gfp)
> + gfp_t gfp,
> + u16 id)
> {
> struct vring_desc_extra *extra;
> struct vring_packed_desc *desc;
> struct scatterlist *sg;
> - unsigned int i, n, err_idx, len;
> - u16 head, id;
> + unsigned int i, n, err_idx, len, total_in_len = 0;
> + u16 head;
> dma_addr_t addr;
>
> head = vq->packed.next_avail_idx;
> @@ -1387,8 +1515,6 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
> }
>
> i = 0;
> - id = vq->free_head;
> - BUG_ON(id == vq->packed.vring.num);
>
> for (n = 0; n < out_sgs + in_sgs; n++) {
> for (sg = sgs[n]; sg; sg = sg_next(sg)) {
> @@ -1408,6 +1534,8 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
> extra[i].flags = n < out_sgs ? 0 : VRING_DESC_F_WRITE;
> }
>
> + if (n >= out_sgs)
> + total_in_len += len;
> i++;
> }
> }
> @@ -1454,13 +1582,15 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
> 1 << VRING_PACKED_DESC_F_USED;
> }
> vq->packed.next_avail_idx = n;
> - vq->free_head = vq->packed.desc_extra[id].next;
> + if (!virtqueue_is_in_order(vq))
> + vq->free_head = vq->packed.desc_extra[id].next;
>
> /* Store token and indirect buffer state. */
> vq->packed.desc_state[id].num = 1;
> vq->packed.desc_state[id].data = data;
> vq->packed.desc_state[id].indir_desc = desc;
> vq->packed.desc_state[id].last = id;
> + vq->packed.desc_state[id].total_in_len = total_in_len;
>
> vq->num_added += 1;
>
> @@ -1513,8 +1643,11 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
> BUG_ON(total_sg == 0);
>
> if (virtqueue_use_indirect(vq, total_sg)) {
> + id = vq->free_head;
> + BUG_ON(id == vq->packed.vring.num);
> err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
> - in_sgs, data, premapped, gfp);
> + in_sgs, data, premapped,
> + gfp, id);
> if (err != -ENOMEM) {
> END_USE(vq);
> return err;
> @@ -1635,6 +1768,157 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
> return -EIO;
> }
>
> +static inline int virtqueue_add_packed_in_order(struct vring_virtqueue *vq,
> + struct scatterlist *sgs[],
> + unsigned int total_sg,
> + unsigned int out_sgs,
> + unsigned int in_sgs,
> + void *data,
> + void *ctx,
> + bool premapped,
> + gfp_t gfp)
> +{
> + struct vring_packed_desc *desc;
> + struct scatterlist *sg;
> + unsigned int i, n, sg_count, err_idx, total_in_len = 0;
> + __le16 head_flags, flags;
> + u16 head, avail_used_flags;
> + int err;
> +
> + START_USE(vq);
> +
> + BUG_ON(data == NULL);
> + BUG_ON(ctx && vq->indirect);
> +
> + if (unlikely(vq->broken)) {
> + END_USE(vq);
> + return -EIO;
> + }
> +
> + LAST_ADD_TIME_UPDATE(vq);
> +
> + BUG_ON(total_sg == 0);
> +
> + if (virtqueue_use_indirect(vq, total_sg)) {
> + err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
> + in_sgs, data, premapped, gfp,
> + vq->packed.next_avail_idx);
> + if (err != -ENOMEM) {
> + END_USE(vq);
> + return err;
> + }
> +
> + /* fall back on direct */
> + }
> +
> + head = vq->packed.next_avail_idx;
> + avail_used_flags = vq->packed.avail_used_flags;
> +
> + WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
> +
> + desc = vq->packed.vring.desc;
> + i = head;
> +
> + if (unlikely(vq->vq.num_free < total_sg)) {
> + pr_debug("Can't add buf len %i - avail = %i\n",
> + total_sg, vq->vq.num_free);
> + END_USE(vq);
> + return -ENOSPC;
> + }
> +
> + sg_count = 0;
> + for (n = 0; n < out_sgs + in_sgs; n++) {
> + for (sg = sgs[n]; sg; sg = sg_next(sg)) {
> + dma_addr_t addr;
> + u32 len;
> +
> + flags = 0;
> + if (++sg_count != total_sg)
> + flags |= cpu_to_le16(VRING_DESC_F_NEXT);
> + if (n >= out_sgs)
> + flags |= cpu_to_le16(VRING_DESC_F_WRITE);
> +
> + if (vring_map_one_sg(vq, sg, n < out_sgs ?
> + DMA_TO_DEVICE : DMA_FROM_DEVICE,
> + &addr, &len, premapped))
> + goto unmap_release;
> +
> + flags |= cpu_to_le16(vq->packed.avail_used_flags);
> +
> + if (i == head)
> + head_flags = flags;
> + else
> + desc[i].flags = flags;
> +
> + desc[i].addr = cpu_to_le64(addr);
> + desc[i].len = cpu_to_le32(len);
> + desc[i].id = cpu_to_le16(head);
> +
> + if (unlikely(vq->use_map_api)) {
> + vq->packed.desc_extra[i].addr = premapped ?
> + DMA_MAPPING_ERROR: addr;
> + vq->packed.desc_extra[i].len = len;
> + vq->packed.desc_extra[i].flags =
> + le16_to_cpu(flags);
> + }
> +
> + if ((unlikely(++i >= vq->packed.vring.num))) {
> + i = 0;
> + vq->packed.avail_used_flags ^=
> + 1 << VRING_PACKED_DESC_F_AVAIL |
> + 1 << VRING_PACKED_DESC_F_USED;
> + vq->packed.avail_wrap_counter ^= 1;
> + }
> +
> + if (n >= out_sgs)
> + total_in_len += len;
> + }
> + }
> +
> + /* We're using some buffers from the free list. */
> + vq->vq.num_free -= total_sg;
> +
> + /* Update free pointer */
> + vq->packed.next_avail_idx = i;
> +
> + /* Store token. */
> + vq->packed.desc_state[head].num = total_sg;
> + vq->packed.desc_state[head].data = data;
> + vq->packed.desc_state[head].indir_desc = ctx;
> + vq->packed.desc_state[head].total_in_len = total_in_len;
> +
> + /*
> + * A driver MUST NOT make the first descriptor in the list
> + * available before all subsequent descriptors comprising
> + * the list are made available.
> + */
> + virtio_wmb(vq->weak_barriers);
> + vq->packed.vring.desc[head].flags = head_flags;
> + vq->num_added += total_sg;
> +
> + pr_debug("Added buffer head %i to %p\n", head, vq);
> + END_USE(vq);
> +
> + return 0;
> +
> +unmap_release:
> + err_idx = i;
> + i = head;
> + vq->packed.avail_used_flags = avail_used_flags;
> +
> + for (n = 0; n < total_sg; n++) {
> + if (i == err_idx)
> + break;
> + vring_unmap_extra_packed(vq, &vq->packed.desc_extra[i]);
> + i++;
> + if (i >= vq->packed.vring.num)
> + i = 0;
> + }
> +
> + END_USE(vq);
> + return -EIO;
> +}
> +
> static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
> {
> u16 new, old, off_wrap, flags, wrap_counter, event_idx;
> @@ -1796,10 +2080,82 @@ static void update_last_used_idx_packed(struct vring_virtqueue *vq,
> cpu_to_le16(vq->last_used_idx));
> }
>
> +static bool more_used_packed_in_order(const struct vring_virtqueue *vq)
> +{
> + if (vq->batch_last.id != UINT_MAX)
> + return true;
> +
> + return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
> +}
> +
> +static void *virtqueue_get_buf_ctx_packed_in_order(struct vring_virtqueue *vq,
> + unsigned int *len,
> + void **ctx)
> +{
> + unsigned int num = vq->packed.vring.num;
> + u16 last_used, last_used_idx;
> + bool used_wrap_counter;
> + void *ret;
> +
> + START_USE(vq);
> +
> + if (unlikely(vq->broken)) {
> + END_USE(vq);
> + return NULL;
> + }
> +
> + last_used_idx = vq->last_used_idx;
> + used_wrap_counter = packed_used_wrap_counter(last_used_idx);
> + last_used = packed_last_used(last_used_idx);
> +
> + if (vq->batch_last.id == UINT_MAX) {
> + if (!more_used_packed_in_order(vq)) {
> + pr_debug("No more buffers in queue\n");
> + END_USE(vq);
> + return NULL;
> + }
> + /* Only get used elements after they have been exposed by host. */
> + virtio_rmb(vq->weak_barriers);
> + vq->batch_last.id =
> + le16_to_cpu(vq->packed.vring.desc[last_used].id);
> + vq->batch_last.len =
> + le32_to_cpu(vq->packed.vring.desc[last_used].len);
> + }
> +
> + if (vq->batch_last.id == last_used) {
> + vq->batch_last.id = UINT_MAX;
> + *len = vq->batch_last.len;
> + } else {
> + *len = vq->packed.desc_state[last_used].total_in_len;
> + }
> +
> + if (unlikely(last_used >= num)) {
> + BAD_RING(vq, "id %u out of range\n", last_used);
> + return NULL;
> + }
> + if (unlikely(!vq->packed.desc_state[last_used].data)) {
> + BAD_RING(vq, "id %u is not a head!\n", last_used);
> + return NULL;
> + }
> +
> + /* detach_buf_packed clears data, so grab it now. */
> + ret = vq->packed.desc_state[last_used].data;
> + detach_buf_packed_in_order(vq, last_used, ctx);
> +
> + update_last_used_idx_packed(vq, last_used, last_used,
> + used_wrap_counter);
> +
> + LAST_ADD_TIME_INVALID(vq);
> +
> + END_USE(vq);
> + return ret;
> +}
> +
> static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
> unsigned int *len,
> void **ctx)
> {
> + unsigned int num = vq->packed.vring.num;
> u16 last_used, id, last_used_idx;
> bool used_wrap_counter;
> void *ret;
> @@ -1826,7 +2182,7 @@ static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
> id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
> *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
>
> - if (unlikely(id >= vq->packed.vring.num)) {
> + if (unlikely(id >= num)) {
> BAD_RING(vq, "id %u out of range\n", id);
> return NULL;
> }
> @@ -1967,7 +2323,10 @@ static void *virtqueue_detach_unused_buf_packed(struct vring_virtqueue *vq)
> continue;
> /* detach_buf clears data, so grab it now. */
> buf = vq->packed.desc_state[i].data;
> - detach_buf_packed(vq, i, NULL);
> + if (virtqueue_is_in_order(vq))
> + detach_buf_packed_in_order(vq, i, NULL);
> + else
> + detach_buf_packed(vq, i, NULL);
> END_USE(vq);
> return buf;
> }
> @@ -1993,6 +2352,8 @@ static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
> for (i = 0; i < num - 1; i++)
> desc_extra[i].next = i + 1;
>
> + desc_extra[num - 1].next = 0;
> +
> return desc_extra;
> }
>
> @@ -2124,10 +2485,17 @@ static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
> {
> vq->packed = *vring_packed;
>
> - /* Put everything in free lists. */
> - vq->free_head = 0;
> + if (virtqueue_is_in_order(vq)) {
> + vq->batch_last.id = UINT_MAX;
> + } else {
> + /*
> + * Put everything in free lists. Note that
> + * next_avail_idx is sufficient with IN_ORDER so
> + * free_head is unused.
> + */
> + vq->free_head = 0;
> + }
> }
> -
> static void virtqueue_reset_packed(struct vring_virtqueue *vq)
> {
> memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
> @@ -2172,13 +2540,14 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> #else
> vq->broken = false;
> #endif
> - vq->layout = PACKED;
> vq->map = map;
> vq->use_map_api = vring_use_map_api(vdev);
>
> vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
> !context;
> vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
> + vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ?
> + PACKED_IN_ORDER : PACKED;
>
> if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
> vq->weak_barriers = false;
> @@ -2288,9 +2657,39 @@ static const struct virtqueue_ops packed_ops = {
> .reset = virtqueue_reset_packed,
> };
>
> +static const struct virtqueue_ops split_in_order_ops = {
> + .add = virtqueue_add_split,
> + .get = virtqueue_get_buf_ctx_split_in_order,
> + .kick_prepare = virtqueue_kick_prepare_split,
> + .disable_cb = virtqueue_disable_cb_split,
> + .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> + .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> + .poll = virtqueue_poll_split,
> + .detach_unused_buf = virtqueue_detach_unused_buf_split,
> + .more_used = more_used_split_in_order,
> + .resize = virtqueue_resize_split,
> + .reset = virtqueue_reset_split,
> +};
> +
> +static const struct virtqueue_ops packed_in_order_ops = {
> + .add = virtqueue_add_packed_in_order,
> + .get = virtqueue_get_buf_ctx_packed_in_order,
> + .kick_prepare = virtqueue_kick_prepare_packed,
> + .disable_cb = virtqueue_disable_cb_packed,
> + .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> + .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> + .poll = virtqueue_poll_packed,
> + .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> + .more_used = more_used_packed_in_order,
> + .resize = virtqueue_resize_packed,
> + .reset = virtqueue_reset_packed,
> +};
> +
> static const struct virtqueue_ops *const all_ops[VQ_TYPE_MAX] = {
> [SPLIT] = &split_ops,
> - [PACKED] = &packed_ops
> + [PACKED] = &packed_ops,
> + [SPLIT_IN_ORDER] = &split_in_order_ops,
> + [PACKED_IN_ORDER] = &packed_in_order_ops,
> };
>
> static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
> @@ -2346,6 +2745,12 @@ static int virtqueue_enable_after_reset(struct virtqueue *_vq)
> case PACKED: \
> ret = all_ops[PACKED]->op(vq, ##__VA_ARGS__); \
> break; \
> + case SPLIT_IN_ORDER: \
> + ret = all_ops[SPLIT_IN_ORDER]->op(vq, ##__VA_ARGS__); \
> + break; \
> + case PACKED_IN_ORDER: \
> + ret = all_ops[PACKED_IN_ORDER]->op(vq, ##__VA_ARGS__); \
> + break; \
> default: \
> BUG(); \
> break; \
> @@ -2362,10 +2767,16 @@ static int virtqueue_enable_after_reset(struct virtqueue *_vq)
> case PACKED: \
> all_ops[PACKED]->op(vq, ##__VA_ARGS__); \
> break; \
> - default: \
> - BUG(); \
> - break; \
> - } \
> + case SPLIT_IN_ORDER: \
> + all_ops[SPLIT_IN_ORDER]->op(vq, ##__VA_ARGS__); \
> + break; \
> + case PACKED_IN_ORDER: \
> + all_ops[PACKED_IN_ORDER]->op(vq, ##__VA_ARGS__); \
> + break; \
> + default: \
> + BUG(); \
> + break; \
> + } \
> })
>
> static inline int virtqueue_add(struct virtqueue *_vq,
> @@ -3081,6 +3492,8 @@ void vring_transport_features(struct virtio_device *vdev)
> break;
> case VIRTIO_F_NOTIFICATION_DATA:
> break;
> + case VIRTIO_F_IN_ORDER:
> + break;
> default:
> /* We don't understand this bit. */
> __virtio_clear_bit(vdev, i);
> --
> 2.31.1
>
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH V10 19/19] virtio_ring: add in order support
2025-12-24 8:05 ` Eugenio Perez Martin
@ 2025-12-25 4:20 ` Jason Wang
0 siblings, 0 replies; 4+ messages in thread
From: Jason Wang @ 2025-12-25 4:20 UTC (permalink / raw)
To: Eugenio Perez Martin; +Cc: mst, xuanzhuo, virtualization, linux-kernel
On Wed, Dec 24, 2025 at 4:06 PM Eugenio Perez Martin
<eperezma@redhat.com> wrote:
>
> On Wed, Dec 24, 2025 at 1:34 AM Jason Wang <jasowang@redhat.com> wrote:
> >
> > This patch implements in order support for both split virtqueue and
> > packed virtqueue. Performance could be gained for the device where the
> > memory access could be expensive (e.g vhost-net or a real PCI device):
> >
> > Benchmark with KVM guest:
> >
> > Vhost-net on the host: (pktgen + XDP_DROP):
> >
> > in_order=off | in_order=on | +%
> > TX: 4.51Mpps | 5.30Mpps | +17%
> > RX: 3.47Mpps | 3.61Mpps | + 4%
> >
> > Vhost-user(testpmd) on the host: (pktgen/XDP_DROP):
> >
> > For split virtqueue:
> >
> > in_order=off | in_order=on | +%
> > TX: 5.60Mpps | 5.60Mpps | +0.0%
> > RX: 9.16Mpps | 9.61Mpps | +4.9%
> >
> > For packed virtqueue:
> >
> > in_order=off | in_order=on | +%
> > TX: 5.60Mpps | 5.70Mpps | +1.7%
> > RX: 10.6Mpps | 10.8Mpps | +1.8%
> >
> > Benchmark also shows no performance impact for in_order=off for queue
> > size with 256 and 1024.
> >
> > Signed-off-by: Jason Wang <jasowang@redhat.com>
> >
> > fix
>
> Extra "fix" here?
My bad, let me fix that.
>
> Reviewed-by: Eugenio Pérez <eperezma@redhat.com>
>
> Thanks!
[...]
> > +
> > + if (unlikely(last_used >= num)) {
> > + BAD_RING(vq, "id %u out of range\n", last_used);
> > + return NULL;
> > + }
>
> This check cannot be true, but it does not harm so it is totally ok to
> fix on top later.
Will fix this in V11.
Thanks
^ permalink raw reply [flat|nested] 4+ messages in thread
* Re: [PATCH V10 19/19] virtio_ring: add in order support
@ 2025-12-25 4:30 kernel test robot
0 siblings, 0 replies; 4+ messages in thread
From: kernel test robot @ 2025-12-25 4:30 UTC (permalink / raw)
To: oe-kbuild; +Cc: lkp, Dan Carpenter
BCC: lkp@intel.com
CC: oe-kbuild-all@lists.linux.dev
In-Reply-To: <20251224003309.23432-20-jasowang@redhat.com>
References: <20251224003309.23432-20-jasowang@redhat.com>
TO: Jason Wang <jasowang@redhat.com>
Hi Jason,
kernel test robot noticed the following build warnings:
[auto build test WARNING on mst-vhost/linux-next]
[also build test WARNING on linus/master v6.19-rc2]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Jason-Wang/virtio_ring-rename-virtqueue_reinit_xxx-to-virtqueue_reset_xxx/20251224-085755
base: https://git.kernel.org/pub/scm/linux/kernel/git/mst/vhost.git linux-next
patch link: https://lore.kernel.org/r/20251224003309.23432-20-jasowang%40redhat.com
patch subject: [PATCH V10 19/19] virtio_ring: add in order support
:::::: branch date: 27 hours ago
:::::: commit date: 27 hours ago
config: arm-randconfig-r071-20251224 (https://download.01.org/0day-ci/archive/20251225/202512251237.1g0MyG32-lkp@intel.com/config)
compiler: arm-linux-gnueabi-gcc (GCC) 8.5.0
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Reported-by: Dan Carpenter <error27@gmail.com>
| Closes: https://lore.kernel.org/r/202512251237.1g0MyG32-lkp@intel.com/
New smatch warnings:
drivers/virtio/virtio_ring.c:1896 virtqueue_add_packed_in_order() error: uninitialized symbol 'head_flags'.
Old smatch warnings:
drivers/virtio/virtio_ring.c:1734 virtqueue_add_packed() error: uninitialized symbol 'prev'.
drivers/virtio/virtio_ring.c:1742 virtqueue_add_packed() error: uninitialized symbol 'head_flags'.
vim +/head_flags +1896 drivers/virtio/virtio_ring.c
1ce9e6055fa0a9 Tiwei Bie 2018-11-21 1770
b8ff8e25553483 Jason Wang 2025-12-24 1771 static inline int virtqueue_add_packed_in_order(struct vring_virtqueue *vq,
b8ff8e25553483 Jason Wang 2025-12-24 1772 struct scatterlist *sgs[],
b8ff8e25553483 Jason Wang 2025-12-24 1773 unsigned int total_sg,
b8ff8e25553483 Jason Wang 2025-12-24 1774 unsigned int out_sgs,
b8ff8e25553483 Jason Wang 2025-12-24 1775 unsigned int in_sgs,
b8ff8e25553483 Jason Wang 2025-12-24 1776 void *data,
b8ff8e25553483 Jason Wang 2025-12-24 1777 void *ctx,
b8ff8e25553483 Jason Wang 2025-12-24 1778 bool premapped,
b8ff8e25553483 Jason Wang 2025-12-24 1779 gfp_t gfp)
b8ff8e25553483 Jason Wang 2025-12-24 1780 {
b8ff8e25553483 Jason Wang 2025-12-24 1781 struct vring_packed_desc *desc;
b8ff8e25553483 Jason Wang 2025-12-24 1782 struct scatterlist *sg;
b8ff8e25553483 Jason Wang 2025-12-24 1783 unsigned int i, n, sg_count, err_idx, total_in_len = 0;
b8ff8e25553483 Jason Wang 2025-12-24 1784 __le16 head_flags, flags;
b8ff8e25553483 Jason Wang 2025-12-24 1785 u16 head, avail_used_flags;
b8ff8e25553483 Jason Wang 2025-12-24 1786 int err;
b8ff8e25553483 Jason Wang 2025-12-24 1787
b8ff8e25553483 Jason Wang 2025-12-24 1788 START_USE(vq);
b8ff8e25553483 Jason Wang 2025-12-24 1789
b8ff8e25553483 Jason Wang 2025-12-24 1790 BUG_ON(data == NULL);
b8ff8e25553483 Jason Wang 2025-12-24 1791 BUG_ON(ctx && vq->indirect);
b8ff8e25553483 Jason Wang 2025-12-24 1792
b8ff8e25553483 Jason Wang 2025-12-24 1793 if (unlikely(vq->broken)) {
b8ff8e25553483 Jason Wang 2025-12-24 1794 END_USE(vq);
b8ff8e25553483 Jason Wang 2025-12-24 1795 return -EIO;
b8ff8e25553483 Jason Wang 2025-12-24 1796 }
b8ff8e25553483 Jason Wang 2025-12-24 1797
b8ff8e25553483 Jason Wang 2025-12-24 1798 LAST_ADD_TIME_UPDATE(vq);
b8ff8e25553483 Jason Wang 2025-12-24 1799
b8ff8e25553483 Jason Wang 2025-12-24 1800 BUG_ON(total_sg == 0);
b8ff8e25553483 Jason Wang 2025-12-24 1801
b8ff8e25553483 Jason Wang 2025-12-24 1802 if (virtqueue_use_indirect(vq, total_sg)) {
b8ff8e25553483 Jason Wang 2025-12-24 1803 err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
b8ff8e25553483 Jason Wang 2025-12-24 1804 in_sgs, data, premapped, gfp,
b8ff8e25553483 Jason Wang 2025-12-24 1805 vq->packed.next_avail_idx);
b8ff8e25553483 Jason Wang 2025-12-24 1806 if (err != -ENOMEM) {
b8ff8e25553483 Jason Wang 2025-12-24 1807 END_USE(vq);
b8ff8e25553483 Jason Wang 2025-12-24 1808 return err;
b8ff8e25553483 Jason Wang 2025-12-24 1809 }
b8ff8e25553483 Jason Wang 2025-12-24 1810
b8ff8e25553483 Jason Wang 2025-12-24 1811 /* fall back on direct */
b8ff8e25553483 Jason Wang 2025-12-24 1812 }
b8ff8e25553483 Jason Wang 2025-12-24 1813
b8ff8e25553483 Jason Wang 2025-12-24 1814 head = vq->packed.next_avail_idx;
b8ff8e25553483 Jason Wang 2025-12-24 1815 avail_used_flags = vq->packed.avail_used_flags;
b8ff8e25553483 Jason Wang 2025-12-24 1816
b8ff8e25553483 Jason Wang 2025-12-24 1817 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
b8ff8e25553483 Jason Wang 2025-12-24 1818
b8ff8e25553483 Jason Wang 2025-12-24 1819 desc = vq->packed.vring.desc;
b8ff8e25553483 Jason Wang 2025-12-24 1820 i = head;
b8ff8e25553483 Jason Wang 2025-12-24 1821
b8ff8e25553483 Jason Wang 2025-12-24 1822 if (unlikely(vq->vq.num_free < total_sg)) {
b8ff8e25553483 Jason Wang 2025-12-24 1823 pr_debug("Can't add buf len %i - avail = %i\n",
b8ff8e25553483 Jason Wang 2025-12-24 1824 total_sg, vq->vq.num_free);
b8ff8e25553483 Jason Wang 2025-12-24 1825 END_USE(vq);
b8ff8e25553483 Jason Wang 2025-12-24 1826 return -ENOSPC;
b8ff8e25553483 Jason Wang 2025-12-24 1827 }
b8ff8e25553483 Jason Wang 2025-12-24 1828
b8ff8e25553483 Jason Wang 2025-12-24 1829 sg_count = 0;
b8ff8e25553483 Jason Wang 2025-12-24 1830 for (n = 0; n < out_sgs + in_sgs; n++) {
b8ff8e25553483 Jason Wang 2025-12-24 1831 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
b8ff8e25553483 Jason Wang 2025-12-24 1832 dma_addr_t addr;
b8ff8e25553483 Jason Wang 2025-12-24 1833 u32 len;
b8ff8e25553483 Jason Wang 2025-12-24 1834
b8ff8e25553483 Jason Wang 2025-12-24 1835 flags = 0;
b8ff8e25553483 Jason Wang 2025-12-24 1836 if (++sg_count != total_sg)
b8ff8e25553483 Jason Wang 2025-12-24 1837 flags |= cpu_to_le16(VRING_DESC_F_NEXT);
b8ff8e25553483 Jason Wang 2025-12-24 1838 if (n >= out_sgs)
b8ff8e25553483 Jason Wang 2025-12-24 1839 flags |= cpu_to_le16(VRING_DESC_F_WRITE);
b8ff8e25553483 Jason Wang 2025-12-24 1840
b8ff8e25553483 Jason Wang 2025-12-24 1841 if (vring_map_one_sg(vq, sg, n < out_sgs ?
b8ff8e25553483 Jason Wang 2025-12-24 1842 DMA_TO_DEVICE : DMA_FROM_DEVICE,
b8ff8e25553483 Jason Wang 2025-12-24 1843 &addr, &len, premapped))
b8ff8e25553483 Jason Wang 2025-12-24 1844 goto unmap_release;
b8ff8e25553483 Jason Wang 2025-12-24 1845
b8ff8e25553483 Jason Wang 2025-12-24 1846 flags |= cpu_to_le16(vq->packed.avail_used_flags);
b8ff8e25553483 Jason Wang 2025-12-24 1847
b8ff8e25553483 Jason Wang 2025-12-24 1848 if (i == head)
b8ff8e25553483 Jason Wang 2025-12-24 1849 head_flags = flags;
b8ff8e25553483 Jason Wang 2025-12-24 1850 else
b8ff8e25553483 Jason Wang 2025-12-24 1851 desc[i].flags = flags;
b8ff8e25553483 Jason Wang 2025-12-24 1852
b8ff8e25553483 Jason Wang 2025-12-24 1853 desc[i].addr = cpu_to_le64(addr);
b8ff8e25553483 Jason Wang 2025-12-24 1854 desc[i].len = cpu_to_le32(len);
b8ff8e25553483 Jason Wang 2025-12-24 1855 desc[i].id = cpu_to_le16(head);
b8ff8e25553483 Jason Wang 2025-12-24 1856
b8ff8e25553483 Jason Wang 2025-12-24 1857 if (unlikely(vq->use_map_api)) {
b8ff8e25553483 Jason Wang 2025-12-24 1858 vq->packed.desc_extra[i].addr = premapped ?
b8ff8e25553483 Jason Wang 2025-12-24 1859 DMA_MAPPING_ERROR: addr;
b8ff8e25553483 Jason Wang 2025-12-24 1860 vq->packed.desc_extra[i].len = len;
b8ff8e25553483 Jason Wang 2025-12-24 1861 vq->packed.desc_extra[i].flags =
b8ff8e25553483 Jason Wang 2025-12-24 1862 le16_to_cpu(flags);
b8ff8e25553483 Jason Wang 2025-12-24 1863 }
b8ff8e25553483 Jason Wang 2025-12-24 1864
b8ff8e25553483 Jason Wang 2025-12-24 1865 if ((unlikely(++i >= vq->packed.vring.num))) {
b8ff8e25553483 Jason Wang 2025-12-24 1866 i = 0;
b8ff8e25553483 Jason Wang 2025-12-24 1867 vq->packed.avail_used_flags ^=
b8ff8e25553483 Jason Wang 2025-12-24 1868 1 << VRING_PACKED_DESC_F_AVAIL |
b8ff8e25553483 Jason Wang 2025-12-24 1869 1 << VRING_PACKED_DESC_F_USED;
b8ff8e25553483 Jason Wang 2025-12-24 1870 vq->packed.avail_wrap_counter ^= 1;
b8ff8e25553483 Jason Wang 2025-12-24 1871 }
b8ff8e25553483 Jason Wang 2025-12-24 1872
b8ff8e25553483 Jason Wang 2025-12-24 1873 if (n >= out_sgs)
b8ff8e25553483 Jason Wang 2025-12-24 1874 total_in_len += len;
b8ff8e25553483 Jason Wang 2025-12-24 1875 }
b8ff8e25553483 Jason Wang 2025-12-24 1876 }
b8ff8e25553483 Jason Wang 2025-12-24 1877
b8ff8e25553483 Jason Wang 2025-12-24 1878 /* We're using some buffers from the free list. */
b8ff8e25553483 Jason Wang 2025-12-24 1879 vq->vq.num_free -= total_sg;
b8ff8e25553483 Jason Wang 2025-12-24 1880
b8ff8e25553483 Jason Wang 2025-12-24 1881 /* Update free pointer */
b8ff8e25553483 Jason Wang 2025-12-24 1882 vq->packed.next_avail_idx = i;
b8ff8e25553483 Jason Wang 2025-12-24 1883
b8ff8e25553483 Jason Wang 2025-12-24 1884 /* Store token. */
b8ff8e25553483 Jason Wang 2025-12-24 1885 vq->packed.desc_state[head].num = total_sg;
b8ff8e25553483 Jason Wang 2025-12-24 1886 vq->packed.desc_state[head].data = data;
b8ff8e25553483 Jason Wang 2025-12-24 1887 vq->packed.desc_state[head].indir_desc = ctx;
b8ff8e25553483 Jason Wang 2025-12-24 1888 vq->packed.desc_state[head].total_in_len = total_in_len;
b8ff8e25553483 Jason Wang 2025-12-24 1889
b8ff8e25553483 Jason Wang 2025-12-24 1890 /*
b8ff8e25553483 Jason Wang 2025-12-24 1891 * A driver MUST NOT make the first descriptor in the list
b8ff8e25553483 Jason Wang 2025-12-24 1892 * available before all subsequent descriptors comprising
b8ff8e25553483 Jason Wang 2025-12-24 1893 * the list are made available.
b8ff8e25553483 Jason Wang 2025-12-24 1894 */
b8ff8e25553483 Jason Wang 2025-12-24 1895 virtio_wmb(vq->weak_barriers);
b8ff8e25553483 Jason Wang 2025-12-24 @1896 vq->packed.vring.desc[head].flags = head_flags;
b8ff8e25553483 Jason Wang 2025-12-24 1897 vq->num_added += total_sg;
b8ff8e25553483 Jason Wang 2025-12-24 1898
b8ff8e25553483 Jason Wang 2025-12-24 1899 pr_debug("Added buffer head %i to %p\n", head, vq);
b8ff8e25553483 Jason Wang 2025-12-24 1900 END_USE(vq);
b8ff8e25553483 Jason Wang 2025-12-24 1901
b8ff8e25553483 Jason Wang 2025-12-24 1902 return 0;
b8ff8e25553483 Jason Wang 2025-12-24 1903
b8ff8e25553483 Jason Wang 2025-12-24 1904 unmap_release:
b8ff8e25553483 Jason Wang 2025-12-24 1905 err_idx = i;
b8ff8e25553483 Jason Wang 2025-12-24 1906 i = head;
b8ff8e25553483 Jason Wang 2025-12-24 1907 vq->packed.avail_used_flags = avail_used_flags;
b8ff8e25553483 Jason Wang 2025-12-24 1908
b8ff8e25553483 Jason Wang 2025-12-24 1909 for (n = 0; n < total_sg; n++) {
b8ff8e25553483 Jason Wang 2025-12-24 1910 if (i == err_idx)
b8ff8e25553483 Jason Wang 2025-12-24 1911 break;
b8ff8e25553483 Jason Wang 2025-12-24 1912 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[i]);
b8ff8e25553483 Jason Wang 2025-12-24 1913 i++;
b8ff8e25553483 Jason Wang 2025-12-24 1914 if (i >= vq->packed.vring.num)
b8ff8e25553483 Jason Wang 2025-12-24 1915 i = 0;
b8ff8e25553483 Jason Wang 2025-12-24 1916 }
b8ff8e25553483 Jason Wang 2025-12-24 1917
b8ff8e25553483 Jason Wang 2025-12-24 1918 END_USE(vq);
b8ff8e25553483 Jason Wang 2025-12-24 1919 return -EIO;
b8ff8e25553483 Jason Wang 2025-12-24 1920 }
b8ff8e25553483 Jason Wang 2025-12-24 1921
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
^ permalink raw reply [flat|nested] 4+ messages in thread
end of thread, other threads:[~2025-12-25 4:31 UTC | newest]
Thread overview: 4+ messages (download: mbox.gz follow: Atom feed
-- links below jump to the message on this page --
2025-12-25 4:30 [PATCH V10 19/19] virtio_ring: add in order support kernel test robot
-- strict thread matches above, loose matches on Subject: below --
2025-12-24 0:32 [PATCH V10 00/19] virtio_ring " Jason Wang
2025-12-24 0:33 ` [PATCH V10 19/19] virtio_ring: add " Jason Wang
2025-12-24 8:05 ` Eugenio Perez Martin
2025-12-25 4:20 ` Jason Wang
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.