[PATCH 00/19] virtio_ring in order support

[PATCH 00/19] virtio_ring in order support

[Jason Wang]

Hello all:

This sereis tries to implement the VIRTIO_F_IN_ORDER to
virtio_ring. This is done by introducing virtqueue ops so we can
implement separate helpers for different virtqueue layout/features
then the in-order were implmeented on top.

Tests shows 5% imporvemnt in RX PPS with KVM guest + testpmd on the
host.

Please review.

Thanks

Jason Wang (19):
  virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()
  virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants
  virtio_ring: unify logic of virtqueue_poll() and more_used()
  virtio_ring: switch to use vring_virtqueue for virtqueue resize
    variants
  virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare
    variants
  virtio_ring: switch to use vring_virtqueue for virtqueue_add variants
  virtio: switch to use vring_virtqueue for virtqueue_add variants
  virtio_ring: switch to use vring_virtqueue for enable_cb_prepare
    variants
  virtio_ring: use vring_virtqueue for enable_cb_delayed variants
  virtio_ring: switch to use vring_virtqueue for disable_cb variants
  virtio_ring: switch to use vring_virtqueue for detach_unused_buf
    variants
  virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()
  virtio_ring: introduce virtqueue ops
  virtio_ring: determine descriptor flags at one time
  virtio_ring: factor out core logic of buffer detaching
  virtio_ring: factor out core logic for updating last_used_idx
  virtio_ring: move next_avail_idx to vring_virtqueue
  virtio_ring: factor out split indirect detaching logic
  virtio_ring: add in order support

 drivers/virtio/virtio_ring.c | 856 ++++++++++++++++++++++++++---------
 1 file changed, 653 insertions(+), 203 deletions(-)

-- 
2.42.0

[PATCH 01/19] virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()

[Jason Wang]

To be consistent with virtqueue_reset().

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index fdd2d2b07b5a..1bdfd5d617a7 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -1005,7 +1005,7 @@ static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split
 	}
 }
 
-static void virtqueue_reinit_split(struct vring_virtqueue *vq)
+static void virtqueue_reset_split(struct vring_virtqueue *vq)
 {
 	int num;
 
@@ -1248,7 +1248,7 @@ static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
 err_state_extra:
 	vring_free_split(&vring_split, vdev, vring_dma_dev(vq));
 err:
-	virtqueue_reinit_split(vq);
+	virtqueue_reset_split(vq);
 	return -ENOMEM;
 }
 
@@ -2092,7 +2092,7 @@ static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
 	vq->free_head = 0;
 }
 
-static void virtqueue_reinit_packed(struct vring_virtqueue *vq)
+static void virtqueue_reset_packed(struct vring_virtqueue *vq)
 {
 	memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
 	memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
@@ -2219,7 +2219,7 @@ static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
 err_state_extra:
 	vring_free_packed(&vring_packed, vdev, vring_dma_dev(vq));
 err_ring:
-	virtqueue_reinit_packed(vq);
+	virtqueue_reset_packed(vq);
 	return -ENOMEM;
 }
 
@@ -2852,9 +2852,9 @@ int virtqueue_reset(struct virtqueue *_vq,
 		recycle_done(_vq);
 
 	if (vq->packed_ring)
-		virtqueue_reinit_packed(vq);
+		virtqueue_reset_packed(vq);
 	else
-		virtqueue_reinit_split(vq);
+		virtqueue_reset_split(vq);
 
 	return virtqueue_enable_after_reset(_vq);
 }
-- 
2.42.0

[PATCH 02/19] virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 1bdfd5d617a7..1c6b63812bf8 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -915,11 +915,10 @@ static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
 	return last_used_idx;
 }
 
-static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)
+static bool virtqueue_poll_split(struct vring_virtqueue *vq,
+				 unsigned int last_used_idx)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
-
-	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
+	return (u16)last_used_idx != virtio16_to_cpu(vq->vq.vdev,
 			vq->split.vring.used->idx);
 }
 
@@ -1845,9 +1844,8 @@ static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
 	return vq->last_used_idx;
 }
 
-static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
+static bool virtqueue_poll_packed(struct vring_virtqueue *vq, u16 off_wrap)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	bool wrap_counter;
 	u16 used_idx;
 
@@ -2608,8 +2606,8 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
 		return false;
 
 	virtio_mb(vq->weak_barriers);
-	return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
-				 virtqueue_poll_split(_vq, last_used_idx);
+	return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
+				 virtqueue_poll_split(vq, last_used_idx);
 }
 EXPORT_SYMBOL_GPL(virtqueue_poll);
 
-- 
2.42.0

[PATCH 03/19] virtio_ring: unify logic of virtqueue_poll() and more_used()

[Jason Wang]

This patch unifies the logic of virtqueue_poll() and more_used() for
better code reusing and ease the future in order implementation.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 48 +++++++++++++++---------------------
 1 file changed, 20 insertions(+), 28 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 1c6b63812bf8..9172f3a089a0 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -802,12 +802,18 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 	}
 }
 
-static bool more_used_split(const struct vring_virtqueue *vq)
+static bool virtqueue_poll_split(const struct vring_virtqueue *vq,
+				 unsigned int last_used_idx)
 {
-	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
+	return (u16)last_used_idx != virtio16_to_cpu(vq->vq.vdev,
 			vq->split.vring.used->idx);
 }
 
+static bool more_used_split(const struct vring_virtqueue *vq)
+{
+	return virtqueue_poll_split(vq, vq->last_used_idx);
+}
+
 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
 					 unsigned int *len,
 					 void **ctx)
@@ -915,13 +921,6 @@ static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
 	return last_used_idx;
 }
 
-static bool virtqueue_poll_split(struct vring_virtqueue *vq,
-				 unsigned int last_used_idx)
-{
-	return (u16)last_used_idx != virtio16_to_cpu(vq->vq.vdev,
-			vq->split.vring.used->idx);
-}
-
 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
@@ -1711,16 +1710,20 @@ static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
 	return avail == used && used == used_wrap_counter;
 }
 
-static bool more_used_packed(const struct vring_virtqueue *vq)
+static bool virtqueue_poll_packed(const struct vring_virtqueue *vq, u16 off_wrap)
 {
-	u16 last_used;
-	u16 last_used_idx;
-	bool used_wrap_counter;
+	bool wrap_counter;
+	u16 used_idx;
 
-	last_used_idx = READ_ONCE(vq->last_used_idx);
-	last_used = packed_last_used(last_used_idx);
-	used_wrap_counter = packed_used_wrap_counter(last_used_idx);
-	return is_used_desc_packed(vq, last_used, used_wrap_counter);
+	wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
+	used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
+
+	return is_used_desc_packed(vq, used_idx, wrap_counter);
+}
+
+static bool more_used_packed(const struct vring_virtqueue *vq)
+{
+	return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
 }
 
 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
@@ -1844,17 +1847,6 @@ static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
 	return vq->last_used_idx;
 }
 
-static bool virtqueue_poll_packed(struct vring_virtqueue *vq, u16 off_wrap)
-{
-	bool wrap_counter;
-	u16 used_idx;
-
-	wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
-	used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
-
-	return is_used_desc_packed(vq, used_idx, wrap_counter);
-}
-
 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
-- 
2.42.0

[PATCH 04/19] virtio_ring: switch to use vring_virtqueue for virtqueue resize variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 9172f3a089a0..94ce711963e6 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -1216,11 +1216,10 @@ static struct virtqueue *vring_create_virtqueue_split(
 	return vq;
 }
 
-static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
+static int virtqueue_resize_split(struct vring_virtqueue *vq, u32 num)
 {
 	struct vring_virtqueue_split vring_split = {};
-	struct vring_virtqueue *vq = to_vvq(_vq);
-	struct virtio_device *vdev = _vq->vdev;
+	struct virtio_device *vdev = vq->vq.vdev;
 	int err;
 
 	err = vring_alloc_queue_split(&vring_split, vdev, num,
@@ -2183,11 +2182,10 @@ static struct virtqueue *vring_create_virtqueue_packed(
 	return vq;
 }
 
-static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
+static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
 {
 	struct vring_virtqueue_packed vring_packed = {};
-	struct vring_virtqueue *vq = to_vvq(_vq);
-	struct virtio_device *vdev = _vq->vdev;
+	struct virtio_device *vdev = vq->vq.vdev;
 	int err;
 
 	if (vring_alloc_queue_packed(&vring_packed, vdev, num, vring_dma_dev(vq)))
@@ -2805,9 +2803,9 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
 		recycle_done(_vq);
 
 	if (vq->packed_ring)
-		err = virtqueue_resize_packed(_vq, num);
+		err = virtqueue_resize_packed(vq, num);
 	else
-		err = virtqueue_resize_split(_vq, num);
+		err = virtqueue_resize_split(vq, num);
 
 	return virtqueue_enable_after_reset(_vq);
 }
-- 
2.42.0

[PATCH 05/19] virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 94ce711963e6..cf3ab6404698 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -713,9 +713,8 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 	return -ENOMEM;
 }
 
-static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
+static bool virtqueue_kick_prepare_split(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	u16 new, old;
 	bool needs_kick;
 
@@ -732,12 +731,12 @@ static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
 	LAST_ADD_TIME_INVALID(vq);
 
 	if (vq->event) {
-		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
+		needs_kick = vring_need_event(virtio16_to_cpu(vq->vq.vdev,
 					vring_avail_event(&vq->split.vring)),
 					      new, old);
 	} else {
 		needs_kick = !(vq->split.vring.used->flags &
-					cpu_to_virtio16(_vq->vdev,
+					cpu_to_virtio16(vq->vq.vdev,
 						VRING_USED_F_NO_NOTIFY));
 	}
 	END_USE(vq);
@@ -1597,9 +1596,8 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq,
 	return -EIO;
 }
 
-static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
+static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	u16 new, old, off_wrap, flags, wrap_counter, event_idx;
 	bool needs_kick;
 	union {
@@ -2454,8 +2452,8 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
-				 virtqueue_kick_prepare_split(_vq);
+	return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
+				 virtqueue_kick_prepare_split(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
 
-- 
2.42.0

[PATCH 06/19] virtio_ring: switch to use vring_virtqueue for virtqueue_add variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 40 +++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 21 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index cf3ab6404698..619869e80309 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -472,7 +472,7 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
 	return extra->next;
 }
 
-static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
+static struct vring_desc *alloc_indirect_split(struct vring_virtqueue *vq,
 					       unsigned int total_sg,
 					       gfp_t gfp)
 {
@@ -501,7 +501,7 @@ static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
 	return desc;
 }
 
-static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
+static inline unsigned int virtqueue_add_desc_split(struct vring_virtqueue *vq,
 						    struct vring_desc *desc,
 						    struct vring_desc_extra *extra,
 						    unsigned int i,
@@ -509,11 +509,12 @@ static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
 						    unsigned int len,
 						    u16 flags, bool premapped)
 {
+	struct virtio_device *vdev = vq->vq.vdev;
 	u16 next;
 
-	desc[i].flags = cpu_to_virtio16(vq->vdev, flags);
-	desc[i].addr = cpu_to_virtio64(vq->vdev, addr);
-	desc[i].len = cpu_to_virtio32(vq->vdev, len);
+	desc[i].flags = cpu_to_virtio16(vdev, flags);
+	desc[i].addr = cpu_to_virtio64(vdev, addr);
+	desc[i].len = cpu_to_virtio32(vdev, len);
 
 	extra[i].addr = premapped ? DMA_MAPPING_ERROR : addr;
 	extra[i].len = len;
@@ -521,12 +522,12 @@ static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
 
 	next = extra[i].next;
 
-	desc[i].next = cpu_to_virtio16(vq->vdev, next);
+	desc[i].next = cpu_to_virtio16(vdev, next);
 
 	return next;
 }
 
-static inline int virtqueue_add_split(struct virtqueue *_vq,
+static inline int virtqueue_add_split(struct vring_virtqueue *vq,
 				      struct scatterlist *sgs[],
 				      unsigned int total_sg,
 				      unsigned int out_sgs,
@@ -536,7 +537,6 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 				      bool premapped,
 				      gfp_t gfp)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	struct vring_desc_extra *extra;
 	struct scatterlist *sg;
 	struct vring_desc *desc;
@@ -561,7 +561,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 	head = vq->free_head;
 
 	if (virtqueue_use_indirect(vq, total_sg))
-		desc = alloc_indirect_split(_vq, total_sg, gfp);
+		desc = alloc_indirect_split(vq, total_sg, gfp);
 	else {
 		desc = NULL;
 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
@@ -608,7 +608,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 			/* Note that we trust indirect descriptor
 			 * table since it use stream DMA mapping.
 			 */
-			i = virtqueue_add_desc_split(_vq, desc, extra, i, addr, len,
+			i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
 						     VRING_DESC_F_NEXT,
 						     premapped);
 		}
@@ -625,14 +625,14 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 			/* Note that we trust indirect descriptor
 			 * table since it use stream DMA mapping.
 			 */
-			i = virtqueue_add_desc_split(_vq, desc, extra, i, addr, len,
+			i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
 						     VRING_DESC_F_NEXT |
 						     VRING_DESC_F_WRITE,
 						     premapped);
 		}
 	}
 	/* Last one doesn't continue. */
-	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
+	desc[prev].flags &= cpu_to_virtio16(vq->vq.vdev, ~VRING_DESC_F_NEXT);
 	if (!indirect && vring_need_unmap_buffer(vq, &extra[prev]))
 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
 			~VRING_DESC_F_NEXT;
@@ -645,7 +645,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 		if (vring_mapping_error(vq, addr))
 			goto unmap_release;
 
-		virtqueue_add_desc_split(_vq, vq->split.vring.desc,
+		virtqueue_add_desc_split(vq, vq->split.vring.desc,
 					 vq->split.desc_extra,
 					 head, addr,
 					 total_sg * sizeof(struct vring_desc),
@@ -671,13 +671,13 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 	/* Put entry in available array (but don't update avail->idx until they
 	 * do sync). */
 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
-	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
+	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(vq->vq.vdev, head);
 
 	/* Descriptors and available array need to be set before we expose the
 	 * new available array entries. */
 	virtio_wmb(vq->weak_barriers);
 	vq->split.avail_idx_shadow++;
-	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
+	vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
 						vq->split.avail_idx_shadow);
 	vq->num_added++;
 
@@ -687,7 +687,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 	/* This is very unlikely, but theoretically possible.  Kick
 	 * just in case. */
 	if (unlikely(vq->num_added == (1 << 16) - 1))
-		virtqueue_kick(_vq);
+		virtqueue_kick(&vq->vq);
 
 	return 0;
 
@@ -702,7 +702,6 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
 	for (n = 0; n < total_sg; n++) {
 		if (i == err_idx)
 			break;
-
 		i = vring_unmap_one_split(vq, &extra[i]);
 	}
 
@@ -1441,7 +1440,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 	return -ENOMEM;
 }
 
-static inline int virtqueue_add_packed(struct virtqueue *_vq,
+static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
 				       struct scatterlist *sgs[],
 				       unsigned int total_sg,
 				       unsigned int out_sgs,
@@ -1451,7 +1450,6 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq,
 				       bool premapped,
 				       gfp_t gfp)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	struct vring_packed_desc *desc;
 	struct scatterlist *sg;
 	unsigned int i, n, c, descs_used, err_idx, len;
@@ -2263,9 +2261,9 @@ static inline int virtqueue_add(struct virtqueue *_vq,
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
+	return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
 					out_sgs, in_sgs, data, ctx, premapped, gfp) :
-				 virtqueue_add_split(_vq, sgs, total_sg,
+				 virtqueue_add_split(vq, sgs, total_sg,
 					out_sgs, in_sgs, data, ctx, premapped, gfp);
 }
 
-- 
2.42.0

[PATCH 07/19] virtio: switch to use vring_virtqueue for virtqueue_add variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 619869e80309..3fab40648c75 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -812,11 +812,10 @@ static bool more_used_split(const struct vring_virtqueue *vq)
 	return virtqueue_poll_split(vq, vq->last_used_idx);
 }
 
-static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
+static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
 					 unsigned int *len,
 					 void **ctx)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	void *ret;
 	unsigned int i;
 	u16 last_used;
@@ -838,9 +837,9 @@ static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
 	virtio_rmb(vq->weak_barriers);
 
 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
-	i = virtio32_to_cpu(_vq->vdev,
+	i = virtio32_to_cpu(vq->vq.vdev,
 			vq->split.vring.used->ring[last_used].id);
-	*len = virtio32_to_cpu(_vq->vdev,
+	*len = virtio32_to_cpu(vq->vq.vdev,
 			vq->split.vring.used->ring[last_used].len);
 
 	if (unlikely(i >= vq->split.vring.num)) {
@@ -862,7 +861,7 @@ static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
 		virtio_store_mb(vq->weak_barriers,
 				&vring_used_event(&vq->split.vring),
-				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
+				cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx));
 
 	LAST_ADD_TIME_INVALID(vq);
 
@@ -1721,11 +1720,10 @@ static bool more_used_packed(const struct vring_virtqueue *vq)
 	return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
 }
 
-static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
+static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
 					  unsigned int *len,
 					  void **ctx)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	u16 last_used, id, last_used_idx;
 	bool used_wrap_counter;
 	void *ret;
@@ -2521,8 +2519,8 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
-				 virtqueue_get_buf_ctx_split(_vq, len, ctx);
+	return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
+				 virtqueue_get_buf_ctx_split(vq, len, ctx);
 }
 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
 
-- 
2.42.0

[PATCH 08/19] virtio_ring: switch to use vring_virtqueue for enable_cb_prepare variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 3fab40648c75..a414e66915d9 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -893,9 +893,8 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq)
 	}
 }
 
-static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
+static unsigned int virtqueue_enable_cb_prepare_split(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	u16 last_used_idx;
 
 	START_USE(vq);
@@ -909,10 +908,10 @@ static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
 		if (!vq->event)
 			vq->split.vring.avail->flags =
-				cpu_to_virtio16(_vq->vdev,
+				cpu_to_virtio16(vq->vq.vdev,
 						vq->split.avail_flags_shadow);
 	}
-	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
+	vring_used_event(&vq->split.vring) = cpu_to_virtio16(vq->vq.vdev,
 			last_used_idx = vq->last_used_idx);
 	END_USE(vq);
 	return last_used_idx;
@@ -1807,10 +1806,8 @@ static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
 	}
 }
 
-static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
+static unsigned int virtqueue_enable_cb_prepare_packed(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
-
 	START_USE(vq);
 
 	/*
@@ -2568,8 +2565,8 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
 	if (vq->event_triggered)
 		vq->event_triggered = false;
 
-	return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
-				 virtqueue_enable_cb_prepare_split(_vq);
+	return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
+				 virtqueue_enable_cb_prepare_split(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
 
-- 
2.42.0

[PATCH 09/19] virtio_ring: use vring_virtqueue for enable_cb_delayed variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index a414e66915d9..af8ce6e9d892 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -917,9 +917,8 @@ static unsigned int virtqueue_enable_cb_prepare_split(struct vring_virtqueue *vq
 	return last_used_idx;
 }
 
-static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
+static bool virtqueue_enable_cb_delayed_split(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	u16 bufs;
 
 	START_USE(vq);
@@ -933,7 +932,7 @@ static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
 		if (!vq->event)
 			vq->split.vring.avail->flags =
-				cpu_to_virtio16(_vq->vdev,
+				cpu_to_virtio16(vq->vq.vdev,
 						vq->split.avail_flags_shadow);
 	}
 	/* TODO: tune this threshold */
@@ -941,9 +940,9 @@ static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
 
 	virtio_store_mb(vq->weak_barriers,
 			&vring_used_event(&vq->split.vring),
-			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
+			cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx + bufs));
 
-	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
+	if (unlikely((u16)(virtio16_to_cpu(vq->vq.vdev, vq->split.vring.used->idx)
 					- vq->last_used_idx) > bufs)) {
 		END_USE(vq);
 		return false;
@@ -1837,9 +1836,8 @@ static unsigned int virtqueue_enable_cb_prepare_packed(struct vring_virtqueue *v
 	return vq->last_used_idx;
 }
 
-static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
+static bool virtqueue_enable_cb_delayed_packed(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	u16 used_idx, wrap_counter, last_used_idx;
 	u16 bufs;
 
@@ -2631,8 +2629,8 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
 	if (vq->event_triggered)
 		vq->event_triggered = false;
 
-	return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
-				 virtqueue_enable_cb_delayed_split(_vq);
+	return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
+				 virtqueue_enable_cb_delayed_split(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
 
-- 
2.42.0

[PATCH 10/19] virtio_ring: switch to use vring_virtqueue for disable_cb variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index af8ce6e9d892..f754da8c2ac6 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -869,10 +869,8 @@ static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
 	return ret;
 }
 
-static void virtqueue_disable_cb_split(struct virtqueue *_vq)
+static void virtqueue_disable_cb_split(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
-
 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
 		vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
 
@@ -888,7 +886,7 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq)
 			vring_used_event(&vq->split.vring) = 0x0;
 		else
 			vq->split.vring.avail->flags =
-				cpu_to_virtio16(_vq->vdev,
+				cpu_to_virtio16(vq->vq.vdev,
 						vq->split.avail_flags_shadow);
 	}
 }
@@ -1786,10 +1784,8 @@ static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
 	return ret;
 }
 
-static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
+static void virtqueue_disable_cb_packed(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
-
 	if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
 		vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
 
@@ -2538,9 +2534,9 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
 	if (vq->packed_ring)
-		virtqueue_disable_cb_packed(_vq);
+		virtqueue_disable_cb_packed(vq);
 	else
-		virtqueue_disable_cb_split(_vq);
+		virtqueue_disable_cb_split(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
 
-- 
2.42.0

[PATCH 11/19] virtio_ring: switch to use vring_virtqueue for detach_unused_buf variants

[Jason Wang]

Those variants are used internally so let's switch to use
vring_virtqueue as parameter to be consistent with other internal
virtqueue helpers.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index f754da8c2ac6..19aa24d62d20 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -950,9 +950,8 @@ static bool virtqueue_enable_cb_delayed_split(struct vring_virtqueue *vq)
 	return true;
 }
 
-static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
+static void *virtqueue_detach_unused_buf_split(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	unsigned int i;
 	void *buf;
 
@@ -965,7 +964,7 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
 		buf = vq->split.desc_state[i].data;
 		detach_buf_split(vq, i, NULL);
 		vq->split.avail_idx_shadow--;
-		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
+		vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
 				vq->split.avail_idx_shadow);
 		END_USE(vq);
 		return buf;
@@ -1892,9 +1891,8 @@ static bool virtqueue_enable_cb_delayed_packed(struct vring_virtqueue *vq)
 	return true;
 }
 
-static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
+static void *virtqueue_detach_unused_buf_packed(struct vring_virtqueue *vq)
 {
-	struct vring_virtqueue *vq = to_vvq(_vq);
 	unsigned int i;
 	void *buf;
 
@@ -2642,8 +2640,8 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
-				 virtqueue_detach_unused_buf_split(_vq);
+	return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
+				 virtqueue_detach_unused_buf_split(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
 
-- 
2.42.0

[PATCH 12/19] virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()

[Jason Wang]

Use u16 for last_used_idx in virtqueue_poll_split() to align with the
spec.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 19aa24d62d20..a2884eae14d9 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -801,7 +801,7 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 }
 
 static bool virtqueue_poll_split(const struct vring_virtqueue *vq,
-				 unsigned int last_used_idx)
+				 u16 last_used_idx)
 {
 	return (u16)last_used_idx != virtio16_to_cpu(vq->vq.vdev,
 			vq->split.vring.used->idx);
-- 
2.42.0

[PATCH 13/19] virtio_ring: introduce virtqueue ops

[Jason Wang]

This patch introduces virtqueue ops which is a set of the callbacks
that will be called for different queue layout or features. This would
help to avoid branches for split/packed and will ease the future
implementation like in order.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
 1 file changed, 67 insertions(+), 29 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index a2884eae14d9..ce1dc90ee89d 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
 	size_t event_size_in_bytes;
 };
 
+struct vring_virtqueue;
+
+struct virtqueue_ops {
+	int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
+		   unsigned int total_sg, unsigned int out_sgs,
+		   unsigned int in_sgs,	void *data,
+		   void *ctx, bool premapped, gfp_t gfp);
+	void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
+	bool (*kick_prepare)(struct vring_virtqueue *vq);
+	void (*disable_cb)(struct vring_virtqueue *vq);
+	bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
+	unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
+	bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
+	void *(*detach_unused_buf)(struct vring_virtqueue *vq);
+	bool (*more_used)(const struct vring_virtqueue *vq);
+	int (*resize)(struct vring_virtqueue *vq, u32 num);
+	void (*reset)(struct vring_virtqueue *vq);
+};
+
 struct vring_virtqueue {
 	struct virtqueue vq;
 
+	struct virtqueue_ops *ops;
+
 	/* Is this a packed ring? */
 	bool packed_ring;
 
@@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
 	return 0;
 }
 
+struct virtqueue_ops split_ops;
+
 static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
 					       struct vring_virtqueue_split *vring_split,
 					       struct virtio_device *vdev,
@@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
 		return NULL;
 
 	vq->packed_ring = false;
+	vq->ops = &split_ops;
 	vq->vq.callback = callback;
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
@@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
 	virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
 }
 
+struct virtqueue_ops packed_ops;
+
 static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
 					       struct vring_virtqueue_packed *vring_packed,
 					       struct virtio_device *vdev,
@@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
 	vq->broken = false;
 #endif
 	vq->packed_ring = true;
+	vq->ops = &packed_ops;
 	vq->dma_dev = dma_dev;
 	vq->use_dma_api = vring_use_dma_api(vdev);
 
@@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
 	return -ENOMEM;
 }
 
+struct virtqueue_ops split_ops = {
+	.add = virtqueue_add_split,
+	.get = virtqueue_get_buf_ctx_split,
+	.kick_prepare = virtqueue_kick_prepare_split,
+	.disable_cb = virtqueue_disable_cb_split,
+	.enable_cb_delayed = virtqueue_enable_cb_delayed_split,
+	.enable_cb_prepare = virtqueue_enable_cb_prepare_split,
+	.poll = virtqueue_poll_split,
+	.detach_unused_buf = virtqueue_detach_unused_buf_split,
+	.more_used = more_used_split,
+	.resize = virtqueue_resize_split,
+	.reset = virtqueue_reset_split,
+};
+
+struct virtqueue_ops packed_ops = {
+	.add = virtqueue_add_packed,
+	.get = virtqueue_get_buf_ctx_packed,
+	.kick_prepare = virtqueue_kick_prepare_packed,
+	.disable_cb = virtqueue_disable_cb_packed,
+	.enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
+	.enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
+	.poll = virtqueue_poll_packed,
+	.detach_unused_buf = virtqueue_detach_unused_buf_packed,
+	.more_used = more_used_packed,
+	.resize = virtqueue_resize_packed,
+	.reset = virtqueue_reset_packed,
+};
+
 static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
 					 void (*recycle)(struct virtqueue *vq, void *buf))
 {
@@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
-					out_sgs, in_sgs, data, ctx, premapped, gfp) :
-				 virtqueue_add_split(vq, sgs, total_sg,
-					out_sgs, in_sgs, data, ctx, premapped, gfp);
+	return vq->ops->add(vq, sgs, total_sg,
+			    out_sgs, in_sgs, data, ctx, premapped, gfp);
 }
 
 /**
@@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
-				 virtqueue_kick_prepare_split(vq);
+	return vq->ops->kick_prepare(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
 
@@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
-				 virtqueue_get_buf_ctx_split(vq, len, ctx);
+	return vq->ops->get(vq, len, ctx);
 }
 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
 
@@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	if (vq->packed_ring)
-		virtqueue_disable_cb_packed(vq);
-	else
-		virtqueue_disable_cb_split(vq);
+	return vq->ops->disable_cb(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
 
@@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
 	if (vq->event_triggered)
 		vq->event_triggered = false;
 
-	return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
-				 virtqueue_enable_cb_prepare_split(vq);
+	return vq->ops->enable_cb_prepare(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
 
@@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
 		return false;
 
 	virtio_mb(vq->weak_barriers);
-	return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
-				 virtqueue_poll_split(vq, last_used_idx);
+	return vq->ops->poll(vq, last_used_idx);
 }
 EXPORT_SYMBOL_GPL(virtqueue_poll);
 
@@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
 	if (vq->event_triggered)
 		vq->event_triggered = false;
 
-	return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
-				 virtqueue_enable_cb_delayed_split(vq);
+	return vq->ops->enable_cb_delayed(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
 
@@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
 {
 	struct vring_virtqueue *vq = to_vvq(_vq);
 
-	return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
-				 virtqueue_detach_unused_buf_split(vq);
+	return vq->ops->detach_unused_buf(vq);
 }
 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
 
 static inline bool more_used(const struct vring_virtqueue *vq)
 {
-	return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
+	return vq->ops->more_used(vq);
 }
 
 /**
@@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
 	if (recycle_done)
 		recycle_done(_vq);
 
-	if (vq->packed_ring)
-		err = virtqueue_resize_packed(vq, num);
-	else
-		err = virtqueue_resize_split(vq, num);
+	err = vq->ops->resize(vq, num);
 
 	return virtqueue_enable_after_reset(_vq);
 }
@@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
 	if (recycle_done)
 		recycle_done(_vq);
 
-	if (vq->packed_ring)
-		virtqueue_reset_packed(vq);
-	else
-		virtqueue_reset_split(vq);
+	vq->ops->reset(vq);
 
 	return virtqueue_enable_after_reset(_vq);
 }
-- 
2.42.0

[PATCH 14/19] virtio_ring: determine descriptor flags at one time

[Jason Wang]

Let's determine the last descriptor by counting the number of sg. This
would be consistent with packed virtqueue implementation and ease the
future in-order implementation.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index ce1dc90ee89d..31aa4a935c27 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -561,7 +561,7 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
 	struct vring_desc_extra *extra;
 	struct scatterlist *sg;
 	struct vring_desc *desc;
-	unsigned int i, n, avail, descs_used, prev, err_idx;
+	unsigned int i, n, c, avail, descs_used, err_idx;
 	int head;
 	bool indirect;
 
@@ -617,46 +617,47 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
 		return -ENOSPC;
 	}
 
+	c = 0;
 	for (n = 0; n < out_sgs; n++) {
+		sg = sgs[n];
 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
 			dma_addr_t addr;
 			u32 len;
+			u16 flags = 0;
 
 			if (vring_map_one_sg(vq, sg, DMA_TO_DEVICE, &addr, &len, premapped))
 				goto unmap_release;
 
-			prev = i;
+			if (++c != total_sg)
+				flags = VRING_DESC_F_NEXT;
+
 			/* Note that we trust indirect descriptor
 			 * table since it use stream DMA mapping.
 			 */
 			i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
-						     VRING_DESC_F_NEXT,
+						     flags,
 						     premapped);
 		}
 	}
 	for (; n < (out_sgs + in_sgs); n++) {
 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
+			u16 flags = VRING_DESC_F_WRITE;
 			dma_addr_t addr;
 			u32 len;
 
 			if (vring_map_one_sg(vq, sg, DMA_FROM_DEVICE, &addr, &len, premapped))
 				goto unmap_release;
 
-			prev = i;
+			if (++c != total_sg)
+				flags |= VRING_DESC_F_NEXT;
+
 			/* Note that we trust indirect descriptor
 			 * table since it use stream DMA mapping.
 			 */
 			i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
-						     VRING_DESC_F_NEXT |
-						     VRING_DESC_F_WRITE,
-						     premapped);
+						     flags, premapped);
 		}
 	}
-	/* Last one doesn't continue. */
-	desc[prev].flags &= cpu_to_virtio16(vq->vq.vdev, ~VRING_DESC_F_NEXT);
-	if (!indirect && vring_need_unmap_buffer(vq, &extra[prev]))
-		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
-			~VRING_DESC_F_NEXT;
 
 	if (indirect) {
 		/* Now that the indirect table is filled in, map it. */
-- 
2.42.0

[PATCH 15/19] virtio_ring: factor out core logic of buffer detaching

[Jason Wang]

Factor out core logic of buffer detaching and leave the id population
to the caller so in order can just call the core logic.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 31aa4a935c27..fe3e6f3d0f96 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -1661,8 +1661,8 @@ static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
 	return needs_kick;
 }
 
-static void detach_buf_packed(struct vring_virtqueue *vq,
-			      unsigned int id, void **ctx)
+static void __detach_buf_packed(struct vring_virtqueue *vq,
+				unsigned int id, void **ctx)
 {
 	struct vring_desc_state_packed *state = NULL;
 	struct vring_packed_desc *desc;
@@ -1673,8 +1673,6 @@ static void detach_buf_packed(struct vring_virtqueue *vq,
 	/* Clear data ptr. */
 	state->data = NULL;
 
-	vq->packed.desc_extra[state->last].next = vq->free_head;
-	vq->free_head = id;
 	vq->vq.num_free += state->num;
 
 	if (unlikely(vq->use_dma_api)) {
@@ -1711,6 +1709,17 @@ static void detach_buf_packed(struct vring_virtqueue *vq,
 	}
 }
 
+static void detach_buf_packed(struct vring_virtqueue *vq,
+			      unsigned int id, void **ctx)
+{
+	struct vring_desc_state_packed *state = &vq->packed.desc_state[id];
+
+	vq->packed.desc_extra[state->last].next = vq->free_head;
+	vq->free_head = id;
+
+	return __detach_buf_packed(vq, id, ctx);
+}
+
 static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
 				       u16 idx, bool used_wrap_counter)
 {
-- 
2.42.0

[PATCH 16/19] virtio_ring: factor out core logic for updating last_used_idx

[Jason Wang]

Factor out the core logic for updating last_used_idx to be reused by
the packed in order implementation.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 43 +++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index fe3e6f3d0f96..bd4faf04862c 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -1749,6 +1749,30 @@ static bool more_used_packed(const struct vring_virtqueue *vq)
 	return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
 }
 
+static void update_last_used_idx_packed(struct vring_virtqueue *vq,
+					u16 id, u16 last_used,
+					u16 used_wrap_counter)
+{
+	last_used += vq->packed.desc_state[id].num;
+	if (unlikely(last_used >= vq->packed.vring.num)) {
+		last_used -= vq->packed.vring.num;
+		used_wrap_counter ^= 1;
+	}
+
+	last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
+	WRITE_ONCE(vq->last_used_idx, last_used);
+
+	/*
+	 * If we expect an interrupt for the next entry, tell host
+	 * by writing event index and flush out the write before
+	 * the read in the next get_buf call.
+	 */
+	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
+		virtio_store_mb(vq->weak_barriers,
+				&vq->packed.vring.driver->off_wrap,
+				cpu_to_le16(vq->last_used_idx));
+}
+
 static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
 					  unsigned int *len,
 					  void **ctx)
@@ -1792,24 +1816,7 @@ static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
 	ret = vq->packed.desc_state[id].data;
 	detach_buf_packed(vq, id, ctx);
 
-	last_used += vq->packed.desc_state[id].num;
-	if (unlikely(last_used >= vq->packed.vring.num)) {
-		last_used -= vq->packed.vring.num;
-		used_wrap_counter ^= 1;
-	}
-
-	last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
-	WRITE_ONCE(vq->last_used_idx, last_used);
-
-	/*
-	 * If we expect an interrupt for the next entry, tell host
-	 * by writing event index and flush out the write before
-	 * the read in the next get_buf call.
-	 */
-	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
-		virtio_store_mb(vq->weak_barriers,
-				&vq->packed.vring.driver->off_wrap,
-				cpu_to_le16(vq->last_used_idx));
+	update_last_used_idx_packed(vq, id, last_used, used_wrap_counter);
 
 	LAST_ADD_TIME_INVALID(vq);
 
-- 
2.42.0

[PATCH 17/19] virtio_ring: move next_avail_idx to vring_virtqueue

[Jason Wang]

This variable is used by packed virtqueue now, moving it to
vring_virtqueue to make it possible to be reused by split virtqueue
in-order implementation.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index bd4faf04862c..a1a8cd931052 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -138,9 +138,6 @@ struct vring_virtqueue_packed {
 	/* Avail used flags. */
 	u16 avail_used_flags;
 
-	/* Index of the next avail descriptor. */
-	u16 next_avail_idx;
-
 	/*
 	 * Last written value to driver->flags in
 	 * guest byte order.
@@ -214,6 +211,9 @@ struct vring_virtqueue {
 	 */
 	u16 last_used_idx;
 
+	/* Index of the next avail descriptor. */
+	u16 next_avail_idx;
+
 	/* Hint for event idx: already triggered no need to disable. */
 	bool event_triggered;
 
@@ -448,6 +448,7 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
 	else
 		vq->last_used_idx = 0;
 
+	vq->next_avail_idx = 0;
 	vq->event_triggered = false;
 	vq->num_added = 0;
 
@@ -1350,7 +1351,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 	u16 head, id;
 	dma_addr_t addr;
 
-	head = vq->packed.next_avail_idx;
+	head = vq->next_avail_idx;
 	desc = alloc_indirect_packed(total_sg, gfp);
 	if (!desc)
 		return -ENOMEM;
@@ -1431,7 +1432,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 				1 << VRING_PACKED_DESC_F_AVAIL |
 				1 << VRING_PACKED_DESC_F_USED;
 	}
-	vq->packed.next_avail_idx = n;
+	vq->next_avail_idx = n;
 	vq->free_head = vq->packed.desc_extra[id].next;
 
 	/* Store token and indirect buffer state. */
@@ -1501,7 +1502,7 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
 		/* fall back on direct */
 	}
 
-	head = vq->packed.next_avail_idx;
+	head = vq->next_avail_idx;
 	avail_used_flags = vq->packed.avail_used_flags;
 
 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
@@ -1569,7 +1570,7 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
 	vq->vq.num_free -= descs_used;
 
 	/* Update free pointer */
-	vq->packed.next_avail_idx = i;
+	vq->next_avail_idx = i;
 	vq->free_head = curr;
 
 	/* Store token. */
@@ -1633,8 +1634,8 @@ static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
 	 */
 	virtio_mb(vq->weak_barriers);
 
-	old = vq->packed.next_avail_idx - vq->num_added;
-	new = vq->packed.next_avail_idx;
+	old = vq->next_avail_idx - vq->num_added;
+	new = vq->next_avail_idx;
 	vq->num_added = 0;
 
 	snapshot.u32 = *(u32 *)vq->packed.vring.device;
@@ -2083,7 +2084,6 @@ static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_p
 static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
 					bool callback)
 {
-	vring_packed->next_avail_idx = 0;
 	vring_packed->avail_wrap_counter = 1;
 	vring_packed->event_flags_shadow = 0;
 	vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
@@ -2977,7 +2977,7 @@ u32 vring_notification_data(struct virtqueue *_vq)
 	u16 next;
 
 	if (vq->packed_ring)
-		next = (vq->packed.next_avail_idx &
+		next = (vq->next_avail_idx &
 				~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) |
 			vq->packed.avail_wrap_counter <<
 				VRING_PACKED_EVENT_F_WRAP_CTR;
-- 
2.42.0

[PATCH 18/19] virtio_ring: factor out split indirect detaching logic

[Jason Wang]

Factor out the split indirect descriptor detaching logic in order to
make it be reused by the in order support.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 63 ++++++++++++++++++++----------------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index a1a8cd931052..0fad8e8419c8 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -765,11 +765,42 @@ static bool virtqueue_kick_prepare_split(struct vring_virtqueue *vq)
 	return needs_kick;
 }
 
+static void detach_indirect_split(struct vring_virtqueue *vq,
+				  unsigned int head)
+{
+	struct vring_desc_extra *extra = vq->split.desc_extra;
+	struct vring_desc *indir_desc =
+	       vq->split.desc_state[head].indir_desc;
+	unsigned int j;
+	u32 len, num;
+
+	/* Free the indirect table, if any, now that it's unmapped. */
+	if (!indir_desc)
+		return;
+	len = vq->split.desc_extra[head].len;
+
+	BUG_ON(!(vq->split.desc_extra[head].flags &
+			VRING_DESC_F_INDIRECT));
+	BUG_ON(len == 0 || len % sizeof(struct vring_desc));
+
+	num = len / sizeof(struct vring_desc);
+
+	extra = (struct vring_desc_extra *)&indir_desc[num];
+
+	if (vq->use_dma_api) {
+		for (j = 0; j < num; j++)
+			vring_unmap_one_split(vq, &extra[j]);
+	}
+
+	kfree(indir_desc);
+	vq->split.desc_state[head].indir_desc = NULL;
+}
+
 static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 			     void **ctx)
 {
 	struct vring_desc_extra *extra;
-	unsigned int i, j;
+	unsigned int i;
 	__virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
 
 	/* Clear data ptr. */
@@ -793,34 +824,10 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 	/* Plus final descriptor */
 	vq->vq.num_free++;
 
-	if (vq->indirect) {
-		struct vring_desc *indir_desc =
-				vq->split.desc_state[head].indir_desc;
-		u32 len, num;
-
-		/* Free the indirect table, if any, now that it's unmapped. */
-		if (!indir_desc)
-			return;
-		len = vq->split.desc_extra[head].len;
-
-		BUG_ON(!(vq->split.desc_extra[head].flags &
-				VRING_DESC_F_INDIRECT));
-		BUG_ON(len == 0 || len % sizeof(struct vring_desc));
-
-		num = len / sizeof(struct vring_desc);
-
-		extra = (struct vring_desc_extra *)&indir_desc[num];
-
-		if (vq->use_dma_api) {
-			for (j = 0; j < num; j++)
-				vring_unmap_one_split(vq, &extra[j]);
-		}
-
-		kfree(indir_desc);
-		vq->split.desc_state[head].indir_desc = NULL;
-	} else if (ctx) {
+	if (vq->indirect)
+		detach_indirect_split(vq, head);
+	else if (ctx)
 		*ctx = vq->split.desc_state[head].indir_desc;
-	}
 }
 
 static bool virtqueue_poll_split(const struct vring_virtqueue *vq,
-- 
2.42.0

[PATCH 19/19] virtio_ring: add in order support

[Jason Wang]

This patch implements in order support for both split virtqueue and
packed virtqueue. Dedicate virtqueue ops are introduced for the packed
virtqueue. Most of the ops were reused but the ones that have major
difference.

KVM guest + testpmd on the host shows 5% improvement in packed
virtqueue TX PPS. No obvious changes in other test cases.

Signed-off-by: Jason Wang <jasowang@redhat.com>
---
 drivers/virtio/virtio_ring.c | 451 +++++++++++++++++++++++++++++++++--
 1 file changed, 434 insertions(+), 17 deletions(-)

diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 0fad8e8419c8..4341c178d53f 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -74,6 +74,7 @@ struct vring_desc_state_split {
 	 * allocated together. So we won't stress more to the memory allocator.
 	 */
 	struct vring_desc *indir_desc;
+	u32 total_len;			/* Buffer Length */
 };
 
 struct vring_desc_state_packed {
@@ -85,6 +86,7 @@ struct vring_desc_state_packed {
 	struct vring_packed_desc *indir_desc;
 	u16 num;			/* Descriptor list length. */
 	u16 last;			/* The last desc state in a list. */
+	u32 total_len;			/* Buffer Length */
 };
 
 struct vring_desc_extra {
@@ -198,8 +200,15 @@ struct vring_virtqueue {
 	/* Host publishes avail event idx */
 	bool event;
 
+	/* Device supports in order */
+	bool in_order;
+
 	/* Head of free buffer list. */
 	unsigned int free_head;
+
+	/* Head of the batched used buffers, vq->num means no batching */
+	unsigned int batch_head;
+
 	/* Number we've added since last sync. */
 	unsigned int num_added;
 
@@ -562,7 +571,7 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
 	struct vring_desc_extra *extra;
 	struct scatterlist *sg;
 	struct vring_desc *desc;
-	unsigned int i, n, c, avail, descs_used, err_idx;
+	unsigned int i, n, c, avail, descs_used, err_idx, total_len = 0;
 	int head;
 	bool indirect;
 
@@ -638,6 +647,7 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
 			i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
 						     flags,
 						     premapped);
+			total_len += len;
 		}
 	}
 	for (; n < (out_sgs + in_sgs); n++) {
@@ -657,6 +667,7 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
 			 */
 			i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
 						     flags, premapped);
+			total_len += len;
 		}
 	}
 
@@ -679,7 +690,12 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
 	vq->vq.num_free -= descs_used;
 
 	/* Update free pointer */
-	if (indirect)
+	if (vq->in_order) {
+		vq->free_head += descs_used;
+		if (vq->free_head >= vq->split.vring.num)
+			vq->free_head -= vq->split.vring.num;
+		vq->split.desc_state[head].total_len = total_len;;
+	} else if (indirect)
 		vq->free_head = vq->split.desc_extra[head].next;
 	else
 		vq->free_head = i;
@@ -816,8 +832,8 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 		i = vq->split.desc_extra[i].next;
 		vq->vq.num_free++;
 	}
-
 	vring_unmap_one_split(vq, &extra[i]);
+
 	vq->split.desc_extra[i].next = vq->free_head;
 	vq->free_head = head;
 
@@ -830,6 +846,40 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
 		*ctx = vq->split.desc_state[head].indir_desc;
 }
 
+static void detach_buf_split_in_order(struct vring_virtqueue *vq, unsigned int head,
+				      void **ctx)
+{
+	struct vring_desc_extra *extra;
+	unsigned int i;
+	__virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
+
+	/* Clear data ptr. */
+	vq->split.desc_state[head].data = NULL;
+
+	extra = vq->split.desc_extra;
+
+	/* Put back on free list: unmap first-level descriptors and find end */
+	i = head;
+
+	while (vq->split.vring.desc[i].flags & nextflag) {
+		vring_unmap_one_split(vq, &extra[i]);
+		i++;
+		if (i >= vq->split.vring.num)
+			i = 0;
+		vq->vq.num_free++;
+	}
+
+	vring_unmap_one_split(vq, &extra[i]);
+
+	/* Plus final descriptor */
+	vq->vq.num_free++;
+
+	if (vq->indirect)
+		detach_indirect_split(vq, head);
+	else if (ctx)
+		*ctx = vq->split.desc_state[head].indir_desc;
+}
+
 static bool virtqueue_poll_split(const struct vring_virtqueue *vq,
 				 u16 last_used_idx)
 {
@@ -899,6 +949,83 @@ static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
 	return ret;
 }
 
+static void *virtqueue_get_buf_ctx_split_in_order(struct vring_virtqueue *vq,
+						  unsigned int *len,
+						  void **ctx)
+{
+	void *ret;
+	unsigned int i, num = vq->split.vring.num, num_free = vq->vq.num_free;
+	u16 last_used;
+
+	START_USE(vq);
+
+	if (unlikely(vq->broken)) {
+		END_USE(vq);
+		return NULL;
+	}
+
+	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
+
+	if (vq->batch_head == num) {
+		if (!more_used_split(vq)) {
+			pr_debug("No more buffers in queue\n");
+			END_USE(vq);
+			return NULL;
+		}
+
+		/* Only get used array entries after they have been
+		 * exposed by host. */
+		virtio_rmb(vq->weak_barriers);
+
+		i = virtio32_to_cpu(vq->vq.vdev,
+				    vq->split.vring.used->ring[last_used].id);
+		if (i != last_used) {
+			vq->batch_head = i;
+			i = last_used;
+			*len = vq->split.desc_state[i].total_len;
+		} else {
+			*len = virtio32_to_cpu(vq->vq.vdev,
+					vq->split.vring.used->ring[last_used].len);
+		}
+	} else {
+		i = vq->next_avail_idx;
+		*len = vq->split.desc_state[i].total_len;
+		if (i == vq->batch_head)
+			vq->batch_head = num;
+	}
+
+	if (unlikely(i >= num)) {
+		BAD_RING(vq, "id %u out of range\n", i);
+		return NULL;
+	}
+	if (unlikely(!vq->split.desc_state[i].data)) {
+		BAD_RING(vq, "id %u is not a head!\n", i);
+		return NULL;
+	}
+
+	/* detach_buf_split clears data, so grab it now. */
+	ret = vq->split.desc_state[i].data;
+	detach_buf_split_in_order(vq, i, ctx);
+
+	vq->next_avail_idx += vq->vq.num_free - num_free;
+	if (vq->next_avail_idx >= num)
+		vq->next_avail_idx -= num;
+
+	vq->last_used_idx++;
+	/* If we expect an interrupt for the next entry, tell host
+	 * by writing event index and flush out the write before
+	 * the read in the next get_buf call. */
+	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
+		virtio_store_mb(vq->weak_barriers,
+				&vring_used_event(&vq->split.vring),
+				cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx));
+
+	LAST_ADD_TIME_INVALID(vq);
+
+	END_USE(vq);
+	return ret;
+}
+
 static void virtqueue_disable_cb_split(struct vring_virtqueue *vq)
 {
 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
@@ -992,7 +1119,10 @@ static void *virtqueue_detach_unused_buf_split(struct vring_virtqueue *vq)
 			continue;
 		/* detach_buf_split clears data, so grab it now. */
 		buf = vq->split.desc_state[i].data;
-		detach_buf_split(vq, i, NULL);
+		if (vq->in_order)
+			detach_buf_split_in_order(vq, i, NULL);
+		else
+			detach_buf_split(vq, i, NULL);
 		vq->split.avail_idx_shadow--;
 		vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
 				vq->split.avail_idx_shadow);
@@ -1055,6 +1185,7 @@ static void virtqueue_vring_attach_split(struct vring_virtqueue *vq,
 
 	/* Put everything in free lists. */
 	vq->free_head = 0;
+	vq->batch_head = vq->split.vring.num;
 }
 
 static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split)
@@ -1147,6 +1278,7 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
 }
 
 struct virtqueue_ops split_ops;
+struct virtqueue_ops split_in_order_ops;
 
 static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
 					       struct vring_virtqueue_split *vring_split,
@@ -1166,7 +1298,6 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
 		return NULL;
 
 	vq->packed_ring = false;
-	vq->ops = &split_ops;
 	vq->vq.callback = callback;
 	vq->vq.vdev = vdev;
 	vq->vq.name = name;
@@ -1186,6 +1317,9 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
 		!context;
 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
+	vq->in_order = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER);
+
+	vq->ops = vq->in_order ? &split_in_order_ops : &split_ops;
 
 	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
 		vq->weak_barriers = false;
@@ -1349,13 +1483,14 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 					 unsigned int in_sgs,
 					 void *data,
 					 bool premapped,
-					 gfp_t gfp)
+					 gfp_t gfp,
+					 u16 id)
 {
 	struct vring_desc_extra *extra;
 	struct vring_packed_desc *desc;
 	struct scatterlist *sg;
-	unsigned int i, n, err_idx, len;
-	u16 head, id;
+	unsigned int i, n, err_idx, len, total_len = 0;
+	u16 head;
 	dma_addr_t addr;
 
 	head = vq->next_avail_idx;
@@ -1373,8 +1508,6 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 	}
 
 	i = 0;
-	id = vq->free_head;
-	BUG_ON(id == vq->packed.vring.num);
 
 	for (n = 0; n < out_sgs + in_sgs; n++) {
 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
@@ -1394,6 +1527,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 				extra[i].flags = n < out_sgs ?  0 : VRING_DESC_F_WRITE;
 			}
 
+			total_len += len;
 			i++;
 		}
 	}
@@ -1447,6 +1581,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
 	vq->packed.desc_state[id].data = data;
 	vq->packed.desc_state[id].indir_desc = desc;
 	vq->packed.desc_state[id].last = id;
+	vq->packed.desc_state[id].total_len = total_len;
 
 	vq->num_added += 1;
 
@@ -1499,8 +1634,11 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
 	BUG_ON(total_sg == 0);
 
 	if (virtqueue_use_indirect(vq, total_sg)) {
+		id = vq->free_head;
+		BUG_ON(id == vq->packed.vring.num);
 		err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
-						    in_sgs, data, premapped, gfp);
+						    in_sgs, data, premapped,
+						    gfp, id);
 		if (err != -ENOMEM) {
 			END_USE(vq);
 			return err;
@@ -1621,6 +1759,152 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
 	return -EIO;
 }
 
+static inline int virtqueue_add_packed_in_order(struct vring_virtqueue *vq,
+						struct scatterlist *sgs[],
+						unsigned int total_sg,
+						unsigned int out_sgs,
+						unsigned int in_sgs,
+						void *data,
+						void *ctx,
+						bool premapped,
+						gfp_t gfp)
+{
+	struct vring_packed_desc *desc;
+	struct scatterlist *sg;
+	unsigned int i, n, c, err_idx, total_len = 0;
+	__le16 head_flags, flags;
+	u16 head, avail_used_flags;
+	int err;
+
+	START_USE(vq);
+
+	BUG_ON(data == NULL);
+	BUG_ON(ctx && vq->indirect);
+
+	if (unlikely(vq->broken)) {
+		END_USE(vq);
+		return -EIO;
+	}
+
+	LAST_ADD_TIME_UPDATE(vq);
+
+	BUG_ON(total_sg == 0);
+
+	if (virtqueue_use_indirect(vq, total_sg)) {
+		err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
+						    in_sgs, data, premapped, gfp,
+						    vq->next_avail_idx);
+		if (err != -ENOMEM) {
+			END_USE(vq);
+			return err;
+		}
+
+		/* fall back on direct */
+	}
+
+	head = vq->next_avail_idx;
+	avail_used_flags = vq->packed.avail_used_flags;
+
+	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
+
+	desc = vq->packed.vring.desc;
+	i = head;
+
+	if (unlikely(vq->vq.num_free < total_sg)) {
+		pr_debug("Can't add buf len %i - avail = %i\n",
+			 total_sg, vq->vq.num_free);
+		END_USE(vq);
+		return -ENOSPC;
+	}
+
+	c = 0;
+	for (n = 0; n < out_sgs + in_sgs; n++) {
+		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
+			dma_addr_t addr;
+			u32 len;
+
+			if (vring_map_one_sg(vq, sg, n < out_sgs ?
+					     DMA_TO_DEVICE : DMA_FROM_DEVICE,
+					     &addr, &len, premapped))
+				goto unmap_release;
+
+			flags = cpu_to_le16(vq->packed.avail_used_flags |
+				    (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
+				    (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
+			if (i == head)
+				head_flags = flags;
+			else
+				desc[i].flags = flags;
+
+
+			desc[i].addr = cpu_to_le64(addr);
+			desc[i].len = cpu_to_le32(len);
+			desc[i].id = cpu_to_le16(head);
+
+			if (unlikely(vq->use_dma_api)) {
+				vq->packed.desc_extra[i].addr = premapped ?
+				      DMA_MAPPING_ERROR: addr;
+				vq->packed.desc_extra[i].len = len;
+				vq->packed.desc_extra[i].flags =
+					le16_to_cpu(flags);
+			}
+
+			if ((unlikely(++i >= vq->packed.vring.num))) {
+				i = 0;
+				vq->packed.avail_used_flags ^=
+					1 << VRING_PACKED_DESC_F_AVAIL |
+					1 << VRING_PACKED_DESC_F_USED;
+				vq->packed.avail_wrap_counter ^= 1;
+			}
+
+			total_len += len;
+		}
+	}
+
+	/* We're using some buffers from the free list. */
+	vq->vq.num_free -= total_sg;
+
+	/* Update free pointer */
+	vq->next_avail_idx = i;
+
+	/* Store token. */
+	vq->packed.desc_state[head].num = total_sg;
+	vq->packed.desc_state[head].data = data;
+	vq->packed.desc_state[head].indir_desc = ctx;
+	vq->packed.desc_state[head].total_len = total_len;
+
+	/*
+	 * A driver MUST NOT make the first descriptor in the list
+	 * available before all subsequent descriptors comprising
+	 * the list are made available.
+	 */
+	virtio_wmb(vq->weak_barriers);
+	vq->packed.vring.desc[head].flags = head_flags;
+	vq->num_added += total_sg;
+
+	pr_debug("Added buffer head %i to %p\n", head, vq);
+	END_USE(vq);
+
+	return 0;
+
+unmap_release:
+	err_idx = i;
+	i = head;
+	vq->packed.avail_used_flags = avail_used_flags;
+
+	for (n = 0; n < total_sg; n++) {
+		if (i == err_idx)
+			break;
+		vring_unmap_extra_packed(vq, &vq->packed.desc_extra[i]);
+		i++;
+		if (i >= vq->packed.vring.num)
+			i = 0;
+	}
+
+	END_USE(vq);
+	return -EIO;
+}
+
 static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
 {
 	u16 new, old, off_wrap, flags, wrap_counter, event_idx;
@@ -1741,7 +2025,7 @@ static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
 	return avail == used && used == used_wrap_counter;
 }
 
-static bool virtqueue_poll_packed(const struct vring_virtqueue *vq, u16 off_wrap)
+static bool __virtqueue_poll_packed(const struct vring_virtqueue *vq, u16 off_wrap)
 {
 	bool wrap_counter;
 	u16 used_idx;
@@ -1752,6 +2036,20 @@ static bool virtqueue_poll_packed(const struct vring_virtqueue *vq, u16 off_wrap
 	return is_used_desc_packed(vq, used_idx, wrap_counter);
 }
 
+static bool virtqueue_poll_packed(const struct vring_virtqueue *vq, u16 off_wrap)
+{
+	return __virtqueue_poll_packed(vq, off_wrap);
+}
+
+static bool virtqueue_poll_packed_in_order(const struct vring_virtqueue *vq,
+					   u16 off_wrap)
+{
+	if (vq->batch_head != vq->packed.vring.num)
+		return true;
+
+	return __virtqueue_poll_packed(vq, off_wrap);
+}
+
 static bool more_used_packed(const struct vring_virtqueue *vq)
 {
 	return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
@@ -1781,10 +2079,88 @@ static void update_last_used_idx_packed(struct vring_virtqueue *vq,
 				cpu_to_le16(vq->last_used_idx));
 }
 
+static bool more_used_packed_in_order(const struct vring_virtqueue *vq)
+{
+	return virtqueue_poll_packed_in_order(vq, READ_ONCE(vq->last_used_idx));
+}
+
+static bool __more_used_packed(const struct vring_virtqueue *vq)
+{
+	return __virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
+}
+
+static void *virtqueue_get_buf_ctx_packed_in_order(struct vring_virtqueue *vq,
+						   unsigned int *len,
+						   void **ctx)
+{
+	unsigned int num = vq->packed.vring.num;
+	u16 last_used, id, last_used_idx, batch_head;
+	bool used_wrap_counter;
+	void *ret;
+
+	START_USE(vq);
+
+	batch_head = READ_ONCE(vq->batch_head);
+
+	if (unlikely(vq->broken)) {
+		END_USE(vq);
+		return NULL;
+	}
+
+	last_used_idx = vq->last_used_idx;
+	used_wrap_counter = packed_used_wrap_counter(last_used_idx);
+	last_used = packed_last_used(last_used_idx);
+
+	if (batch_head == num) {
+		if (!__more_used_packed(vq)) {
+			pr_debug("No more buffers in queue\n");
+			END_USE(vq);
+			return NULL;
+		}
+		/* Only get used elements after they have been exposed by host. */
+		virtio_rmb(vq->weak_barriers);
+
+		id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
+		if (id != last_used) {
+			vq->batch_head = id;
+			id = last_used;
+			*len = vq->packed.desc_state[id].total_len;
+		} else {
+			*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
+		}
+	} else {
+		id = last_used;
+		*len = vq->packed.desc_state[id].total_len;
+		if (id == batch_head)
+			vq->batch_head = num;
+	}
+
+	if (unlikely(id >= num)) {
+		BAD_RING(vq, "id %u out of range\n", id);
+		return NULL;
+	}
+	if (unlikely(!vq->packed.desc_state[id].data)) {
+		BAD_RING(vq, "id %u is not a head!\n", id);
+		return NULL;
+	}
+
+	/* detach_buf_packed clears data, so grab it now. */
+	ret = vq->packed.desc_state[id].data;
+	__detach_buf_packed(vq, id, ctx);
+
+	update_last_used_idx_packed(vq, id, last_used, used_wrap_counter);
+
+	LAST_ADD_TIME_INVALID(vq);
+
+	END_USE(vq);
+	return ret;
+}
+
 static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
 					  unsigned int *len,
 					  void **ctx)
 {
+	unsigned int num = vq->packed.vring.num;
 	u16 last_used, id, last_used_idx;
 	bool used_wrap_counter;
 	void *ret;
@@ -1796,7 +2172,7 @@ static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
 		return NULL;
 	}
 
-	if (!more_used_packed(vq)) {
+	if (!__more_used_packed(vq)) {
 		pr_debug("No more buffers in queue\n");
 		END_USE(vq);
 		return NULL;
@@ -1811,7 +2187,7 @@ static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
 
-	if (unlikely(id >= vq->packed.vring.num)) {
+	if (unlikely(id >= num)) {
 		BAD_RING(vq, "id %u out of range\n", id);
 		return NULL;
 	}
@@ -1931,6 +2307,7 @@ static bool virtqueue_enable_cb_delayed_packed(struct vring_virtqueue *vq)
 	last_used_idx = READ_ONCE(vq->last_used_idx);
 	wrap_counter = packed_used_wrap_counter(last_used_idx);
 	used_idx = packed_last_used(last_used_idx);
+
 	if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
 		END_USE(vq);
 		return false;
@@ -1952,7 +2329,7 @@ static void *virtqueue_detach_unused_buf_packed(struct vring_virtqueue *vq)
 			continue;
 		/* detach_buf clears data, so grab it now. */
 		buf = vq->packed.desc_state[i].data;
-		detach_buf_packed(vq, i, NULL);
+		__detach_buf_packed(vq, i, NULL);
 		END_USE(vq);
 		return buf;
 	}
@@ -1978,6 +2355,8 @@ static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num)
 	for (i = 0; i < num - 1; i++)
 		desc_extra[i].next = i + 1;
 
+	desc_extra[num - 1].next = 0;
+
 	return desc_extra;
 }
 
@@ -2108,8 +2487,12 @@ static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
 {
 	vq->packed = *vring_packed;
 
-	/* Put everything in free lists. */
-	vq->free_head = 0;
+	if (vq->in_order)
+		vq->batch_head = vq->split.vring.num;
+	else {
+		/* Put everything in free lists. */
+		vq->free_head = 0;
+	}
 }
 
 static void virtqueue_reset_packed(struct vring_virtqueue *vq)
@@ -2125,6 +2508,7 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
 }
 
 struct virtqueue_ops packed_ops;
+struct virtqueue_ops packed_in_order_ops;
 
 static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
 					       struct vring_virtqueue_packed *vring_packed,
@@ -2164,6 +2548,9 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
 		!context;
 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
+	vq->in_order = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER);
+
+	vq->ops = vq->in_order ? &packed_in_order_ops : &packed_ops;
 
 	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
 		vq->weak_barriers = false;
@@ -2259,6 +2646,20 @@ struct virtqueue_ops split_ops = {
 	.reset = virtqueue_reset_split,
 };
 
+struct virtqueue_ops split_in_order_ops = {
+	.add = virtqueue_add_split,
+	.get = virtqueue_get_buf_ctx_split_in_order,
+	.kick_prepare = virtqueue_kick_prepare_split,
+	.disable_cb = virtqueue_disable_cb_split,
+	.enable_cb_delayed = virtqueue_enable_cb_delayed_split,
+	.enable_cb_prepare = virtqueue_enable_cb_prepare_split,
+	.poll = virtqueue_poll_split,
+	.detach_unused_buf = virtqueue_detach_unused_buf_split,
+	.more_used = more_used_split,
+	.resize = virtqueue_resize_split,
+	.reset = virtqueue_reset_split,
+};
+
 struct virtqueue_ops packed_ops = {
 	.add = virtqueue_add_packed,
 	.get = virtqueue_get_buf_ctx_packed,
@@ -2273,6 +2674,20 @@ struct virtqueue_ops packed_ops = {
 	.reset = virtqueue_reset_packed,
 };
 
+struct virtqueue_ops packed_in_order_ops = {
+	.add = virtqueue_add_packed_in_order,
+	.get = virtqueue_get_buf_ctx_packed_in_order,
+	.kick_prepare = virtqueue_kick_prepare_packed,
+	.disable_cb = virtqueue_disable_cb_packed,
+	.enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
+	.enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
+	.poll = virtqueue_poll_packed_in_order,
+	.detach_unused_buf = virtqueue_detach_unused_buf_packed,
+	.more_used = more_used_packed_in_order,
+	.resize = virtqueue_resize_packed,
+	.reset = virtqueue_reset_packed,
+};
+
 static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
 					 void (*recycle)(struct virtqueue *vq, void *buf))
 {
@@ -3016,6 +3431,8 @@ void vring_transport_features(struct virtio_device *vdev)
 			break;
 		case VIRTIO_F_NOTIFICATION_DATA:
 			break;
+		case VIRTIO_F_IN_ORDER:
+			break;
 		default:
 			/* We don't understand this bit. */
 			__virtio_clear_bit(vdev, i);
-- 
2.42.0

Re: [PATCH 00/19] virtio_ring in order support

[Lei Yang]

QE tested this series of patches with virtio-net regression tests,
everything works fine.

Tested-by: Lei Yang <leiyang@redhat.com>

On Mon, Mar 24, 2025 at 1:45 PM Jason Wang <jasowang@redhat.com> wrote:
>
> Hello all:
>
> This sereis tries to implement the VIRTIO_F_IN_ORDER to
> virtio_ring. This is done by introducing virtqueue ops so we can
> implement separate helpers for different virtqueue layout/features
> then the in-order were implmeented on top.
>
> Tests shows 5% imporvemnt in RX PPS with KVM guest + testpmd on the
> host.
>
> Please review.
>
> Thanks
>
> Jason Wang (19):
>   virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()
>   virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants
>   virtio_ring: unify logic of virtqueue_poll() and more_used()
>   virtio_ring: switch to use vring_virtqueue for virtqueue resize
>     variants
>   virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare
>     variants
>   virtio_ring: switch to use vring_virtqueue for virtqueue_add variants
>   virtio: switch to use vring_virtqueue for virtqueue_add variants
>   virtio_ring: switch to use vring_virtqueue for enable_cb_prepare
>     variants
>   virtio_ring: use vring_virtqueue for enable_cb_delayed variants
>   virtio_ring: switch to use vring_virtqueue for disable_cb variants
>   virtio_ring: switch to use vring_virtqueue for detach_unused_buf
>     variants
>   virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()
>   virtio_ring: introduce virtqueue ops
>   virtio_ring: determine descriptor flags at one time
>   virtio_ring: factor out core logic of buffer detaching
>   virtio_ring: factor out core logic for updating last_used_idx
>   virtio_ring: move next_avail_idx to vring_virtqueue
>   virtio_ring: factor out split indirect detaching logic
>   virtio_ring: add in order support
>
>  drivers/virtio/virtio_ring.c | 856 ++++++++++++++++++++++++++---------
>  1 file changed, 653 insertions(+), 203 deletions(-)
>
> --
> 2.42.0
>
>

Re: [PATCH 00/19] virtio_ring in order support

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 3:44 PM Lei Yang <leiyang@redhat.com> wrote:
>
> QE tested this series of patches with virtio-net regression tests,
> everything works fine.
>

Hi Lei,

Is it possible to test this series also with virtio-net-pci,...,in_order=on?

Thanks!

> Tested-by: Lei Yang <leiyang@redhat.com>
>
> On Mon, Mar 24, 2025 at 1:45 PM Jason Wang <jasowang@redhat.com> wrote:
> >
> > Hello all:
> >
> > This sereis tries to implement the VIRTIO_F_IN_ORDER to
> > virtio_ring. This is done by introducing virtqueue ops so we can
> > implement separate helpers for different virtqueue layout/features
> > then the in-order were implmeented on top.
> >
> > Tests shows 5% imporvemnt in RX PPS with KVM guest + testpmd on the
> > host.
> >
> > Please review.
> >
> > Thanks
> >
> > Jason Wang (19):
> >   virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()
> >   virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants
> >   virtio_ring: unify logic of virtqueue_poll() and more_used()
> >   virtio_ring: switch to use vring_virtqueue for virtqueue resize
> >     variants
> >   virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare
> >     variants
> >   virtio_ring: switch to use vring_virtqueue for virtqueue_add variants
> >   virtio: switch to use vring_virtqueue for virtqueue_add variants
> >   virtio_ring: switch to use vring_virtqueue for enable_cb_prepare
> >     variants
> >   virtio_ring: use vring_virtqueue for enable_cb_delayed variants
> >   virtio_ring: switch to use vring_virtqueue for disable_cb variants
> >   virtio_ring: switch to use vring_virtqueue for detach_unused_buf
> >     variants
> >   virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()
> >   virtio_ring: introduce virtqueue ops
> >   virtio_ring: determine descriptor flags at one time
> >   virtio_ring: factor out core logic of buffer detaching
> >   virtio_ring: factor out core logic for updating last_used_idx
> >   virtio_ring: move next_avail_idx to vring_virtqueue
> >   virtio_ring: factor out split indirect detaching logic
> >   virtio_ring: add in order support
> >
> >  drivers/virtio/virtio_ring.c | 856 ++++++++++++++++++++++++++---------
> >  1 file changed, 653 insertions(+), 203 deletions(-)
> >
> > --
> > 2.42.0
> >
> >
>

Re: [PATCH 01/19] virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:43 AM Jason Wang <jasowang@redhat.com> wrote:
>
> To be consistent with virtqueue_reset().
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> ---
>  drivers/virtio/virtio_ring.c | 12 ++++++------
>  1 file changed, 6 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index fdd2d2b07b5a..1bdfd5d617a7 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -1005,7 +1005,7 @@ static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split
>         }
>  }
>
> -static void virtqueue_reinit_split(struct vring_virtqueue *vq)
> +static void virtqueue_reset_split(struct vring_virtqueue *vq)
>  {
>         int num;
>
> @@ -1248,7 +1248,7 @@ static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
>  err_state_extra:
>         vring_free_split(&vring_split, vdev, vring_dma_dev(vq));
>  err:
> -       virtqueue_reinit_split(vq);
> +       virtqueue_reset_split(vq);
>         return -ENOMEM;
>  }
>
> @@ -2092,7 +2092,7 @@ static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq,
>         vq->free_head = 0;
>  }
>
> -static void virtqueue_reinit_packed(struct vring_virtqueue *vq)
> +static void virtqueue_reset_packed(struct vring_virtqueue *vq)
>  {
>         memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
>         memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
> @@ -2219,7 +2219,7 @@ static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
>  err_state_extra:
>         vring_free_packed(&vring_packed, vdev, vring_dma_dev(vq));
>  err_ring:
> -       virtqueue_reinit_packed(vq);
> +       virtqueue_reset_packed(vq);
>         return -ENOMEM;
>  }
>
> @@ -2852,9 +2852,9 @@ int virtqueue_reset(struct virtqueue *_vq,
>                 recycle_done(_vq);
>
>         if (vq->packed_ring)
> -               virtqueue_reinit_packed(vq);
> +               virtqueue_reset_packed(vq);
>         else
> -               virtqueue_reinit_split(vq);
> +               virtqueue_reset_split(vq);
>
>         return virtqueue_enable_after_reset(_vq);
>  }
> --
> 2.42.0
>

Re: [PATCH 02/19] virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:43 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> ---
>  drivers/virtio/virtio_ring.c | 14 ++++++--------
>  1 file changed, 6 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 1bdfd5d617a7..1c6b63812bf8 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -915,11 +915,10 @@ static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
>         return last_used_idx;
>  }
>
> -static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)
> +static bool virtqueue_poll_split(struct vring_virtqueue *vq,
> +                                unsigned int last_used_idx)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
> -
> -       return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
> +       return (u16)last_used_idx != virtio16_to_cpu(vq->vq.vdev,
>                         vq->split.vring.used->idx);
>  }
>
> @@ -1845,9 +1844,8 @@ static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
>         return vq->last_used_idx;
>  }
>
> -static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
> +static bool virtqueue_poll_packed(struct vring_virtqueue *vq, u16 off_wrap)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         bool wrap_counter;
>         u16 used_idx;
>
> @@ -2608,8 +2606,8 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
>                 return false;
>
>         virtio_mb(vq->weak_barriers);
> -       return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
> -                                virtqueue_poll_split(_vq, last_used_idx);
> +       return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> +                                virtqueue_poll_split(vq, last_used_idx);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_poll);
>
> --
> 2.42.0
>

Re: [PATCH 04/19] virtio_ring: switch to use vring_virtqueue for virtqueue resize variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> ---
>  drivers/virtio/virtio_ring.c | 14 ++++++--------
>  1 file changed, 6 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 9172f3a089a0..94ce711963e6 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -1216,11 +1216,10 @@ static struct virtqueue *vring_create_virtqueue_split(
>         return vq;
>  }
>
> -static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
> +static int virtqueue_resize_split(struct vring_virtqueue *vq, u32 num)
>  {
>         struct vring_virtqueue_split vring_split = {};
> -       struct vring_virtqueue *vq = to_vvq(_vq);
> -       struct virtio_device *vdev = _vq->vdev;
> +       struct virtio_device *vdev = vq->vq.vdev;
>         int err;
>
>         err = vring_alloc_queue_split(&vring_split, vdev, num,
> @@ -2183,11 +2182,10 @@ static struct virtqueue *vring_create_virtqueue_packed(
>         return vq;
>  }
>
> -static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
> +static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
>  {
>         struct vring_virtqueue_packed vring_packed = {};
> -       struct vring_virtqueue *vq = to_vvq(_vq);
> -       struct virtio_device *vdev = _vq->vdev;
> +       struct virtio_device *vdev = vq->vq.vdev;
>         int err;
>
>         if (vring_alloc_queue_packed(&vring_packed, vdev, num, vring_dma_dev(vq)))
> @@ -2805,9 +2803,9 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
>                 recycle_done(_vq);
>
>         if (vq->packed_ring)
> -               err = virtqueue_resize_packed(_vq, num);
> +               err = virtqueue_resize_packed(vq, num);
>         else
> -               err = virtqueue_resize_split(_vq, num);
> +               err = virtqueue_resize_split(vq, num);
>
>         return virtqueue_enable_after_reset(_vq);
>  }
> --
> 2.42.0
>

Re: [PATCH 05/19] virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> ---
>  drivers/virtio/virtio_ring.c | 14 ++++++--------
>  1 file changed, 6 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 94ce711963e6..cf3ab6404698 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -713,9 +713,8 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>         return -ENOMEM;
>  }
>
> -static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
> +static bool virtqueue_kick_prepare_split(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         u16 new, old;
>         bool needs_kick;
>
> @@ -732,12 +731,12 @@ static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
>         LAST_ADD_TIME_INVALID(vq);
>
>         if (vq->event) {
> -               needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
> +               needs_kick = vring_need_event(virtio16_to_cpu(vq->vq.vdev,
>                                         vring_avail_event(&vq->split.vring)),
>                                               new, old);
>         } else {
>                 needs_kick = !(vq->split.vring.used->flags &
> -                                       cpu_to_virtio16(_vq->vdev,
> +                                       cpu_to_virtio16(vq->vq.vdev,
>                                                 VRING_USED_F_NO_NOTIFY));
>         }
>         END_USE(vq);
> @@ -1597,9 +1596,8 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq,
>         return -EIO;
>  }
>
> -static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
> +static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         u16 new, old, off_wrap, flags, wrap_counter, event_idx;
>         bool needs_kick;
>         union {
> @@ -2454,8 +2452,8 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
> -                                virtqueue_kick_prepare_split(_vq);
> +       return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> +                                virtqueue_kick_prepare_split(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
>
> --
> 2.42.0
>

Re: [PATCH 06/19] virtio_ring: switch to use vring_virtqueue for virtqueue_add variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> ---
>  drivers/virtio/virtio_ring.c | 40 +++++++++++++++++-------------------
>  1 file changed, 19 insertions(+), 21 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index cf3ab6404698..619869e80309 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -472,7 +472,7 @@ static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
>         return extra->next;
>  }
>
> -static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
> +static struct vring_desc *alloc_indirect_split(struct vring_virtqueue *vq,
>                                                unsigned int total_sg,
>                                                gfp_t gfp)
>  {
> @@ -501,7 +501,7 @@ static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
>         return desc;
>  }
>
> -static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
> +static inline unsigned int virtqueue_add_desc_split(struct vring_virtqueue *vq,
>                                                     struct vring_desc *desc,
>                                                     struct vring_desc_extra *extra,
>                                                     unsigned int i,
> @@ -509,11 +509,12 @@ static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
>                                                     unsigned int len,
>                                                     u16 flags, bool premapped)
>  {
> +       struct virtio_device *vdev = vq->vq.vdev;
>         u16 next;
>
> -       desc[i].flags = cpu_to_virtio16(vq->vdev, flags);
> -       desc[i].addr = cpu_to_virtio64(vq->vdev, addr);
> -       desc[i].len = cpu_to_virtio32(vq->vdev, len);
> +       desc[i].flags = cpu_to_virtio16(vdev, flags);
> +       desc[i].addr = cpu_to_virtio64(vdev, addr);
> +       desc[i].len = cpu_to_virtio32(vdev, len);
>
>         extra[i].addr = premapped ? DMA_MAPPING_ERROR : addr;
>         extra[i].len = len;
> @@ -521,12 +522,12 @@ static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
>
>         next = extra[i].next;
>
> -       desc[i].next = cpu_to_virtio16(vq->vdev, next);
> +       desc[i].next = cpu_to_virtio16(vdev, next);
>
>         return next;
>  }
>
> -static inline int virtqueue_add_split(struct virtqueue *_vq,
> +static inline int virtqueue_add_split(struct vring_virtqueue *vq,
>                                       struct scatterlist *sgs[],
>                                       unsigned int total_sg,
>                                       unsigned int out_sgs,
> @@ -536,7 +537,6 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>                                       bool premapped,
>                                       gfp_t gfp)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         struct vring_desc_extra *extra;
>         struct scatterlist *sg;
>         struct vring_desc *desc;
> @@ -561,7 +561,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>         head = vq->free_head;
>
>         if (virtqueue_use_indirect(vq, total_sg))
> -               desc = alloc_indirect_split(_vq, total_sg, gfp);
> +               desc = alloc_indirect_split(vq, total_sg, gfp);
>         else {
>                 desc = NULL;
>                 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
> @@ -608,7 +608,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>                         /* Note that we trust indirect descriptor
>                          * table since it use stream DMA mapping.
>                          */
> -                       i = virtqueue_add_desc_split(_vq, desc, extra, i, addr, len,
> +                       i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
>                                                      VRING_DESC_F_NEXT,
>                                                      premapped);
>                 }
> @@ -625,14 +625,14 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>                         /* Note that we trust indirect descriptor
>                          * table since it use stream DMA mapping.
>                          */
> -                       i = virtqueue_add_desc_split(_vq, desc, extra, i, addr, len,
> +                       i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
>                                                      VRING_DESC_F_NEXT |
>                                                      VRING_DESC_F_WRITE,
>                                                      premapped);
>                 }
>         }
>         /* Last one doesn't continue. */
> -       desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
> +       desc[prev].flags &= cpu_to_virtio16(vq->vq.vdev, ~VRING_DESC_F_NEXT);
>         if (!indirect && vring_need_unmap_buffer(vq, &extra[prev]))
>                 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
>                         ~VRING_DESC_F_NEXT;
> @@ -645,7 +645,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>                 if (vring_mapping_error(vq, addr))
>                         goto unmap_release;
>
> -               virtqueue_add_desc_split(_vq, vq->split.vring.desc,
> +               virtqueue_add_desc_split(vq, vq->split.vring.desc,
>                                          vq->split.desc_extra,
>                                          head, addr,
>                                          total_sg * sizeof(struct vring_desc),
> @@ -671,13 +671,13 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>         /* Put entry in available array (but don't update avail->idx until they
>          * do sync). */
>         avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
> -       vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
> +       vq->split.vring.avail->ring[avail] = cpu_to_virtio16(vq->vq.vdev, head);
>
>         /* Descriptors and available array need to be set before we expose the
>          * new available array entries. */
>         virtio_wmb(vq->weak_barriers);
>         vq->split.avail_idx_shadow++;
> -       vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
> +       vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
>                                                 vq->split.avail_idx_shadow);
>         vq->num_added++;
>
> @@ -687,7 +687,7 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>         /* This is very unlikely, but theoretically possible.  Kick
>          * just in case. */
>         if (unlikely(vq->num_added == (1 << 16) - 1))
> -               virtqueue_kick(_vq);
> +               virtqueue_kick(&vq->vq);
>
>         return 0;
>
> @@ -702,7 +702,6 @@ static inline int virtqueue_add_split(struct virtqueue *_vq,
>         for (n = 0; n < total_sg; n++) {
>                 if (i == err_idx)
>                         break;
> -
>                 i = vring_unmap_one_split(vq, &extra[i]);
>         }
>
> @@ -1441,7 +1440,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
>         return -ENOMEM;
>  }
>
> -static inline int virtqueue_add_packed(struct virtqueue *_vq,
> +static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
>                                        struct scatterlist *sgs[],
>                                        unsigned int total_sg,
>                                        unsigned int out_sgs,
> @@ -1451,7 +1450,6 @@ static inline int virtqueue_add_packed(struct virtqueue *_vq,
>                                        bool premapped,
>                                        gfp_t gfp)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         struct vring_packed_desc *desc;
>         struct scatterlist *sg;
>         unsigned int i, n, c, descs_used, err_idx, len;
> @@ -2263,9 +2261,9 @@ static inline int virtqueue_add(struct virtqueue *_vq,
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
> +       return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
>                                         out_sgs, in_sgs, data, ctx, premapped, gfp) :
> -                                virtqueue_add_split(_vq, sgs, total_sg,
> +                                virtqueue_add_split(vq, sgs, total_sg,
>                                         out_sgs, in_sgs, data, ctx, premapped, gfp);
>  }
>
> --
> 2.42.0
>

Re: [PATCH 07/19] virtio: switch to use vring_virtqueue for virtqueue_add variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> ---
>  drivers/virtio/virtio_ring.c | 16 +++++++---------
>  1 file changed, 7 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 619869e80309..3fab40648c75 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -812,11 +812,10 @@ static bool more_used_split(const struct vring_virtqueue *vq)
>         return virtqueue_poll_split(vq, vq->last_used_idx);
>  }
>
> -static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
> +static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
>                                          unsigned int *len,
>                                          void **ctx)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         void *ret;
>         unsigned int i;
>         u16 last_used;
> @@ -838,9 +837,9 @@ static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
>         virtio_rmb(vq->weak_barriers);
>
>         last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
> -       i = virtio32_to_cpu(_vq->vdev,
> +       i = virtio32_to_cpu(vq->vq.vdev,
>                         vq->split.vring.used->ring[last_used].id);
> -       *len = virtio32_to_cpu(_vq->vdev,
> +       *len = virtio32_to_cpu(vq->vq.vdev,
>                         vq->split.vring.used->ring[last_used].len);
>
>         if (unlikely(i >= vq->split.vring.num)) {
> @@ -862,7 +861,7 @@ static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
>         if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
>                 virtio_store_mb(vq->weak_barriers,
>                                 &vring_used_event(&vq->split.vring),
> -                               cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
> +                               cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx));
>
>         LAST_ADD_TIME_INVALID(vq);
>
> @@ -1721,11 +1720,10 @@ static bool more_used_packed(const struct vring_virtqueue *vq)
>         return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
>  }
>
> -static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
> +static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
>                                           unsigned int *len,
>                                           void **ctx)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         u16 last_used, id, last_used_idx;
>         bool used_wrap_counter;
>         void *ret;
> @@ -2521,8 +2519,8 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
> -                                virtqueue_get_buf_ctx_split(_vq, len, ctx);
> +       return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> +                                virtqueue_get_buf_ctx_split(vq, len, ctx);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
>
> --
> 2.42.0
>

Re: [PATCH 08/19] virtio_ring: switch to use vring_virtqueue for enable_cb_prepare variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 15 ++++++---------
>  1 file changed, 6 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 3fab40648c75..a414e66915d9 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -893,9 +893,8 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq)
>         }
>  }
>
> -static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
> +static unsigned int virtqueue_enable_cb_prepare_split(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         u16 last_used_idx;
>
>         START_USE(vq);
> @@ -909,10 +908,10 @@ static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
>                 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
>                 if (!vq->event)
>                         vq->split.vring.avail->flags =
> -                               cpu_to_virtio16(_vq->vdev,
> +                               cpu_to_virtio16(vq->vq.vdev,
>                                                 vq->split.avail_flags_shadow);
>         }
> -       vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
> +       vring_used_event(&vq->split.vring) = cpu_to_virtio16(vq->vq.vdev,
>                         last_used_idx = vq->last_used_idx);
>         END_USE(vq);
>         return last_used_idx;
> @@ -1807,10 +1806,8 @@ static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
>         }
>  }
>
> -static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
> +static unsigned int virtqueue_enable_cb_prepare_packed(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
> -
>         START_USE(vq);
>
>         /*
> @@ -2568,8 +2565,8 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
>         if (vq->event_triggered)
>                 vq->event_triggered = false;
>
> -       return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
> -                                virtqueue_enable_cb_prepare_split(_vq);
> +       return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> +                                virtqueue_enable_cb_prepare_split(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
>
> --
> 2.42.0
>

Re: [PATCH 09/19] virtio_ring: use vring_virtqueue for enable_cb_delayed variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 16 +++++++---------
>  1 file changed, 7 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index a414e66915d9..af8ce6e9d892 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -917,9 +917,8 @@ static unsigned int virtqueue_enable_cb_prepare_split(struct vring_virtqueue *vq
>         return last_used_idx;
>  }
>
> -static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
> +static bool virtqueue_enable_cb_delayed_split(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         u16 bufs;
>
>         START_USE(vq);
> @@ -933,7 +932,7 @@ static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
>                 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
>                 if (!vq->event)
>                         vq->split.vring.avail->flags =
> -                               cpu_to_virtio16(_vq->vdev,
> +                               cpu_to_virtio16(vq->vq.vdev,
>                                                 vq->split.avail_flags_shadow);
>         }
>         /* TODO: tune this threshold */
> @@ -941,9 +940,9 @@ static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
>
>         virtio_store_mb(vq->weak_barriers,
>                         &vring_used_event(&vq->split.vring),
> -                       cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
> +                       cpu_to_virtio16(vq->vq.vdev, vq->last_used_idx + bufs));
>
> -       if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
> +       if (unlikely((u16)(virtio16_to_cpu(vq->vq.vdev, vq->split.vring.used->idx)
>                                         - vq->last_used_idx) > bufs)) {
>                 END_USE(vq);
>                 return false;
> @@ -1837,9 +1836,8 @@ static unsigned int virtqueue_enable_cb_prepare_packed(struct vring_virtqueue *v
>         return vq->last_used_idx;
>  }
>
> -static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
> +static bool virtqueue_enable_cb_delayed_packed(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         u16 used_idx, wrap_counter, last_used_idx;
>         u16 bufs;
>
> @@ -2631,8 +2629,8 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
>         if (vq->event_triggered)
>                 vq->event_triggered = false;
>
> -       return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
> -                                virtqueue_enable_cb_delayed_split(_vq);
> +       return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> +                                virtqueue_enable_cb_delayed_split(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
>
> --
> 2.42.0
>

Re: [PATCH 10/19] virtio_ring: switch to use vring_virtqueue for disable_cb variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 14 +++++---------
>  1 file changed, 5 insertions(+), 9 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index af8ce6e9d892..f754da8c2ac6 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -869,10 +869,8 @@ static void *virtqueue_get_buf_ctx_split(struct vring_virtqueue *vq,
>         return ret;
>  }
>
> -static void virtqueue_disable_cb_split(struct virtqueue *_vq)
> +static void virtqueue_disable_cb_split(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
> -
>         if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
>                 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
>
> @@ -888,7 +886,7 @@ static void virtqueue_disable_cb_split(struct virtqueue *_vq)
>                         vring_used_event(&vq->split.vring) = 0x0;
>                 else
>                         vq->split.vring.avail->flags =
> -                               cpu_to_virtio16(_vq->vdev,
> +                               cpu_to_virtio16(vq->vq.vdev,
>                                                 vq->split.avail_flags_shadow);
>         }
>  }
> @@ -1786,10 +1784,8 @@ static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
>         return ret;
>  }
>
> -static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
> +static void virtqueue_disable_cb_packed(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
> -
>         if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
>                 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
>
> @@ -2538,9 +2534,9 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
>         if (vq->packed_ring)
> -               virtqueue_disable_cb_packed(_vq);
> +               virtqueue_disable_cb_packed(vq);
>         else
> -               virtqueue_disable_cb_split(_vq);
> +               virtqueue_disable_cb_split(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
>
> --
> 2.42.0
>

Re: [PATCH 11/19] virtio_ring: switch to use vring_virtqueue for detach_unused_buf variants

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Those variants are used internally so let's switch to use
> vring_virtqueue as parameter to be consistent with other internal
> virtqueue helpers.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 12 +++++-------
>  1 file changed, 5 insertions(+), 7 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index f754da8c2ac6..19aa24d62d20 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -950,9 +950,8 @@ static bool virtqueue_enable_cb_delayed_split(struct vring_virtqueue *vq)
>         return true;
>  }
>
> -static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
> +static void *virtqueue_detach_unused_buf_split(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         unsigned int i;
>         void *buf;
>
> @@ -965,7 +964,7 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
>                 buf = vq->split.desc_state[i].data;
>                 detach_buf_split(vq, i, NULL);
>                 vq->split.avail_idx_shadow--;
> -               vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
> +               vq->split.vring.avail->idx = cpu_to_virtio16(vq->vq.vdev,
>                                 vq->split.avail_idx_shadow);
>                 END_USE(vq);
>                 return buf;
> @@ -1892,9 +1891,8 @@ static bool virtqueue_enable_cb_delayed_packed(struct vring_virtqueue *vq)
>         return true;
>  }
>
> -static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
> +static void *virtqueue_detach_unused_buf_packed(struct vring_virtqueue *vq)
>  {
> -       struct vring_virtqueue *vq = to_vvq(_vq);
>         unsigned int i;
>         void *buf;
>
> @@ -2642,8 +2640,8 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
> -                                virtqueue_detach_unused_buf_split(_vq);
> +       return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> +                                virtqueue_detach_unused_buf_split(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
>
> --
> 2.42.0
>

Re: [PATCH 12/19] virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 6:44 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Use u16 for last_used_idx in virtqueue_poll_split() to align with the
> spec.
>
> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 19aa24d62d20..a2884eae14d9 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -801,7 +801,7 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
>  }
>
>  static bool virtqueue_poll_split(const struct vring_virtqueue *vq,
> -                                unsigned int last_used_idx)
> +                                u16 last_used_idx)
>  {
>         return (u16)last_used_idx != virtio16_to_cpu(vq->vq.vdev,

So this cast to u16 is not needed anymore, isn't it?

>                         vq->split.vring.used->idx);
> --
> 2.42.0
>

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 7:01 AM Jason Wang <jasowang@redhat.com> wrote:
>
> This patch introduces virtqueue ops which is a set of the callbacks
> that will be called for different queue layout or features. This would
> help to avoid branches for split/packed and will ease the future
> implementation like in order.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
>  1 file changed, 67 insertions(+), 29 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index a2884eae14d9..ce1dc90ee89d 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
>         size_t event_size_in_bytes;
>  };
>
> +struct vring_virtqueue;
> +
> +struct virtqueue_ops {
> +       int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> +                  unsigned int total_sg, unsigned int out_sgs,
> +                  unsigned int in_sgs, void *data,
> +                  void *ctx, bool premapped, gfp_t gfp);
> +       void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> +       bool (*kick_prepare)(struct vring_virtqueue *vq);
> +       void (*disable_cb)(struct vring_virtqueue *vq);
> +       bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> +       unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> +       bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> +       void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> +       bool (*more_used)(const struct vring_virtqueue *vq);
> +       int (*resize)(struct vring_virtqueue *vq, u32 num);
> +       void (*reset)(struct vring_virtqueue *vq);
> +};
> +
>  struct vring_virtqueue {
>         struct virtqueue vq;
>
> +       struct virtqueue_ops *ops;
> +
>         /* Is this a packed ring? */
>         bool packed_ring;
>
> @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
>         return 0;
>  }
>
> +struct virtqueue_ops split_ops;
> +
>  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
>                                                struct vring_virtqueue_split *vring_split,
>                                                struct virtio_device *vdev,
> @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
>                 return NULL;
>
>         vq->packed_ring = false;
> +       vq->ops = &split_ops;
>         vq->vq.callback = callback;
>         vq->vq.vdev = vdev;
>         vq->vq.name = name;
> @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
>         virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
>  }
>
> +struct virtqueue_ops packed_ops;
> +
>  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
>                                                struct vring_virtqueue_packed *vring_packed,
>                                                struct virtio_device *vdev,
> @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
>         vq->broken = false;
>  #endif
>         vq->packed_ring = true;
> +       vq->ops = &packed_ops;
>         vq->dma_dev = dma_dev;
>         vq->use_dma_api = vring_use_dma_api(vdev);
>
> @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
>         return -ENOMEM;
>  }
>
> +struct virtqueue_ops split_ops = {
> +       .add = virtqueue_add_split,
> +       .get = virtqueue_get_buf_ctx_split,
> +       .kick_prepare = virtqueue_kick_prepare_split,
> +       .disable_cb = virtqueue_disable_cb_split,
> +       .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> +       .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> +       .poll = virtqueue_poll_split,
> +       .detach_unused_buf = virtqueue_detach_unused_buf_split,
> +       .more_used = more_used_split,
> +       .resize = virtqueue_resize_split,
> +       .reset = virtqueue_reset_split,
> +};
> +
> +struct virtqueue_ops packed_ops = {
> +       .add = virtqueue_add_packed,
> +       .get = virtqueue_get_buf_ctx_packed,
> +       .kick_prepare = virtqueue_kick_prepare_packed,
> +       .disable_cb = virtqueue_disable_cb_packed,
> +       .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> +       .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> +       .poll = virtqueue_poll_packed,
> +       .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> +       .more_used = more_used_packed,
> +       .resize = virtqueue_resize_packed,
> +       .reset = virtqueue_reset_packed,
> +};
> +
>  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
>                                          void (*recycle)(struct virtqueue *vq, void *buf))
>  {
> @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> -                                       out_sgs, in_sgs, data, ctx, premapped, gfp) :
> -                                virtqueue_add_split(vq, sgs, total_sg,
> -                                       out_sgs, in_sgs, data, ctx, premapped, gfp);
> +       return vq->ops->add(vq, sgs, total_sg,
> +                           out_sgs, in_sgs, data, ctx, premapped, gfp);
>  }
>
>  /**
> @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> -                                virtqueue_kick_prepare_split(vq);
> +       return vq->ops->kick_prepare(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
>
> @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> -                                virtqueue_get_buf_ctx_split(vq, len, ctx);
> +       return vq->ops->get(vq, len, ctx);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
>
> @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       if (vq->packed_ring)
> -               virtqueue_disable_cb_packed(vq);
> -       else
> -               virtqueue_disable_cb_split(vq);
> +       return vq->ops->disable_cb(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
>
> @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
>         if (vq->event_triggered)
>                 vq->event_triggered = false;
>
> -       return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> -                                virtqueue_enable_cb_prepare_split(vq);
> +       return vq->ops->enable_cb_prepare(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
>
> @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
>                 return false;
>
>         virtio_mb(vq->weak_barriers);
> -       return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> -                                virtqueue_poll_split(vq, last_used_idx);
> +       return vq->ops->poll(vq, last_used_idx);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_poll);
>
> @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
>         if (vq->event_triggered)
>                 vq->event_triggered = false;
>
> -       return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> -                                virtqueue_enable_cb_delayed_split(vq);
> +       return vq->ops->enable_cb_delayed(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
>
> @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
>  {
>         struct vring_virtqueue *vq = to_vvq(_vq);
>
> -       return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> -                                virtqueue_detach_unused_buf_split(vq);
> +       return vq->ops->detach_unused_buf(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
>
>  static inline bool more_used(const struct vring_virtqueue *vq)
>  {
> -       return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> +       return vq->ops->more_used(vq);
>  }
>
>  /**
> @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
>         if (recycle_done)
>                 recycle_done(_vq);
>
> -       if (vq->packed_ring)
> -               err = virtqueue_resize_packed(vq, num);
> -       else
> -               err = virtqueue_resize_split(vq, num);
> +       err = vq->ops->resize(vq, num);
>
>         return virtqueue_enable_after_reset(_vq);
>  }
> @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
>         if (recycle_done)
>                 recycle_done(_vq);
>
> -       if (vq->packed_ring)
> -               virtqueue_reset_packed(vq);
> -       else
> -               virtqueue_reset_split(vq);
> +       vq->ops->reset(vq);
>
>         return virtqueue_enable_after_reset(_vq);
>  }
> --
> 2.42.0
>

Re: [PATCH 14/19] virtio_ring: determine descriptor flags at one time

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 7:01 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Let's determine the last descriptor by counting the number of sg. This
> would be consistent with packed virtqueue implementation and ease the
> future in-order implementation.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 25 +++++++++++++------------
>  1 file changed, 13 insertions(+), 12 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index ce1dc90ee89d..31aa4a935c27 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -561,7 +561,7 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
>         struct vring_desc_extra *extra;
>         struct scatterlist *sg;
>         struct vring_desc *desc;
> -       unsigned int i, n, avail, descs_used, prev, err_idx;
> +       unsigned int i, n, c, avail, descs_used, err_idx;
>         int head;
>         bool indirect;
>
> @@ -617,46 +617,47 @@ static inline int virtqueue_add_split(struct vring_virtqueue *vq,
>                 return -ENOSPC;
>         }
>
> +       c = 0;
>         for (n = 0; n < out_sgs; n++) {
> +               sg = sgs[n];
>                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
>                         dma_addr_t addr;
>                         u32 len;
> +                       u16 flags = 0;
>
>                         if (vring_map_one_sg(vq, sg, DMA_TO_DEVICE, &addr, &len, premapped))
>                                 goto unmap_release;
>
> -                       prev = i;
> +                       if (++c != total_sg)
> +                               flags = VRING_DESC_F_NEXT;
> +
>                         /* Note that we trust indirect descriptor
>                          * table since it use stream DMA mapping.
>                          */
>                         i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
> -                                                    VRING_DESC_F_NEXT,
> +                                                    flags,
>                                                      premapped);
>                 }
>         }
>         for (; n < (out_sgs + in_sgs); n++) {
>                 for (sg = sgs[n]; sg; sg = sg_next(sg)) {
> +                       u16 flags = VRING_DESC_F_WRITE;
>                         dma_addr_t addr;
>                         u32 len;
>
>                         if (vring_map_one_sg(vq, sg, DMA_FROM_DEVICE, &addr, &len, premapped))
>                                 goto unmap_release;
>
> -                       prev = i;
> +                       if (++c != total_sg)
> +                               flags |= VRING_DESC_F_NEXT;
> +
>                         /* Note that we trust indirect descriptor
>                          * table since it use stream DMA mapping.
>                          */
>                         i = virtqueue_add_desc_split(vq, desc, extra, i, addr, len,
> -                                                    VRING_DESC_F_NEXT |
> -                                                    VRING_DESC_F_WRITE,
> -                                                    premapped);
> +                                                    flags, premapped);
>                 }
>         }
> -       /* Last one doesn't continue. */
> -       desc[prev].flags &= cpu_to_virtio16(vq->vq.vdev, ~VRING_DESC_F_NEXT);
> -       if (!indirect && vring_need_unmap_buffer(vq, &extra[prev]))
> -               vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
> -                       ~VRING_DESC_F_NEXT;
>
>         if (indirect) {
>                 /* Now that the indirect table is filled in, map it. */
> --
> 2.42.0
>

Re: [PATCH 15/19] virtio_ring: factor out core logic of buffer detaching

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 7:01 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Factor out core logic of buffer detaching and leave the id population
> to the caller so in order can just call the core logic.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 17 +++++++++++++----
>  1 file changed, 13 insertions(+), 4 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index 31aa4a935c27..fe3e6f3d0f96 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -1661,8 +1661,8 @@ static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
>         return needs_kick;
>  }
>
> -static void detach_buf_packed(struct vring_virtqueue *vq,
> -                             unsigned int id, void **ctx)
> +static void __detach_buf_packed(struct vring_virtqueue *vq,
> +                               unsigned int id, void **ctx)
>  {
>         struct vring_desc_state_packed *state = NULL;
>         struct vring_packed_desc *desc;
> @@ -1673,8 +1673,6 @@ static void detach_buf_packed(struct vring_virtqueue *vq,
>         /* Clear data ptr. */
>         state->data = NULL;
>
> -       vq->packed.desc_extra[state->last].next = vq->free_head;
> -       vq->free_head = id;
>         vq->vq.num_free += state->num;
>
>         if (unlikely(vq->use_dma_api)) {
> @@ -1711,6 +1709,17 @@ static void detach_buf_packed(struct vring_virtqueue *vq,
>         }
>  }
>
> +static void detach_buf_packed(struct vring_virtqueue *vq,
> +                             unsigned int id, void **ctx)
> +{
> +       struct vring_desc_state_packed *state = &vq->packed.desc_state[id];
> +
> +       vq->packed.desc_extra[state->last].next = vq->free_head;
> +       vq->free_head = id;
> +
> +       return __detach_buf_packed(vq, id, ctx);
> +}
> +
>  static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
>                                        u16 idx, bool used_wrap_counter)
>  {
> --
> 2.42.0
>

Re: [PATCH 16/19] virtio_ring: factor out core logic for updating last_used_idx

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 7:01 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Factor out the core logic for updating last_used_idx to be reused by
> the packed in order implementation.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 43 +++++++++++++++++++++---------------
>  1 file changed, 25 insertions(+), 18 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index fe3e6f3d0f96..bd4faf04862c 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -1749,6 +1749,30 @@ static bool more_used_packed(const struct vring_virtqueue *vq)
>         return virtqueue_poll_packed(vq, READ_ONCE(vq->last_used_idx));
>  }
>
> +static void update_last_used_idx_packed(struct vring_virtqueue *vq,
> +                                       u16 id, u16 last_used,
> +                                       u16 used_wrap_counter)
> +{
> +       last_used += vq->packed.desc_state[id].num;
> +       if (unlikely(last_used >= vq->packed.vring.num)) {
> +               last_used -= vq->packed.vring.num;
> +               used_wrap_counter ^= 1;
> +       }
> +
> +       last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
> +       WRITE_ONCE(vq->last_used_idx, last_used);
> +
> +       /*
> +        * If we expect an interrupt for the next entry, tell host
> +        * by writing event index and flush out the write before
> +        * the read in the next get_buf call.
> +        */
> +       if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
> +               virtio_store_mb(vq->weak_barriers,
> +                               &vq->packed.vring.driver->off_wrap,
> +                               cpu_to_le16(vq->last_used_idx));
> +}
> +
>  static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
>                                           unsigned int *len,
>                                           void **ctx)
> @@ -1792,24 +1816,7 @@ static void *virtqueue_get_buf_ctx_packed(struct vring_virtqueue *vq,
>         ret = vq->packed.desc_state[id].data;
>         detach_buf_packed(vq, id, ctx);
>
> -       last_used += vq->packed.desc_state[id].num;
> -       if (unlikely(last_used >= vq->packed.vring.num)) {
> -               last_used -= vq->packed.vring.num;
> -               used_wrap_counter ^= 1;
> -       }
> -
> -       last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
> -       WRITE_ONCE(vq->last_used_idx, last_used);
> -
> -       /*
> -        * If we expect an interrupt for the next entry, tell host
> -        * by writing event index and flush out the write before
> -        * the read in the next get_buf call.
> -        */
> -       if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
> -               virtio_store_mb(vq->weak_barriers,
> -                               &vq->packed.vring.driver->off_wrap,
> -                               cpu_to_le16(vq->last_used_idx));
> +       update_last_used_idx_packed(vq, id, last_used, used_wrap_counter);
>
>         LAST_ADD_TIME_INVALID(vq);
>
> --
> 2.42.0
>

Re: [PATCH 17/19] virtio_ring: move next_avail_idx to vring_virtqueue

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 7:01 AM Jason Wang <jasowang@redhat.com> wrote:
>
> This variable is used by packed virtqueue now, moving it to
> vring_virtqueue to make it possible to be reused by split virtqueue
> in-order implementation.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 22 +++++++++++-----------
>  1 file changed, 11 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index bd4faf04862c..a1a8cd931052 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -138,9 +138,6 @@ struct vring_virtqueue_packed {
>         /* Avail used flags. */
>         u16 avail_used_flags;
>
> -       /* Index of the next avail descriptor. */
> -       u16 next_avail_idx;
> -
>         /*
>          * Last written value to driver->flags in
>          * guest byte order.
> @@ -214,6 +211,9 @@ struct vring_virtqueue {
>          */
>         u16 last_used_idx;
>
> +       /* Index of the next avail descriptor. */
> +       u16 next_avail_idx;
> +
>         /* Hint for event idx: already triggered no need to disable. */
>         bool event_triggered;
>
> @@ -448,6 +448,7 @@ static void virtqueue_init(struct vring_virtqueue *vq, u32 num)
>         else
>                 vq->last_used_idx = 0;
>
> +       vq->next_avail_idx = 0;
>         vq->event_triggered = false;
>         vq->num_added = 0;
>
> @@ -1350,7 +1351,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
>         u16 head, id;
>         dma_addr_t addr;
>
> -       head = vq->packed.next_avail_idx;
> +       head = vq->next_avail_idx;
>         desc = alloc_indirect_packed(total_sg, gfp);
>         if (!desc)
>                 return -ENOMEM;
> @@ -1431,7 +1432,7 @@ static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
>                                 1 << VRING_PACKED_DESC_F_AVAIL |
>                                 1 << VRING_PACKED_DESC_F_USED;
>         }
> -       vq->packed.next_avail_idx = n;
> +       vq->next_avail_idx = n;
>         vq->free_head = vq->packed.desc_extra[id].next;
>
>         /* Store token and indirect buffer state. */
> @@ -1501,7 +1502,7 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
>                 /* fall back on direct */
>         }
>
> -       head = vq->packed.next_avail_idx;
> +       head = vq->next_avail_idx;
>         avail_used_flags = vq->packed.avail_used_flags;
>
>         WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
> @@ -1569,7 +1570,7 @@ static inline int virtqueue_add_packed(struct vring_virtqueue *vq,
>         vq->vq.num_free -= descs_used;
>
>         /* Update free pointer */
> -       vq->packed.next_avail_idx = i;
> +       vq->next_avail_idx = i;
>         vq->free_head = curr;
>
>         /* Store token. */
> @@ -1633,8 +1634,8 @@ static bool virtqueue_kick_prepare_packed(struct vring_virtqueue *vq)
>          */
>         virtio_mb(vq->weak_barriers);
>
> -       old = vq->packed.next_avail_idx - vq->num_added;
> -       new = vq->packed.next_avail_idx;
> +       old = vq->next_avail_idx - vq->num_added;
> +       new = vq->next_avail_idx;
>         vq->num_added = 0;
>
>         snapshot.u32 = *(u32 *)vq->packed.vring.device;
> @@ -2083,7 +2084,6 @@ static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_p
>  static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed,
>                                         bool callback)
>  {
> -       vring_packed->next_avail_idx = 0;
>         vring_packed->avail_wrap_counter = 1;
>         vring_packed->event_flags_shadow = 0;
>         vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
> @@ -2977,7 +2977,7 @@ u32 vring_notification_data(struct virtqueue *_vq)
>         u16 next;
>
>         if (vq->packed_ring)
> -               next = (vq->packed.next_avail_idx &
> +               next = (vq->next_avail_idx &
>                                 ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) |
>                         vq->packed.avail_wrap_counter <<
>                                 VRING_PACKED_EVENT_F_WRAP_CTR;
> --
> 2.42.0
>

Re: [PATCH 18/19] virtio_ring: factor out split indirect detaching logic

[Eugenio Perez Martin]

On Mon, Mar 24, 2025 at 7:02 AM Jason Wang <jasowang@redhat.com> wrote:
>
> Factor out the split indirect descriptor detaching logic in order to
> make it be reused by the in order support.
>

Acked-by: Eugenio Pérez <eperezma@redhat.com>

> Signed-off-by: Jason Wang <jasowang@redhat.com>
> ---
>  drivers/virtio/virtio_ring.c | 63 ++++++++++++++++++++----------------
>  1 file changed, 35 insertions(+), 28 deletions(-)
>
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index a1a8cd931052..0fad8e8419c8 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -765,11 +765,42 @@ static bool virtqueue_kick_prepare_split(struct vring_virtqueue *vq)
>         return needs_kick;
>  }
>
> +static void detach_indirect_split(struct vring_virtqueue *vq,
> +                                 unsigned int head)
> +{
> +       struct vring_desc_extra *extra = vq->split.desc_extra;
> +       struct vring_desc *indir_desc =
> +              vq->split.desc_state[head].indir_desc;
> +       unsigned int j;
> +       u32 len, num;
> +
> +       /* Free the indirect table, if any, now that it's unmapped. */
> +       if (!indir_desc)
> +               return;
> +       len = vq->split.desc_extra[head].len;
> +
> +       BUG_ON(!(vq->split.desc_extra[head].flags &
> +                       VRING_DESC_F_INDIRECT));
> +       BUG_ON(len == 0 || len % sizeof(struct vring_desc));
> +
> +       num = len / sizeof(struct vring_desc);
> +
> +       extra = (struct vring_desc_extra *)&indir_desc[num];
> +
> +       if (vq->use_dma_api) {
> +               for (j = 0; j < num; j++)
> +                       vring_unmap_one_split(vq, &extra[j]);
> +       }
> +
> +       kfree(indir_desc);
> +       vq->split.desc_state[head].indir_desc = NULL;
> +}
> +
>  static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
>                              void **ctx)
>  {
>         struct vring_desc_extra *extra;
> -       unsigned int i, j;
> +       unsigned int i;
>         __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
>
>         /* Clear data ptr. */
> @@ -793,34 +824,10 @@ static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
>         /* Plus final descriptor */
>         vq->vq.num_free++;
>
> -       if (vq->indirect) {
> -               struct vring_desc *indir_desc =
> -                               vq->split.desc_state[head].indir_desc;
> -               u32 len, num;
> -
> -               /* Free the indirect table, if any, now that it's unmapped. */
> -               if (!indir_desc)
> -                       return;
> -               len = vq->split.desc_extra[head].len;
> -
> -               BUG_ON(!(vq->split.desc_extra[head].flags &
> -                               VRING_DESC_F_INDIRECT));
> -               BUG_ON(len == 0 || len % sizeof(struct vring_desc));
> -
> -               num = len / sizeof(struct vring_desc);
> -
> -               extra = (struct vring_desc_extra *)&indir_desc[num];
> -
> -               if (vq->use_dma_api) {
> -                       for (j = 0; j < num; j++)
> -                               vring_unmap_one_split(vq, &extra[j]);
> -               }
> -
> -               kfree(indir_desc);
> -               vq->split.desc_state[head].indir_desc = NULL;
> -       } else if (ctx) {
> +       if (vq->indirect)
> +               detach_indirect_split(vq, head);
> +       else if (ctx)
>                 *ctx = vq->split.desc_state[head].indir_desc;
> -       }
>  }
>
>  static bool virtqueue_poll_split(const struct vring_virtqueue *vq,
> --
> 2.42.0
>

Re: [PATCH 00/19] virtio_ring in order support

[Xuan Zhuo]

For series:

Reviewed-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>

Thanks.

On Mon, 24 Mar 2025 13:43:14 +0800, Jason Wang <jasowang@redhat.com> wrote:
> Hello all:
>
> This sereis tries to implement the VIRTIO_F_IN_ORDER to
> virtio_ring. This is done by introducing virtqueue ops so we can
> implement separate helpers for different virtqueue layout/features
> then the in-order were implmeented on top.
>
> Tests shows 5% imporvemnt in RX PPS with KVM guest + testpmd on the
> host.
>
> Please review.
>
> Thanks
>
> Jason Wang (19):
>   virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()
>   virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants
>   virtio_ring: unify logic of virtqueue_poll() and more_used()
>   virtio_ring: switch to use vring_virtqueue for virtqueue resize
>     variants
>   virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare
>     variants
>   virtio_ring: switch to use vring_virtqueue for virtqueue_add variants
>   virtio: switch to use vring_virtqueue for virtqueue_add variants
>   virtio_ring: switch to use vring_virtqueue for enable_cb_prepare
>     variants
>   virtio_ring: use vring_virtqueue for enable_cb_delayed variants
>   virtio_ring: switch to use vring_virtqueue for disable_cb variants
>   virtio_ring: switch to use vring_virtqueue for detach_unused_buf
>     variants
>   virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()
>   virtio_ring: introduce virtqueue ops
>   virtio_ring: determine descriptor flags at one time
>   virtio_ring: factor out core logic of buffer detaching
>   virtio_ring: factor out core logic for updating last_used_idx
>   virtio_ring: move next_avail_idx to vring_virtqueue
>   virtio_ring: factor out split indirect detaching logic
>   virtio_ring: add in order support
>
>  drivers/virtio/virtio_ring.c | 856 ++++++++++++++++++++++++++---------
>  1 file changed, 653 insertions(+), 203 deletions(-)
>
> --
> 2.42.0
>

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Michael S. Tsirkin]

On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> This patch introduces virtqueue ops which is a set of the callbacks
> that will be called for different queue layout or features. This would
> help to avoid branches for split/packed and will ease the future
> implementation like in order.
> 
> Signed-off-by: Jason Wang <jasowang@redhat.com>




> ---
>  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
>  1 file changed, 67 insertions(+), 29 deletions(-)
> 
> diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> index a2884eae14d9..ce1dc90ee89d 100644
> --- a/drivers/virtio/virtio_ring.c
> +++ b/drivers/virtio/virtio_ring.c
> @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
>  	size_t event_size_in_bytes;
>  };
>  
> +struct vring_virtqueue;
> +
> +struct virtqueue_ops {
> +	int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> +		   unsigned int total_sg, unsigned int out_sgs,
> +		   unsigned int in_sgs,	void *data,
> +		   void *ctx, bool premapped, gfp_t gfp);
> +	void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> +	bool (*kick_prepare)(struct vring_virtqueue *vq);
> +	void (*disable_cb)(struct vring_virtqueue *vq);
> +	bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> +	unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> +	bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> +	void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> +	bool (*more_used)(const struct vring_virtqueue *vq);
> +	int (*resize)(struct vring_virtqueue *vq, u32 num);
> +	void (*reset)(struct vring_virtqueue *vq);
> +};

I like it that it's organized but
I worry about the overhead of indirect calls here.
How about a switch statement instead?

struct vring_virtqueue {
	enum vring_virtqueue_ops ops;

}


@@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
 {
      struct vring_virtqueue *vq = to_vvq(_vq);

	switch (vq->ops) {
	 VQ_PACKED:
	 VQ_SPLIT:
	 VQ_IN_ORDER:
	}


}


What do you think?



> +
>  struct vring_virtqueue {
>  	struct virtqueue vq;
>  
> +	struct virtqueue_ops *ops;
> +
>  	/* Is this a packed ring? */
>  	bool packed_ring;
>  
> @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
>  	return 0;
>  }
>  
> +struct virtqueue_ops split_ops;
> +
>  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
>  					       struct vring_virtqueue_split *vring_split,
>  					       struct virtio_device *vdev,
> @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
>  		return NULL;
>  
>  	vq->packed_ring = false;
> +	vq->ops = &split_ops;
>  	vq->vq.callback = callback;
>  	vq->vq.vdev = vdev;
>  	vq->vq.name = name;
> @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
>  	virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
>  }
>  
> +struct virtqueue_ops packed_ops;
> +
>  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
>  					       struct vring_virtqueue_packed *vring_packed,
>  					       struct virtio_device *vdev,
> @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
>  	vq->broken = false;
>  #endif
>  	vq->packed_ring = true;
> +	vq->ops = &packed_ops;
>  	vq->dma_dev = dma_dev;
>  	vq->use_dma_api = vring_use_dma_api(vdev);
>  
> @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
>  	return -ENOMEM;
>  }
>  
> +struct virtqueue_ops split_ops = {
> +	.add = virtqueue_add_split,
> +	.get = virtqueue_get_buf_ctx_split,
> +	.kick_prepare = virtqueue_kick_prepare_split,
> +	.disable_cb = virtqueue_disable_cb_split,
> +	.enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> +	.enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> +	.poll = virtqueue_poll_split,
> +	.detach_unused_buf = virtqueue_detach_unused_buf_split,
> +	.more_used = more_used_split,
> +	.resize = virtqueue_resize_split,
> +	.reset = virtqueue_reset_split,
> +};
> +
> +struct virtqueue_ops packed_ops = {
> +	.add = virtqueue_add_packed,
> +	.get = virtqueue_get_buf_ctx_packed,
> +	.kick_prepare = virtqueue_kick_prepare_packed,
> +	.disable_cb = virtqueue_disable_cb_packed,
> +	.enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> +	.enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> +	.poll = virtqueue_poll_packed,
> +	.detach_unused_buf = virtqueue_detach_unused_buf_packed,
> +	.more_used = more_used_packed,
> +	.resize = virtqueue_resize_packed,
> +	.reset = virtqueue_reset_packed,
> +};
> +
>  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
>  					 void (*recycle)(struct virtqueue *vq, void *buf))
>  {
> @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
>  {
>  	struct vring_virtqueue *vq = to_vvq(_vq);
>  
> -	return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> -					out_sgs, in_sgs, data, ctx, premapped, gfp) :
> -				 virtqueue_add_split(vq, sgs, total_sg,
> -					out_sgs, in_sgs, data, ctx, premapped, gfp);
> +	return vq->ops->add(vq, sgs, total_sg,
> +			    out_sgs, in_sgs, data, ctx, premapped, gfp);
>  }
>  
>  /**
> @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
>  {
>  	struct vring_virtqueue *vq = to_vvq(_vq);
>  
> -	return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> -				 virtqueue_kick_prepare_split(vq);
> +	return vq->ops->kick_prepare(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
>  
> @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
>  {
>  	struct vring_virtqueue *vq = to_vvq(_vq);
>  
> -	return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> -				 virtqueue_get_buf_ctx_split(vq, len, ctx);
> +	return vq->ops->get(vq, len, ctx);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
>  
> @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
>  {
>  	struct vring_virtqueue *vq = to_vvq(_vq);
>  
> -	if (vq->packed_ring)
> -		virtqueue_disable_cb_packed(vq);
> -	else
> -		virtqueue_disable_cb_split(vq);
> +	return vq->ops->disable_cb(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
>  
> @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
>  	if (vq->event_triggered)
>  		vq->event_triggered = false;
>  
> -	return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> -				 virtqueue_enable_cb_prepare_split(vq);
> +	return vq->ops->enable_cb_prepare(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
>  
> @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
>  		return false;
>  
>  	virtio_mb(vq->weak_barriers);
> -	return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> -				 virtqueue_poll_split(vq, last_used_idx);
> +	return vq->ops->poll(vq, last_used_idx);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_poll);
>  
> @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
>  	if (vq->event_triggered)
>  		vq->event_triggered = false;
>  
> -	return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> -				 virtqueue_enable_cb_delayed_split(vq);
> +	return vq->ops->enable_cb_delayed(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
>  
> @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
>  {
>  	struct vring_virtqueue *vq = to_vvq(_vq);
>  
> -	return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> -				 virtqueue_detach_unused_buf_split(vq);
> +	return vq->ops->detach_unused_buf(vq);
>  }
>  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
>  
>  static inline bool more_used(const struct vring_virtqueue *vq)
>  {
> -	return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> +	return vq->ops->more_used(vq);
>  }
>  
>  /**
> @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
>  	if (recycle_done)
>  		recycle_done(_vq);
>  
> -	if (vq->packed_ring)
> -		err = virtqueue_resize_packed(vq, num);
> -	else
> -		err = virtqueue_resize_split(vq, num);
> +	err = vq->ops->resize(vq, num);
>  
>  	return virtqueue_enable_after_reset(_vq);
>  }
> @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
>  	if (recycle_done)
>  		recycle_done(_vq);
>  
> -	if (vq->packed_ring)
> -		virtqueue_reset_packed(vq);
> -	else
> -		virtqueue_reset_split(vq);
> +	vq->ops->reset(vq);
>  
>  	return virtqueue_enable_after_reset(_vq);
>  }
> -- 
> 2.42.0

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Jason Wang]

On Mon, Apr 7, 2025 at 4:20 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> > This patch introduces virtqueue ops which is a set of the callbacks
> > that will be called for different queue layout or features. This would
> > help to avoid branches for split/packed and will ease the future
> > implementation like in order.
> >
> > Signed-off-by: Jason Wang <jasowang@redhat.com>
>
>
>
>
> > ---
> >  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
> >  1 file changed, 67 insertions(+), 29 deletions(-)
> >
> > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> > index a2884eae14d9..ce1dc90ee89d 100644
> > --- a/drivers/virtio/virtio_ring.c
> > +++ b/drivers/virtio/virtio_ring.c
> > @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
> >       size_t event_size_in_bytes;
> >  };
> >
> > +struct vring_virtqueue;
> > +
> > +struct virtqueue_ops {
> > +     int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> > +                unsigned int total_sg, unsigned int out_sgs,
> > +                unsigned int in_sgs, void *data,
> > +                void *ctx, bool premapped, gfp_t gfp);
> > +     void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> > +     bool (*kick_prepare)(struct vring_virtqueue *vq);
> > +     void (*disable_cb)(struct vring_virtqueue *vq);
> > +     bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> > +     unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> > +     bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> > +     void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> > +     bool (*more_used)(const struct vring_virtqueue *vq);
> > +     int (*resize)(struct vring_virtqueue *vq, u32 num);
> > +     void (*reset)(struct vring_virtqueue *vq);
> > +};
>
> I like it that it's organized but
> I worry about the overhead of indirect calls here.

We can switch to use INDIRECT_CALL_X() here (but I'm not sure we
should worry about it too much as ndo_ops or qdiscs doesn't use that).

> How about a switch statement instead?
>
> struct vring_virtqueue {
>         enum vring_virtqueue_ops ops;
>
> }
>
>
> @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
>  {
>       struct vring_virtqueue *vq = to_vvq(_vq);
>
>         switch (vq->ops) {
>          VQ_PACKED:
>          VQ_SPLIT:
>          VQ_IN_ORDER:
>         }
>
>
> }
>
>
> What do you think?

Actually, the matrix will be 2*2:

PACKED, SPLIT, PACKED_IN_ORDER, SPLIT_IN_ORDER

And will be doubled if a new layout is implemented.

If we open them such a switch will spread in a lot of places in the code.

Thanks

>
>
>
> > +
> >  struct vring_virtqueue {
> >       struct virtqueue vq;
> >
> > +     struct virtqueue_ops *ops;
> > +
> >       /* Is this a packed ring? */
> >       bool packed_ring;
> >
> > @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
> >       return 0;
> >  }
> >
> > +struct virtqueue_ops split_ops;
> > +
> >  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> >                                              struct vring_virtqueue_split *vring_split,
> >                                              struct virtio_device *vdev,
> > @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> >               return NULL;
> >
> >       vq->packed_ring = false;
> > +     vq->ops = &split_ops;
> >       vq->vq.callback = callback;
> >       vq->vq.vdev = vdev;
> >       vq->vq.name = name;
> > @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
> >       virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
> >  }
> >
> > +struct virtqueue_ops packed_ops;
> > +
> >  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> >                                              struct vring_virtqueue_packed *vring_packed,
> >                                              struct virtio_device *vdev,
> > @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> >       vq->broken = false;
> >  #endif
> >       vq->packed_ring = true;
> > +     vq->ops = &packed_ops;
> >       vq->dma_dev = dma_dev;
> >       vq->use_dma_api = vring_use_dma_api(vdev);
> >
> > @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
> >       return -ENOMEM;
> >  }
> >
> > +struct virtqueue_ops split_ops = {
> > +     .add = virtqueue_add_split,
> > +     .get = virtqueue_get_buf_ctx_split,
> > +     .kick_prepare = virtqueue_kick_prepare_split,
> > +     .disable_cb = virtqueue_disable_cb_split,
> > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> > +     .poll = virtqueue_poll_split,
> > +     .detach_unused_buf = virtqueue_detach_unused_buf_split,
> > +     .more_used = more_used_split,
> > +     .resize = virtqueue_resize_split,
> > +     .reset = virtqueue_reset_split,
> > +};
> > +
> > +struct virtqueue_ops packed_ops = {
> > +     .add = virtqueue_add_packed,
> > +     .get = virtqueue_get_buf_ctx_packed,
> > +     .kick_prepare = virtqueue_kick_prepare_packed,
> > +     .disable_cb = virtqueue_disable_cb_packed,
> > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> > +     .poll = virtqueue_poll_packed,
> > +     .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> > +     .more_used = more_used_packed,
> > +     .resize = virtqueue_resize_packed,
> > +     .reset = virtqueue_reset_packed,
> > +};
> > +
> >  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
> >                                        void (*recycle)(struct virtqueue *vq, void *buf))
> >  {
> > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> >  {
> >       struct vring_virtqueue *vq = to_vvq(_vq);
> >
> > -     return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp) :
> > -                              virtqueue_add_split(vq, sgs, total_sg,
> > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp);
> > +     return vq->ops->add(vq, sgs, total_sg,
> > +                         out_sgs, in_sgs, data, ctx, premapped, gfp);
> >  }
> >
> >  /**
> > @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
> >  {
> >       struct vring_virtqueue *vq = to_vvq(_vq);
> >
> > -     return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> > -                              virtqueue_kick_prepare_split(vq);
> > +     return vq->ops->kick_prepare(vq);
> >  }
> >  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
> >
> > @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
> >  {
> >       struct vring_virtqueue *vq = to_vvq(_vq);
> >
> > -     return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> > -                              virtqueue_get_buf_ctx_split(vq, len, ctx);
> > +     return vq->ops->get(vq, len, ctx);
> >  }
> >  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
> >
> > @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
> >  {
> >       struct vring_virtqueue *vq = to_vvq(_vq);
> >
> > -     if (vq->packed_ring)
> > -             virtqueue_disable_cb_packed(vq);
> > -     else
> > -             virtqueue_disable_cb_split(vq);
> > +     return vq->ops->disable_cb(vq);
> >  }
> >  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
> >
> > @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> >       if (vq->event_triggered)
> >               vq->event_triggered = false;
> >
> > -     return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> > -                              virtqueue_enable_cb_prepare_split(vq);
> > +     return vq->ops->enable_cb_prepare(vq);
> >  }
> >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> >
> > @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
> >               return false;
> >
> >       virtio_mb(vq->weak_barriers);
> > -     return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> > -                              virtqueue_poll_split(vq, last_used_idx);
> > +     return vq->ops->poll(vq, last_used_idx);
> >  }
> >  EXPORT_SYMBOL_GPL(virtqueue_poll);
> >
> > @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
> >       if (vq->event_triggered)
> >               vq->event_triggered = false;
> >
> > -     return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> > -                              virtqueue_enable_cb_delayed_split(vq);
> > +     return vq->ops->enable_cb_delayed(vq);
> >  }
> >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
> >
> > @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
> >  {
> >       struct vring_virtqueue *vq = to_vvq(_vq);
> >
> > -     return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> > -                              virtqueue_detach_unused_buf_split(vq);
> > +     return vq->ops->detach_unused_buf(vq);
> >  }
> >  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
> >
> >  static inline bool more_used(const struct vring_virtqueue *vq)
> >  {
> > -     return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> > +     return vq->ops->more_used(vq);
> >  }
> >
> >  /**
> > @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
> >       if (recycle_done)
> >               recycle_done(_vq);
> >
> > -     if (vq->packed_ring)
> > -             err = virtqueue_resize_packed(vq, num);
> > -     else
> > -             err = virtqueue_resize_split(vq, num);
> > +     err = vq->ops->resize(vq, num);
> >
> >       return virtqueue_enable_after_reset(_vq);
> >  }
> > @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
> >       if (recycle_done)
> >               recycle_done(_vq);
> >
> > -     if (vq->packed_ring)
> > -             virtqueue_reset_packed(vq);
> > -     else
> > -             virtqueue_reset_split(vq);
> > +     vq->ops->reset(vq);
> >
> >       return virtqueue_enable_after_reset(_vq);
> >  }
> > --
> > 2.42.0
>

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Michael S. Tsirkin]

On Tue, Apr 08, 2025 at 03:02:35PM +0800, Jason Wang wrote:
> On Mon, Apr 7, 2025 at 4:20 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> >
> > On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> > > This patch introduces virtqueue ops which is a set of the callbacks
> > > that will be called for different queue layout or features. This would
> > > help to avoid branches for split/packed and will ease the future
> > > implementation like in order.
> > >
> > > Signed-off-by: Jason Wang <jasowang@redhat.com>
> >
> >
> >
> >
> > > ---
> > >  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
> > >  1 file changed, 67 insertions(+), 29 deletions(-)
> > >
> > > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> > > index a2884eae14d9..ce1dc90ee89d 100644
> > > --- a/drivers/virtio/virtio_ring.c
> > > +++ b/drivers/virtio/virtio_ring.c
> > > @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
> > >       size_t event_size_in_bytes;
> > >  };
> > >
> > > +struct vring_virtqueue;
> > > +
> > > +struct virtqueue_ops {
> > > +     int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> > > +                unsigned int total_sg, unsigned int out_sgs,
> > > +                unsigned int in_sgs, void *data,
> > > +                void *ctx, bool premapped, gfp_t gfp);
> > > +     void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> > > +     bool (*kick_prepare)(struct vring_virtqueue *vq);
> > > +     void (*disable_cb)(struct vring_virtqueue *vq);
> > > +     bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> > > +     unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> > > +     bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> > > +     void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> > > +     bool (*more_used)(const struct vring_virtqueue *vq);
> > > +     int (*resize)(struct vring_virtqueue *vq, u32 num);
> > > +     void (*reset)(struct vring_virtqueue *vq);
> > > +};
> >
> > I like it that it's organized but
> > I worry about the overhead of indirect calls here.
> 
> We can switch to use INDIRECT_CALL_X() here

If you think it's cleaner.. but INDIRECT_CALL is all chained
while a switch can do a binary search.


> (but I'm not sure we
> should worry about it too much as ndo_ops or qdiscs doesn't use that).


And that's why we ended up with xdp, no? the stack's too heavy ...

> > How about a switch statement instead?
> >
> > struct vring_virtqueue {
> >         enum vring_virtqueue_ops ops;
> >
> > }
> >
> >
> > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> >  {
> >       struct vring_virtqueue *vq = to_vvq(_vq);
> >
> >         switch (vq->ops) {
> >          VQ_PACKED:
> >          VQ_SPLIT:
> >          VQ_IN_ORDER:
> >         }
> >
> >
> > }
> >
> >
> > What do you think?
> 
> Actually, the matrix will be 2*2:
> 
> PACKED, SPLIT, PACKED_IN_ORDER, SPLIT_IN_ORDER

Confused. Same amount of enums as ops structures in your approach, no?


> And will be doubled if a new layout is implemented.
> 
> If we open them such a switch will spread in a lot of places in the code.
> 
> Thanks
> 
> >
> >
> >
> > > +
> > >  struct vring_virtqueue {
> > >       struct virtqueue vq;
> > >
> > > +     struct virtqueue_ops *ops;
> > > +
> > >       /* Is this a packed ring? */
> > >       bool packed_ring;
> > >
> > > @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
> > >       return 0;
> > >  }
> > >
> > > +struct virtqueue_ops split_ops;
> > > +
> > >  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > >                                              struct vring_virtqueue_split *vring_split,
> > >                                              struct virtio_device *vdev,
> > > @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > >               return NULL;
> > >
> > >       vq->packed_ring = false;
> > > +     vq->ops = &split_ops;
> > >       vq->vq.callback = callback;
> > >       vq->vq.vdev = vdev;
> > >       vq->vq.name = name;
> > > @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
> > >       virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
> > >  }
> > >
> > > +struct virtqueue_ops packed_ops;
> > > +
> > >  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > >                                              struct vring_virtqueue_packed *vring_packed,
> > >                                              struct virtio_device *vdev,
> > > @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > >       vq->broken = false;
> > >  #endif
> > >       vq->packed_ring = true;
> > > +     vq->ops = &packed_ops;
> > >       vq->dma_dev = dma_dev;
> > >       vq->use_dma_api = vring_use_dma_api(vdev);
> > >
> > > @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
> > >       return -ENOMEM;
> > >  }
> > >
> > > +struct virtqueue_ops split_ops = {
> > > +     .add = virtqueue_add_split,
> > > +     .get = virtqueue_get_buf_ctx_split,
> > > +     .kick_prepare = virtqueue_kick_prepare_split,
> > > +     .disable_cb = virtqueue_disable_cb_split,
> > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> > > +     .poll = virtqueue_poll_split,
> > > +     .detach_unused_buf = virtqueue_detach_unused_buf_split,
> > > +     .more_used = more_used_split,
> > > +     .resize = virtqueue_resize_split,
> > > +     .reset = virtqueue_reset_split,
> > > +};
> > > +
> > > +struct virtqueue_ops packed_ops = {
> > > +     .add = virtqueue_add_packed,
> > > +     .get = virtqueue_get_buf_ctx_packed,
> > > +     .kick_prepare = virtqueue_kick_prepare_packed,
> > > +     .disable_cb = virtqueue_disable_cb_packed,
> > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> > > +     .poll = virtqueue_poll_packed,
> > > +     .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> > > +     .more_used = more_used_packed,
> > > +     .resize = virtqueue_resize_packed,
> > > +     .reset = virtqueue_reset_packed,
> > > +};
> > > +
> > >  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
> > >                                        void (*recycle)(struct virtqueue *vq, void *buf))
> > >  {
> > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > >  {
> > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > >
> > > -     return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp) :
> > > -                              virtqueue_add_split(vq, sgs, total_sg,
> > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > +     return vq->ops->add(vq, sgs, total_sg,
> > > +                         out_sgs, in_sgs, data, ctx, premapped, gfp);
> > >  }
> > >
> > >  /**
> > > @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
> > >  {
> > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > >
> > > -     return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> > > -                              virtqueue_kick_prepare_split(vq);
> > > +     return vq->ops->kick_prepare(vq);
> > >  }
> > >  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
> > >
> > > @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
> > >  {
> > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > >
> > > -     return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> > > -                              virtqueue_get_buf_ctx_split(vq, len, ctx);
> > > +     return vq->ops->get(vq, len, ctx);
> > >  }
> > >  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
> > >
> > > @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
> > >  {
> > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > >
> > > -     if (vq->packed_ring)
> > > -             virtqueue_disable_cb_packed(vq);
> > > -     else
> > > -             virtqueue_disable_cb_split(vq);
> > > +     return vq->ops->disable_cb(vq);
> > >  }
> > >  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
> > >
> > > @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> > >       if (vq->event_triggered)
> > >               vq->event_triggered = false;
> > >
> > > -     return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> > > -                              virtqueue_enable_cb_prepare_split(vq);
> > > +     return vq->ops->enable_cb_prepare(vq);
> > >  }
> > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> > >
> > > @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
> > >               return false;
> > >
> > >       virtio_mb(vq->weak_barriers);
> > > -     return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> > > -                              virtqueue_poll_split(vq, last_used_idx);
> > > +     return vq->ops->poll(vq, last_used_idx);
> > >  }
> > >  EXPORT_SYMBOL_GPL(virtqueue_poll);
> > >
> > > @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
> > >       if (vq->event_triggered)
> > >               vq->event_triggered = false;
> > >
> > > -     return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> > > -                              virtqueue_enable_cb_delayed_split(vq);
> > > +     return vq->ops->enable_cb_delayed(vq);
> > >  }
> > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
> > >
> > > @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
> > >  {
> > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > >
> > > -     return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> > > -                              virtqueue_detach_unused_buf_split(vq);
> > > +     return vq->ops->detach_unused_buf(vq);
> > >  }
> > >  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
> > >
> > >  static inline bool more_used(const struct vring_virtqueue *vq)
> > >  {
> > > -     return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> > > +     return vq->ops->more_used(vq);
> > >  }
> > >
> > >  /**
> > > @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
> > >       if (recycle_done)
> > >               recycle_done(_vq);
> > >
> > > -     if (vq->packed_ring)
> > > -             err = virtqueue_resize_packed(vq, num);
> > > -     else
> > > -             err = virtqueue_resize_split(vq, num);
> > > +     err = vq->ops->resize(vq, num);
> > >
> > >       return virtqueue_enable_after_reset(_vq);
> > >  }
> > > @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
> > >       if (recycle_done)
> > >               recycle_done(_vq);
> > >
> > > -     if (vq->packed_ring)
> > > -             virtqueue_reset_packed(vq);
> > > -     else
> > > -             virtqueue_reset_split(vq);
> > > +     vq->ops->reset(vq);
> > >
> > >       return virtqueue_enable_after_reset(_vq);
> > >  }
> > > --
> > > 2.42.0
> >

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Jason Wang]

On Tue, Apr 8, 2025 at 7:37 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Tue, Apr 08, 2025 at 03:02:35PM +0800, Jason Wang wrote:
> > On Mon, Apr 7, 2025 at 4:20 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > >
> > > On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> > > > This patch introduces virtqueue ops which is a set of the callbacks
> > > > that will be called for different queue layout or features. This would
> > > > help to avoid branches for split/packed and will ease the future
> > > > implementation like in order.
> > > >
> > > > Signed-off-by: Jason Wang <jasowang@redhat.com>
> > >
> > >
> > >
> > >
> > > > ---
> > > >  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
> > > >  1 file changed, 67 insertions(+), 29 deletions(-)
> > > >
> > > > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> > > > index a2884eae14d9..ce1dc90ee89d 100644
> > > > --- a/drivers/virtio/virtio_ring.c
> > > > +++ b/drivers/virtio/virtio_ring.c
> > > > @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
> > > >       size_t event_size_in_bytes;
> > > >  };
> > > >
> > > > +struct vring_virtqueue;
> > > > +
> > > > +struct virtqueue_ops {
> > > > +     int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> > > > +                unsigned int total_sg, unsigned int out_sgs,
> > > > +                unsigned int in_sgs, void *data,
> > > > +                void *ctx, bool premapped, gfp_t gfp);
> > > > +     void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> > > > +     bool (*kick_prepare)(struct vring_virtqueue *vq);
> > > > +     void (*disable_cb)(struct vring_virtqueue *vq);
> > > > +     bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> > > > +     unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> > > > +     bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> > > > +     void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> > > > +     bool (*more_used)(const struct vring_virtqueue *vq);
> > > > +     int (*resize)(struct vring_virtqueue *vq, u32 num);
> > > > +     void (*reset)(struct vring_virtqueue *vq);
> > > > +};
> > >
> > > I like it that it's organized but
> > > I worry about the overhead of indirect calls here.
> >
> > We can switch to use INDIRECT_CALL_X() here
>
> If you think it's cleaner.. but INDIRECT_CALL is all chained

Yes, and it would be problematic as the number of ops increased.

> while a switch can do a binary search.
>

Do you mean a nested switch?

>
> > (but I'm not sure we
> > should worry about it too much as ndo_ops or qdiscs doesn't use that).
>
>
> And that's why we ended up with xdp, no? the stack's too heavy ...
>
> > > How about a switch statement instead?
> > >
> > > struct vring_virtqueue {
> > >         enum vring_virtqueue_ops ops;
> > >
> > > }
> > >
> > >
> > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > >  {
> > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > >
> > >         switch (vq->ops) {
> > >          VQ_PACKED:
> > >          VQ_SPLIT:
> > >          VQ_IN_ORDER:
> > >         }
> > >
> > >
> > > }
> > >
> > >
> > > What do you think?
> >
> > Actually, the matrix will be 2*2:
> >
> > PACKED, SPLIT, PACKED_IN_ORDER, SPLIT_IN_ORDER
>
> Confused. Same amount of enums as ops structures in your approach, no?

I meant in this series, we will have 4 ops not 3.

>
>
> > And will be doubled if a new layout is implemented.
> >
> > If we open them such a switch will spread in a lot of places in the code.
> >
> > Thanks

Thanks

> >
> > >
> > >
> > >
> > > > +
> > > >  struct vring_virtqueue {
> > > >       struct virtqueue vq;
> > > >
> > > > +     struct virtqueue_ops *ops;
> > > > +
> > > >       /* Is this a packed ring? */
> > > >       bool packed_ring;
> > > >
> > > > @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
> > > >       return 0;
> > > >  }
> > > >
> > > > +struct virtqueue_ops split_ops;
> > > > +
> > > >  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > >                                              struct vring_virtqueue_split *vring_split,
> > > >                                              struct virtio_device *vdev,
> > > > @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > >               return NULL;
> > > >
> > > >       vq->packed_ring = false;
> > > > +     vq->ops = &split_ops;
> > > >       vq->vq.callback = callback;
> > > >       vq->vq.vdev = vdev;
> > > >       vq->vq.name = name;
> > > > @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
> > > >       virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
> > > >  }
> > > >
> > > > +struct virtqueue_ops packed_ops;
> > > > +
> > > >  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > >                                              struct vring_virtqueue_packed *vring_packed,
> > > >                                              struct virtio_device *vdev,
> > > > @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > >       vq->broken = false;
> > > >  #endif
> > > >       vq->packed_ring = true;
> > > > +     vq->ops = &packed_ops;
> > > >       vq->dma_dev = dma_dev;
> > > >       vq->use_dma_api = vring_use_dma_api(vdev);
> > > >
> > > > @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
> > > >       return -ENOMEM;
> > > >  }
> > > >
> > > > +struct virtqueue_ops split_ops = {
> > > > +     .add = virtqueue_add_split,
> > > > +     .get = virtqueue_get_buf_ctx_split,
> > > > +     .kick_prepare = virtqueue_kick_prepare_split,
> > > > +     .disable_cb = virtqueue_disable_cb_split,
> > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> > > > +     .poll = virtqueue_poll_split,
> > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_split,
> > > > +     .more_used = more_used_split,
> > > > +     .resize = virtqueue_resize_split,
> > > > +     .reset = virtqueue_reset_split,
> > > > +};
> > > > +
> > > > +struct virtqueue_ops packed_ops = {
> > > > +     .add = virtqueue_add_packed,
> > > > +     .get = virtqueue_get_buf_ctx_packed,
> > > > +     .kick_prepare = virtqueue_kick_prepare_packed,
> > > > +     .disable_cb = virtqueue_disable_cb_packed,
> > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> > > > +     .poll = virtqueue_poll_packed,
> > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> > > > +     .more_used = more_used_packed,
> > > > +     .resize = virtqueue_resize_packed,
> > > > +     .reset = virtqueue_reset_packed,
> > > > +};
> > > > +
> > > >  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
> > > >                                        void (*recycle)(struct virtqueue *vq, void *buf))
> > > >  {
> > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > >  {
> > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > >
> > > > -     return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp) :
> > > > -                              virtqueue_add_split(vq, sgs, total_sg,
> > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > +     return vq->ops->add(vq, sgs, total_sg,
> > > > +                         out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > >  }
> > > >
> > > >  /**
> > > > @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
> > > >  {
> > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > >
> > > > -     return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> > > > -                              virtqueue_kick_prepare_split(vq);
> > > > +     return vq->ops->kick_prepare(vq);
> > > >  }
> > > >  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
> > > >
> > > > @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
> > > >  {
> > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > >
> > > > -     return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> > > > -                              virtqueue_get_buf_ctx_split(vq, len, ctx);
> > > > +     return vq->ops->get(vq, len, ctx);
> > > >  }
> > > >  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
> > > >
> > > > @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
> > > >  {
> > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > >
> > > > -     if (vq->packed_ring)
> > > > -             virtqueue_disable_cb_packed(vq);
> > > > -     else
> > > > -             virtqueue_disable_cb_split(vq);
> > > > +     return vq->ops->disable_cb(vq);
> > > >  }
> > > >  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
> > > >
> > > > @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> > > >       if (vq->event_triggered)
> > > >               vq->event_triggered = false;
> > > >
> > > > -     return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> > > > -                              virtqueue_enable_cb_prepare_split(vq);
> > > > +     return vq->ops->enable_cb_prepare(vq);
> > > >  }
> > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> > > >
> > > > @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
> > > >               return false;
> > > >
> > > >       virtio_mb(vq->weak_barriers);
> > > > -     return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> > > > -                              virtqueue_poll_split(vq, last_used_idx);
> > > > +     return vq->ops->poll(vq, last_used_idx);
> > > >  }
> > > >  EXPORT_SYMBOL_GPL(virtqueue_poll);
> > > >
> > > > @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
> > > >       if (vq->event_triggered)
> > > >               vq->event_triggered = false;
> > > >
> > > > -     return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> > > > -                              virtqueue_enable_cb_delayed_split(vq);
> > > > +     return vq->ops->enable_cb_delayed(vq);
> > > >  }
> > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
> > > >
> > > > @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
> > > >  {
> > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > >
> > > > -     return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> > > > -                              virtqueue_detach_unused_buf_split(vq);
> > > > +     return vq->ops->detach_unused_buf(vq);
> > > >  }
> > > >  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
> > > >
> > > >  static inline bool more_used(const struct vring_virtqueue *vq)
> > > >  {
> > > > -     return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> > > > +     return vq->ops->more_used(vq);
> > > >  }
> > > >
> > > >  /**
> > > > @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
> > > >       if (recycle_done)
> > > >               recycle_done(_vq);
> > > >
> > > > -     if (vq->packed_ring)
> > > > -             err = virtqueue_resize_packed(vq, num);
> > > > -     else
> > > > -             err = virtqueue_resize_split(vq, num);
> > > > +     err = vq->ops->resize(vq, num);
> > > >
> > > >       return virtqueue_enable_after_reset(_vq);
> > > >  }
> > > > @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
> > > >       if (recycle_done)
> > > >               recycle_done(_vq);
> > > >
> > > > -     if (vq->packed_ring)
> > > > -             virtqueue_reset_packed(vq);
> > > > -     else
> > > > -             virtqueue_reset_split(vq);
> > > > +     vq->ops->reset(vq);
> > > >
> > > >       return virtqueue_enable_after_reset(_vq);
> > > >  }
> > > > --
> > > > 2.42.0
> > >
>

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Michael S. Tsirkin]

On Wed, Apr 09, 2025 at 12:06:03PM +0800, Jason Wang wrote:
> On Tue, Apr 8, 2025 at 7:37 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> >
> > On Tue, Apr 08, 2025 at 03:02:35PM +0800, Jason Wang wrote:
> > > On Mon, Apr 7, 2025 at 4:20 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > > >
> > > > On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> > > > > This patch introduces virtqueue ops which is a set of the callbacks
> > > > > that will be called for different queue layout or features. This would
> > > > > help to avoid branches for split/packed and will ease the future
> > > > > implementation like in order.
> > > > >
> > > > > Signed-off-by: Jason Wang <jasowang@redhat.com>
> > > >
> > > >
> > > >
> > > >
> > > > > ---
> > > > >  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
> > > > >  1 file changed, 67 insertions(+), 29 deletions(-)
> > > > >
> > > > > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> > > > > index a2884eae14d9..ce1dc90ee89d 100644
> > > > > --- a/drivers/virtio/virtio_ring.c
> > > > > +++ b/drivers/virtio/virtio_ring.c
> > > > > @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
> > > > >       size_t event_size_in_bytes;
> > > > >  };
> > > > >
> > > > > +struct vring_virtqueue;
> > > > > +
> > > > > +struct virtqueue_ops {
> > > > > +     int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> > > > > +                unsigned int total_sg, unsigned int out_sgs,
> > > > > +                unsigned int in_sgs, void *data,
> > > > > +                void *ctx, bool premapped, gfp_t gfp);
> > > > > +     void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> > > > > +     bool (*kick_prepare)(struct vring_virtqueue *vq);
> > > > > +     void (*disable_cb)(struct vring_virtqueue *vq);
> > > > > +     bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> > > > > +     unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> > > > > +     bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> > > > > +     void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> > > > > +     bool (*more_used)(const struct vring_virtqueue *vq);
> > > > > +     int (*resize)(struct vring_virtqueue *vq, u32 num);
> > > > > +     void (*reset)(struct vring_virtqueue *vq);
> > > > > +};
> > > >
> > > > I like it that it's organized but
> > > > I worry about the overhead of indirect calls here.
> > >
> > > We can switch to use INDIRECT_CALL_X() here
> >
> > If you think it's cleaner.. but INDIRECT_CALL is all chained
> 
> Yes, and it would be problematic as the number of ops increased.
> 
> > while a switch can do a binary search.
> >
> 
> Do you mean a nested switch?

Not sure what is nested. gcc does a decent job of optimizing
switches. You have 4 types of ops:
packed/packed in order/split/split in order

So:

enum {
	VQ_SPLIT,
	VQ_SPLIT_IN_ORDER,
	VQ_PACKED,
	VQ_PACKED_IN_ORDER,
}


I do not see how it is worse?









> >
> > > (but I'm not sure we
> > > should worry about it too much as ndo_ops or qdiscs doesn't use that).
> >
> >
> > And that's why we ended up with xdp, no? the stack's too heavy ...
> >
> > > > How about a switch statement instead?
> > > >
> > > > struct vring_virtqueue {
> > > >         enum vring_virtqueue_ops ops;
> > > >
> > > > }
> > > >
> > > >
> > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > >  {
> > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > >
> > > >         switch (vq->ops) {
> > > >          VQ_PACKED:
> > > >          VQ_SPLIT:
> > > >          VQ_IN_ORDER:
> > > >         }
> > > >
> > > >
> > > > }
> > > >
> > > >
> > > > What do you think?
> > >
> > > Actually, the matrix will be 2*2:
> > >
> > > PACKED, SPLIT, PACKED_IN_ORDER, SPLIT_IN_ORDER
> >
> > Confused. Same amount of enums as ops structures in your approach, no?
> 
> I meant in this series, we will have 4 ops not 3.
> 
> >
> >
> > > And will be doubled if a new layout is implemented.
> > >
> > > If we open them such a switch will spread in a lot of places in the code.
> > >
> > > Thanks
> 
> Thanks
> 
> > >
> > > >
> > > >
> > > >
> > > > > +
> > > > >  struct vring_virtqueue {
> > > > >       struct virtqueue vq;
> > > > >
> > > > > +     struct virtqueue_ops *ops;
> > > > > +
> > > > >       /* Is this a packed ring? */
> > > > >       bool packed_ring;
> > > > >
> > > > > @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
> > > > >       return 0;
> > > > >  }
> > > > >
> > > > > +struct virtqueue_ops split_ops;
> > > > > +
> > > > >  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > > >                                              struct vring_virtqueue_split *vring_split,
> > > > >                                              struct virtio_device *vdev,
> > > > > @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > > >               return NULL;
> > > > >
> > > > >       vq->packed_ring = false;
> > > > > +     vq->ops = &split_ops;
> > > > >       vq->vq.callback = callback;
> > > > >       vq->vq.vdev = vdev;
> > > > >       vq->vq.name = name;
> > > > > @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
> > > > >       virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
> > > > >  }
> > > > >
> > > > > +struct virtqueue_ops packed_ops;
> > > > > +
> > > > >  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > > >                                              struct vring_virtqueue_packed *vring_packed,
> > > > >                                              struct virtio_device *vdev,
> > > > > @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > > >       vq->broken = false;
> > > > >  #endif
> > > > >       vq->packed_ring = true;
> > > > > +     vq->ops = &packed_ops;
> > > > >       vq->dma_dev = dma_dev;
> > > > >       vq->use_dma_api = vring_use_dma_api(vdev);
> > > > >
> > > > > @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
> > > > >       return -ENOMEM;
> > > > >  }
> > > > >
> > > > > +struct virtqueue_ops split_ops = {
> > > > > +     .add = virtqueue_add_split,
> > > > > +     .get = virtqueue_get_buf_ctx_split,
> > > > > +     .kick_prepare = virtqueue_kick_prepare_split,
> > > > > +     .disable_cb = virtqueue_disable_cb_split,
> > > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> > > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> > > > > +     .poll = virtqueue_poll_split,
> > > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_split,
> > > > > +     .more_used = more_used_split,
> > > > > +     .resize = virtqueue_resize_split,
> > > > > +     .reset = virtqueue_reset_split,
> > > > > +};
> > > > > +
> > > > > +struct virtqueue_ops packed_ops = {
> > > > > +     .add = virtqueue_add_packed,
> > > > > +     .get = virtqueue_get_buf_ctx_packed,
> > > > > +     .kick_prepare = virtqueue_kick_prepare_packed,
> > > > > +     .disable_cb = virtqueue_disable_cb_packed,
> > > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> > > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> > > > > +     .poll = virtqueue_poll_packed,
> > > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> > > > > +     .more_used = more_used_packed,
> > > > > +     .resize = virtqueue_resize_packed,
> > > > > +     .reset = virtqueue_reset_packed,
> > > > > +};
> > > > > +
> > > > >  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
> > > > >                                        void (*recycle)(struct virtqueue *vq, void *buf))
> > > > >  {
> > > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > > >  {
> > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > >
> > > > > -     return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> > > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp) :
> > > > > -                              virtqueue_add_split(vq, sgs, total_sg,
> > > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > > +     return vq->ops->add(vq, sgs, total_sg,
> > > > > +                         out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > >  }
> > > > >
> > > > >  /**
> > > > > @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
> > > > >  {
> > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > >
> > > > > -     return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> > > > > -                              virtqueue_kick_prepare_split(vq);
> > > > > +     return vq->ops->kick_prepare(vq);
> > > > >  }
> > > > >  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
> > > > >
> > > > > @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
> > > > >  {
> > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > >
> > > > > -     return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> > > > > -                              virtqueue_get_buf_ctx_split(vq, len, ctx);
> > > > > +     return vq->ops->get(vq, len, ctx);
> > > > >  }
> > > > >  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
> > > > >
> > > > > @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
> > > > >  {
> > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > >
> > > > > -     if (vq->packed_ring)
> > > > > -             virtqueue_disable_cb_packed(vq);
> > > > > -     else
> > > > > -             virtqueue_disable_cb_split(vq);
> > > > > +     return vq->ops->disable_cb(vq);
> > > > >  }
> > > > >  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
> > > > >
> > > > > @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> > > > >       if (vq->event_triggered)
> > > > >               vq->event_triggered = false;
> > > > >
> > > > > -     return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> > > > > -                              virtqueue_enable_cb_prepare_split(vq);
> > > > > +     return vq->ops->enable_cb_prepare(vq);
> > > > >  }
> > > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> > > > >
> > > > > @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
> > > > >               return false;
> > > > >
> > > > >       virtio_mb(vq->weak_barriers);
> > > > > -     return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> > > > > -                              virtqueue_poll_split(vq, last_used_idx);
> > > > > +     return vq->ops->poll(vq, last_used_idx);
> > > > >  }
> > > > >  EXPORT_SYMBOL_GPL(virtqueue_poll);
> > > > >
> > > > > @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
> > > > >       if (vq->event_triggered)
> > > > >               vq->event_triggered = false;
> > > > >
> > > > > -     return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> > > > > -                              virtqueue_enable_cb_delayed_split(vq);
> > > > > +     return vq->ops->enable_cb_delayed(vq);
> > > > >  }
> > > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
> > > > >
> > > > > @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
> > > > >  {
> > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > >
> > > > > -     return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> > > > > -                              virtqueue_detach_unused_buf_split(vq);
> > > > > +     return vq->ops->detach_unused_buf(vq);
> > > > >  }
> > > > >  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
> > > > >
> > > > >  static inline bool more_used(const struct vring_virtqueue *vq)
> > > > >  {
> > > > > -     return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> > > > > +     return vq->ops->more_used(vq);
> > > > >  }
> > > > >
> > > > >  /**
> > > > > @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
> > > > >       if (recycle_done)
> > > > >               recycle_done(_vq);
> > > > >
> > > > > -     if (vq->packed_ring)
> > > > > -             err = virtqueue_resize_packed(vq, num);
> > > > > -     else
> > > > > -             err = virtqueue_resize_split(vq, num);
> > > > > +     err = vq->ops->resize(vq, num);
> > > > >
> > > > >       return virtqueue_enable_after_reset(_vq);
> > > > >  }
> > > > > @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
> > > > >       if (recycle_done)
> > > > >               recycle_done(_vq);
> > > > >
> > > > > -     if (vq->packed_ring)
> > > > > -             virtqueue_reset_packed(vq);
> > > > > -     else
> > > > > -             virtqueue_reset_split(vq);
> > > > > +     vq->ops->reset(vq);
> > > > >
> > > > >       return virtqueue_enable_after_reset(_vq);
> > > > >  }
> > > > > --
> > > > > 2.42.0
> > > >
> >

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Michael S. Tsirkin]

On Wed, May 14, 2025 at 10:19:05AM -0400, Michael S. Tsirkin wrote:
> On Wed, Apr 09, 2025 at 12:06:03PM +0800, Jason Wang wrote:
> > On Tue, Apr 8, 2025 at 7:37 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > >
> > > On Tue, Apr 08, 2025 at 03:02:35PM +0800, Jason Wang wrote:
> > > > On Mon, Apr 7, 2025 at 4:20 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > > > >
> > > > > On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> > > > > > This patch introduces virtqueue ops which is a set of the callbacks
> > > > > > that will be called for different queue layout or features. This would
> > > > > > help to avoid branches for split/packed and will ease the future
> > > > > > implementation like in order.
> > > > > >
> > > > > > Signed-off-by: Jason Wang <jasowang@redhat.com>
> > > > >
> > > > >
> > > > >
> > > > >
> > > > > > ---
> > > > > >  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
> > > > > >  1 file changed, 67 insertions(+), 29 deletions(-)
> > > > > >
> > > > > > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> > > > > > index a2884eae14d9..ce1dc90ee89d 100644
> > > > > > --- a/drivers/virtio/virtio_ring.c
> > > > > > +++ b/drivers/virtio/virtio_ring.c
> > > > > > @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
> > > > > >       size_t event_size_in_bytes;
> > > > > >  };
> > > > > >
> > > > > > +struct vring_virtqueue;
> > > > > > +
> > > > > > +struct virtqueue_ops {
> > > > > > +     int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> > > > > > +                unsigned int total_sg, unsigned int out_sgs,
> > > > > > +                unsigned int in_sgs, void *data,
> > > > > > +                void *ctx, bool premapped, gfp_t gfp);
> > > > > > +     void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> > > > > > +     bool (*kick_prepare)(struct vring_virtqueue *vq);
> > > > > > +     void (*disable_cb)(struct vring_virtqueue *vq);
> > > > > > +     bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> > > > > > +     unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> > > > > > +     bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> > > > > > +     void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> > > > > > +     bool (*more_used)(const struct vring_virtqueue *vq);
> > > > > > +     int (*resize)(struct vring_virtqueue *vq, u32 num);
> > > > > > +     void (*reset)(struct vring_virtqueue *vq);
> > > > > > +};
> > > > >
> > > > > I like it that it's organized but
> > > > > I worry about the overhead of indirect calls here.
> > > >
> > > > We can switch to use INDIRECT_CALL_X() here
> > >
> > > If you think it's cleaner.. but INDIRECT_CALL is all chained
> > 
> > Yes, and it would be problematic as the number of ops increased.
> > 
> > > while a switch can do a binary search.
> > >
> > 
> > Do you mean a nested switch?
> 
> Not sure what is nested. gcc does a decent job of optimizing
> switches. You have 4 types of ops:
> packed/packed in order/split/split in order
> 
> So:
> 
> enum {
> 	VQ_SPLIT,
> 	VQ_SPLIT_IN_ORDER,
> 	VQ_PACKED,
> 	VQ_PACKED_IN_ORDER,
> }
> 
> 
> I do not see how it is worse?
> 
> 



Actually, here is an idea - create an array of ops:



enum vqtype {
        SPLIT,
        SPLIT_IN_ORDER,
        PACKED,
        PACKED_IN_ORDER,
        MAX
};


struct ops {
        int (*add)(int bar);
};

extern int packed(int);
extern int packedinorder(int);
extern int split(int);
extern int splitinorder(int);

const struct ops allops[MAX] = { [SPLIT] = {split}, [SPLIT_IN_ORDER] = { splitinorder}, [PACKED] = { packed }, [PACKED_IN_ORDER] = {packedinorder}};

int main(int argc, char **argv)
{
        switch (argc) {
                case 0:
			return allops[PACKED].foo(argc);
                case 1:
			return allops[SPLIT].foo(argc);
                default:
			return allops[PACKED_IN_ORDER].foo(argc);
        }
}


I tested this and compiler is able to elide the indirect calls.




















> 
> 
> 
> 
> 
> 
> > >
> > > > (but I'm not sure we
> > > > should worry about it too much as ndo_ops or qdiscs doesn't use that).
> > >
> > >
> > > And that's why we ended up with xdp, no? the stack's too heavy ...
> > >
> > > > > How about a switch statement instead?
> > > > >
> > > > > struct vring_virtqueue {
> > > > >         enum vring_virtqueue_ops ops;
> > > > >
> > > > > }
> > > > >
> > > > >
> > > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > > >  {
> > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > >
> > > > >         switch (vq->ops) {
> > > > >          VQ_PACKED:
> > > > >          VQ_SPLIT:
> > > > >          VQ_IN_ORDER:
> > > > >         }
> > > > >
> > > > >
> > > > > }
> > > > >
> > > > >
> > > > > What do you think?
> > > >
> > > > Actually, the matrix will be 2*2:
> > > >
> > > > PACKED, SPLIT, PACKED_IN_ORDER, SPLIT_IN_ORDER
> > >
> > > Confused. Same amount of enums as ops structures in your approach, no?
> > 
> > I meant in this series, we will have 4 ops not 3.
> > 
> > >
> > >
> > > > And will be doubled if a new layout is implemented.
> > > >
> > > > If we open them such a switch will spread in a lot of places in the code.
> > > >
> > > > Thanks
> > 
> > Thanks
> > 
> > > >
> > > > >
> > > > >
> > > > >
> > > > > > +
> > > > > >  struct vring_virtqueue {
> > > > > >       struct virtqueue vq;
> > > > > >
> > > > > > +     struct virtqueue_ops *ops;
> > > > > > +
> > > > > >       /* Is this a packed ring? */
> > > > > >       bool packed_ring;
> > > > > >
> > > > > > @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
> > > > > >       return 0;
> > > > > >  }
> > > > > >
> > > > > > +struct virtqueue_ops split_ops;
> > > > > > +
> > > > > >  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > > > >                                              struct vring_virtqueue_split *vring_split,
> > > > > >                                              struct virtio_device *vdev,
> > > > > > @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > > > >               return NULL;
> > > > > >
> > > > > >       vq->packed_ring = false;
> > > > > > +     vq->ops = &split_ops;
> > > > > >       vq->vq.callback = callback;
> > > > > >       vq->vq.vdev = vdev;
> > > > > >       vq->vq.name = name;
> > > > > > @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
> > > > > >       virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
> > > > > >  }
> > > > > >
> > > > > > +struct virtqueue_ops packed_ops;
> > > > > > +
> > > > > >  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > > > >                                              struct vring_virtqueue_packed *vring_packed,
> > > > > >                                              struct virtio_device *vdev,
> > > > > > @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > > > >       vq->broken = false;
> > > > > >  #endif
> > > > > >       vq->packed_ring = true;
> > > > > > +     vq->ops = &packed_ops;
> > > > > >       vq->dma_dev = dma_dev;
> > > > > >       vq->use_dma_api = vring_use_dma_api(vdev);
> > > > > >
> > > > > > @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
> > > > > >       return -ENOMEM;
> > > > > >  }
> > > > > >
> > > > > > +struct virtqueue_ops split_ops = {
> > > > > > +     .add = virtqueue_add_split,
> > > > > > +     .get = virtqueue_get_buf_ctx_split,
> > > > > > +     .kick_prepare = virtqueue_kick_prepare_split,
> > > > > > +     .disable_cb = virtqueue_disable_cb_split,
> > > > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> > > > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> > > > > > +     .poll = virtqueue_poll_split,
> > > > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_split,
> > > > > > +     .more_used = more_used_split,
> > > > > > +     .resize = virtqueue_resize_split,
> > > > > > +     .reset = virtqueue_reset_split,
> > > > > > +};
> > > > > > +
> > > > > > +struct virtqueue_ops packed_ops = {
> > > > > > +     .add = virtqueue_add_packed,
> > > > > > +     .get = virtqueue_get_buf_ctx_packed,
> > > > > > +     .kick_prepare = virtqueue_kick_prepare_packed,
> > > > > > +     .disable_cb = virtqueue_disable_cb_packed,
> > > > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> > > > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> > > > > > +     .poll = virtqueue_poll_packed,
> > > > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> > > > > > +     .more_used = more_used_packed,
> > > > > > +     .resize = virtqueue_resize_packed,
> > > > > > +     .reset = virtqueue_reset_packed,
> > > > > > +};
> > > > > > +
> > > > > >  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
> > > > > >                                        void (*recycle)(struct virtqueue *vq, void *buf))
> > > > > >  {
> > > > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > > > >  {
> > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > >
> > > > > > -     return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> > > > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp) :
> > > > > > -                              virtqueue_add_split(vq, sgs, total_sg,
> > > > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > > > +     return vq->ops->add(vq, sgs, total_sg,
> > > > > > +                         out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > > >  }
> > > > > >
> > > > > >  /**
> > > > > > @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
> > > > > >  {
> > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > >
> > > > > > -     return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> > > > > > -                              virtqueue_kick_prepare_split(vq);
> > > > > > +     return vq->ops->kick_prepare(vq);
> > > > > >  }
> > > > > >  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
> > > > > >
> > > > > > @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
> > > > > >  {
> > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > >
> > > > > > -     return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> > > > > > -                              virtqueue_get_buf_ctx_split(vq, len, ctx);
> > > > > > +     return vq->ops->get(vq, len, ctx);
> > > > > >  }
> > > > > >  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
> > > > > >
> > > > > > @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
> > > > > >  {
> > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > >
> > > > > > -     if (vq->packed_ring)
> > > > > > -             virtqueue_disable_cb_packed(vq);
> > > > > > -     else
> > > > > > -             virtqueue_disable_cb_split(vq);
> > > > > > +     return vq->ops->disable_cb(vq);
> > > > > >  }
> > > > > >  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
> > > > > >
> > > > > > @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> > > > > >       if (vq->event_triggered)
> > > > > >               vq->event_triggered = false;
> > > > > >
> > > > > > -     return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> > > > > > -                              virtqueue_enable_cb_prepare_split(vq);
> > > > > > +     return vq->ops->enable_cb_prepare(vq);
> > > > > >  }
> > > > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> > > > > >
> > > > > > @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
> > > > > >               return false;
> > > > > >
> > > > > >       virtio_mb(vq->weak_barriers);
> > > > > > -     return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> > > > > > -                              virtqueue_poll_split(vq, last_used_idx);
> > > > > > +     return vq->ops->poll(vq, last_used_idx);
> > > > > >  }
> > > > > >  EXPORT_SYMBOL_GPL(virtqueue_poll);
> > > > > >
> > > > > > @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
> > > > > >       if (vq->event_triggered)
> > > > > >               vq->event_triggered = false;
> > > > > >
> > > > > > -     return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> > > > > > -                              virtqueue_enable_cb_delayed_split(vq);
> > > > > > +     return vq->ops->enable_cb_delayed(vq);
> > > > > >  }
> > > > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
> > > > > >
> > > > > > @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
> > > > > >  {
> > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > >
> > > > > > -     return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> > > > > > -                              virtqueue_detach_unused_buf_split(vq);
> > > > > > +     return vq->ops->detach_unused_buf(vq);
> > > > > >  }
> > > > > >  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
> > > > > >
> > > > > >  static inline bool more_used(const struct vring_virtqueue *vq)
> > > > > >  {
> > > > > > -     return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> > > > > > +     return vq->ops->more_used(vq);
> > > > > >  }
> > > > > >
> > > > > >  /**
> > > > > > @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
> > > > > >       if (recycle_done)
> > > > > >               recycle_done(_vq);
> > > > > >
> > > > > > -     if (vq->packed_ring)
> > > > > > -             err = virtqueue_resize_packed(vq, num);
> > > > > > -     else
> > > > > > -             err = virtqueue_resize_split(vq, num);
> > > > > > +     err = vq->ops->resize(vq, num);
> > > > > >
> > > > > >       return virtqueue_enable_after_reset(_vq);
> > > > > >  }
> > > > > > @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
> > > > > >       if (recycle_done)
> > > > > >               recycle_done(_vq);
> > > > > >
> > > > > > -     if (vq->packed_ring)
> > > > > > -             virtqueue_reset_packed(vq);
> > > > > > -     else
> > > > > > -             virtqueue_reset_split(vq);
> > > > > > +     vq->ops->reset(vq);
> > > > > >
> > > > > >       return virtqueue_enable_after_reset(_vq);
> > > > > >  }
> > > > > > --
> > > > > > 2.42.0
> > > > >
> > >

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Jason Wang]

On Wed, May 14, 2025 at 10:24 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Wed, May 14, 2025 at 10:19:05AM -0400, Michael S. Tsirkin wrote:
> > On Wed, Apr 09, 2025 at 12:06:03PM +0800, Jason Wang wrote:
> > > On Tue, Apr 8, 2025 at 7:37 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > > >
> > > > On Tue, Apr 08, 2025 at 03:02:35PM +0800, Jason Wang wrote:
> > > > > On Mon, Apr 7, 2025 at 4:20 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > > > > >
> > > > > > On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> > > > > > > This patch introduces virtqueue ops which is a set of the callbacks
> > > > > > > that will be called for different queue layout or features. This would
> > > > > > > help to avoid branches for split/packed and will ease the future
> > > > > > > implementation like in order.
> > > > > > >
> > > > > > > Signed-off-by: Jason Wang <jasowang@redhat.com>
> > > > > >
> > > > > >
> > > > > >
> > > > > >
> > > > > > > ---
> > > > > > >  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
> > > > > > >  1 file changed, 67 insertions(+), 29 deletions(-)
> > > > > > >
> > > > > > > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> > > > > > > index a2884eae14d9..ce1dc90ee89d 100644
> > > > > > > --- a/drivers/virtio/virtio_ring.c
> > > > > > > +++ b/drivers/virtio/virtio_ring.c
> > > > > > > @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
> > > > > > >       size_t event_size_in_bytes;
> > > > > > >  };
> > > > > > >
> > > > > > > +struct vring_virtqueue;
> > > > > > > +
> > > > > > > +struct virtqueue_ops {
> > > > > > > +     int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> > > > > > > +                unsigned int total_sg, unsigned int out_sgs,
> > > > > > > +                unsigned int in_sgs, void *data,
> > > > > > > +                void *ctx, bool premapped, gfp_t gfp);
> > > > > > > +     void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> > > > > > > +     bool (*kick_prepare)(struct vring_virtqueue *vq);
> > > > > > > +     void (*disable_cb)(struct vring_virtqueue *vq);
> > > > > > > +     bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> > > > > > > +     unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> > > > > > > +     bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> > > > > > > +     void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> > > > > > > +     bool (*more_used)(const struct vring_virtqueue *vq);
> > > > > > > +     int (*resize)(struct vring_virtqueue *vq, u32 num);
> > > > > > > +     void (*reset)(struct vring_virtqueue *vq);
> > > > > > > +};
> > > > > >
> > > > > > I like it that it's organized but
> > > > > > I worry about the overhead of indirect calls here.
> > > > >
> > > > > We can switch to use INDIRECT_CALL_X() here
> > > >
> > > > If you think it's cleaner.. but INDIRECT_CALL is all chained
> > >
> > > Yes, and it would be problematic as the number of ops increased.
> > >
> > > > while a switch can do a binary search.
> > > >
> > >
> > > Do you mean a nested switch?
> >
> > Not sure what is nested. gcc does a decent job of optimizing
> > switches. You have 4 types of ops:
> > packed/packed in order/split/split in order
> >
> > So:
> >
> > enum {
> >       VQ_SPLIT,
> >       VQ_SPLIT_IN_ORDER,
> >       VQ_PACKED,
> >       VQ_PACKED_IN_ORDER,
> > }
> >
> >
> > I do not see how it is worse?
> >
> >
>
>
>
> Actually, here is an idea - create an array of ops:
>
>
>
> enum vqtype {
>         SPLIT,
>         SPLIT_IN_ORDER,
>         PACKED,
>         PACKED_IN_ORDER,
>         MAX
> };
>
>
> struct ops {
>         int (*add)(int bar);
> };
>
> extern int packed(int);
> extern int packedinorder(int);
> extern int split(int);
> extern int splitinorder(int);
>
> const struct ops allops[MAX] = { [SPLIT] = {split}, [SPLIT_IN_ORDER] = { splitinorder}, [PACKED] = { packed }, [PACKED_IN_ORDER] = {packedinorder}};
>
> int main(int argc, char **argv)
> {
>         switch (argc) {
>                 case 0:
>                         return allops[PACKED].foo(argc);
>                 case 1:
>                         return allops[SPLIT].foo(argc);
>                 default:
>                         return allops[PACKED_IN_ORDER].foo(argc);

This still looks like an indirection call as we don't call the symbol
directly but need to load the function address into a register.

>         }
> }
>
>
> I tested this and compiler is able to elide the indirect calls.

I've tried the following:

struct virtqueue_ops split_ops = {
        .add = virtqueue_add_split,
        .get = virtqueue_get_buf_ctx_split,
        .kick_prepare = virtqueue_kick_prepare_split,
        .disable_cb = virtqueue_disable_cb_split,
        .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
        .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
        .poll = virtqueue_poll_split,
        .detach_unused_buf = virtqueue_detach_unused_buf_split,
        .more_used = more_used_split,
        .resize = virtqueue_resize_split,
        .reset = virtqueue_reset_split,
};

struct virtqueue_ops packed_ops = {
        .add = virtqueue_add_packed,
        .get = virtqueue_get_buf_ctx_packed,
        .kick_prepare = virtqueue_kick_prepare_packed,
        .disable_cb = virtqueue_disable_cb_packed,
        .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
        .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
        .poll = virtqueue_poll_packed,
        .detach_unused_buf = virtqueue_detach_unused_buf_packed,
        .more_used = more_used_packed,
        .resize = virtqueue_resize_packed,
        .reset = virtqueue_reset_packed,
};

const struct virtqueue_ops *all_ops[VQ_TYPE_MAX] = { [SPLIT] = &split_ops,
                                                     [PACKED] = &packed_ops};

unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
{
        struct vring_virtqueue *vq = to_vvq(_vq);

        if (vq->event_triggered)
                vq->event_triggered = false;

        switch (vq->layout) {
case SPLIT:
                return all_ops[SPLIT]->enable_cb_prepare(vq);
                break;
        case PACKED:
                return all_ops[PACKED]->enable_cb_prepare(vq);
                break;
        default:
                BUG();
                break;
        }

return -EFAULT;
}
EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);

Compilers gives me (when RETPOLINE is enabled):

ffffffff8193a870 <virtqueue_enable_cb_prepare>:
ffffffff8193a870:       f3 0f 1e fa             endbr64
ffffffff8193a874:       e8 47 68 93 ff          callq
ffffffff812710c0 <__fentry__>
ffffffff8193a879:       80 bf 8e 00 00 00 00    cmpb   $0x0,0x8e(%rdi)
ffffffff8193a880:       74 07                   je
ffffffff8193a889 <virtqueue_enable_cb_prepare+0x19>
ffffffff8193a882:       c6 87 8e 00 00 00 00    movb   $0x0,0x8e(%rdi)
ffffffff8193a889:       8b 87 80 00 00 00       mov    0x80(%rdi),%eax
ffffffff8193a88f:       85 c0                   test   %eax,%eax
ffffffff8193a891:       74 15                   je
ffffffff8193a8a8 <virtqueue_enable_cb_prepare+0x38>
ffffffff8193a893:       83 f8 01                cmp    $0x1,%eax
ffffffff8193a896:       75 20                   jne
ffffffff8193a8b8 <virtqueue_enable_cb_prepare+0x48>
ffffffff8193a898:       48 8b 05 49 03 4a 01    mov
0x14a0349(%rip),%rax        # ffffffff82ddabe8 <all_ops+0x8>
ffffffff8193a89f:       48 8b 40 28             mov    0x28(%rax),%rax
ffffffff8193a8a3:       e9 b8 d8 9b 00          jmpq
ffffffff822f8160 <__x86_indirect_thunk_array>
ffffffff8193a8a8:       48 8b 05 31 03 4a 01    mov
0x14a0331(%rip),%rax        # ffffffff82ddabe0 <all_ops>
ffffffff8193a8af:       48 8b 40 28             mov    0x28(%rax),%rax
ffffffff8193a8b3:       e9 a8 d8 9b 00          jmpq
ffffffff822f8160 <__x86_indirect_thunk_array>
ffffffff8193a8b8:       0f 0b                   ud2
ffffffff8193a8ba:       66 0f 1f 44 00 00       nopw   0x0(%rax,%rax,1)

indirection call is still being mitigated via thunk.

The way I can think so far is something like this that passess the
function symbol to the macro:

#define VIRTQUEUE_CALL(vq, split_fn, packed_fn, ...)    \
        ({                                              \
        typeof(split_fn(vq, ##__VA_ARGS__)) ret;        \
        switch ((vq)->layout) {                 \
        case SPLIT:                                     \
                ret = split_fn(vq, ##__VA_ARGS__);      \
                break;                                  \
        case PACKED:                                    \
                ret = packed_fn(vq, ##__VA_ARGS__);     \
                break;                                  \
        default:                                        \
                BUG();                                  \
                ret = (typeof(ret))-EFAULT;             \
                break;                                  \
        }                                               \
        ret;                                            \
})

Then I can't see RETPOLINE for indirect calls.

Thanks

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
> >
> >
> >
> >
> >
> >
> > > >
> > > > > (but I'm not sure we
> > > > > should worry about it too much as ndo_ops or qdiscs doesn't use that).
> > > >
> > > >
> > > > And that's why we ended up with xdp, no? the stack's too heavy ...
> > > >
> > > > > > How about a switch statement instead?
> > > > > >
> > > > > > struct vring_virtqueue {
> > > > > >         enum vring_virtqueue_ops ops;
> > > > > >
> > > > > > }
> > > > > >
> > > > > >
> > > > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > > > >  {
> > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > >
> > > > > >         switch (vq->ops) {
> > > > > >          VQ_PACKED:
> > > > > >          VQ_SPLIT:
> > > > > >          VQ_IN_ORDER:
> > > > > >         }
> > > > > >
> > > > > >
> > > > > > }
> > > > > >
> > > > > >
> > > > > > What do you think?
> > > > >
> > > > > Actually, the matrix will be 2*2:
> > > > >
> > > > > PACKED, SPLIT, PACKED_IN_ORDER, SPLIT_IN_ORDER
> > > >
> > > > Confused. Same amount of enums as ops structures in your approach, no?
> > >
> > > I meant in this series, we will have 4 ops not 3.
> > >
> > > >
> > > >
> > > > > And will be doubled if a new layout is implemented.
> > > > >
> > > > > If we open them such a switch will spread in a lot of places in the code.
> > > > >
> > > > > Thanks
> > >
> > > Thanks
> > >
> > > > >
> > > > > >
> > > > > >
> > > > > >
> > > > > > > +
> > > > > > >  struct vring_virtqueue {
> > > > > > >       struct virtqueue vq;
> > > > > > >
> > > > > > > +     struct virtqueue_ops *ops;
> > > > > > > +
> > > > > > >       /* Is this a packed ring? */
> > > > > > >       bool packed_ring;
> > > > > > >
> > > > > > > @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
> > > > > > >       return 0;
> > > > > > >  }
> > > > > > >
> > > > > > > +struct virtqueue_ops split_ops;
> > > > > > > +
> > > > > > >  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > > > > >                                              struct vring_virtqueue_split *vring_split,
> > > > > > >                                              struct virtio_device *vdev,
> > > > > > > @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > > > > >               return NULL;
> > > > > > >
> > > > > > >       vq->packed_ring = false;
> > > > > > > +     vq->ops = &split_ops;
> > > > > > >       vq->vq.callback = callback;
> > > > > > >       vq->vq.vdev = vdev;
> > > > > > >       vq->vq.name = name;
> > > > > > > @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
> > > > > > >       virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
> > > > > > >  }
> > > > > > >
> > > > > > > +struct virtqueue_ops packed_ops;
> > > > > > > +
> > > > > > >  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > > > > >                                              struct vring_virtqueue_packed *vring_packed,
> > > > > > >                                              struct virtio_device *vdev,
> > > > > > > @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > > > > >       vq->broken = false;
> > > > > > >  #endif
> > > > > > >       vq->packed_ring = true;
> > > > > > > +     vq->ops = &packed_ops;
> > > > > > >       vq->dma_dev = dma_dev;
> > > > > > >       vq->use_dma_api = vring_use_dma_api(vdev);
> > > > > > >
> > > > > > > @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
> > > > > > >       return -ENOMEM;
> > > > > > >  }
> > > > > > >
> > > > > > > +struct virtqueue_ops split_ops = {
> > > > > > > +     .add = virtqueue_add_split,
> > > > > > > +     .get = virtqueue_get_buf_ctx_split,
> > > > > > > +     .kick_prepare = virtqueue_kick_prepare_split,
> > > > > > > +     .disable_cb = virtqueue_disable_cb_split,
> > > > > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> > > > > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> > > > > > > +     .poll = virtqueue_poll_split,
> > > > > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_split,
> > > > > > > +     .more_used = more_used_split,
> > > > > > > +     .resize = virtqueue_resize_split,
> > > > > > > +     .reset = virtqueue_reset_split,
> > > > > > > +};
> > > > > > > +
> > > > > > > +struct virtqueue_ops packed_ops = {
> > > > > > > +     .add = virtqueue_add_packed,
> > > > > > > +     .get = virtqueue_get_buf_ctx_packed,
> > > > > > > +     .kick_prepare = virtqueue_kick_prepare_packed,
> > > > > > > +     .disable_cb = virtqueue_disable_cb_packed,
> > > > > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> > > > > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> > > > > > > +     .poll = virtqueue_poll_packed,
> > > > > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> > > > > > > +     .more_used = more_used_packed,
> > > > > > > +     .resize = virtqueue_resize_packed,
> > > > > > > +     .reset = virtqueue_reset_packed,
> > > > > > > +};
> > > > > > > +
> > > > > > >  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
> > > > > > >                                        void (*recycle)(struct virtqueue *vq, void *buf))
> > > > > > >  {
> > > > > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > > > > >  {
> > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > >
> > > > > > > -     return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> > > > > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp) :
> > > > > > > -                              virtqueue_add_split(vq, sgs, total_sg,
> > > > > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > > > > +     return vq->ops->add(vq, sgs, total_sg,
> > > > > > > +                         out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > > > >  }
> > > > > > >
> > > > > > >  /**
> > > > > > > @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
> > > > > > >  {
> > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > >
> > > > > > > -     return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> > > > > > > -                              virtqueue_kick_prepare_split(vq);
> > > > > > > +     return vq->ops->kick_prepare(vq);
> > > > > > >  }
> > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
> > > > > > >
> > > > > > > @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
> > > > > > >  {
> > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > >
> > > > > > > -     return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> > > > > > > -                              virtqueue_get_buf_ctx_split(vq, len, ctx);
> > > > > > > +     return vq->ops->get(vq, len, ctx);
> > > > > > >  }
> > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
> > > > > > >
> > > > > > > @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
> > > > > > >  {
> > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > >
> > > > > > > -     if (vq->packed_ring)
> > > > > > > -             virtqueue_disable_cb_packed(vq);
> > > > > > > -     else
> > > > > > > -             virtqueue_disable_cb_split(vq);
> > > > > > > +     return vq->ops->disable_cb(vq);
> > > > > > >  }
> > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
> > > > > > >
> > > > > > > @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> > > > > > >       if (vq->event_triggered)
> > > > > > >               vq->event_triggered = false;
> > > > > > >
> > > > > > > -     return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> > > > > > > -                              virtqueue_enable_cb_prepare_split(vq);
> > > > > > > +     return vq->ops->enable_cb_prepare(vq);
> > > > > > >  }
> > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> > > > > > >
> > > > > > > @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
> > > > > > >               return false;
> > > > > > >
> > > > > > >       virtio_mb(vq->weak_barriers);
> > > > > > > -     return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> > > > > > > -                              virtqueue_poll_split(vq, last_used_idx);
> > > > > > > +     return vq->ops->poll(vq, last_used_idx);
> > > > > > >  }
> > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_poll);
> > > > > > >
> > > > > > > @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
> > > > > > >       if (vq->event_triggered)
> > > > > > >               vq->event_triggered = false;
> > > > > > >
> > > > > > > -     return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> > > > > > > -                              virtqueue_enable_cb_delayed_split(vq);
> > > > > > > +     return vq->ops->enable_cb_delayed(vq);
> > > > > > >  }
> > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
> > > > > > >
> > > > > > > @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
> > > > > > >  {
> > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > >
> > > > > > > -     return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> > > > > > > -                              virtqueue_detach_unused_buf_split(vq);
> > > > > > > +     return vq->ops->detach_unused_buf(vq);
> > > > > > >  }
> > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
> > > > > > >
> > > > > > >  static inline bool more_used(const struct vring_virtqueue *vq)
> > > > > > >  {
> > > > > > > -     return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> > > > > > > +     return vq->ops->more_used(vq);
> > > > > > >  }
> > > > > > >
> > > > > > >  /**
> > > > > > > @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
> > > > > > >       if (recycle_done)
> > > > > > >               recycle_done(_vq);
> > > > > > >
> > > > > > > -     if (vq->packed_ring)
> > > > > > > -             err = virtqueue_resize_packed(vq, num);
> > > > > > > -     else
> > > > > > > -             err = virtqueue_resize_split(vq, num);
> > > > > > > +     err = vq->ops->resize(vq, num);
> > > > > > >
> > > > > > >       return virtqueue_enable_after_reset(_vq);
> > > > > > >  }
> > > > > > > @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
> > > > > > >       if (recycle_done)
> > > > > > >               recycle_done(_vq);
> > > > > > >
> > > > > > > -     if (vq->packed_ring)
> > > > > > > -             virtqueue_reset_packed(vq);
> > > > > > > -     else
> > > > > > > -             virtqueue_reset_split(vq);
> > > > > > > +     vq->ops->reset(vq);
> > > > > > >
> > > > > > >       return virtqueue_enable_after_reset(_vq);
> > > > > > >  }
> > > > > > > --
> > > > > > > 2.42.0
> > > > > >
> > > >
>

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Michael S. Tsirkin]

On Fri, May 16, 2025 at 09:30:01AM +0800, Jason Wang wrote:
> On Wed, May 14, 2025 at 10:24 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> >
> > On Wed, May 14, 2025 at 10:19:05AM -0400, Michael S. Tsirkin wrote:
> > > On Wed, Apr 09, 2025 at 12:06:03PM +0800, Jason Wang wrote:
> > > > On Tue, Apr 8, 2025 at 7:37 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > > > >
> > > > > On Tue, Apr 08, 2025 at 03:02:35PM +0800, Jason Wang wrote:
> > > > > > On Mon, Apr 7, 2025 at 4:20 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > > > > > >
> > > > > > > On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> > > > > > > > This patch introduces virtqueue ops which is a set of the callbacks
> > > > > > > > that will be called for different queue layout or features. This would
> > > > > > > > help to avoid branches for split/packed and will ease the future
> > > > > > > > implementation like in order.
> > > > > > > >
> > > > > > > > Signed-off-by: Jason Wang <jasowang@redhat.com>
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > > > ---
> > > > > > > >  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
> > > > > > > >  1 file changed, 67 insertions(+), 29 deletions(-)
> > > > > > > >
> > > > > > > > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> > > > > > > > index a2884eae14d9..ce1dc90ee89d 100644
> > > > > > > > --- a/drivers/virtio/virtio_ring.c
> > > > > > > > +++ b/drivers/virtio/virtio_ring.c
> > > > > > > > @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
> > > > > > > >       size_t event_size_in_bytes;
> > > > > > > >  };
> > > > > > > >
> > > > > > > > +struct vring_virtqueue;
> > > > > > > > +
> > > > > > > > +struct virtqueue_ops {
> > > > > > > > +     int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> > > > > > > > +                unsigned int total_sg, unsigned int out_sgs,
> > > > > > > > +                unsigned int in_sgs, void *data,
> > > > > > > > +                void *ctx, bool premapped, gfp_t gfp);
> > > > > > > > +     void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> > > > > > > > +     bool (*kick_prepare)(struct vring_virtqueue *vq);
> > > > > > > > +     void (*disable_cb)(struct vring_virtqueue *vq);
> > > > > > > > +     bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> > > > > > > > +     unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> > > > > > > > +     bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> > > > > > > > +     void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> > > > > > > > +     bool (*more_used)(const struct vring_virtqueue *vq);
> > > > > > > > +     int (*resize)(struct vring_virtqueue *vq, u32 num);
> > > > > > > > +     void (*reset)(struct vring_virtqueue *vq);
> > > > > > > > +};
> > > > > > >
> > > > > > > I like it that it's organized but
> > > > > > > I worry about the overhead of indirect calls here.
> > > > > >
> > > > > > We can switch to use INDIRECT_CALL_X() here
> > > > >
> > > > > If you think it's cleaner.. but INDIRECT_CALL is all chained
> > > >
> > > > Yes, and it would be problematic as the number of ops increased.
> > > >
> > > > > while a switch can do a binary search.
> > > > >
> > > >
> > > > Do you mean a nested switch?
> > >
> > > Not sure what is nested. gcc does a decent job of optimizing
> > > switches. You have 4 types of ops:
> > > packed/packed in order/split/split in order
> > >
> > > So:
> > >
> > > enum {
> > >       VQ_SPLIT,
> > >       VQ_SPLIT_IN_ORDER,
> > >       VQ_PACKED,
> > >       VQ_PACKED_IN_ORDER,
> > > }
> > >
> > >
> > > I do not see how it is worse?
> > >
> > >
> >
> >
> >
> > Actually, here is an idea - create an array of ops:
> >
> >
> >
> > enum vqtype {
> >         SPLIT,
> >         SPLIT_IN_ORDER,
> >         PACKED,
> >         PACKED_IN_ORDER,
> >         MAX
> > };
> >
> >
> > struct ops {
> >         int (*add)(int bar);
> > };
> >
> > extern int packed(int);
> > extern int packedinorder(int);
> > extern int split(int);
> > extern int splitinorder(int);
> >
> > const struct ops allops[MAX] = { [SPLIT] = {split}, [SPLIT_IN_ORDER] = { splitinorder}, [PACKED] = { packed }, [PACKED_IN_ORDER] = {packedinorder}};
> >
> > int main(int argc, char **argv)
> > {
> >         switch (argc) {
> >                 case 0:
> >                         return allops[PACKED].foo(argc);
> >                 case 1:
> >                         return allops[SPLIT].foo(argc);
> >                 default:
> >                         return allops[PACKED_IN_ORDER].foo(argc);
> 
> This still looks like an indirection call as we don't call the symbol
> directly but need to load the function address into a register.

See below.



> >         }
> > }
> >
> >
> > I tested this and compiler is able to elide the indirect calls.
> 
> I've tried the following:
> 
> struct virtqueue_ops split_ops = {
>         .add = virtqueue_add_split,
>         .get = virtqueue_get_buf_ctx_split,
>         .kick_prepare = virtqueue_kick_prepare_split,
>         .disable_cb = virtqueue_disable_cb_split,
>         .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
>         .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
>         .poll = virtqueue_poll_split,
>         .detach_unused_buf = virtqueue_detach_unused_buf_split,
>         .more_used = more_used_split,
>         .resize = virtqueue_resize_split,
>         .reset = virtqueue_reset_split,
> };
> 
> struct virtqueue_ops packed_ops = {
>         .add = virtqueue_add_packed,
>         .get = virtqueue_get_buf_ctx_packed,
>         .kick_prepare = virtqueue_kick_prepare_packed,
>         .disable_cb = virtqueue_disable_cb_packed,
>         .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
>         .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
>         .poll = virtqueue_poll_packed,
>         .detach_unused_buf = virtqueue_detach_unused_buf_packed,
>         .more_used = more_used_packed,
>         .resize = virtqueue_resize_packed,
>         .reset = virtqueue_reset_packed,
> };
> 
> const struct virtqueue_ops *all_ops[VQ_TYPE_MAX] = { [SPLIT] = &split_ops,
>                                                      [PACKED] = &packed_ops};
> 
> unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> {
>         struct vring_virtqueue *vq = to_vvq(_vq);
> 
>         if (vq->event_triggered)
>                 vq->event_triggered = false;
> 
>         switch (vq->layout) {
> case SPLIT:
>                 return all_ops[SPLIT]->enable_cb_prepare(vq);
>                 break;
>         case PACKED:
>                 return all_ops[PACKED]->enable_cb_prepare(vq);
>                 break;
>         default:
>                 BUG();
>                 break;
>         }
> 
> return -EFAULT;
> }
> EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> 
> Compilers gives me (when RETPOLINE is enabled):
> 
> ffffffff8193a870 <virtqueue_enable_cb_prepare>:
> ffffffff8193a870:       f3 0f 1e fa             endbr64
> ffffffff8193a874:       e8 47 68 93 ff          callq
> ffffffff812710c0 <__fentry__>
> ffffffff8193a879:       80 bf 8e 00 00 00 00    cmpb   $0x0,0x8e(%rdi)
> ffffffff8193a880:       74 07                   je
> ffffffff8193a889 <virtqueue_enable_cb_prepare+0x19>
> ffffffff8193a882:       c6 87 8e 00 00 00 00    movb   $0x0,0x8e(%rdi)
> ffffffff8193a889:       8b 87 80 00 00 00       mov    0x80(%rdi),%eax
> ffffffff8193a88f:       85 c0                   test   %eax,%eax
> ffffffff8193a891:       74 15                   je
> ffffffff8193a8a8 <virtqueue_enable_cb_prepare+0x38>
> ffffffff8193a893:       83 f8 01                cmp    $0x1,%eax
> ffffffff8193a896:       75 20                   jne
> ffffffff8193a8b8 <virtqueue_enable_cb_prepare+0x48>
> ffffffff8193a898:       48 8b 05 49 03 4a 01    mov
> 0x14a0349(%rip),%rax        # ffffffff82ddabe8 <all_ops+0x8>
> ffffffff8193a89f:       48 8b 40 28             mov    0x28(%rax),%rax
> ffffffff8193a8a3:       e9 b8 d8 9b 00          jmpq
> ffffffff822f8160 <__x86_indirect_thunk_array>
> ffffffff8193a8a8:       48 8b 05 31 03 4a 01    mov
> 0x14a0331(%rip),%rax        # ffffffff82ddabe0 <all_ops>
> ffffffff8193a8af:       48 8b 40 28             mov    0x28(%rax),%rax
> ffffffff8193a8b3:       e9 a8 d8 9b 00          jmpq
> ffffffff822f8160 <__x86_indirect_thunk_array>
> ffffffff8193a8b8:       0f 0b                   ud2
> ffffffff8193a8ba:       66 0f 1f 44 00 00       nopw   0x0(%rax,%rax,1)
> 
> indirection call is still being mitigated via thunk.



This is because you put the pointers there, and the pointers
are not const themselves.


const struct virtqueue_ops * const all_ops[VQ_TYPE_MAX] 

should do the trick.



Here is the example I wrote:

enum vqtype {
        SPLIT,
        SPLIT_IN_ORDER,
        PACKED,
        PACKED_IN_ORDER,
        MAX
};


struct ops {
        int (*foo)(int bar);
};

extern int packed(int);
extern int packedinorder(int);
extern int split(int);
extern int splitinorder(int);

const struct ops splitops = { split };
const struct ops packedops = { packed };
const struct ops packedinorderops = { packedinorder };
const struct ops splitinorderops = { splitinorder };

const struct ops *const allopsp[MAX] = { [SPLIT] = &splitops, [SPLIT_IN_ORDER] = &splitinorderops, [PACKED] = &packedops, [PACKED_IN_ORDER] = &packedinorderops};
const struct ops allops[MAX] = { [SPLIT] = {split}, [SPLIT_IN_ORDER] = { splitinorder}, [PACKED] = { packed }, [PACKED_IN_ORDER] = {packedinorder}};

int foo(int argc)
{
        switch (argc) {
                case 0:
                return allops[PACKED].foo(argc);
                case 1:
                return allops[SPLIT].foo(argc);
                case 2:
                return allops[PACKED_IN_ORDER].foo(argc);
                case 3:
                return allops[SPLIT_IN_ORDER].foo(argc);
                default:
                return 0;
        }
}
extern int foo(int argc);
int bar(int argc)
{
        switch (argc) {
                case 0:
                return allopsp[PACKED]->foo(argc);
                case 1:
                return allopsp[SPLIT]->foo(argc);
                case 2:
                return allopsp[PACKED_IN_ORDER]->foo(argc);
                case 3:
                return allopsp[SPLIT_IN_ORDER]->foo(argc);
                default:
                return 0;
        }
}
extern int bar(int argc);
int main(int argc, char **argv)
{
        foo(argc);
        bar(argc);
        return 0;
}



then:
$ gcc -c -O2 -ggdb main.c
$ objdump -r -ld main.o

main.o:     file format elf64-x86-64
mst@tuck:~$ objdump -r -ld main.o

main.o:     file format elf64-x86-64


Disassembly of section .text:

0000000000000000 <foo>:
foo():
/home/mst/main.c:29
   0:   83 ff 02                cmp    $0x2,%edi
   3:   74 2a                   je     2f <foo+0x2f>
   5:   7f 12                   jg     19 <foo+0x19>
   7:   85 ff                   test   %edi,%edi
   9:   74 1d                   je     28 <foo+0x28>
   b:   ff cf                   dec    %edi
   d:   75 2a                   jne    39 <foo+0x39>
/home/mst/main.c:33
   f:   bf 01 00 00 00          mov    $0x1,%edi
  14:   e9 00 00 00 00          jmp    19 <foo+0x19>
                        15: R_X86_64_PLT32      split-0x4
/home/mst/main.c:29
  19:   83 ff 03                cmp    $0x3,%edi
  1c:   75 1b                   jne    39 <foo+0x39>
/home/mst/main.c:37
  1e:   bf 03 00 00 00          mov    $0x3,%edi
  23:   e9 00 00 00 00          jmp    28 <foo+0x28>
                        24: R_X86_64_PLT32      splitinorder-0x4
/home/mst/main.c:31
  28:   31 ff                   xor    %edi,%edi
  2a:   e9 00 00 00 00          jmp    2f <foo+0x2f>
                        2b: R_X86_64_PLT32      packed-0x4
/home/mst/main.c:35
  2f:   bf 02 00 00 00          mov    $0x2,%edi
  34:   e9 00 00 00 00          jmp    39 <foo+0x39>
                        35: R_X86_64_PLT32      packedinorder-0x4
/home/mst/main.c:41
  39:   31 c0                   xor    %eax,%eax
  3b:   c3                      ret

000000000000003c <bar>:
bar():
/home/mst/main.c:43
  3c:   eb c2                   jmp    0 <foo>

Disassembly of section .text.startup:

0000000000000000 <main>:
main():
/home/mst/main.c:60
   0:   48 83 ec 18             sub    $0x18,%rsp
/home/mst/main.c:61
   4:   89 7c 24 0c             mov    %edi,0xc(%rsp)
   8:   e8 00 00 00 00          call   d <main+0xd>
                        9: R_X86_64_PLT32       foo-0x4
/home/mst/main.c:62
   d:   8b 7c 24 0c             mov    0xc(%rsp),%edi
  11:   e8 00 00 00 00          call   16 <main+0x16>
                        12: R_X86_64_PLT32      foo-0x4
/home/mst/main.c:64
  16:   31 c0                   xor    %eax,%eax
  18:   48 83 c4 18             add    $0x18,%rsp


compiler was able to figure out they are the same.



> The way I can think so far is something like this that passess the
> function symbol to the macro:
> 
> #define VIRTQUEUE_CALL(vq, split_fn, packed_fn, ...)    \
>         ({                                              \
>         typeof(split_fn(vq, ##__VA_ARGS__)) ret;        \
>         switch ((vq)->layout) {                 \
>         case SPLIT:                                     \
>                 ret = split_fn(vq, ##__VA_ARGS__);      \
>                 break;                                  \
>         case PACKED:                                    \
>                 ret = packed_fn(vq, ##__VA_ARGS__);     \
>                 break;                                  \
>         default:                                        \
>                 BUG();                                  \
>                 ret = (typeof(ret))-EFAULT;             \
>                 break;                                  \
>         }                                               \
>         ret;                                            \
> })
> 
> Then I can't see RETPOLINE for indirect calls.
> 
> Thanks

So check out my hack I think a bit cleaner?
Or really we can do e.g. two INDIRECT calls chained,
I will think if we can combine it with the array hack
maybe.






> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> >
> > >
> > >
> > >
> > >
> > >
> > >
> > > > >
> > > > > > (but I'm not sure we
> > > > > > should worry about it too much as ndo_ops or qdiscs doesn't use that).
> > > > >
> > > > >
> > > > > And that's why we ended up with xdp, no? the stack's too heavy ...
> > > > >
> > > > > > > How about a switch statement instead?
> > > > > > >
> > > > > > > struct vring_virtqueue {
> > > > > > >         enum vring_virtqueue_ops ops;
> > > > > > >
> > > > > > > }
> > > > > > >
> > > > > > >
> > > > > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > > > > >  {
> > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > >
> > > > > > >         switch (vq->ops) {
> > > > > > >          VQ_PACKED:
> > > > > > >          VQ_SPLIT:
> > > > > > >          VQ_IN_ORDER:
> > > > > > >         }
> > > > > > >
> > > > > > >
> > > > > > > }
> > > > > > >
> > > > > > >
> > > > > > > What do you think?
> > > > > >
> > > > > > Actually, the matrix will be 2*2:
> > > > > >
> > > > > > PACKED, SPLIT, PACKED_IN_ORDER, SPLIT_IN_ORDER
> > > > >
> > > > > Confused. Same amount of enums as ops structures in your approach, no?
> > > >
> > > > I meant in this series, we will have 4 ops not 3.
> > > >
> > > > >
> > > > >
> > > > > > And will be doubled if a new layout is implemented.
> > > > > >
> > > > > > If we open them such a switch will spread in a lot of places in the code.
> > > > > >
> > > > > > Thanks
> > > >
> > > > Thanks
> > > >
> > > > > >
> > > > > > >
> > > > > > >
> > > > > > >
> > > > > > > > +
> > > > > > > >  struct vring_virtqueue {
> > > > > > > >       struct virtqueue vq;
> > > > > > > >
> > > > > > > > +     struct virtqueue_ops *ops;
> > > > > > > > +
> > > > > > > >       /* Is this a packed ring? */
> > > > > > > >       bool packed_ring;
> > > > > > > >
> > > > > > > > @@ -1116,6 +1137,8 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split,
> > > > > > > >       return 0;
> > > > > > > >  }
> > > > > > > >
> > > > > > > > +struct virtqueue_ops split_ops;
> > > > > > > > +
> > > > > > > >  static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > > > > > >                                              struct vring_virtqueue_split *vring_split,
> > > > > > > >                                              struct virtio_device *vdev,
> > > > > > > > @@ -1134,6 +1157,7 @@ static struct virtqueue *__vring_new_virtqueue_split(unsigned int index,
> > > > > > > >               return NULL;
> > > > > > > >
> > > > > > > >       vq->packed_ring = false;
> > > > > > > > +     vq->ops = &split_ops;
> > > > > > > >       vq->vq.callback = callback;
> > > > > > > >       vq->vq.vdev = vdev;
> > > > > > > >       vq->vq.name = name;
> > > > > > > > @@ -2076,6 +2100,8 @@ static void virtqueue_reset_packed(struct vring_virtqueue *vq)
> > > > > > > >       virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
> > > > > > > >  }
> > > > > > > >
> > > > > > > > +struct virtqueue_ops packed_ops;
> > > > > > > > +
> > > > > > > >  static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > > > > > >                                              struct vring_virtqueue_packed *vring_packed,
> > > > > > > >                                              struct virtio_device *vdev,
> > > > > > > > @@ -2107,6 +2133,7 @@ static struct virtqueue *__vring_new_virtqueue_packed(unsigned int index,
> > > > > > > >       vq->broken = false;
> > > > > > > >  #endif
> > > > > > > >       vq->packed_ring = true;
> > > > > > > > +     vq->ops = &packed_ops;
> > > > > > > >       vq->dma_dev = dma_dev;
> > > > > > > >       vq->use_dma_api = vring_use_dma_api(vdev);
> > > > > > > >
> > > > > > > > @@ -2194,6 +2221,34 @@ static int virtqueue_resize_packed(struct vring_virtqueue *vq, u32 num)
> > > > > > > >       return -ENOMEM;
> > > > > > > >  }
> > > > > > > >
> > > > > > > > +struct virtqueue_ops split_ops = {
> > > > > > > > +     .add = virtqueue_add_split,
> > > > > > > > +     .get = virtqueue_get_buf_ctx_split,
> > > > > > > > +     .kick_prepare = virtqueue_kick_prepare_split,
> > > > > > > > +     .disable_cb = virtqueue_disable_cb_split,
> > > > > > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> > > > > > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> > > > > > > > +     .poll = virtqueue_poll_split,
> > > > > > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_split,
> > > > > > > > +     .more_used = more_used_split,
> > > > > > > > +     .resize = virtqueue_resize_split,
> > > > > > > > +     .reset = virtqueue_reset_split,
> > > > > > > > +};
> > > > > > > > +
> > > > > > > > +struct virtqueue_ops packed_ops = {
> > > > > > > > +     .add = virtqueue_add_packed,
> > > > > > > > +     .get = virtqueue_get_buf_ctx_packed,
> > > > > > > > +     .kick_prepare = virtqueue_kick_prepare_packed,
> > > > > > > > +     .disable_cb = virtqueue_disable_cb_packed,
> > > > > > > > +     .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> > > > > > > > +     .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> > > > > > > > +     .poll = virtqueue_poll_packed,
> > > > > > > > +     .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> > > > > > > > +     .more_used = more_used_packed,
> > > > > > > > +     .resize = virtqueue_resize_packed,
> > > > > > > > +     .reset = virtqueue_reset_packed,
> > > > > > > > +};
> > > > > > > > +
> > > > > > > >  static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
> > > > > > > >                                        void (*recycle)(struct virtqueue *vq, void *buf))
> > > > > > > >  {
> > > > > > > > @@ -2248,10 +2303,8 @@ static inline int virtqueue_add(struct virtqueue *_vq,
> > > > > > > >  {
> > > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > > >
> > > > > > > > -     return vq->packed_ring ? virtqueue_add_packed(vq, sgs, total_sg,
> > > > > > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp) :
> > > > > > > > -                              virtqueue_add_split(vq, sgs, total_sg,
> > > > > > > > -                                     out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > > > > > +     return vq->ops->add(vq, sgs, total_sg,
> > > > > > > > +                         out_sgs, in_sgs, data, ctx, premapped, gfp);
> > > > > > > >  }
> > > > > > > >
> > > > > > > >  /**
> > > > > > > > @@ -2437,8 +2490,7 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
> > > > > > > >  {
> > > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > > >
> > > > > > > > -     return vq->packed_ring ? virtqueue_kick_prepare_packed(vq) :
> > > > > > > > -                              virtqueue_kick_prepare_split(vq);
> > > > > > > > +     return vq->ops->kick_prepare(vq);
> > > > > > > >  }
> > > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
> > > > > > > >
> > > > > > > > @@ -2508,8 +2560,7 @@ void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
> > > > > > > >  {
> > > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > > >
> > > > > > > > -     return vq->packed_ring ? virtqueue_get_buf_ctx_packed(vq, len, ctx) :
> > > > > > > > -                              virtqueue_get_buf_ctx_split(vq, len, ctx);
> > > > > > > > +     return vq->ops->get(vq, len, ctx);
> > > > > > > >  }
> > > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
> > > > > > > >
> > > > > > > > @@ -2531,10 +2582,7 @@ void virtqueue_disable_cb(struct virtqueue *_vq)
> > > > > > > >  {
> > > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > > >
> > > > > > > > -     if (vq->packed_ring)
> > > > > > > > -             virtqueue_disable_cb_packed(vq);
> > > > > > > > -     else
> > > > > > > > -             virtqueue_disable_cb_split(vq);
> > > > > > > > +     return vq->ops->disable_cb(vq);
> > > > > > > >  }
> > > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
> > > > > > > >
> > > > > > > > @@ -2557,8 +2605,7 @@ unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> > > > > > > >       if (vq->event_triggered)
> > > > > > > >               vq->event_triggered = false;
> > > > > > > >
> > > > > > > > -     return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(vq) :
> > > > > > > > -                              virtqueue_enable_cb_prepare_split(vq);
> > > > > > > > +     return vq->ops->enable_cb_prepare(vq);
> > > > > > > >  }
> > > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> > > > > > > >
> > > > > > > > @@ -2579,8 +2626,7 @@ bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
> > > > > > > >               return false;
> > > > > > > >
> > > > > > > >       virtio_mb(vq->weak_barriers);
> > > > > > > > -     return vq->packed_ring ? virtqueue_poll_packed(vq, last_used_idx) :
> > > > > > > > -                              virtqueue_poll_split(vq, last_used_idx);
> > > > > > > > +     return vq->ops->poll(vq, last_used_idx);
> > > > > > > >  }
> > > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_poll);
> > > > > > > >
> > > > > > > > @@ -2623,8 +2669,7 @@ bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
> > > > > > > >       if (vq->event_triggered)
> > > > > > > >               vq->event_triggered = false;
> > > > > > > >
> > > > > > > > -     return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(vq) :
> > > > > > > > -                              virtqueue_enable_cb_delayed_split(vq);
> > > > > > > > +     return vq->ops->enable_cb_delayed(vq);
> > > > > > > >  }
> > > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
> > > > > > > >
> > > > > > > > @@ -2640,14 +2685,13 @@ void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
> > > > > > > >  {
> > > > > > > >       struct vring_virtqueue *vq = to_vvq(_vq);
> > > > > > > >
> > > > > > > > -     return vq->packed_ring ? virtqueue_detach_unused_buf_packed(vq) :
> > > > > > > > -                              virtqueue_detach_unused_buf_split(vq);
> > > > > > > > +     return vq->ops->detach_unused_buf(vq);
> > > > > > > >  }
> > > > > > > >  EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
> > > > > > > >
> > > > > > > >  static inline bool more_used(const struct vring_virtqueue *vq)
> > > > > > > >  {
> > > > > > > > -     return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
> > > > > > > > +     return vq->ops->more_used(vq);
> > > > > > > >  }
> > > > > > > >
> > > > > > > >  /**
> > > > > > > > @@ -2785,10 +2829,7 @@ int virtqueue_resize(struct virtqueue *_vq, u32 num,
> > > > > > > >       if (recycle_done)
> > > > > > > >               recycle_done(_vq);
> > > > > > > >
> > > > > > > > -     if (vq->packed_ring)
> > > > > > > > -             err = virtqueue_resize_packed(vq, num);
> > > > > > > > -     else
> > > > > > > > -             err = virtqueue_resize_split(vq, num);
> > > > > > > > +     err = vq->ops->resize(vq, num);
> > > > > > > >
> > > > > > > >       return virtqueue_enable_after_reset(_vq);
> > > > > > > >  }
> > > > > > > > @@ -2822,10 +2863,7 @@ int virtqueue_reset(struct virtqueue *_vq,
> > > > > > > >       if (recycle_done)
> > > > > > > >               recycle_done(_vq);
> > > > > > > >
> > > > > > > > -     if (vq->packed_ring)
> > > > > > > > -             virtqueue_reset_packed(vq);
> > > > > > > > -     else
> > > > > > > > -             virtqueue_reset_split(vq);
> > > > > > > > +     vq->ops->reset(vq);
> > > > > > > >
> > > > > > > >       return virtqueue_enable_after_reset(_vq);
> > > > > > > >  }
> > > > > > > > --
> > > > > > > > 2.42.0
> > > > > > >
> > > > >
> >

Re: [PATCH 00/19] virtio_ring in order support

[Michael S. Tsirkin]

On Wed, Mar 26, 2025 at 07:39:47AM +0100, Eugenio Perez Martin wrote:
> On Mon, Mar 24, 2025 at 3:44 PM Lei Yang <leiyang@redhat.com> wrote:
> >
> > QE tested this series of patches with virtio-net regression tests,
> > everything works fine.
> >
> 
> Hi Lei,
> 
> Is it possible to test this series also with virtio-net-pci,...,in_order=on?
> 
> Thanks!


Lei, what do you think?

> > Tested-by: Lei Yang <leiyang@redhat.com>
> >
> > On Mon, Mar 24, 2025 at 1:45 PM Jason Wang <jasowang@redhat.com> wrote:
> > >
> > > Hello all:
> > >
> > > This sereis tries to implement the VIRTIO_F_IN_ORDER to
> > > virtio_ring. This is done by introducing virtqueue ops so we can
> > > implement separate helpers for different virtqueue layout/features
> > > then the in-order were implmeented on top.
> > >
> > > Tests shows 5% imporvemnt in RX PPS with KVM guest + testpmd on the
> > > host.
> > >
> > > Please review.
> > >
> > > Thanks
> > >
> > > Jason Wang (19):
> > >   virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()
> > >   virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants
> > >   virtio_ring: unify logic of virtqueue_poll() and more_used()
> > >   virtio_ring: switch to use vring_virtqueue for virtqueue resize
> > >     variants
> > >   virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare
> > >     variants
> > >   virtio_ring: switch to use vring_virtqueue for virtqueue_add variants
> > >   virtio: switch to use vring_virtqueue for virtqueue_add variants
> > >   virtio_ring: switch to use vring_virtqueue for enable_cb_prepare
> > >     variants
> > >   virtio_ring: use vring_virtqueue for enable_cb_delayed variants
> > >   virtio_ring: switch to use vring_virtqueue for disable_cb variants
> > >   virtio_ring: switch to use vring_virtqueue for detach_unused_buf
> > >     variants
> > >   virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()
> > >   virtio_ring: introduce virtqueue ops
> > >   virtio_ring: determine descriptor flags at one time
> > >   virtio_ring: factor out core logic of buffer detaching
> > >   virtio_ring: factor out core logic for updating last_used_idx
> > >   virtio_ring: move next_avail_idx to vring_virtqueue
> > >   virtio_ring: factor out split indirect detaching logic
> > >   virtio_ring: add in order support
> > >
> > >  drivers/virtio/virtio_ring.c | 856 ++++++++++++++++++++++++++---------
> > >  1 file changed, 653 insertions(+), 203 deletions(-)
> > >
> > > --
> > > 2.42.0
> > >
> > >
> >

Re: [PATCH 13/19] virtio_ring: introduce virtqueue ops

[Jason Wang]

On Fri, May 16, 2025 at 6:35 PM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Fri, May 16, 2025 at 09:30:01AM +0800, Jason Wang wrote:
> > On Wed, May 14, 2025 at 10:24 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > >
> > > On Wed, May 14, 2025 at 10:19:05AM -0400, Michael S. Tsirkin wrote:
> > > > On Wed, Apr 09, 2025 at 12:06:03PM +0800, Jason Wang wrote:
> > > > > On Tue, Apr 8, 2025 at 7:37 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > > > > >
> > > > > > On Tue, Apr 08, 2025 at 03:02:35PM +0800, Jason Wang wrote:
> > > > > > > On Mon, Apr 7, 2025 at 4:20 PM Michael S. Tsirkin <mst@redhat.com> wrote:
> > > > > > > >
> > > > > > > > On Mon, Mar 24, 2025 at 02:01:21PM +0800, Jason Wang wrote:
> > > > > > > > > This patch introduces virtqueue ops which is a set of the callbacks
> > > > > > > > > that will be called for different queue layout or features. This would
> > > > > > > > > help to avoid branches for split/packed and will ease the future
> > > > > > > > > implementation like in order.
> > > > > > > > >
> > > > > > > > > Signed-off-by: Jason Wang <jasowang@redhat.com>
> > > > > > > >
> > > > > > > >
> > > > > > > >
> > > > > > > >
> > > > > > > > > ---
> > > > > > > > >  drivers/virtio/virtio_ring.c | 96 +++++++++++++++++++++++++-----------
> > > > > > > > >  1 file changed, 67 insertions(+), 29 deletions(-)
> > > > > > > > >
> > > > > > > > > diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
> > > > > > > > > index a2884eae14d9..ce1dc90ee89d 100644
> > > > > > > > > --- a/drivers/virtio/virtio_ring.c
> > > > > > > > > +++ b/drivers/virtio/virtio_ring.c
> > > > > > > > > @@ -159,9 +159,30 @@ struct vring_virtqueue_packed {
> > > > > > > > >       size_t event_size_in_bytes;
> > > > > > > > >  };
> > > > > > > > >
> > > > > > > > > +struct vring_virtqueue;
> > > > > > > > > +
> > > > > > > > > +struct virtqueue_ops {
> > > > > > > > > +     int (*add)(struct vring_virtqueue *_vq, struct scatterlist *sgs[],
> > > > > > > > > +                unsigned int total_sg, unsigned int out_sgs,
> > > > > > > > > +                unsigned int in_sgs, void *data,
> > > > > > > > > +                void *ctx, bool premapped, gfp_t gfp);
> > > > > > > > > +     void *(*get)(struct vring_virtqueue *vq, unsigned int *len, void **ctx);
> > > > > > > > > +     bool (*kick_prepare)(struct vring_virtqueue *vq);
> > > > > > > > > +     void (*disable_cb)(struct vring_virtqueue *vq);
> > > > > > > > > +     bool (*enable_cb_delayed)(struct vring_virtqueue *vq);
> > > > > > > > > +     unsigned int (*enable_cb_prepare)(struct vring_virtqueue *vq);
> > > > > > > > > +     bool (*poll)(const struct vring_virtqueue *vq, u16 last_used_idx);
> > > > > > > > > +     void *(*detach_unused_buf)(struct vring_virtqueue *vq);
> > > > > > > > > +     bool (*more_used)(const struct vring_virtqueue *vq);
> > > > > > > > > +     int (*resize)(struct vring_virtqueue *vq, u32 num);
> > > > > > > > > +     void (*reset)(struct vring_virtqueue *vq);
> > > > > > > > > +};
> > > > > > > >
> > > > > > > > I like it that it's organized but
> > > > > > > > I worry about the overhead of indirect calls here.
> > > > > > >
> > > > > > > We can switch to use INDIRECT_CALL_X() here
> > > > > >
> > > > > > If you think it's cleaner.. but INDIRECT_CALL is all chained
> > > > >
> > > > > Yes, and it would be problematic as the number of ops increased.
> > > > >
> > > > > > while a switch can do a binary search.
> > > > > >
> > > > >
> > > > > Do you mean a nested switch?
> > > >
> > > > Not sure what is nested. gcc does a decent job of optimizing
> > > > switches. You have 4 types of ops:
> > > > packed/packed in order/split/split in order
> > > >
> > > > So:
> > > >
> > > > enum {
> > > >       VQ_SPLIT,
> > > >       VQ_SPLIT_IN_ORDER,
> > > >       VQ_PACKED,
> > > >       VQ_PACKED_IN_ORDER,
> > > > }
> > > >
> > > >
> > > > I do not see how it is worse?
> > > >
> > > >
> > >
> > >
> > >
> > > Actually, here is an idea - create an array of ops:
> > >
> > >
> > >
> > > enum vqtype {
> > >         SPLIT,
> > >         SPLIT_IN_ORDER,
> > >         PACKED,
> > >         PACKED_IN_ORDER,
> > >         MAX
> > > };
> > >
> > >
> > > struct ops {
> > >         int (*add)(int bar);
> > > };
> > >
> > > extern int packed(int);
> > > extern int packedinorder(int);
> > > extern int split(int);
> > > extern int splitinorder(int);
> > >
> > > const struct ops allops[MAX] = { [SPLIT] = {split}, [SPLIT_IN_ORDER] = { splitinorder}, [PACKED] = { packed }, [PACKED_IN_ORDER] = {packedinorder}};
> > >
> > > int main(int argc, char **argv)
> > > {
> > >         switch (argc) {
> > >                 case 0:
> > >                         return allops[PACKED].foo(argc);
> > >                 case 1:
> > >                         return allops[SPLIT].foo(argc);
> > >                 default:
> > >                         return allops[PACKED_IN_ORDER].foo(argc);
> >
> > This still looks like an indirection call as we don't call the symbol
> > directly but need to load the function address into a register.
>
> See below.
>
>
>
> > >         }
> > > }
> > >
> > >
> > > I tested this and compiler is able to elide the indirect calls.
> >
> > I've tried the following:
> >
> > struct virtqueue_ops split_ops = {
> >         .add = virtqueue_add_split,
> >         .get = virtqueue_get_buf_ctx_split,
> >         .kick_prepare = virtqueue_kick_prepare_split,
> >         .disable_cb = virtqueue_disable_cb_split,
> >         .enable_cb_delayed = virtqueue_enable_cb_delayed_split,
> >         .enable_cb_prepare = virtqueue_enable_cb_prepare_split,
> >         .poll = virtqueue_poll_split,
> >         .detach_unused_buf = virtqueue_detach_unused_buf_split,
> >         .more_used = more_used_split,
> >         .resize = virtqueue_resize_split,
> >         .reset = virtqueue_reset_split,
> > };
> >
> > struct virtqueue_ops packed_ops = {
> >         .add = virtqueue_add_packed,
> >         .get = virtqueue_get_buf_ctx_packed,
> >         .kick_prepare = virtqueue_kick_prepare_packed,
> >         .disable_cb = virtqueue_disable_cb_packed,
> >         .enable_cb_delayed = virtqueue_enable_cb_delayed_packed,
> >         .enable_cb_prepare = virtqueue_enable_cb_prepare_packed,
> >         .poll = virtqueue_poll_packed,
> >         .detach_unused_buf = virtqueue_detach_unused_buf_packed,
> >         .more_used = more_used_packed,
> >         .resize = virtqueue_resize_packed,
> >         .reset = virtqueue_reset_packed,
> > };
> >
> > const struct virtqueue_ops *all_ops[VQ_TYPE_MAX] = { [SPLIT] = &split_ops,
> >                                                      [PACKED] = &packed_ops};
> >
> > unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
> > {
> >         struct vring_virtqueue *vq = to_vvq(_vq);
> >
> >         if (vq->event_triggered)
> >                 vq->event_triggered = false;
> >
> >         switch (vq->layout) {
> > case SPLIT:
> >                 return all_ops[SPLIT]->enable_cb_prepare(vq);
> >                 break;
> >         case PACKED:
> >                 return all_ops[PACKED]->enable_cb_prepare(vq);
> >                 break;
> >         default:
> >                 BUG();
> >                 break;
> >         }
> >
> > return -EFAULT;
> > }
> > EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
> >
> > Compilers gives me (when RETPOLINE is enabled):
> >
> > ffffffff8193a870 <virtqueue_enable_cb_prepare>:
> > ffffffff8193a870:       f3 0f 1e fa             endbr64
> > ffffffff8193a874:       e8 47 68 93 ff          callq
> > ffffffff812710c0 <__fentry__>
> > ffffffff8193a879:       80 bf 8e 00 00 00 00    cmpb   $0x0,0x8e(%rdi)
> > ffffffff8193a880:       74 07                   je
> > ffffffff8193a889 <virtqueue_enable_cb_prepare+0x19>
> > ffffffff8193a882:       c6 87 8e 00 00 00 00    movb   $0x0,0x8e(%rdi)
> > ffffffff8193a889:       8b 87 80 00 00 00       mov    0x80(%rdi),%eax
> > ffffffff8193a88f:       85 c0                   test   %eax,%eax
> > ffffffff8193a891:       74 15                   je
> > ffffffff8193a8a8 <virtqueue_enable_cb_prepare+0x38>
> > ffffffff8193a893:       83 f8 01                cmp    $0x1,%eax
> > ffffffff8193a896:       75 20                   jne
> > ffffffff8193a8b8 <virtqueue_enable_cb_prepare+0x48>
> > ffffffff8193a898:       48 8b 05 49 03 4a 01    mov
> > 0x14a0349(%rip),%rax        # ffffffff82ddabe8 <all_ops+0x8>
> > ffffffff8193a89f:       48 8b 40 28             mov    0x28(%rax),%rax
> > ffffffff8193a8a3:       e9 b8 d8 9b 00          jmpq
> > ffffffff822f8160 <__x86_indirect_thunk_array>
> > ffffffff8193a8a8:       48 8b 05 31 03 4a 01    mov
> > 0x14a0331(%rip),%rax        # ffffffff82ddabe0 <all_ops>
> > ffffffff8193a8af:       48 8b 40 28             mov    0x28(%rax),%rax
> > ffffffff8193a8b3:       e9 a8 d8 9b 00          jmpq
> > ffffffff822f8160 <__x86_indirect_thunk_array>
> > ffffffff8193a8b8:       0f 0b                   ud2
> > ffffffff8193a8ba:       66 0f 1f 44 00 00       nopw   0x0(%rax,%rax,1)
> >
> > indirection call is still being mitigated via thunk.
>
>
>
> This is because you put the pointers there, and the pointers
> are not const themselves.
>
>
> const struct virtqueue_ops * const all_ops[VQ_TYPE_MAX]
>
> should do the trick.
>
>
>
> Here is the example I wrote:
>
> enum vqtype {
>         SPLIT,
>         SPLIT_IN_ORDER,
>         PACKED,
>         PACKED_IN_ORDER,
>         MAX
> };
>
>
> struct ops {
>         int (*foo)(int bar);
> };
>
> extern int packed(int);
> extern int packedinorder(int);
> extern int split(int);
> extern int splitinorder(int);
>
> const struct ops splitops = { split };
> const struct ops packedops = { packed };
> const struct ops packedinorderops = { packedinorder };
> const struct ops splitinorderops = { splitinorder };
>
> const struct ops *const allopsp[MAX] = { [SPLIT] = &splitops, [SPLIT_IN_ORDER] = &splitinorderops, [PACKED] = &packedops, [PACKED_IN_ORDER] = &packedinorderops};
> const struct ops allops[MAX] = { [SPLIT] = {split}, [SPLIT_IN_ORDER] = { splitinorder}, [PACKED] = { packed }, [PACKED_IN_ORDER] = {packedinorder}};
>
> int foo(int argc)
> {
>         switch (argc) {
>                 case 0:
>                 return allops[PACKED].foo(argc);
>                 case 1:
>                 return allops[SPLIT].foo(argc);
>                 case 2:
>                 return allops[PACKED_IN_ORDER].foo(argc);
>                 case 3:
>                 return allops[SPLIT_IN_ORDER].foo(argc);
>                 default:
>                 return 0;
>         }
> }
> extern int foo(int argc);
> int bar(int argc)
> {
>         switch (argc) {
>                 case 0:
>                 return allopsp[PACKED]->foo(argc);
>                 case 1:
>                 return allopsp[SPLIT]->foo(argc);
>                 case 2:
>                 return allopsp[PACKED_IN_ORDER]->foo(argc);
>                 case 3:
>                 return allopsp[SPLIT_IN_ORDER]->foo(argc);
>                 default:
>                 return 0;
>         }
> }
> extern int bar(int argc);
> int main(int argc, char **argv)
> {
>         foo(argc);
>         bar(argc);
>         return 0;
> }
>
>
>
> then:
> $ gcc -c -O2 -ggdb main.c
> $ objdump -r -ld main.o
>
> main.o:     file format elf64-x86-64
> mst@tuck:~$ objdump -r -ld main.o
>
> main.o:     file format elf64-x86-64
>
>
> Disassembly of section .text:
>
> 0000000000000000 <foo>:
> foo():
> /home/mst/main.c:29
>    0:   83 ff 02                cmp    $0x2,%edi
>    3:   74 2a                   je     2f <foo+0x2f>
>    5:   7f 12                   jg     19 <foo+0x19>
>    7:   85 ff                   test   %edi,%edi
>    9:   74 1d                   je     28 <foo+0x28>
>    b:   ff cf                   dec    %edi
>    d:   75 2a                   jne    39 <foo+0x39>
> /home/mst/main.c:33
>    f:   bf 01 00 00 00          mov    $0x1,%edi
>   14:   e9 00 00 00 00          jmp    19 <foo+0x19>
>                         15: R_X86_64_PLT32      split-0x4
> /home/mst/main.c:29
>   19:   83 ff 03                cmp    $0x3,%edi
>   1c:   75 1b                   jne    39 <foo+0x39>
> /home/mst/main.c:37
>   1e:   bf 03 00 00 00          mov    $0x3,%edi
>   23:   e9 00 00 00 00          jmp    28 <foo+0x28>
>                         24: R_X86_64_PLT32      splitinorder-0x4
> /home/mst/main.c:31
>   28:   31 ff                   xor    %edi,%edi
>   2a:   e9 00 00 00 00          jmp    2f <foo+0x2f>
>                         2b: R_X86_64_PLT32      packed-0x4
> /home/mst/main.c:35
>   2f:   bf 02 00 00 00          mov    $0x2,%edi
>   34:   e9 00 00 00 00          jmp    39 <foo+0x39>
>                         35: R_X86_64_PLT32      packedinorder-0x4
> /home/mst/main.c:41
>   39:   31 c0                   xor    %eax,%eax
>   3b:   c3                      ret
>
> 000000000000003c <bar>:
> bar():
> /home/mst/main.c:43
>   3c:   eb c2                   jmp    0 <foo>
>
> Disassembly of section .text.startup:
>
> 0000000000000000 <main>:
> main():
> /home/mst/main.c:60
>    0:   48 83 ec 18             sub    $0x18,%rsp
> /home/mst/main.c:61
>    4:   89 7c 24 0c             mov    %edi,0xc(%rsp)
>    8:   e8 00 00 00 00          call   d <main+0xd>
>                         9: R_X86_64_PLT32       foo-0x4
> /home/mst/main.c:62
>    d:   8b 7c 24 0c             mov    0xc(%rsp),%edi
>   11:   e8 00 00 00 00          call   16 <main+0x16>
>                         12: R_X86_64_PLT32      foo-0x4
> /home/mst/main.c:64
>   16:   31 c0                   xor    %eax,%eax
>   18:   48 83 c4 18             add    $0x18,%rsp
>
>
> compiler was able to figure out they are the same.
>
>

Exactly, it works.

Let me redo the benchmark and post a new version.

Thanks

Re: [PATCH 00/19] virtio_ring in order support

[Lei Yang]

On Mon, May 19, 2025 at 5:35 AM Michael S. Tsirkin <mst@redhat.com> wrote:
>
> On Wed, Mar 26, 2025 at 07:39:47AM +0100, Eugenio Perez Martin wrote:
> > On Mon, Mar 24, 2025 at 3:44 PM Lei Yang <leiyang@redhat.com> wrote:
> > >
> > > QE tested this series of patches with virtio-net regression tests,
> > > everything works fine.
> > >
> >
> > Hi Lei,
> >
> > Is it possible to test this series also with virtio-net-pci,...,in_order=on?
> >
> > Thanks!
>
>
> Lei, what do you think?


Sure,  I will test it and provide test results ASAP.

Thanks
Lei
>
>
> > > Tested-by: Lei Yang <leiyang@redhat.com>
> > >
> > > On Mon, Mar 24, 2025 at 1:45 PM Jason Wang <jasowang@redhat.com> wrote:
> > > >
> > > > Hello all:
> > > >
> > > > This sereis tries to implement the VIRTIO_F_IN_ORDER to
> > > > virtio_ring. This is done by introducing virtqueue ops so we can
> > > > implement separate helpers for different virtqueue layout/features
> > > > then the in-order were implmeented on top.
> > > >
> > > > Tests shows 5% imporvemnt in RX PPS with KVM guest + testpmd on the
> > > > host.
> > > >
> > > > Please review.
> > > >
> > > > Thanks
> > > >
> > > > Jason Wang (19):
> > > >   virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()
> > > >   virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants
> > > >   virtio_ring: unify logic of virtqueue_poll() and more_used()
> > > >   virtio_ring: switch to use vring_virtqueue for virtqueue resize
> > > >     variants
> > > >   virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare
> > > >     variants
> > > >   virtio_ring: switch to use vring_virtqueue for virtqueue_add variants
> > > >   virtio: switch to use vring_virtqueue for virtqueue_add variants
> > > >   virtio_ring: switch to use vring_virtqueue for enable_cb_prepare
> > > >     variants
> > > >   virtio_ring: use vring_virtqueue for enable_cb_delayed variants
> > > >   virtio_ring: switch to use vring_virtqueue for disable_cb variants
> > > >   virtio_ring: switch to use vring_virtqueue for detach_unused_buf
> > > >     variants
> > > >   virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()
> > > >   virtio_ring: introduce virtqueue ops
> > > >   virtio_ring: determine descriptor flags at one time
> > > >   virtio_ring: factor out core logic of buffer detaching
> > > >   virtio_ring: factor out core logic for updating last_used_idx
> > > >   virtio_ring: move next_avail_idx to vring_virtqueue
> > > >   virtio_ring: factor out split indirect detaching logic
> > > >   virtio_ring: add in order support
> > > >
> > > >  drivers/virtio/virtio_ring.c | 856 ++++++++++++++++++++++++++---------
> > > >  1 file changed, 653 insertions(+), 203 deletions(-)
> > > >
> > > > --
> > > > 2.42.0
> > > >
> > > >
> > >
>

Re: [PATCH 00/19] virtio_ring in order support

[Lei Yang]

On Thu, May 22, 2025 at 3:01 PM Lei Yang <leiyang@redhat.com> wrote:
>
> On Mon, May 19, 2025 at 5:35 AM Michael S. Tsirkin <mst@redhat.com> wrote:
> >
> > On Wed, Mar 26, 2025 at 07:39:47AM +0100, Eugenio Perez Martin wrote:
> > > On Mon, Mar 24, 2025 at 3:44 PM Lei Yang <leiyang@redhat.com> wrote:
> > > >
> > > > QE tested this series of patches with virtio-net regression tests,
> > > > everything works fine.
> > > >
> > >
> > > Hi Lei,
> > >
> > > Is it possible to test this series also with virtio-net-pci,...,in_order=on?

Virtio-net regression tested pass with virtio-net-pci,...,in_order=on.

Tested-by: Lei Yang <leiyang@redhat.com>
> > >
> > > Thanks!
> >
> >
> > Lei, what do you think?
>
>
> Sure,  I will test it and provide test results ASAP.
>
> Thanks
> Lei
> >
> >
> > > > Tested-by: Lei Yang <leiyang@redhat.com>
> > > >
> > > > On Mon, Mar 24, 2025 at 1:45 PM Jason Wang <jasowang@redhat.com> wrote:
> > > > >
> > > > > Hello all:
> > > > >
> > > > > This sereis tries to implement the VIRTIO_F_IN_ORDER to
> > > > > virtio_ring. This is done by introducing virtqueue ops so we can
> > > > > implement separate helpers for different virtqueue layout/features
> > > > > then the in-order were implmeented on top.
> > > > >
> > > > > Tests shows 5% imporvemnt in RX PPS with KVM guest + testpmd on the
> > > > > host.
> > > > >
> > > > > Please review.
> > > > >
> > > > > Thanks
> > > > >
> > > > > Jason Wang (19):
> > > > >   virtio_ring: rename virtqueue_reinit_xxx to virtqueue_reset_xxx()
> > > > >   virtio_ring: switch to use vring_virtqueue in virtqueue_poll variants
> > > > >   virtio_ring: unify logic of virtqueue_poll() and more_used()
> > > > >   virtio_ring: switch to use vring_virtqueue for virtqueue resize
> > > > >     variants
> > > > >   virtio_ring: switch to use vring_virtqueue for virtqueue_kick_prepare
> > > > >     variants
> > > > >   virtio_ring: switch to use vring_virtqueue for virtqueue_add variants
> > > > >   virtio: switch to use vring_virtqueue for virtqueue_add variants
> > > > >   virtio_ring: switch to use vring_virtqueue for enable_cb_prepare
> > > > >     variants
> > > > >   virtio_ring: use vring_virtqueue for enable_cb_delayed variants
> > > > >   virtio_ring: switch to use vring_virtqueue for disable_cb variants
> > > > >   virtio_ring: switch to use vring_virtqueue for detach_unused_buf
> > > > >     variants
> > > > >   virtio_ring: use u16 for last_used_idx in virtqueue_poll_split()
> > > > >   virtio_ring: introduce virtqueue ops
> > > > >   virtio_ring: determine descriptor flags at one time
> > > > >   virtio_ring: factor out core logic of buffer detaching
> > > > >   virtio_ring: factor out core logic for updating last_used_idx
> > > > >   virtio_ring: move next_avail_idx to vring_virtqueue
> > > > >   virtio_ring: factor out split indirect detaching logic
> > > > >   virtio_ring: add in order support
> > > > >
> > > > >  drivers/virtio/virtio_ring.c | 856 ++++++++++++++++++++++++++---------
> > > > >  1 file changed, 653 insertions(+), 203 deletions(-)
> > > > >
> > > > > --
> > > > > 2.42.0
> > > > >
> > > > >
> > > >
> >