[PATCH 00/10] vDPA: Allocate each vq with its size individually

[PATCH 00/10] vDPA: Allocate each vq with its size individually

[Zhu Lingshan]

The size of a virtqueue is a per vq configuration.

This series introduces a new operation function
get_vq_size to vdpa_config_ops, to help
reporting per vq size.

vhost_vdpa and virtio_vdpa then initialize each
virtqueue buffer individually with its actually
size.

Please help review

Thanks
Zhu Lingshan

Zhu Lingshan (10):
  vhost-vdpa: uapi to support reporting per vq size
  vDPA: introduce get_vq_size to vdpa_config_ops
  vDPA/ifcvf: implement vdpa_config_ops.get_vq_size
  vp_vdpa: implement vdpa_config_ops.get_vq_size
  eni_vdpa: implement vdpa_config_ops.get_vq_size
  vdpa_sim: implement vdpa_config_ops.get_vq_size for vDPA simulator
  vduse: implement vdpa_config_ops.get_vq_size for vduse
  virtio_vdpa: create vqs with the actual size
  vDPA/ifcvf: get_max_vq_size to return max size
  vDPA/ifcvf: implement vdpa_config_ops.get_vq_num_min

 drivers/vdpa/alibaba/eni_vdpa.c    |  8 ++++++++
 drivers/vdpa/ifcvf/ifcvf_base.c    | 11 +++++------
 drivers/vdpa/ifcvf/ifcvf_base.h    |  2 ++
 drivers/vdpa/ifcvf/ifcvf_main.c    | 15 +++++++++++++++
 drivers/vdpa/vdpa_sim/vdpa_sim.c   | 12 ++++++++++++
 drivers/vdpa/vdpa_user/vduse_dev.c | 12 ++++++++++++
 drivers/vdpa/virtio_pci/vp_vdpa.c  |  8 ++++++++
 drivers/vhost/vdpa.c               |  8 ++++++++
 drivers/virtio/virtio_vdpa.c       |  5 ++++-
 include/linux/vdpa.h               |  5 +++++
 include/uapi/linux/vhost.h         |  7 +++++++
 11 files changed, 86 insertions(+), 7 deletions(-)

-- 
2.39.3

[PATCH 01/10] vhost-vdpa: uapi to support reporting per vq size

[Zhu Lingshan]

The size of a virtqueue is a per vq configuration.
This commit introduce a new ioctl uAPI to support this flexibility.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 include/uapi/linux/vhost.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
index 649560c685f1..bea697390613 100644
--- a/include/uapi/linux/vhost.h
+++ b/include/uapi/linux/vhost.h
@@ -227,4 +227,11 @@
  */
 #define VHOST_VDPA_GET_VRING_DESC_GROUP	_IOWR(VHOST_VIRTIO, 0x7F,	\
 					      struct vhost_vring_state)
+
+/* Get the queue size of a specific virtqueue.
+ * userspace set the vring index in vhost_vring_state.index
+ * kernel set the queue size in vhost_vring_state.num
+ */
+#define VHOST_VDPA_GET_VRING_SIZE	_IOWR(VHOST_VIRTIO, 0x80,	\
+					      struct vhost_vring_state)
 #endif
-- 
2.39.3

[PATCH 02/10] vDPA: introduce get_vq_size to vdpa_config_ops

[Zhu Lingshan]

This commit introduces a new interface get_vq_size to
vDPA config ops, this new interface intends to report
the size of a specific virtqueue

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vhost/vdpa.c | 8 ++++++++
 include/linux/vdpa.h | 5 +++++
 2 files changed, 13 insertions(+)

diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c
index bc4a51e4638b..7c2d7c8edb90 100644
--- a/drivers/vhost/vdpa.c
+++ b/drivers/vhost/vdpa.c
@@ -681,6 +681,14 @@ static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,
 		if (!ops->set_group_asid)
 			return -EOPNOTSUPP;
 		return ops->set_group_asid(vdpa, idx, s.num);
+	case VHOST_VDPA_GET_VRING_SIZE:
+		if (!ops->get_vq_size)
+			return -EOPNOTSUPP;
+		s.index = idx;
+		s.num = ops->get_vq_size(vdpa, idx);
+		if (copy_to_user(argp, &s, sizeof(s)))
+			return -EFAULT;
+		return 0;
 	case VHOST_GET_VRING_BASE:
 		r = ops->get_vq_state(v->vdpa, idx, &vq_state);
 		if (r)
diff --git a/include/linux/vdpa.h b/include/linux/vdpa.h
index db15ac07f8a6..4097e8e92860 100644
--- a/include/linux/vdpa.h
+++ b/include/linux/vdpa.h
@@ -195,6 +195,10 @@ struct vdpa_map_file {
  *				@idx: virtqueue index
  *				Returns int: irq number of a virtqueue,
  *				negative number if no irq assigned.
+ * @get_vq_size:		Get the size of a specific virtqueue (optional)
+ *				@vdev: vdpa device
+ *				@idx: virtqueue index
+ *				Return u16: the size of the virtqueue
  * @get_vq_align:		Get the virtqueue align requirement
  *				for the device
  *				@vdev: vdpa device
@@ -386,6 +390,7 @@ struct vdpa_config_ops {
 	(*get_vq_notification)(struct vdpa_device *vdev, u16 idx);
 	/* vq irq is not expected to be changed once DRIVER_OK is set */
 	int (*get_vq_irq)(struct vdpa_device *vdev, u16 idx);
+	u16 (*get_vq_size)(struct vdpa_device *vdev, u16 idx);
 
 	/* Device ops */
 	u32 (*get_vq_align)(struct vdpa_device *vdev);
-- 
2.39.3

[PATCH 03/10] vDPA/ifcvf: implement vdpa_config_ops.get_vq_size

[Zhu Lingshan]

This commit implements vdpa_ops.get_vq_size to report
the size of a specific virtqueue.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_base.c | 5 ++++-
 drivers/vdpa/ifcvf/ifcvf_base.h | 1 +
 drivers/vdpa/ifcvf/ifcvf_main.c | 9 +++++++++
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index 060f837a4f9f..d78621fdd621 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c
@@ -69,10 +69,13 @@ static int ifcvf_read_config_range(struct pci_dev *dev,
 	return 0;
 }
 
-static u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid)
+u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid)
 {
 	u16 queue_size;
 
+	if (qid >= hw->nr_vring)
+		return 0;
+
 	vp_iowrite16(qid, &hw->common_cfg->queue_select);
 	queue_size = vp_ioread16(&hw->common_cfg->queue_size);
 
diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index b57849c643f6..5a4be3a4f939 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h
@@ -131,4 +131,5 @@ void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready);
 void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features);
 u64 ifcvf_get_driver_features(struct ifcvf_hw *hw);
 u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw);
+u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid);
 #endif /* _IFCVF_H_ */
diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index e98fa8100f3c..cd4e9cb020c4 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -597,6 +597,14 @@ static int ifcvf_vdpa_get_vq_irq(struct vdpa_device *vdpa_dev,
 		return -EINVAL;
 }
 
+static u16 ifcvf_vdpa_get_vq_size(struct vdpa_device *vdpa_dev,
+			     u16 qid)
+{
+	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+
+	return ifcvf_get_vq_size(vf, qid);
+}
+
 static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_device *vdpa_dev,
 							       u16 idx)
 {
@@ -632,6 +640,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
 	.set_vq_num	= ifcvf_vdpa_set_vq_num,
 	.set_vq_address	= ifcvf_vdpa_set_vq_address,
 	.get_vq_irq	= ifcvf_vdpa_get_vq_irq,
+	.get_vq_size	= ifcvf_vdpa_get_vq_size,
 	.kick_vq	= ifcvf_vdpa_kick_vq,
 	.get_generation	= ifcvf_vdpa_get_generation,
 	.get_device_id	= ifcvf_vdpa_get_device_id,
-- 
2.39.3

[PATCH 04/10] vp_vdpa: implement vdpa_config_ops.get_vq_size

[Zhu Lingshan]

This commit implements vdpa_config_ops.get_vq_size in
vp_vdpa, which reports per virtqueue size.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/virtio_pci/vp_vdpa.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/drivers/vdpa/virtio_pci/vp_vdpa.c b/drivers/vdpa/virtio_pci/vp_vdpa.c
index 281287fae89f..df5f4a3bccb5 100644
--- a/drivers/vdpa/virtio_pci/vp_vdpa.c
+++ b/drivers/vdpa/virtio_pci/vp_vdpa.c
@@ -328,6 +328,13 @@ static void vp_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 qid,
 	vp_modern_set_queue_size(mdev, qid, num);
 }
 
+static u16 vp_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 qid)
+{
+	struct virtio_pci_modern_device *mdev = vdpa_to_mdev(vdpa);
+
+	return vp_modern_get_queue_size(mdev, qid);
+}
+
 static int vp_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 qid,
 				  u64 desc_area, u64 driver_area,
 				  u64 device_area)
@@ -449,6 +456,7 @@ static const struct vdpa_config_ops vp_vdpa_ops = {
 	.set_vq_ready	= vp_vdpa_set_vq_ready,
 	.get_vq_ready	= vp_vdpa_get_vq_ready,
 	.set_vq_num	= vp_vdpa_set_vq_num,
+	.get_vq_size	= vp_vdpa_get_vq_size,
 	.set_vq_address	= vp_vdpa_set_vq_address,
 	.kick_vq	= vp_vdpa_kick_vq,
 	.get_generation	= vp_vdpa_get_generation,
-- 
2.39.3

[PATCH 05/10] eni_vdpa: implement vdpa_config_ops.get_vq_size

[Zhu Lingshan]

This commit implements get_vq_size which report
per vq size in vdpa_config_ops

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/alibaba/eni_vdpa.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/drivers/vdpa/alibaba/eni_vdpa.c b/drivers/vdpa/alibaba/eni_vdpa.c
index cce3d1837104..ad7f3447fe90 100644
--- a/drivers/vdpa/alibaba/eni_vdpa.c
+++ b/drivers/vdpa/alibaba/eni_vdpa.c
@@ -254,6 +254,13 @@ static u16 eni_vdpa_get_vq_num_min(struct vdpa_device *vdpa)
 	return vp_legacy_get_queue_size(ldev, 0);
 }
 
+static u16 eni_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 qid)
+{
+	struct virtio_pci_legacy_device *ldev = vdpa_to_ldev(vdpa);
+
+	return vp_legacy_get_queue_size(ldev, qid);
+}
+
 static int eni_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 qid,
 				struct vdpa_vq_state *state)
 {
@@ -416,6 +423,7 @@ static const struct vdpa_config_ops eni_vdpa_ops = {
 	.reset		= eni_vdpa_reset,
 	.get_vq_num_max	= eni_vdpa_get_vq_num_max,
 	.get_vq_num_min	= eni_vdpa_get_vq_num_min,
+	.get_vq_size	= eni_vdpa_get_vq_size,
 	.get_vq_state	= eni_vdpa_get_vq_state,
 	.set_vq_state	= eni_vdpa_set_vq_state,
 	.set_vq_cb	= eni_vdpa_set_vq_cb,
-- 
2.39.3

[PATCH 06/10] vdpa_sim: implement vdpa_config_ops.get_vq_size for vDPA simulator

[Zhu Lingshan]

This commit implements vdpa_config_ops.get_vq_size for vDPA
simulator, this new interface can help report per vq size.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/vdpa_sim/vdpa_sim.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
index be2925d0d283..a870c983d89f 100644
--- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
+++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
@@ -311,6 +311,17 @@ static void vdpasim_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
 	vq->num = num;
 }
 
+static u16 vdpasim_get_vq_size(struct vdpa_device *vdpa, u16 idx)
+{
+	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
+	struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
+
+	if (vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)
+		return vq->num;
+	else
+		return VDPASIM_QUEUE_MAX;
+}
+
 static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
 {
 	struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
@@ -774,6 +785,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
 	.get_driver_features    = vdpasim_get_driver_features,
 	.set_config_cb          = vdpasim_set_config_cb,
 	.get_vq_num_max         = vdpasim_get_vq_num_max,
+	.get_vq_size		= vdpasim_get_vq_size,
 	.get_device_id          = vdpasim_get_device_id,
 	.get_vendor_id          = vdpasim_get_vendor_id,
 	.get_status             = vdpasim_get_status,
-- 
2.39.3

[PATCH 07/10] vduse: implement vdpa_config_ops.get_vq_size for vduse

[Zhu Lingshan]

This commit implements get_vq_size for vdpa_config_ops. This
new interface is used to report per vq size.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/vdpa_user/vduse_dev.c | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c
index 1d24da79c399..200eb260cc4c 100644
--- a/drivers/vdpa/vdpa_user/vduse_dev.c
+++ b/drivers/vdpa/vdpa_user/vduse_dev.c
@@ -541,6 +541,17 @@ static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
 	vq->num = num;
 }
 
+static u16 vduse_vdpa_get_vq_size(struct vdpa_device *vdpa, u16 idx)
+{
+	struct vduse_dev *dev = vdpa_to_vduse(vdpa);
+	struct vduse_virtqueue *vq = dev->vqs[idx];
+
+	if (vq->num)
+		return vq->num;
+	else
+		return vq->num_max;
+}
+
 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
 					u16 idx, bool ready)
 {
@@ -773,6 +784,7 @@ static const struct vdpa_config_ops vduse_vdpa_config_ops = {
 	.kick_vq		= vduse_vdpa_kick_vq,
 	.set_vq_cb		= vduse_vdpa_set_vq_cb,
 	.set_vq_num             = vduse_vdpa_set_vq_num,
+	.get_vq_size		= vduse_vdpa_get_vq_size,
 	.set_vq_ready		= vduse_vdpa_set_vq_ready,
 	.get_vq_ready		= vduse_vdpa_get_vq_ready,
 	.set_vq_state		= vduse_vdpa_set_vq_state,
-- 
2.39.3

[PATCH 08/10] virtio_vdpa: create vqs with the actual size

[Zhu Lingshan]

The size of a virtqueue is a per vq configuration,
this commit allows virtio_vdpa to create
virtqueues with the actual size of a specific
vq size that supported by the backend device.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/virtio/virtio_vdpa.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c
index 8d63e5923d24..e803db0da307 100644
--- a/drivers/virtio/virtio_vdpa.c
+++ b/drivers/virtio/virtio_vdpa.c
@@ -183,8 +183,11 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index,
 	info = kmalloc(sizeof(*info), GFP_KERNEL);
 	if (!info)
 		return ERR_PTR(-ENOMEM);
+	if (ops->get_vq_size)
+		max_num = ops->get_vq_size(vdpa, index);
+	else
+		max_num = ops->get_vq_num_max(vdpa);
 
-	max_num = ops->get_vq_num_max(vdpa);
 	if (max_num == 0) {
 		err = -ENOENT;
 		goto error_new_virtqueue;
-- 
2.39.3

[PATCH 09/10] vDPA/ifcvf: get_max_vq_size to return max size

[Zhu Lingshan]

Since we already implemented vdpa_config_ops.get_vq_size,
so get_max_vq_size can return the acutal max size of the
virtqueues other than the max allowed safe size.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_base.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index d78621fdd621..472daa588a9d 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c
@@ -82,10 +82,6 @@ u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid)
 	return queue_size;
 }
 
-/* This function returns the max allowed safe size for
- * all virtqueues. It is the minimal size that can be
- * suppprted by all virtqueues.
- */
 u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw)
 {
 	u16 queue_size, max_size, qid;
@@ -97,7 +93,7 @@ u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw)
 		if (!queue_size)
 			continue;
 
-		max_size = min(queue_size, max_size);
+		max_size = max(queue_size, max_size);
 	}
 
 	return max_size;
-- 
2.39.3

[PATCH 10/10] vDPA/ifcvf: implement vdpa_config_ops.get_vq_num_min

[Zhu Lingshan]

IFCVF HW supports operation with vq size less than the max size,
as the spec required.

This commit implements vdpa_config_ops.get_vq_num_min to report
the minimal size of the virtqueues, which gives vDPA framework
a chance to reduce the vring size.

We need at least one descriptor to be functional, but it is better
no less than 64 to meet ceratin performance requirements.
Actually the framework would allocate at least a PAGE for the vq.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_base.h | 1 +
 drivers/vdpa/ifcvf/ifcvf_main.c | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index 5a4be3a4f939..0f347717021a 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h
@@ -28,6 +28,7 @@
 #define IFCVF_PCI_MAX_RESOURCE	6
 
 #define IFCVF_LM_BAR			4
+#define IFCVF_MIN_VQ_SIZE		64
 
 #define IFCVF_ERR(pdev, fmt, ...)	dev_err(&pdev->dev, fmt, ##__VA_ARGS__)
 #define IFCVF_DBG(pdev, fmt, ...)	dev_dbg(&pdev->dev, fmt, ##__VA_ARGS__)
diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index cd4e9cb020c4..80d0a0460885 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -456,6 +456,11 @@ static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
 	return ifcvf_get_max_vq_size(vf);
 }
 
+static u16 ifcvf_vdpa_get_vq_num_min(struct vdpa_device *vdpa_dev)
+{
+	return IFCVF_MIN_VQ_SIZE;
+}
+
 static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
 				   struct vdpa_vq_state *state)
 {
@@ -632,6 +637,7 @@ static const struct vdpa_config_ops ifc_vdpa_ops = {
 	.set_status	= ifcvf_vdpa_set_status,
 	.reset		= ifcvf_vdpa_reset,
 	.get_vq_num_max	= ifcvf_vdpa_get_vq_num_max,
+	.get_vq_num_min	= ifcvf_vdpa_get_vq_num_min,
 	.get_vq_state	= ifcvf_vdpa_get_vq_state,
 	.set_vq_state	= ifcvf_vdpa_set_vq_state,
 	.set_vq_cb	= ifcvf_vdpa_set_vq_cb,
-- 
2.39.3