[PATCH V2 0/3] vDPA/ifcvf: enable virtio-net on Intel F2000X-PL

[PATCH V2 0/3] vDPA/ifcvf: enable virtio-net on Intel F2000X-PL

[Zhu Lingshan]

Hi

This series enables vDPA/ifcvf support for virtio-net devices
on Intel F2000X-PL Infrastructure Processing Unit.

This series is rebased on MST's vhost tree, linux-next branch

Please help review

Thanks!

Changes from V1:
1)Rebased on MST's tree
2)Remove IFCVF_MAX_QUEUES and other minor improvements

Zhu Lingshan (3):
  vDPA/ifcvf: dynamic allocate vq data stores
  vDPA/ifcvf: detect and report max allowed vq size
  vDPA/ifcvf: implement new accessors for vq_state

 drivers/vdpa/ifcvf/ifcvf_base.c | 55 +++++++++++++++++++++++----------
 drivers/vdpa/ifcvf/ifcvf_base.h | 29 +++++++----------
 drivers/vdpa/ifcvf/ifcvf_main.c | 10 ++++--
 3 files changed, 58 insertions(+), 36 deletions(-)

-- 
2.39.1

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

[PATCH V2 1/3] vDPA/ifcvf: dynamic allocate vq data stores

[Zhu Lingshan]

This commit dynamically allocates the data
stores for the virtqueues based on
virtio_pci_common_cfg.num_queues.

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_base.c | 3 +++
 drivers/vdpa/ifcvf/ifcvf_base.h | 2 +-
 drivers/vdpa/ifcvf/ifcvf_main.c | 2 ++
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index 1b5da11f5403..f86495ace825 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c
@@ -134,6 +134,9 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
 	}
 
 	hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
+	hw->vring = kzalloc(sizeof(struct vring_info) * hw->nr_vring, GFP_KERNEL);
+	if (!hw->vring)
+		return -ENOMEM;
 
 	for (i = 0; i < hw->nr_vring; i++) {
 		vp_iowrite16(i, &hw->common_cfg->queue_select);
diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index 3110ffc50caf..fa797184056b 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h
@@ -74,7 +74,7 @@ struct ifcvf_hw {
 	u64 dev_features;
 	struct virtio_pci_common_cfg __iomem *common_cfg;
 	void __iomem *dev_cfg;
-	struct vring_info vring[IFCVF_MAX_QUEUES];
+	struct vring_info *vring;
 	void __iomem * const *base;
 	char config_msix_name[256];
 	struct vdpa_callback config_cb;
diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index 6e47ac2c669a..2af0de771b49 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -830,6 +830,7 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
 	return 0;
 
 err:
+	kfree(ifcvf_mgmt_dev->vf.vring);
 	kfree(ifcvf_mgmt_dev);
 	return ret;
 }
@@ -840,6 +841,7 @@ static void ifcvf_remove(struct pci_dev *pdev)
 
 	ifcvf_mgmt_dev = pci_get_drvdata(pdev);
 	vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
+	kfree(ifcvf_mgmt_dev->vf.vring);
 	kfree(ifcvf_mgmt_dev);
 }
 
-- 
2.39.1

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

[PATCH V2 2/3] vDPA/ifcvf: detect and report max allowed vq size

[Zhu Lingshan]

Rather than a hardcode, this commit detects
and reports the max value of allowed size
of the virtqueues

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_base.c | 31 +++++++++++++++++++++++++++++++
 drivers/vdpa/ifcvf/ifcvf_base.h |  2 +-
 drivers/vdpa/ifcvf/ifcvf_main.c |  4 +++-
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index f86495ace825..f4d7d96c4c86 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c
@@ -69,6 +69,37 @@ static int ifcvf_read_config_range(struct pci_dev *dev,
 	return 0;
 }
 
+static u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid)
+{
+	u16 queue_size;
+
+	vp_iowrite16(qid, &hw->common_cfg->queue_select);
+	queue_size = vp_ioread16(&hw->common_cfg->queue_size);
+
+	return queue_size;
+}
+
+/* This function returns the max allowed safe size for
+ * all virtqueues. It is the minimal size that can be
+ * suppprted by all virtqueues.
+ */
+u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw)
+{
+	u16 queue_size, max_size, qid;
+
+	max_size = ifcvf_get_vq_size(hw, 0);
+	for (qid = 1; qid < hw->nr_vring; qid++) {
+		queue_size = ifcvf_get_vq_size(hw, qid);
+		/* 0 means the queue is unavailable */
+		if (!queue_size)
+			continue;
+
+		max_size = min(queue_size, max_size);
+	}
+
+	return max_size;
+}
+
 int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
 {
 	struct virtio_pci_cap cap;
diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index fa797184056b..30935a95b672 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h
@@ -28,7 +28,6 @@
 #define IFCVF_MAX_QUEUES	17
 
 #define IFCVF_QUEUE_ALIGNMENT	PAGE_SIZE
-#define IFCVF_QUEUE_MAX		32768
 #define IFCVF_PCI_MAX_RESOURCE	6
 
 #define IFCVF_LM_CFG_SIZE		0x40
@@ -138,4 +137,5 @@ bool ifcvf_get_vq_ready(struct ifcvf_hw *hw, u16 qid);
 void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready);
 void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features);
 u64 ifcvf_get_driver_features(struct ifcvf_hw *hw);
+u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw);
 #endif /* _IFCVF_H_ */
diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index 2af0de771b49..c3ece395caf7 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -451,7 +451,9 @@ static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev)
 
 static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
 {
-	return IFCVF_QUEUE_MAX;
+	struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
+
+	return ifcvf_get_max_vq_size(vf);
 }
 
 static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
-- 
2.39.1

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

[PATCH V2 3/3] vDPA/ifcvf: implement new accessors for vq_state

[Zhu Lingshan]

This commit implements a better layout of the
live migration bar, therefore the accessors for virtqueue
state have been refactored.

This commit also add a comment to the probing-ids list,
indicating this driver drives F2000X-PL virtio-net

Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
---
 drivers/vdpa/ifcvf/ifcvf_base.c | 21 +++++----------------
 drivers/vdpa/ifcvf/ifcvf_base.h | 25 +++++++++----------------
 drivers/vdpa/ifcvf/ifcvf_main.c |  4 +++-
 3 files changed, 17 insertions(+), 33 deletions(-)

diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
index f4d7d96c4c86..060f837a4f9f 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.c
+++ b/drivers/vdpa/ifcvf/ifcvf_base.c
@@ -328,30 +328,19 @@ void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features)
 
 u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid)
 {
-	struct ifcvf_lm_cfg __iomem *ifcvf_lm;
-	void __iomem *avail_idx_addr;
+	struct ifcvf_lm_cfg  __iomem *lm_cfg = hw->lm_cfg;
 	u16 last_avail_idx;
-	u32 q_pair_id;
 
-	ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
-	q_pair_id = qid / 2;
-	avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
-	last_avail_idx = vp_ioread16(avail_idx_addr);
+	last_avail_idx = vp_ioread16(&lm_cfg->vq_state_region + qid * 2);
 
 	return last_avail_idx;
 }
 
 int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num)
 {
-	struct ifcvf_lm_cfg __iomem *ifcvf_lm;
-	void __iomem *avail_idx_addr;
-	u32 q_pair_id;
-
-	ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
-	q_pair_id = qid / 2;
-	avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
-	hw->vring[qid].last_avail_idx = num;
-	vp_iowrite16(num, avail_idx_addr);
+	struct ifcvf_lm_cfg  __iomem *lm_cfg = hw->lm_cfg;
+
+	vp_iowrite16(num, &lm_cfg->vq_state_region + qid * 2);
 
 	return 0;
 }
diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
index 30935a95b672..b57849c643f6 100644
--- a/drivers/vdpa/ifcvf/ifcvf_base.h
+++ b/drivers/vdpa/ifcvf/ifcvf_base.h
@@ -24,14 +24,9 @@
 #define N3000_DEVICE_ID		0x1041
 #define N3000_SUBSYS_DEVICE_ID	0x001A
 
-/* Max 8 data queue pairs(16 queues) and one control vq for now. */
-#define IFCVF_MAX_QUEUES	17
-
 #define IFCVF_QUEUE_ALIGNMENT	PAGE_SIZE
 #define IFCVF_PCI_MAX_RESOURCE	6
 
-#define IFCVF_LM_CFG_SIZE		0x40
-#define IFCVF_LM_RING_STATE_OFFSET	0x20
 #define IFCVF_LM_BAR			4
 
 #define IFCVF_ERR(pdev, fmt, ...)	dev_err(&pdev->dev, fmt, ##__VA_ARGS__)
@@ -54,10 +49,18 @@ struct vring_info {
 	char msix_name[256];
 };
 
+struct ifcvf_lm_cfg {
+	__le64 control;
+	__le64 status;
+	__le64 lm_mem_log_start_addr;
+	__le64 lm_mem_log_end_addr;
+	__le16 vq_state_region;
+};
+
 struct ifcvf_hw {
 	u8 __iomem *isr;
 	/* Live migration */
-	u8 __iomem *lm_cfg;
+	struct ifcvf_lm_cfg  __iomem *lm_cfg;
 	/* Notification bar number */
 	u8 notify_bar;
 	u8 msix_vector_status;
@@ -92,16 +95,6 @@ struct ifcvf_adapter {
 	struct ifcvf_hw *vf;
 };
 
-struct ifcvf_vring_lm_cfg {
-	u32 idx_addr[2];
-	u8 reserved[IFCVF_LM_CFG_SIZE - 8];
-};
-
-struct ifcvf_lm_cfg {
-	u8 reserved[IFCVF_LM_RING_STATE_OFFSET];
-	struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUES];
-};
-
 struct ifcvf_vdpa_mgmt_dev {
 	struct vdpa_mgmt_dev mdev;
 	struct ifcvf_hw vf;
diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
index c3ece395caf7..e98fa8100f3c 100644
--- a/drivers/vdpa/ifcvf/ifcvf_main.c
+++ b/drivers/vdpa/ifcvf/ifcvf_main.c
@@ -853,7 +853,9 @@ static struct pci_device_id ifcvf_pci_ids[] = {
 			 N3000_DEVICE_ID,
 			 PCI_VENDOR_ID_INTEL,
 			 N3000_SUBSYS_DEVICE_ID) },
-	/* C5000X-PL network device */
+	/* C5000X-PL network device
+	 * F2000X-PL network device
+	 */
 	{ PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,
 			 VIRTIO_TRANS_ID_NET,
 			 PCI_VENDOR_ID_INTEL,
-- 
2.39.1

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH V2 1/3] vDPA/ifcvf: dynamic allocate vq data stores

[Jason Wang]

On Mon, Jun 12, 2023 at 3:14 PM Zhu Lingshan <lingshan.zhu@intel.com> wrote:
>
> This commit dynamically allocates the data
> stores for the virtqueues based on
> virtio_pci_common_cfg.num_queues.

While at it, it's better to allocate vring_lm_cfg as well and drop
IFCVF_MAX_QUEUES.

Thanks

>
> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
> ---
>  drivers/vdpa/ifcvf/ifcvf_base.c | 3 +++
>  drivers/vdpa/ifcvf/ifcvf_base.h | 2 +-
>  drivers/vdpa/ifcvf/ifcvf_main.c | 2 ++
>  3 files changed, 6 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
> index 1b5da11f5403..f86495ace825 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.c
> @@ -134,6 +134,9 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
>         }
>
>         hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
> +       hw->vring = kzalloc(sizeof(struct vring_info) * hw->nr_vring, GFP_KERNEL);
> +       if (!hw->vring)
> +               return -ENOMEM;
>
>         for (i = 0; i < hw->nr_vring; i++) {
>                 vp_iowrite16(i, &hw->common_cfg->queue_select);
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
> index 3110ffc50caf..fa797184056b 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
> @@ -74,7 +74,7 @@ struct ifcvf_hw {
>         u64 dev_features;
>         struct virtio_pci_common_cfg __iomem *common_cfg;
>         void __iomem *dev_cfg;
> -       struct vring_info vring[IFCVF_MAX_QUEUES];
> +       struct vring_info *vring;
>         void __iomem * const *base;
>         char config_msix_name[256];
>         struct vdpa_callback config_cb;
> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
> index 6e47ac2c669a..2af0de771b49 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
> @@ -830,6 +830,7 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>         return 0;
>
>  err:
> +       kfree(ifcvf_mgmt_dev->vf.vring);
>         kfree(ifcvf_mgmt_dev);
>         return ret;
>  }
> @@ -840,6 +841,7 @@ static void ifcvf_remove(struct pci_dev *pdev)
>
>         ifcvf_mgmt_dev = pci_get_drvdata(pdev);
>         vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
> +       kfree(ifcvf_mgmt_dev->vf.vring);
>         kfree(ifcvf_mgmt_dev);
>  }
>
> --
> 2.39.1
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH V2 2/3] vDPA/ifcvf: detect and report max allowed vq size

[Jason Wang]

On Mon, Jun 12, 2023 at 3:14 PM Zhu Lingshan <lingshan.zhu@intel.com> wrote:
>
> Rather than a hardcode, this commit detects
> and reports the max value of allowed size
> of the virtqueues
>
> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>

Acked-by: Jason Wang <jasowang@redhat.com>

Thanks

> ---
>  drivers/vdpa/ifcvf/ifcvf_base.c | 31 +++++++++++++++++++++++++++++++
>  drivers/vdpa/ifcvf/ifcvf_base.h |  2 +-
>  drivers/vdpa/ifcvf/ifcvf_main.c |  4 +++-
>  3 files changed, 35 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
> index f86495ace825..f4d7d96c4c86 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.c
> @@ -69,6 +69,37 @@ static int ifcvf_read_config_range(struct pci_dev *dev,
>         return 0;
>  }
>
> +static u16 ifcvf_get_vq_size(struct ifcvf_hw *hw, u16 qid)
> +{
> +       u16 queue_size;
> +
> +       vp_iowrite16(qid, &hw->common_cfg->queue_select);
> +       queue_size = vp_ioread16(&hw->common_cfg->queue_size);
> +
> +       return queue_size;
> +}
> +
> +/* This function returns the max allowed safe size for
> + * all virtqueues. It is the minimal size that can be
> + * suppprted by all virtqueues.
> + */
> +u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw)
> +{
> +       u16 queue_size, max_size, qid;
> +
> +       max_size = ifcvf_get_vq_size(hw, 0);
> +       for (qid = 1; qid < hw->nr_vring; qid++) {
> +               queue_size = ifcvf_get_vq_size(hw, qid);
> +               /* 0 means the queue is unavailable */
> +               if (!queue_size)
> +                       continue;
> +
> +               max_size = min(queue_size, max_size);
> +       }
> +
> +       return max_size;
> +}
> +
>  int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
>  {
>         struct virtio_pci_cap cap;
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
> index fa797184056b..30935a95b672 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
> @@ -28,7 +28,6 @@
>  #define IFCVF_MAX_QUEUES       17
>
>  #define IFCVF_QUEUE_ALIGNMENT  PAGE_SIZE
> -#define IFCVF_QUEUE_MAX                32768
>  #define IFCVF_PCI_MAX_RESOURCE 6
>
>  #define IFCVF_LM_CFG_SIZE              0x40
> @@ -138,4 +137,5 @@ bool ifcvf_get_vq_ready(struct ifcvf_hw *hw, u16 qid);
>  void ifcvf_set_vq_ready(struct ifcvf_hw *hw, u16 qid, bool ready);
>  void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features);
>  u64 ifcvf_get_driver_features(struct ifcvf_hw *hw);
> +u16 ifcvf_get_max_vq_size(struct ifcvf_hw *hw);
>  #endif /* _IFCVF_H_ */
> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
> index 2af0de771b49..c3ece395caf7 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
> @@ -451,7 +451,9 @@ static int ifcvf_vdpa_reset(struct vdpa_device *vdpa_dev)
>
>  static u16 ifcvf_vdpa_get_vq_num_max(struct vdpa_device *vdpa_dev)
>  {
> -       return IFCVF_QUEUE_MAX;
> +       struct ifcvf_hw *vf = vdpa_to_vf(vdpa_dev);
> +
> +       return ifcvf_get_max_vq_size(vf);
>  }
>
>  static int ifcvf_vdpa_get_vq_state(struct vdpa_device *vdpa_dev, u16 qid,
> --
> 2.39.1
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH V2 1/3] vDPA/ifcvf: dynamic allocate vq data stores

[Zhu, Lingshan]

On 6/26/2023 10:32 AM, Jason Wang wrote:
> On Mon, Jun 12, 2023 at 3:14 PM Zhu Lingshan <lingshan.zhu@intel.com> wrote:
>> This commit dynamically allocates the data
>> stores for the virtqueues based on
>> virtio_pci_common_cfg.num_queues.
> While at it, it's better to allocate vring_lm_cfg as well and drop
> IFCVF_MAX_QUEUES.
Yes, this has been done in 3/3 patch in this series.

Thanks
Zhu Lingshan
>
> Thanks
>
>> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
>> ---
>>   drivers/vdpa/ifcvf/ifcvf_base.c | 3 +++
>>   drivers/vdpa/ifcvf/ifcvf_base.h | 2 +-
>>   drivers/vdpa/ifcvf/ifcvf_main.c | 2 ++
>>   3 files changed, 6 insertions(+), 1 deletion(-)
>>
>> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
>> index 1b5da11f5403..f86495ace825 100644
>> --- a/drivers/vdpa/ifcvf/ifcvf_base.c
>> +++ b/drivers/vdpa/ifcvf/ifcvf_base.c
>> @@ -134,6 +134,9 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
>>          }
>>
>>          hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
>> +       hw->vring = kzalloc(sizeof(struct vring_info) * hw->nr_vring, GFP_KERNEL);
>> +       if (!hw->vring)
>> +               return -ENOMEM;
>>
>>          for (i = 0; i < hw->nr_vring; i++) {
>>                  vp_iowrite16(i, &hw->common_cfg->queue_select);
>> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
>> index 3110ffc50caf..fa797184056b 100644
>> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
>> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
>> @@ -74,7 +74,7 @@ struct ifcvf_hw {
>>          u64 dev_features;
>>          struct virtio_pci_common_cfg __iomem *common_cfg;
>>          void __iomem *dev_cfg;
>> -       struct vring_info vring[IFCVF_MAX_QUEUES];
>> +       struct vring_info *vring;
>>          void __iomem * const *base;
>>          char config_msix_name[256];
>>          struct vdpa_callback config_cb;
>> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
>> index 6e47ac2c669a..2af0de771b49 100644
>> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
>> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
>> @@ -830,6 +830,7 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>          return 0;
>>
>>   err:
>> +       kfree(ifcvf_mgmt_dev->vf.vring);
>>          kfree(ifcvf_mgmt_dev);
>>          return ret;
>>   }
>> @@ -840,6 +841,7 @@ static void ifcvf_remove(struct pci_dev *pdev)
>>
>>          ifcvf_mgmt_dev = pci_get_drvdata(pdev);
>>          vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
>> +       kfree(ifcvf_mgmt_dev->vf.vring);
>>          kfree(ifcvf_mgmt_dev);
>>   }
>>
>> --
>> 2.39.1
>>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH V2 3/3] vDPA/ifcvf: implement new accessors for vq_state

[Jason Wang]

On Mon, Jun 12, 2023 at 3:14 PM Zhu Lingshan <lingshan.zhu@intel.com> wrote:
>
> This commit implements a better layout of the
> live migration bar, therefore the accessors for virtqueue
> state have been refactored.

I guess the reason for F2000X is that it can report a #vq which is
greater than IFCVF_MAX_QUEUES. If yes, let's explain it in the
changelog.

Thanks


>
> This commit also add a comment to the probing-ids list,
> indicating this driver drives F2000X-PL virtio-net
>
> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
> ---
>  drivers/vdpa/ifcvf/ifcvf_base.c | 21 +++++----------------
>  drivers/vdpa/ifcvf/ifcvf_base.h | 25 +++++++++----------------
>  drivers/vdpa/ifcvf/ifcvf_main.c |  4 +++-
>  3 files changed, 17 insertions(+), 33 deletions(-)
>
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
> index f4d7d96c4c86..060f837a4f9f 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.c
> @@ -328,30 +328,19 @@ void ifcvf_set_driver_features(struct ifcvf_hw *hw, u64 features)
>
>  u16 ifcvf_get_vq_state(struct ifcvf_hw *hw, u16 qid)
>  {
> -       struct ifcvf_lm_cfg __iomem *ifcvf_lm;
> -       void __iomem *avail_idx_addr;
> +       struct ifcvf_lm_cfg  __iomem *lm_cfg = hw->lm_cfg;
>         u16 last_avail_idx;
> -       u32 q_pair_id;
>
> -       ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
> -       q_pair_id = qid / 2;
> -       avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
> -       last_avail_idx = vp_ioread16(avail_idx_addr);
> +       last_avail_idx = vp_ioread16(&lm_cfg->vq_state_region + qid * 2);
>
>         return last_avail_idx;
>  }
>
>  int ifcvf_set_vq_state(struct ifcvf_hw *hw, u16 qid, u16 num)
>  {
> -       struct ifcvf_lm_cfg __iomem *ifcvf_lm;
> -       void __iomem *avail_idx_addr;
> -       u32 q_pair_id;
> -
> -       ifcvf_lm = (struct ifcvf_lm_cfg __iomem *)hw->lm_cfg;
> -       q_pair_id = qid / 2;
> -       avail_idx_addr = &ifcvf_lm->vring_lm_cfg[q_pair_id].idx_addr[qid % 2];
> -       hw->vring[qid].last_avail_idx = num;
> -       vp_iowrite16(num, avail_idx_addr);
> +       struct ifcvf_lm_cfg  __iomem *lm_cfg = hw->lm_cfg;
> +
> +       vp_iowrite16(num, &lm_cfg->vq_state_region + qid * 2);
>
>         return 0;
>  }
> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
> index 30935a95b672..b57849c643f6 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
> @@ -24,14 +24,9 @@
>  #define N3000_DEVICE_ID                0x1041
>  #define N3000_SUBSYS_DEVICE_ID 0x001A
>
> -/* Max 8 data queue pairs(16 queues) and one control vq for now. */
> -#define IFCVF_MAX_QUEUES       17
> -
>  #define IFCVF_QUEUE_ALIGNMENT  PAGE_SIZE
>  #define IFCVF_PCI_MAX_RESOURCE 6
>
> -#define IFCVF_LM_CFG_SIZE              0x40
> -#define IFCVF_LM_RING_STATE_OFFSET     0x20
>  #define IFCVF_LM_BAR                   4
>
>  #define IFCVF_ERR(pdev, fmt, ...)      dev_err(&pdev->dev, fmt, ##__VA_ARGS__)
> @@ -54,10 +49,18 @@ struct vring_info {
>         char msix_name[256];
>  };
>
> +struct ifcvf_lm_cfg {
> +       __le64 control;
> +       __le64 status;
> +       __le64 lm_mem_log_start_addr;
> +       __le64 lm_mem_log_end_addr;
> +       __le16 vq_state_region;
> +};
> +
>  struct ifcvf_hw {
>         u8 __iomem *isr;
>         /* Live migration */
> -       u8 __iomem *lm_cfg;
> +       struct ifcvf_lm_cfg  __iomem *lm_cfg;
>         /* Notification bar number */
>         u8 notify_bar;
>         u8 msix_vector_status;
> @@ -92,16 +95,6 @@ struct ifcvf_adapter {
>         struct ifcvf_hw *vf;
>  };
>
> -struct ifcvf_vring_lm_cfg {
> -       u32 idx_addr[2];
> -       u8 reserved[IFCVF_LM_CFG_SIZE - 8];
> -};
> -
> -struct ifcvf_lm_cfg {
> -       u8 reserved[IFCVF_LM_RING_STATE_OFFSET];
> -       struct ifcvf_vring_lm_cfg vring_lm_cfg[IFCVF_MAX_QUEUES];
> -};
> -
>  struct ifcvf_vdpa_mgmt_dev {
>         struct vdpa_mgmt_dev mdev;
>         struct ifcvf_hw vf;
> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
> index c3ece395caf7..e98fa8100f3c 100644
> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
> @@ -853,7 +853,9 @@ static struct pci_device_id ifcvf_pci_ids[] = {
>                          N3000_DEVICE_ID,
>                          PCI_VENDOR_ID_INTEL,
>                          N3000_SUBSYS_DEVICE_ID) },
> -       /* C5000X-PL network device */
> +       /* C5000X-PL network device
> +        * F2000X-PL network device
> +        */
>         { PCI_DEVICE_SUB(PCI_VENDOR_ID_REDHAT_QUMRANET,
>                          VIRTIO_TRANS_ID_NET,
>                          PCI_VENDOR_ID_INTEL,
> --
> 2.39.1
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH V2 1/3] vDPA/ifcvf: dynamic allocate vq data stores

[Jason Wang]

On Mon, Jun 26, 2023 at 10:38 AM Zhu, Lingshan <lingshan.zhu@intel.com> wrote:
>
>
>
> On 6/26/2023 10:32 AM, Jason Wang wrote:
> > On Mon, Jun 12, 2023 at 3:14 PM Zhu Lingshan <lingshan.zhu@intel.com> wrote:
> >> This commit dynamically allocates the data
> >> stores for the virtqueues based on
> >> virtio_pci_common_cfg.num_queues.
> > While at it, it's better to allocate vring_lm_cfg as well and drop
> > IFCVF_MAX_QUEUES.
> Yes, this has been done in 3/3 patch in this series.

Ok, yes, but it seems patch 3 implements a lot of logic so I suggest
moving it to patch 1.

Not sure it's too late since I see the patch has been merged by Michael.

Thanks

>
> Thanks
> Zhu Lingshan
> >
> > Thanks
> >
> >> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
> >> ---
> >>   drivers/vdpa/ifcvf/ifcvf_base.c | 3 +++
> >>   drivers/vdpa/ifcvf/ifcvf_base.h | 2 +-
> >>   drivers/vdpa/ifcvf/ifcvf_main.c | 2 ++
> >>   3 files changed, 6 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
> >> index 1b5da11f5403..f86495ace825 100644
> >> --- a/drivers/vdpa/ifcvf/ifcvf_base.c
> >> +++ b/drivers/vdpa/ifcvf/ifcvf_base.c
> >> @@ -134,6 +134,9 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
> >>          }
> >>
> >>          hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
> >> +       hw->vring = kzalloc(sizeof(struct vring_info) * hw->nr_vring, GFP_KERNEL);
> >> +       if (!hw->vring)
> >> +               return -ENOMEM;
> >>
> >>          for (i = 0; i < hw->nr_vring; i++) {
> >>                  vp_iowrite16(i, &hw->common_cfg->queue_select);
> >> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
> >> index 3110ffc50caf..fa797184056b 100644
> >> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
> >> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
> >> @@ -74,7 +74,7 @@ struct ifcvf_hw {
> >>          u64 dev_features;
> >>          struct virtio_pci_common_cfg __iomem *common_cfg;
> >>          void __iomem *dev_cfg;
> >> -       struct vring_info vring[IFCVF_MAX_QUEUES];
> >> +       struct vring_info *vring;
> >>          void __iomem * const *base;
> >>          char config_msix_name[256];
> >>          struct vdpa_callback config_cb;
> >> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
> >> index 6e47ac2c669a..2af0de771b49 100644
> >> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
> >> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
> >> @@ -830,6 +830,7 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
> >>          return 0;
> >>
> >>   err:
> >> +       kfree(ifcvf_mgmt_dev->vf.vring);
> >>          kfree(ifcvf_mgmt_dev);
> >>          return ret;
> >>   }
> >> @@ -840,6 +841,7 @@ static void ifcvf_remove(struct pci_dev *pdev)
> >>
> >>          ifcvf_mgmt_dev = pci_get_drvdata(pdev);
> >>          vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
> >> +       kfree(ifcvf_mgmt_dev->vf.vring);
> >>          kfree(ifcvf_mgmt_dev);
> >>   }
> >>
> >> --
> >> 2.39.1
> >>
>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization

Re: [PATCH V2 1/3] vDPA/ifcvf: dynamic allocate vq data stores

[Zhu, Lingshan]

On 6/26/2023 10:49 AM, Jason Wang wrote:
> On Mon, Jun 26, 2023 at 10:38 AM Zhu, Lingshan <lingshan.zhu@intel.com> wrote:
>>
>>
>> On 6/26/2023 10:32 AM, Jason Wang wrote:
>>> On Mon, Jun 12, 2023 at 3:14 PM Zhu Lingshan <lingshan.zhu@intel.com> wrote:
>>>> This commit dynamically allocates the data
>>>> stores for the virtqueues based on
>>>> virtio_pci_common_cfg.num_queues.
>>> While at it, it's better to allocate vring_lm_cfg as well and drop
>>> IFCVF_MAX_QUEUES.
>> Yes, this has been done in 3/3 patch in this series.
> Ok, yes, but it seems patch 3 implements a lot of logic so I suggest
> moving it to patch 1.
>
> Not sure it's too late since I see the patch has been merged by Michael.
I am not sure, I think we need two patches handle their respective 
changes anyway,
and can not drop IFCVF_MAX_QUEUES in the first one.
>
> Thanks
>
>> Thanks
>> Zhu Lingshan
>>> Thanks
>>>
>>>> Signed-off-by: Zhu Lingshan <lingshan.zhu@intel.com>
>>>> ---
>>>>    drivers/vdpa/ifcvf/ifcvf_base.c | 3 +++
>>>>    drivers/vdpa/ifcvf/ifcvf_base.h | 2 +-
>>>>    drivers/vdpa/ifcvf/ifcvf_main.c | 2 ++
>>>>    3 files changed, 6 insertions(+), 1 deletion(-)
>>>>
>>>> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c
>>>> index 1b5da11f5403..f86495ace825 100644
>>>> --- a/drivers/vdpa/ifcvf/ifcvf_base.c
>>>> +++ b/drivers/vdpa/ifcvf/ifcvf_base.c
>>>> @@ -134,6 +134,9 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev)
>>>>           }
>>>>
>>>>           hw->nr_vring = vp_ioread16(&hw->common_cfg->num_queues);
>>>> +       hw->vring = kzalloc(sizeof(struct vring_info) * hw->nr_vring, GFP_KERNEL);
>>>> +       if (!hw->vring)
>>>> +               return -ENOMEM;
>>>>
>>>>           for (i = 0; i < hw->nr_vring; i++) {
>>>>                   vp_iowrite16(i, &hw->common_cfg->queue_select);
>>>> diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h
>>>> index 3110ffc50caf..fa797184056b 100644
>>>> --- a/drivers/vdpa/ifcvf/ifcvf_base.h
>>>> +++ b/drivers/vdpa/ifcvf/ifcvf_base.h
>>>> @@ -74,7 +74,7 @@ struct ifcvf_hw {
>>>>           u64 dev_features;
>>>>           struct virtio_pci_common_cfg __iomem *common_cfg;
>>>>           void __iomem *dev_cfg;
>>>> -       struct vring_info vring[IFCVF_MAX_QUEUES];
>>>> +       struct vring_info *vring;
>>>>           void __iomem * const *base;
>>>>           char config_msix_name[256];
>>>>           struct vdpa_callback config_cb;
>>>> diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c
>>>> index 6e47ac2c669a..2af0de771b49 100644
>>>> --- a/drivers/vdpa/ifcvf/ifcvf_main.c
>>>> +++ b/drivers/vdpa/ifcvf/ifcvf_main.c
>>>> @@ -830,6 +830,7 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>>>>           return 0;
>>>>
>>>>    err:
>>>> +       kfree(ifcvf_mgmt_dev->vf.vring);
>>>>           kfree(ifcvf_mgmt_dev);
>>>>           return ret;
>>>>    }
>>>> @@ -840,6 +841,7 @@ static void ifcvf_remove(struct pci_dev *pdev)
>>>>
>>>>           ifcvf_mgmt_dev = pci_get_drvdata(pdev);
>>>>           vdpa_mgmtdev_unregister(&ifcvf_mgmt_dev->mdev);
>>>> +       kfree(ifcvf_mgmt_dev->vf.vring);
>>>>           kfree(ifcvf_mgmt_dev);
>>>>    }
>>>>
>>>> --
>>>> 2.39.1
>>>>

_______________________________________________
Virtualization mailing list
Virtualization@lists.linux-foundation.org
https://lists.linuxfoundation.org/mailman/listinfo/virtualization