[PATCH for-next v1 0/2] RDMA/vmw_pvrdma: Add RoCEv2 support

[PATCH for-next v1 0/2] RDMA/vmw_pvrdma: Add RoCEv2 support

[Adit Ranadive]

Hi Doug,

These are a set of patches to add RoCEv2 support for the VMware PVRDMA
driver as well as exposing some device caps. Please add these to your
4.14 queue.

Thanks,
Adit
---
Changes v0 -> v1:
 - Consolidated the version macros.
 - Added macros to check for compatibility.
 - Squashed a previous patch #3 into patch #2.
 - Removed an unused function.
 - Updated tag lines correctly.
---
Adit Ranadive (1):
  RDMA/vmw_pvrdma: Update device query parameters and port caps

Bryan Tan (1):
  RDMA/vmw_pvrdma: Add RoCEv2 support

 drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 37 +++++++++++++++++++++--
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 +++++++++-------------
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   |  9 ++++++
 5 files changed, 68 insertions(+), 23 deletions(-)

-- 
2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-next v1 1/2] RDMA/vmw_pvrdma: Add RoCEv2 support

[Adit Ranadive]

From: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>

The driver version is bumped for compatibility purposes. Also, send correct
GID type during register to device. Added compatibility check macros for
the device.

Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
Signed-off-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 28 +++++++++++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 ++++++++++-------------
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
 4 files changed, 51 insertions(+), 22 deletions(-)

diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
index 8e2f0a1..663a0c3 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
@@ -194,6 +194,7 @@ struct pvrdma_dev {
 	void *resp_slot;
 	unsigned long flags;
 	struct list_head device_link;
+	unsigned int dsr_version;
 
 	/* Locking and interrupt information. */
 	spinlock_t cmd_lock; /* Command lock. */
@@ -444,6 +445,7 @@ void pvrdma_ah_attr_to_rdma(struct rdma_ah_attr *dst,
 			    const struct pvrdma_ah_attr *src);
 void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
 			    const struct rdma_ah_attr *src);
+u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type);
 
 int pvrdma_uar_table_init(struct pvrdma_dev *dev);
 void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev);
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
index 09078cc..3a308ff 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
@@ -50,7 +50,15 @@
 
 #include "pvrdma_verbs.h"
 
-#define PVRDMA_VERSION			17
+/*
+ * PVRDMA version macros. Some new features require updates to PVRDMA_VERSION.
+ * These macros allow us to check for different features if necessary.
+ */
+
+#define PVRDMA_ROCEV1_VERSION		17
+#define PVRDMA_ROCEV2_VERSION		18
+#define PVRDMA_VERSION			PVRDMA_ROCEV2_VERSION
+
 #define PVRDMA_BOARD_ID			1
 #define PVRDMA_REV_ID			1
 
@@ -123,6 +131,24 @@
 #define PVRDMA_GID_TYPE_FLAG_ROCE_V1	BIT(0)
 #define PVRDMA_GID_TYPE_FLAG_ROCE_V2	BIT(1)
 
+/*
+ * Version checks. This checks whether each version supports specific
+ * capabilities from the device.
+ */
+
+#define PVRDMA_IS_VERSION17(_dev)					\
+	(_dev->dsr_version == PVRDMA_ROCEV1_VERSION &&			\
+	 _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
+
+#define PVRDMA_IS_VERSION18(_dev)					\
+	(_dev->dsr_version >= PVRDMA_ROCEV2_VERSION &&			\
+	 (_dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1 ||  \
+	  _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2))	\
+
+#define PVRDMA_SUPPORTED(_dev)						\
+	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
+	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
+
 enum pvrdma_pci_resource {
 	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
 	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
index 5b00156..6ce709a 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
@@ -128,10 +128,14 @@ static int pvrdma_init_device(struct pvrdma_dev *dev)
 static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
 				 struct ib_port_immutable *immutable)
 {
+	struct pvrdma_dev *dev = to_vdev(ibdev);
 	struct ib_port_attr attr;
 	int err;
 
-	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
+	if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
+		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE;
+	else if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2)
+		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
 
 	err = ib_query_port(ibdev, port_num, &attr);
 	if (err)
@@ -569,6 +573,7 @@ static void pvrdma_free_slots(struct pvrdma_dev *dev)
 
 static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
 				   const union ib_gid *gid,
+				   u8 gid_type,
 				   int index)
 {
 	int ret;
@@ -586,7 +591,7 @@ static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
 	cmd_bind->mtu = ib_mtu_enum_to_int(IB_MTU_1024);
 	cmd_bind->vlan = 0xfff;
 	cmd_bind->index = index;
-	cmd_bind->gid_type = PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+	cmd_bind->gid_type = gid_type;
 
 	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
 	if (ret < 0) {
@@ -607,7 +612,9 @@ static int pvrdma_add_gid(struct ib_device *ibdev,
 {
 	struct pvrdma_dev *dev = to_vdev(ibdev);
 
-	return pvrdma_add_gid_at_index(dev, gid, index);
+	return pvrdma_add_gid_at_index(dev, gid,
+				       ib_gid_type_to_pvrdma(attr->gid_type),
+				       index);
 }
 
 static int pvrdma_del_gid_at_index(struct pvrdma_dev *dev, int index)
@@ -722,7 +729,6 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 	int ret;
 	unsigned long start;
 	unsigned long len;
-	unsigned int version;
 	dma_addr_t slot_dma = 0;
 
 	dev_dbg(&pdev->dev, "initializing driver %s\n", pci_name(pdev));
@@ -819,13 +825,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 		goto err_unmap_regs;
 	}
 
-	version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
+	dev->dsr_version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
 	dev_info(&pdev->dev, "device version %d, driver version %d\n",
-		 version, PVRDMA_VERSION);
-	if (version < PVRDMA_VERSION) {
-		dev_err(&pdev->dev, "incompatible device version\n");
-		goto err_uar_unmap;
-	}
+		 dev->dsr_version, PVRDMA_VERSION);
 
 	dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
 				      &dev->dsrbase, GFP_KERNEL);
@@ -896,17 +898,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
 	/* Make sure the write is complete before reading status. */
 	mb();
 
-	/* Currently, the driver only supports RoCE mode. */
-	if (dev->dsr->caps.mode != PVRDMA_DEVICE_MODE_ROCE) {
-		dev_err(&pdev->dev, "unsupported transport %d\n",
-			dev->dsr->caps.mode);
-		ret = -EFAULT;
-		goto err_free_cq_ring;
-	}
-
-	/* Currently, the driver only supports RoCE V1. */
-	if (!(dev->dsr->caps.gid_types & PVRDMA_GID_TYPE_FLAG_ROCE_V1)) {
-		dev_err(&pdev->dev, "driver needs RoCE v1 support\n");
+	/* The driver supports RoCE V1 and V2. */
+	if (!PVRDMA_SUPPORTED(dev)) {
+		dev_err(&pdev->dev, "driver needs RoCE v1 or v2 support\n");
 		ret = -EFAULT;
 		goto err_free_cq_ring;
 	}
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
index ec6a4ca..fb0c5c0 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
@@ -303,3 +303,10 @@ void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
 	dst->port_num = rdma_ah_get_port_num(src);
 	memcpy(&dst->dmac, src->roce.dmac, sizeof(dst->dmac));
 }
+
+u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type)
+{
+	return (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
+		PVRDMA_GID_TYPE_FLAG_ROCE_V2 :
+		PVRDMA_GID_TYPE_FLAG_ROCE_V1;
+}
-- 
2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

[PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Adit Ranadive]

Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
and the IP_BASED_GIDS port cap flag.

Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
---
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
 drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
index 3a308ff..df0a6b5 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
@@ -149,6 +149,13 @@
 	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
 	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
 
+/*
+ * Get capability values based on device version.
+ */
+
+#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
+	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
+
 enum pvrdma_pci_resource {
 	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
 	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
@@ -251,7 +258,7 @@ struct pvrdma_device_caps {
 	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
 	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
 	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
-	u8  reserved[4];
+	u32 max_fast_reg_page_list_len;
 };
 
 struct pvrdma_ring_page_info {
diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
index 2851704..48776f5 100644
--- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
+++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
@@ -83,6 +83,8 @@ int pvrdma_query_device(struct ib_device *ibdev,
 	props->max_qp_wr = dev->dsr->caps.max_qp_wr;
 	props->device_cap_flags = dev->dsr->caps.device_cap_flags;
 	props->max_sge = dev->dsr->caps.max_sge;
+	props->max_sge_rd = PVRDMA_GET_CAP(dev, dev->dsr->caps.max_sge,
+					   dev->dsr->caps.max_sge_rd);
 	props->max_cq = dev->dsr->caps.max_cq;
 	props->max_cqe = dev->dsr->caps.max_cqe;
 	props->max_mr = dev->dsr->caps.max_mr;
@@ -101,8 +103,14 @@ int pvrdma_query_device(struct ib_device *ibdev,
 	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&
 	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_FAST_REG_WR)) {
 		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
+		props->max_fast_reg_page_list_len = PVRDMA_GET_CAP(dev,
+				PVRDMA_MAX_FAST_REG_PAGES,
+				dev->dsr->caps.max_fast_reg_page_list_len);
 	}
 
+	props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT |
+				   IB_DEVICE_RC_RNR_NAK_GEN;
+
 	return 0;
 }
 
@@ -143,6 +151,7 @@ int pvrdma_query_port(struct ib_device *ibdev, u8 port,
 	props->gid_tbl_len = resp->attrs.gid_tbl_len;
 	props->port_cap_flags =
 		pvrdma_port_cap_flags_to_ib(resp->attrs.port_cap_flags);
+	props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;
 	props->max_msg_sz = resp->attrs.max_msg_sz;
 	props->bad_pkey_cntr = resp->attrs.bad_pkey_cntr;
 	props->qkey_viol_cntr = resp->attrs.qkey_viol_cntr;
-- 
2.7.4

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 4116 bytes --]

On Tue, Aug 22, 2017 at 11:19:01PM -0700, Adit Ranadive wrote:
> Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
> and the IP_BASED_GIDS port cap flag.
>
> Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> ---
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
>  2 files changed, 17 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> index 3a308ff..df0a6b5 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> @@ -149,6 +149,13 @@
>  	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
>  	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
>
> +/*
> + * Get capability values based on device version.
> + */
> +
> +#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
> +	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
> +

The current macro implementation will require you to go and update all
the places once you will move to new version. In simple case, everything
is supported and you will change your check of PVRDMA_IS_VERSION18 to
something PVRDMA_IS_VERSIONXXX and it will work.

In more complex case, when one of the features is supported in one version
but isn't supported in another you will need to add "if(specific_feature)" magic
into it.

>  enum pvrdma_pci_resource {
>  	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
>  	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
> @@ -251,7 +258,7 @@ struct pvrdma_device_caps {
>  	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
>  	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
>  	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
> -	u8  reserved[4];
> +	u32 max_fast_reg_page_list_len;
>  };
>
>  struct pvrdma_ring_page_info {
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> index 2851704..48776f5 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> @@ -83,6 +83,8 @@ int pvrdma_query_device(struct ib_device *ibdev,
>  	props->max_qp_wr = dev->dsr->caps.max_qp_wr;
>  	props->device_cap_flags = dev->dsr->caps.device_cap_flags;
>  	props->max_sge = dev->dsr->caps.max_sge;
> +	props->max_sge_rd = PVRDMA_GET_CAP(dev, dev->dsr->caps.max_sge,
> +					   dev->dsr->caps.max_sge_rd);
>  	props->max_cq = dev->dsr->caps.max_cq;
>  	props->max_cqe = dev->dsr->caps.max_cqe;
>  	props->max_mr = dev->dsr->caps.max_mr;
> @@ -101,8 +103,14 @@ int pvrdma_query_device(struct ib_device *ibdev,
>  	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&
>  	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_FAST_REG_WR)) {
>  		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
> +		props->max_fast_reg_page_list_len = PVRDMA_GET_CAP(dev,
> +				PVRDMA_MAX_FAST_REG_PAGES,
> +				dev->dsr->caps.max_fast_reg_page_list_len);
>  	}
>
> +	props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT |
> +				   IB_DEVICE_RC_RNR_NAK_GEN;
> +
>  	return 0;
>  }
>
> @@ -143,6 +151,7 @@ int pvrdma_query_port(struct ib_device *ibdev, u8 port,
>  	props->gid_tbl_len = resp->attrs.gid_tbl_len;
>  	props->port_cap_flags =
>  		pvrdma_port_cap_flags_to_ib(resp->attrs.port_cap_flags);
> +	props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;
>  	props->max_msg_sz = resp->attrs.max_msg_sz;
>  	props->bad_pkey_cntr = resp->attrs.bad_pkey_cntr;
>  	props->qkey_viol_cntr = resp->attrs.qkey_viol_cntr;
> --
> 2.7.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Adit Ranadive]

On Wed, Aug 23, 2017 at 12:09:20AM +0300, Leon Romanovsky wrote:
> On Tue, Aug 22, 2017 at 11:19:01PM -0700, Adit Ranadive wrote:
> > Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
> > and the IP_BASED_GIDS port cap flag.
> >
> > Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > ---
> >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
> >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
> >  2 files changed, 17 insertions(+), 1 deletion(-)
> >
> > diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > index 3a308ff..df0a6b5 100644
> > --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > @@ -149,6 +149,13 @@
> >  	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
> >  	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
> >
> > +/*
> > + * Get capability values based on device version.
> > + */
> > +
> > +#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
> > +	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
> > +
> 
> The current macro implementation will require you to go and update all
> the places once you will move to new version. In simple case, everything
> is supported and you will change your check of PVRDMA_IS_VERSION18 to
> something PVRDMA_IS_VERSIONXXX and it will work.

Thanks for taking a look. That's the intention I wrote the macro with so it
should pick up the value based on the current device version. If you are
concerned that we will change device caps without changing the PVRDMA
version, that's something we avoid for compatibility reasons.

> In more complex case, when one of the features is supported in one version
> but isn't supported in another you will need to add "if(specific_feature)" magic
> into it.

This will not happen based on how our version model currently works.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 2889 bytes --]

On Wed, Aug 23, 2017 at 11:08:01AM -0700, Adit Ranadive wrote:
> On Wed, Aug 23, 2017 at 12:09:20AM +0300, Leon Romanovsky wrote:
> > On Tue, Aug 22, 2017 at 11:19:01PM -0700, Adit Ranadive wrote:
> > > Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
> > > and the IP_BASED_GIDS port cap flag.
> > >
> > > Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > ---
> > >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
> > >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
> > >  2 files changed, 17 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > index 3a308ff..df0a6b5 100644
> > > --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > @@ -149,6 +149,13 @@
> > >  	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
> > >  	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
> > >
> > > +/*
> > > + * Get capability values based on device version.
> > > + */
> > > +
> > > +#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
> > > +	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
> > > +
> >
> > The current macro implementation will require you to go and update all
> > the places once you will move to new version. In simple case, everything
> > is supported and you will change your check of PVRDMA_IS_VERSION18 to
> > something PVRDMA_IS_VERSIONXXX and it will work.
>
> Thanks for taking a look. That's the intention I wrote the macro with so it
> should pick up the value based on the current device version. If you are
> concerned that we will change device caps without changing the PVRDMA
> version, that's something we avoid for compatibility reasons.

I have hard time to understand how will you extend this macro once new
feature is introduced.

Let's for an example conduct the following experiment:
1. IB/core gets new exciting ROCE_V3
2. You are adding support to it and increasing the version to be VERSION19.
3. You now have code for ROCE_V2 and ROCE_V3.

So the question is: how will your VRDMA_GET_CAP() look in such case?
RECE_V2 is supported in VERSION18 and above, while ROCE_V3 is supported
in VERSION19 only.

And if we add more than one new feature, how will it look?

Thanks


>
> > In more complex case, when one of the features is supported in one version
> > but isn't supported in another you will need to add "if(specific_feature)" magic
> > into it.
>
> This will not happen based on how our version model currently works.

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Adit Ranadive]

On Wed, Aug 23, 2017 at 11:53:26AM -0700, Leon Romanovsky wrote:
> On Wed, Aug 23, 2017 at 11:08:01AM -0700, Adit Ranadive wrote:
> > On Wed, Aug 23, 2017 at 12:09:20AM +0300, Leon Romanovsky wrote:
> > > On Tue, Aug 22, 2017 at 11:19:01PM -0700, Adit Ranadive wrote:
> > > > Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
> > > > and the IP_BASED_GIDS port cap flag.
> > > >
> > > > Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > > Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > > Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > > Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > > ---
> > > >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
> > > >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
> > > >  2 files changed, 17 insertions(+), 1 deletion(-)
> > > >
> > > > diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > > index 3a308ff..df0a6b5 100644
> > > > --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > > +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > > @@ -149,6 +149,13 @@
> > > >  	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
> > > >  	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
> > > >
> > > > +/*
> > > > + * Get capability values based on device version.
> > > > + */
> > > > +
> > > > +#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
> > > > +	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
> > > > +
> > >
> > > The current macro implementation will require you to go and update all
> > > the places once you will move to new version. In simple case, everything
> > > is supported and you will change your check of PVRDMA_IS_VERSION18 to
> > > something PVRDMA_IS_VERSIONXXX and it will work.
> >
> > Thanks for taking a look. That's the intention I wrote the macro with so it
> > should pick up the value based on the current device version. If you are
> > concerned that we will change device caps without changing the PVRDMA
> > version, that's something we avoid for compatibility reasons.
> 
> I have hard time to understand how will you extend this macro once new
> feature is introduced.
> 
> Let's for an example conduct the following experiment:
> 1. IB/core gets new exciting ROCE_V3
> 2. You are adding support to it and increasing the version to be VERSION19.
> 3. You now have code for ROCE_V2 and ROCE_V3.
> 
> So the question is: how will your VRDMA_GET_CAP() look in such case?
> RECE_V2 is supported in VERSION18 and above, while ROCE_V3 is supported
> in VERSION19 only.

The values for the capabilities come from our DSR (device shared region) itself.
This would have nothing to do with the exact RoCE version but is based
on the device version itself. So, if we add a VERSION19, we can change the
macro to something like PVRDMA_IS_VERSION_NEXT_GEN, it will just pick up the
new values defined in the DSR rather than based on RoCE versions.
If we have removed support for that capability for VERSION19, that should
simply be 0.

To make this more clear, I can put the RoCE v1/v2 type checks, each in own
macro. Drop them from the PVRDMA_IS_VERSION checks. Rename the
PVRDMA_IS_VERSION18 to PVRDMA_IS_NEXTVERSION so we will always pick up the
cap values correctly for future versions and we are not dependent on the
RoCE version itself.

> And if we add more than one new feature, how will it look?
> 
> Thanks



--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 4133 bytes --]

On Wed, Aug 23, 2017 at 12:29:21PM -0700, Adit Ranadive wrote:
> On Wed, Aug 23, 2017 at 11:53:26AM -0700, Leon Romanovsky wrote:
> > On Wed, Aug 23, 2017 at 11:08:01AM -0700, Adit Ranadive wrote:
> > > On Wed, Aug 23, 2017 at 12:09:20AM +0300, Leon Romanovsky wrote:
> > > > On Tue, Aug 22, 2017 at 11:19:01PM -0700, Adit Ranadive wrote:
> > > > > Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
> > > > > and the IP_BASED_GIDS port cap flag.
> > > > >
> > > > > Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > > > Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > > > Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > > > Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > > > > ---
> > > > >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
> > > > >  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
> > > > >  2 files changed, 17 insertions(+), 1 deletion(-)
> > > > >
> > > > > diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > > > index 3a308ff..df0a6b5 100644
> > > > > --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > > > +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > > > > @@ -149,6 +149,13 @@
> > > > >  	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
> > > > >  	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
> > > > >
> > > > > +/*
> > > > > + * Get capability values based on device version.
> > > > > + */
> > > > > +
> > > > > +#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
> > > > > +	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
> > > > > +
> > > >
> > > > The current macro implementation will require you to go and update all
> > > > the places once you will move to new version. In simple case, everything
> > > > is supported and you will change your check of PVRDMA_IS_VERSION18 to
> > > > something PVRDMA_IS_VERSIONXXX and it will work.
> > >
> > > Thanks for taking a look. That's the intention I wrote the macro with so it
> > > should pick up the value based on the current device version. If you are
> > > concerned that we will change device caps without changing the PVRDMA
> > > version, that's something we avoid for compatibility reasons.
> >
> > I have hard time to understand how will you extend this macro once new
> > feature is introduced.
> >
> > Let's for an example conduct the following experiment:
> > 1. IB/core gets new exciting ROCE_V3
> > 2. You are adding support to it and increasing the version to be VERSION19.
> > 3. You now have code for ROCE_V2 and ROCE_V3.
> >
> > So the question is: how will your VRDMA_GET_CAP() look in such case?
> > RECE_V2 is supported in VERSION18 and above, while ROCE_V3 is supported
> > in VERSION19 only.
>
> The values for the capabilities come from our DSR (device shared region) itself.
> This would have nothing to do with the exact RoCE version but is based
> on the device version itself. So, if we add a VERSION19, we can change the
> macro to something like PVRDMA_IS_VERSION_NEXT_GEN, it will just pick up the
> new values defined in the DSR rather than based on RoCE versions.
> If we have removed support for that capability for VERSION19, that should
> simply be 0.

Thanks, I got the idea and looking forward to see how it handles over time.

>
> To make this more clear, I can put the RoCE v1/v2 type checks, each in own
> macro. Drop them from the PVRDMA_IS_VERSION checks. Rename the
> PVRDMA_IS_VERSION18 to PVRDMA_IS_NEXTVERSION so we will always pick up the
> cap values correctly for future versions and we are not dependent on the
> RoCE version itself.

I don't think that it is needed at this stage.

>
> > And if we add more than one new feature, how will it look?
> >
> > Thanks
>
>
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 777 bytes --]

On Tue, Aug 22, 2017 at 11:19:01PM -0700, Adit Ranadive wrote:
> Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
> and the IP_BASED_GIDS port cap flag.
>
> Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> ---
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
>  2 files changed, 17 insertions(+), 1 deletion(-)
>

Thanks,
Reviewed-by: Leon Romanovsky <leonro-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH for-next v1 1/2] RDMA/vmw_pvrdma: Add RoCEv2 support

[Leon Romanovsky]

[-- Attachment #1: Type: text/plain, Size: 1053 bytes --]

On Tue, Aug 22, 2017 at 11:19:00PM -0700, Adit Ranadive wrote:
> From: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
>
> The driver version is bumped for compatibility purposes. Also, send correct
> GID type during register to device. Added compatibility check macros for
> the device.
>
> Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Signed-off-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> ---
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 28 +++++++++++++++++-
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 ++++++++++-------------
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
>  4 files changed, 51 insertions(+), 22 deletions(-)
>

Thanks,
Reviewed-by: Leon Romanovsky <leonro-VPRAkNaXOzVWk0Htik3J/w@public.gmane.org>

[-- Attachment #2: signature.asc --]
[-- Type: application/pgp-signature, Size: 833 bytes --]

Re: [PATCH for-next v1 1/2] RDMA/vmw_pvrdma: Add RoCEv2 support

[Yuval Shaia]

On Tue, Aug 22, 2017 at 11:19:00PM -0700, Adit Ranadive wrote:
> From: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> 
> The driver version is bumped for compatibility purposes. Also, send correct
> GID type during register to device. Added compatibility check macros for
> the device.
> 
> Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Signed-off-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> ---
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 28 +++++++++++++++++-
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 ++++++++++-------------
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
>  4 files changed, 51 insertions(+), 22 deletions(-)
> 
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
> index 8e2f0a1..663a0c3 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h
> @@ -194,6 +194,7 @@ struct pvrdma_dev {
>  	void *resp_slot;
>  	unsigned long flags;
>  	struct list_head device_link;
> +	unsigned int dsr_version;
>  
>  	/* Locking and interrupt information. */
>  	spinlock_t cmd_lock; /* Command lock. */
> @@ -444,6 +445,7 @@ void pvrdma_ah_attr_to_rdma(struct rdma_ah_attr *dst,
>  			    const struct pvrdma_ah_attr *src);
>  void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
>  			    const struct rdma_ah_attr *src);
> +u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type);
>  
>  int pvrdma_uar_table_init(struct pvrdma_dev *dev);
>  void pvrdma_uar_table_cleanup(struct pvrdma_dev *dev);
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> index 09078cc..3a308ff 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> @@ -50,7 +50,15 @@
>  
>  #include "pvrdma_verbs.h"
>  
> -#define PVRDMA_VERSION			17
> +/*
> + * PVRDMA version macros. Some new features require updates to PVRDMA_VERSION.
> + * These macros allow us to check for different features if necessary.
> + */
> +
> +#define PVRDMA_ROCEV1_VERSION		17
> +#define PVRDMA_ROCEV2_VERSION		18
> +#define PVRDMA_VERSION			PVRDMA_ROCEV2_VERSION
> +
>  #define PVRDMA_BOARD_ID			1
>  #define PVRDMA_REV_ID			1
>  
> @@ -123,6 +131,24 @@
>  #define PVRDMA_GID_TYPE_FLAG_ROCE_V1	BIT(0)
>  #define PVRDMA_GID_TYPE_FLAG_ROCE_V2	BIT(1)
>  
> +/*
> + * Version checks. This checks whether each version supports specific
> + * capabilities from the device.
> + */
> +
> +#define PVRDMA_IS_VERSION17(_dev)					\
> +	(_dev->dsr_version == PVRDMA_ROCEV1_VERSION &&			\
> +	 _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
> +
> +#define PVRDMA_IS_VERSION18(_dev)					\
> +	(_dev->dsr_version >= PVRDMA_ROCEV2_VERSION &&			\
> +	 (_dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1 ||  \
> +	  _dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2))	\
> +
> +#define PVRDMA_SUPPORTED(_dev)						\
> +	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
> +	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
> +
>  enum pvrdma_pci_resource {
>  	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
>  	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
> index 5b00156..6ce709a 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c
> @@ -128,10 +128,14 @@ static int pvrdma_init_device(struct pvrdma_dev *dev)
>  static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num,
>  				 struct ib_port_immutable *immutable)
>  {
> +	struct pvrdma_dev *dev = to_vdev(ibdev);
>  	struct ib_port_attr attr;
>  	int err;
>  
> -	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
> +	if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V1)
> +		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE;
> +	else if (dev->dsr->caps.gid_types == PVRDMA_GID_TYPE_FLAG_ROCE_V2)
> +		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
>  
>  	err = ib_query_port(ibdev, port_num, &attr);
>  	if (err)
> @@ -569,6 +573,7 @@ static void pvrdma_free_slots(struct pvrdma_dev *dev)
>  
>  static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
>  				   const union ib_gid *gid,
> +				   u8 gid_type,
>  				   int index)
>  {
>  	int ret;
> @@ -586,7 +591,7 @@ static int pvrdma_add_gid_at_index(struct pvrdma_dev *dev,
>  	cmd_bind->mtu = ib_mtu_enum_to_int(IB_MTU_1024);
>  	cmd_bind->vlan = 0xfff;
>  	cmd_bind->index = index;
> -	cmd_bind->gid_type = PVRDMA_GID_TYPE_FLAG_ROCE_V1;
> +	cmd_bind->gid_type = gid_type;
>  
>  	ret = pvrdma_cmd_post(dev, &req, NULL, 0);
>  	if (ret < 0) {
> @@ -607,7 +612,9 @@ static int pvrdma_add_gid(struct ib_device *ibdev,
>  {
>  	struct pvrdma_dev *dev = to_vdev(ibdev);
>  
> -	return pvrdma_add_gid_at_index(dev, gid, index);
> +	return pvrdma_add_gid_at_index(dev, gid,
> +				       ib_gid_type_to_pvrdma(attr->gid_type),
> +				       index);
>  }
>  
>  static int pvrdma_del_gid_at_index(struct pvrdma_dev *dev, int index)
> @@ -722,7 +729,6 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
>  	int ret;
>  	unsigned long start;
>  	unsigned long len;
> -	unsigned int version;
>  	dma_addr_t slot_dma = 0;
>  
>  	dev_dbg(&pdev->dev, "initializing driver %s\n", pci_name(pdev));
> @@ -819,13 +825,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
>  		goto err_unmap_regs;
>  	}
>  
> -	version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
> +	dev->dsr_version = pvrdma_read_reg(dev, PVRDMA_REG_VERSION);
>  	dev_info(&pdev->dev, "device version %d, driver version %d\n",
> -		 version, PVRDMA_VERSION);
> -	if (version < PVRDMA_VERSION) {
> -		dev_err(&pdev->dev, "incompatible device version\n");
> -		goto err_uar_unmap;
> -	}
> +		 dev->dsr_version, PVRDMA_VERSION);
>  
>  	dev->dsr = dma_alloc_coherent(&pdev->dev, sizeof(*dev->dsr),
>  				      &dev->dsrbase, GFP_KERNEL);
> @@ -896,17 +898,9 @@ static int pvrdma_pci_probe(struct pci_dev *pdev,
>  	/* Make sure the write is complete before reading status. */
>  	mb();
>  
> -	/* Currently, the driver only supports RoCE mode. */
> -	if (dev->dsr->caps.mode != PVRDMA_DEVICE_MODE_ROCE) {
> -		dev_err(&pdev->dev, "unsupported transport %d\n",
> -			dev->dsr->caps.mode);
> -		ret = -EFAULT;
> -		goto err_free_cq_ring;
> -	}
> -
> -	/* Currently, the driver only supports RoCE V1. */
> -	if (!(dev->dsr->caps.gid_types & PVRDMA_GID_TYPE_FLAG_ROCE_V1)) {
> -		dev_err(&pdev->dev, "driver needs RoCE v1 support\n");
> +	/* The driver supports RoCE V1 and V2. */
> +	if (!PVRDMA_SUPPORTED(dev)) {
> +		dev_err(&pdev->dev, "driver needs RoCE v1 or v2 support\n");
>  		ret = -EFAULT;
>  		goto err_free_cq_ring;
>  	}
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
> index ec6a4ca..fb0c5c0 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c
> @@ -303,3 +303,10 @@ void rdma_ah_attr_to_pvrdma(struct pvrdma_ah_attr *dst,
>  	dst->port_num = rdma_ah_get_port_num(src);
>  	memcpy(&dst->dmac, src->roce.dmac, sizeof(dst->dmac));
>  }
> +
> +u8 ib_gid_type_to_pvrdma(enum ib_gid_type gid_type)
> +{
> +	return (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) ?
> +		PVRDMA_GID_TYPE_FLAG_ROCE_V2 :
> +		PVRDMA_GID_TYPE_FLAG_ROCE_V1;
> +}
> -- 
> 2.7.4

Reviewed-by: Yuval Shaia <yuval.shaia-QHcLZuEGTsvQT0dZR+AlfA@public.gmane.org>

> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Yuval Shaia]

On Tue, Aug 22, 2017 at 11:19:01PM -0700, Adit Ranadive wrote:
> Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
> and the IP_BASED_GIDS port cap flag.

And IB_PORT_CM_SUP

> 
> Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> ---
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
>  2 files changed, 17 insertions(+), 1 deletion(-)
> 
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> index 3a308ff..df0a6b5 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> @@ -149,6 +149,13 @@
>  	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
>  	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
>  
> +/*
> + * Get capability values based on device version.
> + */
> +
> +#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
> +	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
> +
>  enum pvrdma_pci_resource {
>  	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
>  	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
> @@ -251,7 +258,7 @@ struct pvrdma_device_caps {
>  	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
>  	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
>  	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
> -	u8  reserved[4];
> +	u32 max_fast_reg_page_list_len;
>  };
>  
>  struct pvrdma_ring_page_info {
> diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> index 2851704..48776f5 100644
> --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> @@ -83,6 +83,8 @@ int pvrdma_query_device(struct ib_device *ibdev,
>  	props->max_qp_wr = dev->dsr->caps.max_qp_wr;
>  	props->device_cap_flags = dev->dsr->caps.device_cap_flags;
>  	props->max_sge = dev->dsr->caps.max_sge;
> +	props->max_sge_rd = PVRDMA_GET_CAP(dev, dev->dsr->caps.max_sge,
> +					   dev->dsr->caps.max_sge_rd);

I tend to agree with Leon on the other thread, i think that the following
code is much more readable:

	props->max_sge_rd = dev->dsr->caps.max_sge;
	if (PVRDMA_IS_VERSION18(dev))
		props->max_sge_rd = dev->dsr->caps.max_sge_rd;

And then in the future add something like this:
	if (PVRDMA_IS_VERSION19(dev))
		props->max_sge_rd = dev->dsr->caps.max_sge_19;

But it is totally up to you.

>  	props->max_cq = dev->dsr->caps.max_cq;
>  	props->max_cqe = dev->dsr->caps.max_cqe;
>  	props->max_mr = dev->dsr->caps.max_mr;
> @@ -101,8 +103,14 @@ int pvrdma_query_device(struct ib_device *ibdev,
>  	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&
>  	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_FAST_REG_WR)) {
>  		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
> +		props->max_fast_reg_page_list_len = PVRDMA_GET_CAP(dev,
> +				PVRDMA_MAX_FAST_REG_PAGES,
> +				dev->dsr->caps.max_fast_reg_page_list_len);
>  	}
>  
> +	props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT |
> +				   IB_DEVICE_RC_RNR_NAK_GEN;
> +

Is it left over from version 17 or a new HW support?

>  	return 0;
>  }
>  
> @@ -143,6 +151,7 @@ int pvrdma_query_port(struct ib_device *ibdev, u8 port,
>  	props->gid_tbl_len = resp->attrs.gid_tbl_len;
>  	props->port_cap_flags =
>  		pvrdma_port_cap_flags_to_ib(resp->attrs.port_cap_flags);
> +	props->port_cap_flags |= IB_PORT_CM_SUP | IB_PORT_IP_BASED_GIDS;

Ditto.

>  	props->max_msg_sz = resp->attrs.max_msg_sz;
>  	props->bad_pkey_cntr = resp->attrs.bad_pkey_cntr;
>  	props->qkey_viol_cntr = resp->attrs.qkey_viol_cntr;
> -- 
> 2.7.4
> 
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
> More majordomo info at  http://vger.kernel.org/majordomo-info.html
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-next v1 0/2] RDMA/vmw_pvrdma: Add RoCEv2 support

[Doug Ledford]

On Tue, 2017-08-22 at 23:18 -0700, Adit Ranadive wrote:
> Hi Doug,
> 
> These are a set of patches to add RoCEv2 support for the VMware
> PVRDMA
> driver as well as exposing some device caps. Please add these to your
> 4.14 queue.
> 
> Thanks,
> Adit
> ---
> Changes v0 -> v1:
>  - Consolidated the version macros.
>  - Added macros to check for compatibility.
>  - Squashed a previous patch #3 into patch #2.
>  - Removed an unused function.
>  - Updated tag lines correctly.
> ---
> Adit Ranadive (1):
>   RDMA/vmw_pvrdma: Update device query parameters and port caps
> 
> Bryan Tan (1):
>   RDMA/vmw_pvrdma: Add RoCEv2 support
> 
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma.h         |  2 ++
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 37
> +++++++++++++++++++++--
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c    | 36 +++++++++--
> -----------
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_misc.c    |  7 +++++
>  drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   |  9 ++++++
>  5 files changed, 68 insertions(+), 23 deletions(-)

Thanks, applied.

-- 
Doug Ledford <dledford-H+wXaHxf7aLQT0dZR+AlfA@public.gmane.org>
    GPG KeyID: B826A3330E572FDD
    Key fingerprint = AE6B 1BDA 122B 23B4 265B  1274 B826 A333 0E57 2FDD

--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html

Re: [PATCH for-next v1 2/2] RDMA/vmw_pvrdma: Update device query parameters and port caps

[Adit Ranadive]

On Thur, Aug 24, 2017 at 07:33:50AM 0700, Yuval Shaia wrote:
> On Tue, Aug 22, 2017 at 11:19:01PM -0700, Adit Ranadive wrote:
> > Added support for two device caps - max_sge_rd, max_fast_reg_page_list_len
> > and the IP_BASED_GIDS port cap flag.
> 
> And IB_PORT_CM_SUP
> 
> >
> > Reviewed-by: Jorgen Hansen <jhansen-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > Reviewed-by: Bryan Tan <bryantan-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > Reviewed-by: Aditya Sarwade <asarwade-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > Signed-off-by: Adit Ranadive <aditr-pghWNbHTmq7QT0dZR+AlfA@public.gmane.org>
> > ---
> >   drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h | 9 ++++++++-
> >   drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c   | 9 +++++++++
> >   2 files changed, 17 insertions(+), 1 deletion(-)
> >
> > diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > index 3a308ff..df0a6b5 100644
> > --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_dev_api.h
> > @@ -149,6 +149,13 @@
> >   	((_dev->dsr->caps.mode == PVRDMA_DEVICE_MODE_ROCE) &&		\
> >   	 (PVRDMA_IS_VERSION17(_dev) || PVRDMA_IS_VERSION18(_dev)))
> >   
> > +/*
> > + * Get capability values based on device version.
> > + */
> > +
> > +#define PVRDMA_GET_CAP(_dev, _old_val, _val) \
> > +	((PVRDMA_IS_VERSION18(_dev)) ? _val : _old_val)
> > +
> >   enum pvrdma_pci_resource {
> >   	PVRDMA_PCI_RESOURCE_MSIX,	/* BAR0: MSI-X, MMIO. */
> >   	PVRDMA_PCI_RESOURCE_REG,	/* BAR1: Registers, MMIO. */
> > @@ -251,7 +258,7 @@ struct pvrdma_device_caps {
> >   	u8  atomic_ops;				/* PVRDMA_ATOMIC_OP_* bits */
> >   	u8  bmme_flags;				/* FRWR Mem Mgmt Extensions */
> >   	u8  gid_types;				/* PVRDMA_GID_TYPE_FLAG_ */
> > -	u8  reserved[4];
> > +	u32 max_fast_reg_page_list_len;
> >   };
> >   
> >   struct pvrdma_ring_page_info {
> > diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> > index 2851704..48776f5 100644
> > --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> > +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.c
> > @@ -83,6 +83,8 @@ int pvrdma_query_device(struct ib_device *ibdev,
> >   	props->max_qp_wr = dev->dsr->caps.max_qp_wr;
> >   	props->device_cap_flags = dev->dsr->caps.device_cap_flags;
> >   	props->max_sge = dev->dsr->caps.max_sge;
> > +	props->max_sge_rd = PVRDMA_GET_CAP(dev, dev->dsr->caps.max_sge,
> > +					   dev->dsr->caps.max_sge_rd);
> 
> I tend to agree with Leon on the other thread, i think that the following
> code is much more readable:
> 
> 	props->max_sge_rd = dev->dsr->caps.max_sge;
> 	if (PVRDMA_IS_VERSION18(dev))
> 		props->max_sge_rd = dev->dsr->caps.max_sge_rd;
> 
> And then in the future add something like this:
> 	if (PVRDMA_IS_VERSION19(dev))
> 		props->max_sge_rd = dev->dsr->caps.max_sge_19;
> 
> But it is totally up to you.

Thanks for taking a look and your suggestion.

> 
> >   	props->max_cq = dev->dsr->caps.max_cq;
> >   	props->max_cqe = dev->dsr->caps.max_cqe;
> >   	props->max_mr = dev->dsr->caps.max_mr;
> > @@ -101,8 +103,14 @@ int pvrdma_query_device(struct ib_device *ibdev,
> >   	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_REMOTE_INV) &&
> >   	    (dev->dsr->caps.bmme_flags & PVRDMA_BMME_FLAG_FAST_REG_WR)) {
> >   		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
> > +		props->max_fast_reg_page_list_len = PVRDMA_GET_CAP(dev,
> > +				PVRDMA_MAX_FAST_REG_PAGES,
> > +				dev->dsr->caps.max_fast_reg_page_list_len);
> >   	}
> >   
> > +	props->device_cap_flags |= IB_DEVICE_PORT_ACTIVE_EVENT |
> > +				   IB_DEVICE_RC_RNR_NAK_GEN;
> > +
> 
> Is it left over from version 17 or a new HW support?

No, we have always implicitly supported those. This just makes it more 
explicit to users.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo-u79uwXL29TY76Z2rM5mHXA@public.gmane.org
More majordomo info at  http://vger.kernel.org/majordomo-info.html