[PATCH rdma-core v2 0/2] vmw_pvrdma: Use physical resource ids from device

[PATCH rdma-core v2 0/2] vmw_pvrdma: Use physical resource ids from device

[Adit Ranadive]

Changelog:
v2:
 - Used kernel-headers/update script to generate headers commit
v1:
 - Added a separate patch for the kernel header
 - Dropped the ABI version check
 - Added a create qp flag and an in-band check to indicate support
v0:
 - https://patchwork.kernel.org/patch/10946987/

Hi,

Here is a patchset for enabling exposing physical resource ids for vmw_pvrdma
userspace provider.

rdma-core PR:
 - https://github.com/linux-rdma/rdma-core/pull/581

Thanks,
Adit

Adit Ranadive (1):
  Update kernel headers

Bryan Tan (1):
  vmw_pvrdma: Use resource ids from physical device if available

 kernel-headers/CMakeLists.txt        |  1 +
 kernel-headers/rdma/rvt-abi.h        | 66 ++++++++++++++++++++++++++++
 kernel-headers/rdma/vmw_pvrdma-abi.h | 13 ++++++
 providers/vmw_pvrdma/pvrdma-abi.h    |  2 +-
 providers/vmw_pvrdma/pvrdma.h        |  1 +
 providers/vmw_pvrdma/qp.c            | 24 +++++-----
 6 files changed, 95 insertions(+), 12 deletions(-)
 create mode 100644 kernel-headers/rdma/rvt-abi.h

-- 
2.18.1

[PATCH rdma-core v2 1/2] Update kernel headers

[Adit Ranadive]

To commit ?? ("RDMA/vmw_pvrdma: Use resource ids from physical device if
available")

Signed-off-by: Adit Ranadive <aditr@vmware.com>
---
 kernel-headers/CMakeLists.txt        |  1 +
 kernel-headers/rdma/rvt-abi.h        | 66 ++++++++++++++++++++++++++++
 kernel-headers/rdma/vmw_pvrdma-abi.h | 13 ++++++
 3 files changed, 80 insertions(+)
 create mode 100644 kernel-headers/rdma/rvt-abi.h

diff --git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt
index 50bc77e6ab6e..13624b22d81b 100644
--- a/kernel-headers/CMakeLists.txt
+++ b/kernel-headers/CMakeLists.txt
@@ -23,6 +23,7 @@ publish_internal_headers(rdma
   rdma/rdma_user_ioctl.h
   rdma/rdma_user_ioctl_cmds.h
   rdma/rdma_user_rxe.h
+  rdma/rvt-abi.h
   rdma/siw-abi.h
   rdma/vmw_pvrdma-abi.h
   )
diff --git a/kernel-headers/rdma/rvt-abi.h b/kernel-headers/rdma/rvt-abi.h
new file mode 100644
index 000000000000..7c05a02d2be5
--- /dev/null
+++ b/kernel-headers/rdma/rvt-abi.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+
+/*
+ * This file contains defines, structures, etc. that are used
+ * to communicate between kernel and user code.
+ */
+
+#ifndef RVT_ABI_USER_H
+#define RVT_ABI_USER_H
+
+#include <linux/types.h>
+#include <rdma/ib_user_verbs.h>
+#ifndef RDMA_ATOMIC_UAPI
+#define RDMA_ATOMIC_UAPI(_type, _name) struct{ _type val; } _name
+#endif
+
+struct rvt_wqe_sge {
+	__aligned_u64 addr;
+	__u32 length;
+	__u32 lkey;
+};
+
+/*
+ * This structure is used to contain the head pointer, tail pointer,
+ * and completion queue entries as a single memory allocation so
+ * it can be mmap'ed into user space.
+ */
+struct rvt_cq_wc {
+	/* index of next entry to fill */
+	RDMA_ATOMIC_UAPI(__u32, head);
+	/* index of next ib_poll_cq() entry */
+	RDMA_ATOMIC_UAPI(__u32, tail);
+
+	/* these are actually size ibcq.cqe + 1 */
+	struct ib_uverbs_wc uqueue[];
+};
+
+/*
+ * Receive work request queue entry.
+ * The size of the sg_list is determined when the QP (or SRQ) is created
+ * and stored in qp->r_rq.max_sge (or srq->rq.max_sge).
+ */
+struct rvt_rwqe {
+	__u64 wr_id;
+	__u8 num_sge;
+	__u8 padding[7];
+	struct rvt_wqe_sge sg_list[];
+};
+
+/*
+ * This structure is used to contain the head pointer, tail pointer,
+ * and receive work queue entries as a single memory allocation so
+ * it can be mmap'ed into user space.
+ * Note that the wq array elements are variable size so you can't
+ * just index into the array to get the N'th element;
+ * use get_rwqe_ptr() for user space and rvt_get_rwqe_ptr()
+ * for kernel space.
+ */
+struct rvt_rwq {
+	/* new work requests posted to the head */
+	RDMA_ATOMIC_UAPI(__u32, head);
+	/* receives pull requests from here. */
+	RDMA_ATOMIC_UAPI(__u32, tail);
+	struct rvt_rwqe wq[];
+};
+#endif /* RVT_ABI_USER_H */
diff --git a/kernel-headers/rdma/vmw_pvrdma-abi.h b/kernel-headers/rdma/vmw_pvrdma-abi.h
index 6e73f0274e41..1d339285550e 100644
--- a/kernel-headers/rdma/vmw_pvrdma-abi.h
+++ b/kernel-headers/rdma/vmw_pvrdma-abi.h
@@ -133,6 +133,10 @@ enum pvrdma_wc_flags {
 	PVRDMA_WC_FLAGS_MAX		= PVRDMA_WC_WITH_NETWORK_HDR_TYPE,
 };
 
+enum pvrdma_user_qp_create_flags {
+	PVRDMA_USER_QP_CREATE_USE_RESP	= 1 << 0,
+};
+
 struct pvrdma_alloc_ucontext_resp {
 	__u32 qp_tab_size;
 	__u32 reserved;
@@ -177,6 +181,15 @@ struct pvrdma_create_qp {
 	__u32 rbuf_size;
 	__u32 sbuf_size;
 	__aligned_u64 qp_addr;
+	__u32 flags;
+	__u32 reserved;
+};
+
+struct pvrdma_create_qp_resp {
+	__u32 qpn;
+	__u32 qp_handle;
+	__u32 qpn_valid;
+	__u32 reserved;
 };
 
 /* PVRDMA masked atomic compare and swap */
-- 
2.18.1

[PATCH rdma-core v2 2/2] vmw_pvrdma: Use resource ids from physical device if available

[Adit Ranadive]

From: Bryan Tan <bryantan@vmware.com>

This is the accompanying userspace change to allow applications
use physical resource ids if provided by the driver/device.

The create QP command is modified to specify a flag that tells
the driver to send back the physical HCA's QPN and the hypervisor-
generated QP handle separately.

Reviewed-by: Jorgen Hansen <jhansen@vmware.com>
Signed-off-by: Adit Ranadive <aditr@vmware.com>
Signed-off-by: Bryan Tan <bryantan@vmware.com>
---
 providers/vmw_pvrdma/pvrdma-abi.h |  2 +-
 providers/vmw_pvrdma/pvrdma.h     |  1 +
 providers/vmw_pvrdma/qp.c         | 24 +++++++++++++-----------
 3 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/providers/vmw_pvrdma/pvrdma-abi.h b/providers/vmw_pvrdma/pvrdma-abi.h
index 77db9ddd1bb7..1a4c3c8a98f2 100644
--- a/providers/vmw_pvrdma/pvrdma-abi.h
+++ b/providers/vmw_pvrdma/pvrdma-abi.h
@@ -55,7 +55,7 @@ DECLARE_DRV_CMD(user_pvrdma_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
 DECLARE_DRV_CMD(user_pvrdma_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
 		pvrdma_create_cq, pvrdma_create_cq_resp);
 DECLARE_DRV_CMD(user_pvrdma_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
-		pvrdma_create_qp, empty);
+		pvrdma_create_qp, pvrdma_create_qp_resp);
 DECLARE_DRV_CMD(user_pvrdma_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
 		pvrdma_create_srq, pvrdma_create_srq_resp);
 DECLARE_DRV_CMD(user_pvrdma_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
diff --git a/providers/vmw_pvrdma/pvrdma.h b/providers/vmw_pvrdma/pvrdma.h
index d90bd8096664..bb4a2db08d14 100644
--- a/providers/vmw_pvrdma/pvrdma.h
+++ b/providers/vmw_pvrdma/pvrdma.h
@@ -170,6 +170,7 @@ struct pvrdma_qp {
 	struct pvrdma_wq		sq;
 	struct pvrdma_wq		rq;
 	int				is_srq;
+	uint32_t			qp_handle;
 };
 
 struct pvrdma_ah {
diff --git a/providers/vmw_pvrdma/qp.c b/providers/vmw_pvrdma/qp.c
index ef429db93a43..966480f5abaa 100644
--- a/providers/vmw_pvrdma/qp.c
+++ b/providers/vmw_pvrdma/qp.c
@@ -211,9 +211,8 @@ struct ibv_qp *pvrdma_create_qp(struct ibv_pd *pd,
 {
 	struct pvrdma_device *dev = to_vdev(pd->context->device);
 	struct user_pvrdma_create_qp cmd;
-	struct ib_uverbs_create_qp_resp resp;
+	struct user_pvrdma_create_qp_resp resp = {};
 	struct pvrdma_qp *qp;
-	int ret;
 	int is_srq = !!(attr->srq);
 
 	attr->cap.max_send_sge = max_t(uint32_t, 1U, attr->cap.max_send_sge);
@@ -281,15 +280,18 @@ struct ibv_qp *pvrdma_create_qp(struct ibv_pd *pd,
 	cmd.rbuf_addr = (uintptr_t)qp->rbuf.buf;
 	cmd.rbuf_size = qp->rbuf.length;
 	cmd.qp_addr = (uintptr_t) qp;
+	cmd.flags = PVRDMA_USER_QP_CREATE_USE_RESP;
 
-	ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr,
-				&cmd.ibv_cmd, sizeof(cmd),
-				&resp, sizeof(resp));
-
-	if (ret)
+	if (ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof(cmd),
+			      &resp.ibv_resp, sizeof(resp)))
 		goto err_free;
 
-	to_vctx(pd->context)->qp_tbl[qp->ibv_qp.qp_num & 0xFFFF] = qp;
+	if (resp.drv_payload.qpn_valid == PVRDMA_USER_QP_CREATE_USE_RESP)
+		qp->qp_handle = resp.drv_payload.qp_handle;
+	else
+		qp->qp_handle = qp->ibv_qp.qp_num;
+
+	to_vctx(pd->context)->qp_tbl[qp->qp_handle & 0xFFFF] = qp;
 
 	/* If set, each WR submitted to the SQ generate a completion entry */
 	if (attr->sq_sig_all)
@@ -414,7 +416,7 @@ int pvrdma_destroy_qp(struct ibv_qp *ibqp)
 	free(qp->rq.wrid);
 	pvrdma_free_buf(&qp->rbuf);
 	pvrdma_free_buf(&qp->sbuf);
-	ctx->qp_tbl[ibqp->qp_num & 0xFFFF] = NULL;
+	ctx->qp_tbl[qp->qp_handle & 0xFFFF] = NULL;
 	free(qp);
 
 	return 0;
@@ -547,7 +549,7 @@ out:
 	if (nreq) {
 		udma_to_device_barrier();
 		pvrdma_write_uar_qp(ctx->uar,
-				    PVRDMA_UAR_QP_SEND | ibqp->qp_num);
+				    PVRDMA_UAR_QP_SEND | qp->qp_handle);
 	}
 
 	pthread_spin_unlock(&qp->sq.lock);
@@ -630,7 +632,7 @@ int pvrdma_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
 out:
 	if (nreq)
 		pvrdma_write_uar_qp(ctx->uar,
-				    PVRDMA_UAR_QP_RECV | ibqp->qp_num);
+				    PVRDMA_UAR_QP_RECV | qp->qp_handle);
 
 	pthread_spin_unlock(&qp->rq.lock);
 	return ret;
-- 
2.18.1