* [PATCH rdma-core v3 1/3] Update kernel headers
2019-10-22 20:05 [PATCH rdma-core v3 0/3] vmw_pvrdma: Use physical resource ids from device Adit Ranadive
@ 2019-10-22 20:05 ` Adit Ranadive
2019-10-22 20:05 ` [PATCH rdma-core v3 2/3] kernel-headers: Remove cxgb3-abi.h Adit Ranadive
2019-10-22 20:05 ` [PATCH rdma-core v3 3/3] vmw_pvrdma: Use resource ids from physical device if available Adit Ranadive
2 siblings, 0 replies; 4+ messages in thread
From: Adit Ranadive @ 2019-10-22 20:05 UTC (permalink / raw)
To: jgg@mellanox.com, leon@kernel.org, linux-rdma@vger.kernel.org
Cc: Adit Ranadive
To commit ?? ("RDMA/vmw_pvrdma: Use resource ids from physical device if
available")
Signed-off-by: Adit Ranadive <aditr@vmware.com>
---
| 2 +-
| 82 ----------------------
| 22 ++++++
| 22 ------
| 66 +++++++++++++++++
| 5 ++
6 files changed, 94 insertions(+), 105 deletions(-)
delete mode 100644 kernel-headers/rdma/cxgb3-abi.h
create mode 100644 kernel-headers/rdma/rvt-abi.h
--git a/kernel-headers/CMakeLists.txt b/kernel-headers/CMakeLists.txt
index 50bc77e6ab6e..543e0ea1408a 100644
--- a/kernel-headers/CMakeLists.txt
+++ b/kernel-headers/CMakeLists.txt
@@ -1,6 +1,5 @@
publish_internal_headers(rdma
rdma/bnxt_re-abi.h
- rdma/cxgb3-abi.h
rdma/cxgb4-abi.h
rdma/efa-abi.h
rdma/hns-abi.h
@@ -23,6 +22,7 @@ publish_internal_headers(rdma
rdma/rdma_user_ioctl.h
rdma/rdma_user_ioctl_cmds.h
rdma/rdma_user_rxe.h
+ rdma/rvt-abi.h
rdma/siw-abi.h
rdma/vmw_pvrdma-abi.h
)
diff --git a/kernel-headers/rdma/cxgb3-abi.h b/kernel-headers/rdma/cxgb3-abi.h
deleted file mode 100644
index 85aed672f43e..000000000000
--- a/kernel-headers/rdma/cxgb3-abi.h
+++ /dev/null
@@ -1,82 +0,0 @@
-/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
-/*
- * Copyright (c) 2006 Chelsio, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- * Redistribution and use in source and binary forms, with or
- * without modification, are permitted provided that the following
- * conditions are met:
- *
- * - Redistributions of source code must retain the above
- * copyright notice, this list of conditions and the following
- * disclaimer.
- *
- * - Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following
- * disclaimer in the documentation and/or other materials
- * provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-#ifndef CXGB3_ABI_USER_H
-#define CXGB3_ABI_USER_H
-
-#include <linux/types.h>
-
-#define IWCH_UVERBS_ABI_VERSION 1
-
-/*
- * Make sure that all structs defined in this file remain laid out so
- * that they pack the same way on 32-bit and 64-bit architectures (to
- * avoid incompatibility between 32-bit userspace and 64-bit kernels).
- * In particular do not use pointer types -- pass pointers in __aligned_u64
- * instead.
- */
-struct iwch_create_cq_req {
- __aligned_u64 user_rptr_addr;
-};
-
-struct iwch_create_cq_resp_v0 {
- __aligned_u64 key;
- __u32 cqid;
- __u32 size_log2;
-};
-
-struct iwch_create_cq_resp {
- __aligned_u64 key;
- __u32 cqid;
- __u32 size_log2;
- __u32 memsize;
- __u32 reserved;
-};
-
-struct iwch_create_qp_resp {
- __aligned_u64 key;
- __aligned_u64 db_key;
- __u32 qpid;
- __u32 size_log2;
- __u32 sq_size_log2;
- __u32 rq_size_log2;
-};
-
-struct iwch_reg_user_mr_resp {
- __u32 pbl_addr;
-};
-
-struct iwch_alloc_pd_resp {
- __u32 pdid;
-};
-
-#endif /* CXGB3_ABI_USER_H */
--git a/kernel-headers/rdma/ib_user_ioctl_verbs.h b/kernel-headers/rdma/ib_user_ioctl_verbs.h
index 72c7fc75f960..9019b2d906ea 100644
--- a/kernel-headers/rdma/ib_user_ioctl_verbs.h
+++ b/kernel-headers/rdma/ib_user_ioctl_verbs.h
@@ -173,4 +173,26 @@ struct ib_uverbs_query_port_resp_ex {
__u8 reserved[6];
};
+enum rdma_driver_id {
+ RDMA_DRIVER_UNKNOWN,
+ RDMA_DRIVER_MLX5,
+ RDMA_DRIVER_MLX4,
+ RDMA_DRIVER_CXGB3,
+ RDMA_DRIVER_CXGB4,
+ RDMA_DRIVER_MTHCA,
+ RDMA_DRIVER_BNXT_RE,
+ RDMA_DRIVER_OCRDMA,
+ RDMA_DRIVER_NES,
+ RDMA_DRIVER_I40IW,
+ RDMA_DRIVER_VMW_PVRDMA,
+ RDMA_DRIVER_QEDR,
+ RDMA_DRIVER_HNS,
+ RDMA_DRIVER_USNIC,
+ RDMA_DRIVER_RXE,
+ RDMA_DRIVER_HFI1,
+ RDMA_DRIVER_QIB,
+ RDMA_DRIVER_EFA,
+ RDMA_DRIVER_SIW,
+};
+
#endif
--git a/kernel-headers/rdma/rdma_user_ioctl_cmds.h b/kernel-headers/rdma/rdma_user_ioctl_cmds.h
index b8bb285f6b2a..7b1ec806f8f9 100644
--- a/kernel-headers/rdma/rdma_user_ioctl_cmds.h
+++ b/kernel-headers/rdma/rdma_user_ioctl_cmds.h
@@ -84,26 +84,4 @@ struct ib_uverbs_ioctl_hdr {
struct ib_uverbs_attr attrs[0];
};
-enum rdma_driver_id {
- RDMA_DRIVER_UNKNOWN,
- RDMA_DRIVER_MLX5,
- RDMA_DRIVER_MLX4,
- RDMA_DRIVER_CXGB3,
- RDMA_DRIVER_CXGB4,
- RDMA_DRIVER_MTHCA,
- RDMA_DRIVER_BNXT_RE,
- RDMA_DRIVER_OCRDMA,
- RDMA_DRIVER_NES,
- RDMA_DRIVER_I40IW,
- RDMA_DRIVER_VMW_PVRDMA,
- RDMA_DRIVER_QEDR,
- RDMA_DRIVER_HNS,
- RDMA_DRIVER_USNIC,
- RDMA_DRIVER_RXE,
- RDMA_DRIVER_HFI1,
- RDMA_DRIVER_QIB,
- RDMA_DRIVER_EFA,
- RDMA_DRIVER_SIW,
-};
-
#endif
--git a/kernel-headers/rdma/rvt-abi.h b/kernel-headers/rdma/rvt-abi.h
new file mode 100644
index 000000000000..7c05a02d2be5
--- /dev/null
+++ b/kernel-headers/rdma/rvt-abi.h
@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR BSD-3-Clause) */
+
+/*
+ * This file contains defines, structures, etc. that are used
+ * to communicate between kernel and user code.
+ */
+
+#ifndef RVT_ABI_USER_H
+#define RVT_ABI_USER_H
+
+#include <linux/types.h>
+#include <rdma/ib_user_verbs.h>
+#ifndef RDMA_ATOMIC_UAPI
+#define RDMA_ATOMIC_UAPI(_type, _name) struct{ _type val; } _name
+#endif
+
+struct rvt_wqe_sge {
+ __aligned_u64 addr;
+ __u32 length;
+ __u32 lkey;
+};
+
+/*
+ * This structure is used to contain the head pointer, tail pointer,
+ * and completion queue entries as a single memory allocation so
+ * it can be mmap'ed into user space.
+ */
+struct rvt_cq_wc {
+ /* index of next entry to fill */
+ RDMA_ATOMIC_UAPI(__u32, head);
+ /* index of next ib_poll_cq() entry */
+ RDMA_ATOMIC_UAPI(__u32, tail);
+
+ /* these are actually size ibcq.cqe + 1 */
+ struct ib_uverbs_wc uqueue[];
+};
+
+/*
+ * Receive work request queue entry.
+ * The size of the sg_list is determined when the QP (or SRQ) is created
+ * and stored in qp->r_rq.max_sge (or srq->rq.max_sge).
+ */
+struct rvt_rwqe {
+ __u64 wr_id;
+ __u8 num_sge;
+ __u8 padding[7];
+ struct rvt_wqe_sge sg_list[];
+};
+
+/*
+ * This structure is used to contain the head pointer, tail pointer,
+ * and receive work queue entries as a single memory allocation so
+ * it can be mmap'ed into user space.
+ * Note that the wq array elements are variable size so you can't
+ * just index into the array to get the N'th element;
+ * use get_rwqe_ptr() for user space and rvt_get_rwqe_ptr()
+ * for kernel space.
+ */
+struct rvt_rwq {
+ /* new work requests posted to the head */
+ RDMA_ATOMIC_UAPI(__u32, head);
+ /* receives pull requests from here. */
+ RDMA_ATOMIC_UAPI(__u32, tail);
+ struct rvt_rwqe wq[];
+};
+#endif /* RVT_ABI_USER_H */
--git a/kernel-headers/rdma/vmw_pvrdma-abi.h b/kernel-headers/rdma/vmw_pvrdma-abi.h
index 6e73f0274e41..f8b638c73371 100644
--- a/kernel-headers/rdma/vmw_pvrdma-abi.h
+++ b/kernel-headers/rdma/vmw_pvrdma-abi.h
@@ -179,6 +179,11 @@ struct pvrdma_create_qp {
__aligned_u64 qp_addr;
};
+struct pvrdma_create_qp_resp {
+ __u32 qpn;
+ __u32 qp_handle;
+};
+
/* PVRDMA masked atomic compare and swap */
struct pvrdma_ex_cmp_swap {
__aligned_u64 swap_val;
--
2.18.1
^ permalink raw reply related [flat|nested] 4+ messages in thread* [PATCH rdma-core v3 3/3] vmw_pvrdma: Use resource ids from physical device if available
2019-10-22 20:05 [PATCH rdma-core v3 0/3] vmw_pvrdma: Use physical resource ids from device Adit Ranadive
2019-10-22 20:05 ` [PATCH rdma-core v3 1/3] Update kernel headers Adit Ranadive
2019-10-22 20:05 ` [PATCH rdma-core v3 2/3] kernel-headers: Remove cxgb3-abi.h Adit Ranadive
@ 2019-10-22 20:05 ` Adit Ranadive
2 siblings, 0 replies; 4+ messages in thread
From: Adit Ranadive @ 2019-10-22 20:05 UTC (permalink / raw)
To: jgg@mellanox.com, leon@kernel.org, linux-rdma@vger.kernel.org
Cc: Bryan Tan, Adit Ranadive
From: Bryan Tan <bryantan@vmware.com>
This is the accompanying userspace change to allow applications
use physical resource ids if provided by the driver/device.
Reviewed-by: Jorgen Hansen <jhansen@vmware.com>
Signed-off-by: Adit Ranadive <aditr@vmware.com>
Signed-off-by: Bryan Tan <bryantan@vmware.com>
---
providers/vmw_pvrdma/pvrdma-abi.h | 2 +-
providers/vmw_pvrdma/pvrdma.h | 1 +
providers/vmw_pvrdma/qp.c | 23 ++++++++++++-----------
3 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/providers/vmw_pvrdma/pvrdma-abi.h b/providers/vmw_pvrdma/pvrdma-abi.h
index 77db9ddd1bb7..1a4c3c8a98f2 100644
--- a/providers/vmw_pvrdma/pvrdma-abi.h
+++ b/providers/vmw_pvrdma/pvrdma-abi.h
@@ -55,7 +55,7 @@ DECLARE_DRV_CMD(user_pvrdma_alloc_pd, IB_USER_VERBS_CMD_ALLOC_PD,
DECLARE_DRV_CMD(user_pvrdma_create_cq, IB_USER_VERBS_CMD_CREATE_CQ,
pvrdma_create_cq, pvrdma_create_cq_resp);
DECLARE_DRV_CMD(user_pvrdma_create_qp, IB_USER_VERBS_CMD_CREATE_QP,
- pvrdma_create_qp, empty);
+ pvrdma_create_qp, pvrdma_create_qp_resp);
DECLARE_DRV_CMD(user_pvrdma_create_srq, IB_USER_VERBS_CMD_CREATE_SRQ,
pvrdma_create_srq, pvrdma_create_srq_resp);
DECLARE_DRV_CMD(user_pvrdma_alloc_ucontext, IB_USER_VERBS_CMD_GET_CONTEXT,
diff --git a/providers/vmw_pvrdma/pvrdma.h b/providers/vmw_pvrdma/pvrdma.h
index d90bd8096664..bb4a2db08d14 100644
--- a/providers/vmw_pvrdma/pvrdma.h
+++ b/providers/vmw_pvrdma/pvrdma.h
@@ -170,6 +170,7 @@ struct pvrdma_qp {
struct pvrdma_wq sq;
struct pvrdma_wq rq;
int is_srq;
+ uint32_t qp_handle;
};
struct pvrdma_ah {
diff --git a/providers/vmw_pvrdma/qp.c b/providers/vmw_pvrdma/qp.c
index ef429db93a43..10784314f5b5 100644
--- a/providers/vmw_pvrdma/qp.c
+++ b/providers/vmw_pvrdma/qp.c
@@ -211,9 +211,8 @@ struct ibv_qp *pvrdma_create_qp(struct ibv_pd *pd,
{
struct pvrdma_device *dev = to_vdev(pd->context->device);
struct user_pvrdma_create_qp cmd;
- struct ib_uverbs_create_qp_resp resp;
+ struct user_pvrdma_create_qp_resp resp = {};
struct pvrdma_qp *qp;
- int ret;
int is_srq = !!(attr->srq);
attr->cap.max_send_sge = max_t(uint32_t, 1U, attr->cap.max_send_sge);
@@ -282,14 +281,16 @@ struct ibv_qp *pvrdma_create_qp(struct ibv_pd *pd,
cmd.rbuf_size = qp->rbuf.length;
cmd.qp_addr = (uintptr_t) qp;
- ret = ibv_cmd_create_qp(pd, &qp->ibv_qp, attr,
- &cmd.ibv_cmd, sizeof(cmd),
- &resp, sizeof(resp));
-
- if (ret)
+ if (ibv_cmd_create_qp(pd, &qp->ibv_qp, attr, &cmd.ibv_cmd, sizeof(cmd),
+ &resp.ibv_resp, sizeof(resp)))
goto err_free;
- to_vctx(pd->context)->qp_tbl[qp->ibv_qp.qp_num & 0xFFFF] = qp;
+ if (resp.drv_payload.qp_handle != 0)
+ qp->qp_handle = resp.drv_payload.qp_handle;
+ else
+ qp->qp_handle = qp->ibv_qp.qp_num;
+
+ to_vctx(pd->context)->qp_tbl[qp->qp_handle & 0xFFFF] = qp;
/* If set, each WR submitted to the SQ generate a completion entry */
if (attr->sq_sig_all)
@@ -414,7 +415,7 @@ int pvrdma_destroy_qp(struct ibv_qp *ibqp)
free(qp->rq.wrid);
pvrdma_free_buf(&qp->rbuf);
pvrdma_free_buf(&qp->sbuf);
- ctx->qp_tbl[ibqp->qp_num & 0xFFFF] = NULL;
+ ctx->qp_tbl[qp->qp_handle & 0xFFFF] = NULL;
free(qp);
return 0;
@@ -547,7 +548,7 @@ out:
if (nreq) {
udma_to_device_barrier();
pvrdma_write_uar_qp(ctx->uar,
- PVRDMA_UAR_QP_SEND | ibqp->qp_num);
+ PVRDMA_UAR_QP_SEND | qp->qp_handle);
}
pthread_spin_unlock(&qp->sq.lock);
@@ -630,7 +631,7 @@ int pvrdma_post_recv(struct ibv_qp *ibqp, struct ibv_recv_wr *wr,
out:
if (nreq)
pvrdma_write_uar_qp(ctx->uar,
- PVRDMA_UAR_QP_RECV | ibqp->qp_num);
+ PVRDMA_UAR_QP_RECV | qp->qp_handle);
pthread_spin_unlock(&qp->rq.lock);
return ret;
--
2.18.1
^ permalink raw reply related [flat|nested] 4+ messages in thread