* [PATCH rdma-next v2 1/3] RDMA/mana_ib: Create and destroy RC QP
2024-05-21 8:34 [PATCH rdma-next v2 0/3] RDMA/mana_ib: Add support of RC QPs Konstantin Taranov
@ 2024-05-21 8:34 ` Konstantin Taranov
2024-05-21 8:34 ` [PATCH rdma-next v2 2/3] RDMA/mana_ib: Implement uapi to create " Konstantin Taranov
2024-05-21 8:34 ` [PATCH rdma-next v2 3/3] RDMA/mana_ib: Modify QP state Konstantin Taranov
2 siblings, 0 replies; 6+ messages in thread
From: Konstantin Taranov @ 2024-05-21 8:34 UTC (permalink / raw)
To: kotaranov, sharmaajay, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement HW requests to create and destroy an RC QP.
An RC QP may have 5 queues.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
Reviewed-by: Long Li <longli@microsoft.com>
---
drivers/infiniband/hw/mana/main.c | 59 ++++++++++++++++++++++++++++
drivers/infiniband/hw/mana/mana_ib.h | 58 ++++++++++++++++++++++++++-
2 files changed, 116 insertions(+), 1 deletion(-)
diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana/main.c
index 2a4113576..6bd60729e 100644
--- a/drivers/infiniband/hw/mana/main.c
+++ b/drivers/infiniband/hw/mana/main.c
@@ -888,3 +888,62 @@ int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq)
return 0;
}
+
+int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
+ struct ib_qp_init_attr *attr, u32 doorbell, u64 flags)
+{
+ struct mana_ib_cq *send_cq = container_of(qp->ibqp.send_cq, struct mana_ib_cq, ibcq);
+ struct mana_ib_cq *recv_cq = container_of(qp->ibqp.recv_cq, struct mana_ib_cq, ibcq);
+ struct mana_ib_pd *pd = container_of(qp->ibqp.pd, struct mana_ib_pd, ibpd);
+ struct gdma_context *gc = mdev_to_gc(mdev);
+ struct mana_rnic_create_qp_resp resp = {};
+ struct mana_rnic_create_qp_req req = {};
+ int err, i;
+
+ mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_RC_QP, sizeof(req), sizeof(resp));
+ req.hdr.dev_id = gc->mana_ib.dev_id;
+ req.adapter = mdev->adapter_handle;
+ req.pd_handle = pd->pd_handle;
+ req.send_cq_handle = send_cq->cq_handle;
+ req.recv_cq_handle = recv_cq->cq_handle;
+ for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; i++)
+ req.dma_region[i] = qp->rc_qp.queues[i].gdma_region;
+ req.doorbell_page = doorbell;
+ req.max_send_wr = attr->cap.max_send_wr;
+ req.max_recv_wr = attr->cap.max_recv_wr;
+ req.max_send_sge = attr->cap.max_send_sge;
+ req.max_recv_sge = attr->cap.max_recv_sge;
+ req.flags = flags;
+
+ err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
+ if (err) {
+ ibdev_err(&mdev->ib_dev, "Failed to create rc qp err %d", err);
+ return err;
+ }
+ qp->qp_handle = resp.rc_qp_handle;
+ for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; i++) {
+ qp->rc_qp.queues[i].id = resp.queue_ids[i];
+ /* The GDMA regions are now owned by the RNIC QP handle */
+ qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+ }
+ return 0;
+}
+
+int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp)
+{
+ struct mana_rnic_destroy_rc_qp_resp resp = {0};
+ struct mana_rnic_destroy_rc_qp_req req = {0};
+ struct gdma_context *gc = mdev_to_gc(mdev);
+ int err;
+
+ mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_RC_QP, sizeof(req), sizeof(resp));
+ req.hdr.dev_id = gc->mana_ib.dev_id;
+ req.adapter = mdev->adapter_handle;
+ req.rc_qp_handle = qp->qp_handle;
+ err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
+ if (err) {
+ ibdev_err(&mdev->ib_dev, "Failed to destroy rc qp err %d", err);
+ return err;
+ }
+ return 0;
+}
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 68c3b4f0f..a3e229c83 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -95,11 +95,27 @@ struct mana_ib_cq {
mana_handle_t cq_handle;
};
+enum mana_rc_queue_type {
+ MANA_RC_SEND_QUEUE_REQUESTER = 0,
+ MANA_RC_SEND_QUEUE_RESPONDER,
+ MANA_RC_SEND_QUEUE_FMR,
+ MANA_RC_RECV_QUEUE_REQUESTER,
+ MANA_RC_RECV_QUEUE_RESPONDER,
+ MANA_RC_QUEUE_TYPE_MAX,
+};
+
+struct mana_ib_rc_qp {
+ struct mana_ib_queue queues[MANA_RC_QUEUE_TYPE_MAX];
+};
+
struct mana_ib_qp {
struct ib_qp ibqp;
mana_handle_t qp_handle;
- struct mana_ib_queue raw_sq;
+ union {
+ struct mana_ib_queue raw_sq;
+ struct mana_ib_rc_qp rc_qp;
+ };
/* The port on the IB device, starting with 1 */
u32 port;
@@ -122,6 +138,8 @@ enum mana_ib_command_code {
MANA_IB_CONFIG_MAC_ADDR = 0x30005,
MANA_IB_CREATE_CQ = 0x30008,
MANA_IB_DESTROY_CQ = 0x30009,
+ MANA_IB_CREATE_RC_QP = 0x3000a,
+ MANA_IB_DESTROY_RC_QP = 0x3000b,
};
struct mana_ib_query_adapter_caps_req {
@@ -230,6 +248,40 @@ struct mana_rnic_destroy_cq_resp {
struct gdma_resp_hdr hdr;
}; /* HW Data */
+struct mana_rnic_create_qp_req {
+ struct gdma_req_hdr hdr;
+ mana_handle_t adapter;
+ mana_handle_t pd_handle;
+ mana_handle_t send_cq_handle;
+ mana_handle_t recv_cq_handle;
+ u64 dma_region[MANA_RC_QUEUE_TYPE_MAX];
+ u64 deprecated[2];
+ u64 flags;
+ u32 doorbell_page;
+ u32 max_send_wr;
+ u32 max_recv_wr;
+ u32 max_send_sge;
+ u32 max_recv_sge;
+ u32 reserved;
+}; /* HW Data */
+
+struct mana_rnic_create_qp_resp {
+ struct gdma_resp_hdr hdr;
+ mana_handle_t rc_qp_handle;
+ u32 queue_ids[MANA_RC_QUEUE_TYPE_MAX];
+ u32 reserved;
+}; /* HW Data*/
+
+struct mana_rnic_destroy_rc_qp_req {
+ struct gdma_req_hdr hdr;
+ mana_handle_t adapter;
+ mana_handle_t rc_qp_handle;
+}; /* HW Data */
+
+struct mana_rnic_destroy_rc_qp_resp {
+ struct gdma_resp_hdr hdr;
+}; /* HW Data */
+
static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
{
return mdev->gdma_dev->gdma_context;
@@ -354,4 +406,8 @@ int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enum mana_ib_addr_op op, u8
int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq, u32 doorbell);
int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, struct mana_ib_cq *cq);
+
+int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp,
+ struct ib_qp_init_attr *attr, u32 doorbell, u64 flags);
+int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev, struct mana_ib_qp *qp);
#endif
--
2.43.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* [PATCH rdma-next v2 2/3] RDMA/mana_ib: Implement uapi to create and destroy RC QP
2024-05-21 8:34 [PATCH rdma-next v2 0/3] RDMA/mana_ib: Add support of RC QPs Konstantin Taranov
2024-05-21 8:34 ` [PATCH rdma-next v2 1/3] RDMA/mana_ib: Create and destroy RC QP Konstantin Taranov
@ 2024-05-21 8:34 ` Konstantin Taranov
2024-05-21 9:02 ` Zhu Yanjun
2024-05-21 8:34 ` [PATCH rdma-next v2 3/3] RDMA/mana_ib: Modify QP state Konstantin Taranov
2 siblings, 1 reply; 6+ messages in thread
From: Konstantin Taranov @ 2024-05-21 8:34 UTC (permalink / raw)
To: kotaranov, sharmaajay, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement user requests to create and destroy an RC QP.
As the user does not have an FMR queue, it is skipped and NO_FMR flag
is used.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
---
drivers/infiniband/hw/mana/mana_ib.h | 4 ++
drivers/infiniband/hw/mana/qp.c | 94 +++++++++++++++++++++++++++-
include/uapi/rdma/mana-abi.h | 9 +++
3 files changed, 105 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index a3e229c83..5cccbe397 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -248,6 +248,10 @@ struct mana_rnic_destroy_cq_resp {
struct gdma_resp_hdr hdr;
}; /* HW Data */
+enum mana_rnic_create_rc_flags {
+ MANA_RC_FLAG_NO_FMR = 2,
+};
+
struct mana_rnic_create_qp_req {
struct gdma_req_hdr hdr;
mana_handle_t adapter;
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index ba13c5abf..e04e5e778 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -398,6 +398,78 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
return err;
}
+static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
+ struct ib_qp_init_attr *attr, struct ib_udata *udata)
+{
+ struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev);
+ struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+ struct mana_ib_create_rc_qp_resp resp = {};
+ struct mana_ib_ucontext *mana_ucontext;
+ struct mana_ib_create_rc_qp ucmd = {};
+ int i, err, j;
+ u64 flags = 0;
+ u32 doorbell;
+
+ if (!udata || udata->inlen < sizeof(ucmd))
+ return -EINVAL;
+
+ mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext);
+ doorbell = mana_ucontext->doorbell;
+ flags = MANA_RC_FLAG_NO_FMR;
+ err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
+ if (err) {
+ ibdev_dbg(&mdev->ib_dev, "Failed to copy from udata, %d\n", err);
+ return err;
+ }
+
+ for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
+ // skip FMR for user-level RC QPs
+ if (i == MANA_RC_SEND_QUEUE_FMR) {
+ qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
+ qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
+ continue;
+ }
+ err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],
+ &qp->rc_qp.queues[i]);
+ if (err) {
+ ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err);
+ goto destroy_queues;
+ }
+ j++;
+ }
+
+ err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
+ if (err) {
+ ibdev_err(&mdev->ib_dev, "Failed to create rc qp %d\n", err);
+ goto destroy_queues;
+ }
+ qp->ibqp.qp_num = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER].id;
+ qp->port = attr->port_num;
+
+ if (udata) {
+ for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
+ if (i == MANA_RC_SEND_QUEUE_FMR)
+ continue;
+ resp.queue_id[j] = qp->rc_qp.queues[i].id;
+ j++;
+ }
+ err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
+ if (err) {
+ ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
+ goto destroy_qp;
+ }
+ }
+
+ return 0;
+
+destroy_qp:
+ mana_ib_gd_destroy_rc_qp(mdev, qp);
+destroy_queues:
+ while (i-- > 0)
+ mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]);
+ return err;
+}
+
int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
struct ib_udata *udata)
{
@@ -409,8 +481,9 @@ int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
udata);
return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
+ case IB_QPT_RC:
+ return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata);
default:
- /* Creating QP other than IB_QPT_RAW_PACKET is not supported */
ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n",
attr->qp_type);
}
@@ -473,6 +546,22 @@ static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata)
return 0;
}
+static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
+{
+ struct mana_ib_dev *mdev =
+ container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
+ int i;
+
+ /* Ignore return code as there is not much we can do about it.
+ * The error message is printed inside.
+ */
+ mana_ib_gd_destroy_rc_qp(mdev, qp);
+ for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i)
+ mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]);
+
+ return 0;
+}
+
int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
{
struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
@@ -484,7 +573,8 @@ int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
udata);
return mana_ib_destroy_qp_raw(qp, udata);
-
+ case IB_QPT_RC:
+ return mana_ib_destroy_rc_qp(qp, udata);
default:
ibdev_dbg(ibqp->device, "Unexpected QP type %u\n",
ibqp->qp_type);
diff --git a/include/uapi/rdma/mana-abi.h b/include/uapi/rdma/mana-abi.h
index 2c41cc315..45c2df619 100644
--- a/include/uapi/rdma/mana-abi.h
+++ b/include/uapi/rdma/mana-abi.h
@@ -45,6 +45,15 @@ struct mana_ib_create_qp_resp {
__u32 reserved;
};
+struct mana_ib_create_rc_qp {
+ __aligned_u64 queue_buf[4];
+ __u32 queue_size[4];
+};
+
+struct mana_ib_create_rc_qp_resp {
+ __u32 queue_id[4];
+};
+
struct mana_ib_create_wq {
__aligned_u64 wq_buf_addr;
__u32 wq_buf_size;
--
2.43.0
^ permalink raw reply related [flat|nested] 6+ messages in thread* Re: [PATCH rdma-next v2 2/3] RDMA/mana_ib: Implement uapi to create and destroy RC QP
2024-05-21 8:34 ` [PATCH rdma-next v2 2/3] RDMA/mana_ib: Implement uapi to create " Konstantin Taranov
@ 2024-05-21 9:02 ` Zhu Yanjun
2024-05-21 9:13 ` Konstantin Taranov
0 siblings, 1 reply; 6+ messages in thread
From: Zhu Yanjun @ 2024-05-21 9:02 UTC (permalink / raw)
To: Konstantin Taranov, kotaranov, sharmaajay, longli, jgg, leon
Cc: linux-rdma, linux-kernel
On 21.05.24 10:34, Konstantin Taranov wrote:
> From: Konstantin Taranov <kotaranov@microsoft.com>
>
> Implement user requests to create and destroy an RC QP.
> As the user does not have an FMR queue, it is skipped and NO_FMR flag
> is used.
>
> Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
> ---
> drivers/infiniband/hw/mana/mana_ib.h | 4 ++
> drivers/infiniband/hw/mana/qp.c | 94 +++++++++++++++++++++++++++-
> include/uapi/rdma/mana-abi.h | 9 +++
> 3 files changed, 105 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
> index a3e229c83..5cccbe397 100644
> --- a/drivers/infiniband/hw/mana/mana_ib.h
> +++ b/drivers/infiniband/hw/mana/mana_ib.h
> @@ -248,6 +248,10 @@ struct mana_rnic_destroy_cq_resp {
> struct gdma_resp_hdr hdr;
> }; /* HW Data */
>
> +enum mana_rnic_create_rc_flags {
> + MANA_RC_FLAG_NO_FMR = 2,
> +};
> +
> struct mana_rnic_create_qp_req {
> struct gdma_req_hdr hdr;
> mana_handle_t adapter;
> diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
> index ba13c5abf..e04e5e778 100644
> --- a/drivers/infiniband/hw/mana/qp.c
> +++ b/drivers/infiniband/hw/mana/qp.c
> @@ -398,6 +398,78 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
> return err;
> }
>
> +static int mana_ib_create_rc_qp(struct ib_qp *ibqp, struct ib_pd *ibpd,
> + struct ib_qp_init_attr *attr, struct ib_udata *udata)
> +{
> + struct mana_ib_dev *mdev = container_of(ibpd->device, struct mana_ib_dev, ib_dev);
> + struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
> + struct mana_ib_create_rc_qp_resp resp = {};
> + struct mana_ib_ucontext *mana_ucontext;
> + struct mana_ib_create_rc_qp ucmd = {};
> + int i, err, j;
> + u64 flags = 0;
> + u32 doorbell;
> +
> + if (!udata || udata->inlen < sizeof(ucmd))
> + return -EINVAL;
> +
> + mana_ucontext = rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext);
> + doorbell = mana_ucontext->doorbell;
> + flags = MANA_RC_FLAG_NO_FMR;
> + err = ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen));
> + if (err) {
> + ibdev_dbg(&mdev->ib_dev, "Failed to copy from udata, %d\n", err);
> + return err;
> + }
> +
> + for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
> + // skip FMR for user-level RC QPs
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/dev-tools/checkpatch.rst?h=v6.9#n497
"
**C99_COMMENTS**
C99 style single line comments (//) should not be used.
Prefer the block comment style instead.
See:
https://www.kernel.org/doc/html/latest/process/coding-style.html#commenting
"
Zhu Yanjun
> + if (i == MANA_RC_SEND_QUEUE_FMR) {
> + qp->rc_qp.queues[i].id = INVALID_QUEUE_ID;
> + qp->rc_qp.queues[i].gdma_region = GDMA_INVALID_DMA_REGION;
> + continue;
> + }
> + err = mana_ib_create_queue(mdev, ucmd.queue_buf[j], ucmd.queue_size[j],
> + &qp->rc_qp.queues[i]);
> + if (err) {
> + ibdev_err(&mdev->ib_dev, "Failed to create queue %d, err %d\n", i, err);
> + goto destroy_queues;
> + }
> + j++;
> + }
> +
> + err = mana_ib_gd_create_rc_qp(mdev, qp, attr, doorbell, flags);
> + if (err) {
> + ibdev_err(&mdev->ib_dev, "Failed to create rc qp %d\n", err);
> + goto destroy_queues;
> + }
> + qp->ibqp.qp_num = qp->rc_qp.queues[MANA_RC_RECV_QUEUE_RESPONDER].id;
> + qp->port = attr->port_num;
> +
> + if (udata) {
> + for (i = 0, j = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i) {
> + if (i == MANA_RC_SEND_QUEUE_FMR)
> + continue;
> + resp.queue_id[j] = qp->rc_qp.queues[i].id;
> + j++;
> + }
> + err = ib_copy_to_udata(udata, &resp, min(sizeof(resp), udata->outlen));
> + if (err) {
> + ibdev_dbg(&mdev->ib_dev, "Failed to copy to udata, %d\n", err);
> + goto destroy_qp;
> + }
> + }
> +
> + return 0;
> +
> +destroy_qp:
> + mana_ib_gd_destroy_rc_qp(mdev, qp);
> +destroy_queues:
> + while (i-- > 0)
> + mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]);
> + return err;
> +}
> +
> int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
> struct ib_udata *udata)
> {
> @@ -409,8 +481,9 @@ int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
> udata);
>
> return mana_ib_create_qp_raw(ibqp, ibqp->pd, attr, udata);
> + case IB_QPT_RC:
> + return mana_ib_create_rc_qp(ibqp, ibqp->pd, attr, udata);
> default:
> - /* Creating QP other than IB_QPT_RAW_PACKET is not supported */
> ibdev_dbg(ibqp->device, "Creating QP type %u not supported\n",
> attr->qp_type);
> }
> @@ -473,6 +546,22 @@ static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp, struct ib_udata *udata)
> return 0;
> }
>
> +static int mana_ib_destroy_rc_qp(struct mana_ib_qp *qp, struct ib_udata *udata)
> +{
> + struct mana_ib_dev *mdev =
> + container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev);
> + int i;
> +
> + /* Ignore return code as there is not much we can do about it.
> + * The error message is printed inside.
> + */
> + mana_ib_gd_destroy_rc_qp(mdev, qp);
> + for (i = 0; i < MANA_RC_QUEUE_TYPE_MAX; ++i)
> + mana_ib_destroy_queue(mdev, &qp->rc_qp.queues[i]);
> +
> + return 0;
> +}
> +
> int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
> {
> struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
> @@ -484,7 +573,8 @@ int mana_ib_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
> udata);
>
> return mana_ib_destroy_qp_raw(qp, udata);
> -
> + case IB_QPT_RC:
> + return mana_ib_destroy_rc_qp(qp, udata);
> default:
> ibdev_dbg(ibqp->device, "Unexpected QP type %u\n",
> ibqp->qp_type);
> diff --git a/include/uapi/rdma/mana-abi.h b/include/uapi/rdma/mana-abi.h
> index 2c41cc315..45c2df619 100644
> --- a/include/uapi/rdma/mana-abi.h
> +++ b/include/uapi/rdma/mana-abi.h
> @@ -45,6 +45,15 @@ struct mana_ib_create_qp_resp {
> __u32 reserved;
> };
>
> +struct mana_ib_create_rc_qp {
> + __aligned_u64 queue_buf[4];
> + __u32 queue_size[4];
> +};
> +
> +struct mana_ib_create_rc_qp_resp {
> + __u32 queue_id[4];
> +};
> +
> struct mana_ib_create_wq {
> __aligned_u64 wq_buf_addr;
> __u32 wq_buf_size;
^ permalink raw reply [flat|nested] 6+ messages in thread
* [PATCH rdma-next v2 3/3] RDMA/mana_ib: Modify QP state
2024-05-21 8:34 [PATCH rdma-next v2 0/3] RDMA/mana_ib: Add support of RC QPs Konstantin Taranov
2024-05-21 8:34 ` [PATCH rdma-next v2 1/3] RDMA/mana_ib: Create and destroy RC QP Konstantin Taranov
2024-05-21 8:34 ` [PATCH rdma-next v2 2/3] RDMA/mana_ib: Implement uapi to create " Konstantin Taranov
@ 2024-05-21 8:34 ` Konstantin Taranov
2 siblings, 0 replies; 6+ messages in thread
From: Konstantin Taranov @ 2024-05-21 8:34 UTC (permalink / raw)
To: kotaranov, sharmaajay, longli, jgg, leon; +Cc: linux-rdma, linux-kernel
From: Konstantin Taranov <kotaranov@microsoft.com>
Implement modify QP state for RC QPs.
Signed-off-by: Konstantin Taranov <kotaranov@microsoft.com>
Reviewed-by: Long Li <longli@microsoft.com>
---
drivers/infiniband/hw/mana/mana_ib.h | 37 ++++++++++++++
drivers/infiniband/hw/mana/qp.c | 72 +++++++++++++++++++++++++++-
2 files changed, 107 insertions(+), 2 deletions(-)
diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/mana/mana_ib.h
index 5cccbe397..d29dee7b5 100644
--- a/drivers/infiniband/hw/mana/mana_ib.h
+++ b/drivers/infiniband/hw/mana/mana_ib.h
@@ -140,6 +140,7 @@ enum mana_ib_command_code {
MANA_IB_DESTROY_CQ = 0x30009,
MANA_IB_CREATE_RC_QP = 0x3000a,
MANA_IB_DESTROY_RC_QP = 0x3000b,
+ MANA_IB_SET_QP_STATE = 0x3000d,
};
struct mana_ib_query_adapter_caps_req {
@@ -286,6 +287,42 @@ struct mana_rnic_destroy_rc_qp_resp {
struct gdma_resp_hdr hdr;
}; /* HW Data */
+struct mana_ib_ah_attr {
+ u8 src_addr[16];
+ u8 dest_addr[16];
+ u8 src_mac[ETH_ALEN];
+ u8 dest_mac[ETH_ALEN];
+ u8 src_addr_type;
+ u8 dest_addr_type;
+ u8 hop_limit;
+ u8 traffic_class;
+ u16 src_port;
+ u16 dest_port;
+ u32 reserved;
+};
+
+struct mana_rnic_set_qp_state_req {
+ struct gdma_req_hdr hdr;
+ mana_handle_t adapter;
+ mana_handle_t qp_handle;
+ u64 attr_mask;
+ u32 qp_state;
+ u32 path_mtu;
+ u32 rq_psn;
+ u32 sq_psn;
+ u32 dest_qpn;
+ u32 max_dest_rd_atomic;
+ u32 retry_cnt;
+ u32 rnr_retry;
+ u32 min_rnr_timer;
+ u32 reserved;
+ struct mana_ib_ah_attr ah_attr;
+}; /* HW Data */
+
+struct mana_rnic_set_qp_state_resp {
+ struct gdma_resp_hdr hdr;
+}; /* HW Data */
+
static inline struct gdma_context *mdev_to_gc(struct mana_ib_dev *mdev)
{
return mdev->gdma_dev->gdma_context;
diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/qp.c
index e04e5e778..99ba8c080 100644
--- a/drivers/infiniband/hw/mana/qp.c
+++ b/drivers/infiniband/hw/mana/qp.c
@@ -491,11 +491,79 @@ int mana_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
return -EINVAL;
}
+static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+ int attr_mask, struct ib_udata *udata)
+{
+ struct mana_ib_dev *mdev = container_of(ibqp->device, struct mana_ib_dev, ib_dev);
+ struct mana_ib_qp *qp = container_of(ibqp, struct mana_ib_qp, ibqp);
+ struct mana_rnic_set_qp_state_resp resp = {};
+ struct mana_rnic_set_qp_state_req req = {};
+ struct gdma_context *gc = mdev_to_gc(mdev);
+ struct mana_port_context *mpc;
+ struct net_device *ndev;
+ int err;
+
+ mana_gd_init_req_hdr(&req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(resp));
+ req.hdr.dev_id = gc->mana_ib.dev_id;
+ req.adapter = mdev->adapter_handle;
+ req.qp_handle = qp->qp_handle;
+ req.qp_state = attr->qp_state;
+ req.attr_mask = attr_mask;
+ req.path_mtu = attr->path_mtu;
+ req.rq_psn = attr->rq_psn;
+ req.sq_psn = attr->sq_psn;
+ req.dest_qpn = attr->dest_qp_num;
+ req.max_dest_rd_atomic = attr->max_dest_rd_atomic;
+ req.retry_cnt = attr->retry_cnt;
+ req.rnr_retry = attr->rnr_retry;
+ req.min_rnr_timer = attr->min_rnr_timer;
+ if (attr_mask & IB_QP_AV) {
+ ndev = mana_ib_get_netdev(&mdev->ib_dev, ibqp->port);
+ if (!ndev) {
+ ibdev_dbg(&mdev->ib_dev, "Invalid port %u in QP %u\n",
+ ibqp->port, ibqp->qp_num);
+ return -EINVAL;
+ }
+ mpc = netdev_priv(ndev);
+ copy_in_reverse(req.ah_attr.src_mac, mpc->mac_addr, ETH_ALEN);
+ copy_in_reverse(req.ah_attr.dest_mac, attr->ah_attr.roce.dmac, ETH_ALEN);
+ copy_in_reverse(req.ah_attr.src_addr, attr->ah_attr.grh.sgid_attr->gid.raw,
+ sizeof(union ib_gid));
+ copy_in_reverse(req.ah_attr.dest_addr, attr->ah_attr.grh.dgid.raw,
+ sizeof(union ib_gid));
+ if (rdma_gid_attr_network_type(attr->ah_attr.grh.sgid_attr) == RDMA_NETWORK_IPV4) {
+ req.ah_attr.src_addr_type = SGID_TYPE_IPV4;
+ req.ah_attr.dest_addr_type = SGID_TYPE_IPV4;
+ } else {
+ req.ah_attr.src_addr_type = SGID_TYPE_IPV6;
+ req.ah_attr.dest_addr_type = SGID_TYPE_IPV6;
+ }
+ req.ah_attr.dest_port = ROCE_V2_UDP_DPORT;
+ req.ah_attr.src_port = rdma_get_udp_sport(attr->ah_attr.grh.flow_label,
+ ibqp->qp_num, attr->dest_qp_num);
+ req.ah_attr.traffic_class = attr->ah_attr.grh.traffic_class;
+ req.ah_attr.hop_limit = attr->ah_attr.grh.hop_limit;
+ }
+
+ err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);
+ if (err) {
+ ibdev_err(&mdev->ib_dev, "Failed modify qp err %d", err);
+ return err;
+ }
+
+ return 0;
+}
+
int mana_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
- /* modify_qp is not supported by this version of the driver */
- return -EOPNOTSUPP;
+ switch (ibqp->qp_type) {
+ case IB_QPT_RC:
+ return mana_ib_gd_modify_qp(ibqp, attr, attr_mask, udata);
+ default:
+ ibdev_dbg(ibqp->device, "Modify QP type %u not supported", ibqp->qp_type);
+ return -EOPNOTSUPP;
+ }
}
static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp,
--
2.43.0
^ permalink raw reply related [flat|nested] 6+ messages in thread