From: Bob Pearson <rpearsonhpe@gmail.com>
To: jgg@nvidia.com, zyjzyj2000@gmail.com, linux-rdma@vger.kernel.org
Cc: Bob Pearson <rpearsonhpe@gmail.com>
Subject: [PATCH for-next 09/13] RDMA/rxe: Extend SRQs to support extensions
Date: Thu, 29 Jul 2021 17:49:12 -0500 [thread overview]
Message-ID: <20210729224915.38986-10-rpearsonhpe@gmail.com> (raw)
In-Reply-To: <20210729224915.38986-1-rpearsonhpe@gmail.com>
Extend create_srq to support basic and xrc SRQs. Drop srq->pd in favor
of the PD referenced by rdma-core.
Signed-off-by: Bob Pearson <rpearsonhpe@gmail.com>
---
drivers/infiniband/sw/rxe/rxe_loc.h | 5 +-
drivers/infiniband/sw/rxe/rxe_srq.c | 71 ++++++++++++++-------------
drivers/infiniband/sw/rxe/rxe_verbs.c | 10 ++--
drivers/infiniband/sw/rxe/rxe_verbs.h | 22 ++++++++-
4 files changed, 66 insertions(+), 42 deletions(-)
diff --git a/drivers/infiniband/sw/rxe/rxe_loc.h b/drivers/infiniband/sw/rxe/rxe_loc.h
index b4d45c592bd7..eac56e0c64ba 100644
--- a/drivers/infiniband/sw/rxe/rxe_loc.h
+++ b/drivers/infiniband/sw/rxe/rxe_loc.h
@@ -161,15 +161,12 @@ void retransmit_timer(struct timer_list *t);
void rnr_nak_timer(struct timer_list *t);
/* rxe_srq.c */
-#define IB_SRQ_INIT_MASK (~IB_SRQ_LIMIT)
-
+int rxe_srq_chk_init_attr(struct rxe_dev *rxe, struct ib_srq_init_attr *init);
int rxe_srq_chk_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
struct ib_srq_attr *attr, enum ib_srq_attr_mask mask);
-
int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,
struct ib_srq_init_attr *init, struct ib_udata *udata,
struct rxe_create_srq_resp __user *uresp);
-
int rxe_srq_from_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
struct ib_srq_attr *attr, enum ib_srq_attr_mask mask,
struct rxe_modify_srq_cmd *ucmd, struct ib_udata *udata);
diff --git a/drivers/infiniband/sw/rxe/rxe_srq.c b/drivers/infiniband/sw/rxe/rxe_srq.c
index a9e7817e2732..edbfda0cc242 100644
--- a/drivers/infiniband/sw/rxe/rxe_srq.c
+++ b/drivers/infiniband/sw/rxe/rxe_srq.c
@@ -9,6 +9,32 @@
#include "rxe_loc.h"
#include "rxe_queue.h"
+int rxe_srq_chk_init_attr(struct rxe_dev *rxe, struct ib_srq_init_attr *init)
+{
+ switch (init->srq_type) {
+ case IB_SRQT_BASIC:
+ case IB_SRQT_XRC:
+ break;
+ case IB_SRQT_TM:
+ pr_warn("Tag matching SRQ not supported\n");
+ return -EOPNOTSUPP;
+ default:
+ pr_warn("Unexpected SRQ type (%d)\n", init->srq_type);
+ return -EINVAL;
+ }
+
+ if (init->attr.max_sge > rxe->attr.max_srq_sge) {
+ pr_warn("max_sge(%d) > max_srq_sge(%d)\n",
+ init->attr.max_sge, rxe->attr.max_srq_sge);
+ return -EINVAL;
+ }
+
+ if (init->attr.max_sge < RXE_MIN_SRQ_SGE)
+ init->attr.max_sge = RXE_MIN_SRQ_SGE;
+
+ return rxe_srq_chk_attr(rxe, NULL, &init->attr, IB_SRQ_MAX_WR);
+}
+
int rxe_srq_chk_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
struct ib_srq_attr *attr, enum ib_srq_attr_mask mask)
{
@@ -48,23 +74,12 @@ int rxe_srq_chk_attr(struct rxe_dev *rxe, struct rxe_srq *srq,
if (srq && (attr->srq_limit > srq->rq.queue->buf->index_mask)) {
pr_warn("srq_limit (%d) > cur limit(%d)\n",
- attr->srq_limit,
- srq->rq.queue->buf->index_mask);
+ attr->srq_limit,
+ srq->rq.queue->buf->index_mask);
goto err1;
}
}
- if (mask == IB_SRQ_INIT_MASK) {
- if (attr->max_sge > rxe->attr.max_srq_sge) {
- pr_warn("max_sge(%d) > max_srq_sge(%d)\n",
- attr->max_sge, rxe->attr.max_srq_sge);
- goto err1;
- }
-
- if (attr->max_sge < RXE_MIN_SRQ_SGE)
- attr->max_sge = RXE_MIN_SRQ_SGE;
- }
-
return 0;
err1:
@@ -78,24 +93,22 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,
int err;
int srq_wqe_size;
struct rxe_queue *q;
- enum queue_type type;
- srq->ibsrq.event_handler = init->event_handler;
- srq->ibsrq.srq_context = init->srq_context;
- srq->limit = init->attr.srq_limit;
- srq->srq_num = srq->pelem.index;
- srq->rq.max_wr = init->attr.max_wr;
- srq->rq.max_sge = init->attr.max_sge;
- srq->rq.is_user = srq->is_user;
+ srq->limit = init->attr.srq_limit;
+ srq->rq.max_wr = init->attr.max_wr;
+ srq->rq.max_sge = init->attr.max_sge;
+ srq->rq.is_user = srq->is_user;
- srq_wqe_size = rcv_wqe_size(srq->rq.max_sge);
+ if (init->srq_type == IB_SRQT_XRC)
+ srq->ibsrq.ext.xrc.srq_num = srq->pelem.index;
+
+ srq_wqe_size = rcv_wqe_size(srq->rq.max_sge);
spin_lock_init(&srq->rq.producer_lock);
spin_lock_init(&srq->rq.consumer_lock);
- type = QUEUE_TYPE_FROM_CLIENT;
- q = rxe_queue_init(rxe, &srq->rq.max_wr,
- srq_wqe_size, type);
+ q = rxe_queue_init(rxe, &srq->rq.max_wr, srq_wqe_size,
+ QUEUE_TYPE_FROM_CLIENT);
if (!q) {
pr_warn("unable to allocate queue for srq\n");
return -ENOMEM;
@@ -111,14 +124,6 @@ int rxe_srq_from_init(struct rxe_dev *rxe, struct rxe_srq *srq,
return err;
}
- if (uresp) {
- if (copy_to_user(&uresp->srq_num, &srq->srq_num,
- sizeof(uresp->srq_num))) {
- rxe_queue_cleanup(q);
- return -EFAULT;
- }
- }
-
return 0;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
index b4b993f1ce92..fbd1e2d70682 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.c
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
@@ -307,9 +307,6 @@ static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
struct rxe_srq *srq = to_rsrq(ibsrq);
struct rxe_create_srq_resp __user *uresp = NULL;
- if (init->srq_type != IB_SRQT_BASIC)
- return -EOPNOTSUPP;
-
if (udata) {
if (udata->outlen < sizeof(*uresp))
return -EINVAL;
@@ -319,7 +316,7 @@ static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
srq->is_user = false;
}
- err = rxe_srq_chk_attr(rxe, NULL, &init->attr, IB_SRQ_INIT_MASK);
+ err = rxe_srq_chk_init_attr(rxe, init);
if (err)
goto err_out;
@@ -327,6 +324,8 @@ static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
if (err)
goto err_out;
+ rxe_add_index(srq);
+
err = rxe_srq_from_init(rxe, srq, init, udata, uresp);
if (err)
goto err_drop_srq_ref;
@@ -334,6 +333,7 @@ static int rxe_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init,
return 0;
err_drop_srq_ref:
+ rxe_drop_index(srq);
rxe_drop_ref(srq);
err_out:
return err;
@@ -391,7 +391,9 @@ static int rxe_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
if (srq->rq.queue)
rxe_queue_cleanup(srq->rq.queue);
+ rxe_drop_index(srq);
rxe_drop_ref(srq);
+
return 0;
}
diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
index 5b75de74a992..52599f398ddd 100644
--- a/drivers/infiniband/sw/rxe/rxe_verbs.h
+++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
@@ -104,7 +104,6 @@ struct rxe_srq {
struct ib_srq ibsrq;
struct rxe_pool_entry pelem;
struct rxe_rq rq;
- u32 srq_num;
bool is_user;
int limit;
@@ -542,11 +541,32 @@ static inline enum ib_qp_type rxe_qp_type(struct rxe_qp *qp)
return qp->ibqp.qp_type;
}
+/* SRQ extractors */
+static inline struct rxe_cq *rxe_srq_cq(struct rxe_srq *srq)
+{
+ return to_rcq(srq->ibsrq.ext.cq);
+}
+
+static inline int rxe_srq_num(struct rxe_srq *srq)
+{
+ return srq->ibsrq.ext.xrc.srq_num;
+}
+
static inline struct rxe_pd *rxe_srq_pd(struct rxe_srq *srq)
{
return to_rpd(srq->ibsrq.pd);
}
+static inline enum ib_srq_type rxe_srq_type(struct rxe_srq *srq)
+{
+ return srq->ibsrq.srq_type;
+}
+
+static inline struct rxe_xrcd *rxe_srq_xrcd(struct rxe_srq *srq)
+{
+ return to_rxrcd(srq->ibsrq.ext.xrc.xrcd);
+}
+
int rxe_register_device(struct rxe_dev *rxe, const char *ibdev_name);
void rxe_mc_cleanup(struct rxe_pool_entry *arg);
--
2.30.2
next prev parent reply other threads:[~2021-07-29 22:50 UTC|newest]
Thread overview: 15+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-07-29 22:49 [PATCH for-next 00/13] RDMA:rxe: Implement XRC for rxe Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 01/13] RDMA/rxe: Decouple rxe_pkt_info from sk_buff Bob Pearson
2021-08-27 13:01 ` Jason Gunthorpe
2021-07-29 22:49 ` [PATCH for-next 02/13] IB/core: Add xrc opcodes to ib_pack.h Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 03/13] RDMA/rxe: Extend rxe_send_wr to support xrceth Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 04/13] RDMA/rxe: Extend rxe_opcode.h to support xrc Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 05/13] RDMA/rxe: Add XRC ETH to rxe_hdr.h Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 06/13] RDMA/rxe: Add XRC QP type to rxe_wr_opcode_info Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 07/13] RDMA/rxe: Add XRC opcodes to rxe_opcode_info Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 08/13] RDMA/rxe: Support alloc/dealloc xrcd Bob Pearson
2021-07-29 22:49 ` Bob Pearson [this message]
2021-07-29 22:49 ` [PATCH for-next 10/13] RDMA/rxe: Compute next opcode for XRC Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 11/13] RDMA/rxe: Extend rxe_verbs and rxe_qp to support XRC Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 12/13] RDMA/rxe: Extend rxe send XRC packets Bob Pearson
2021-07-29 22:49 ` [PATCH for-next 13/13] RDMA/rxe: Enable receiving " Bob Pearson
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20210729224915.38986-10-rpearsonhpe@gmail.com \
--to=rpearsonhpe@gmail.com \
--cc=jgg@nvidia.com \
--cc=linux-rdma@vger.kernel.org \
--cc=zyjzyj2000@gmail.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.