From: Yuval Shaia <yuval.shaia@oracle.com>
To: Kamal Heib <kamalheib1@gmail.com>
Cc: qemu-devel@nongnu.org
Subject: Re: [Qemu-devel] [PATCH v3 1/4] hw/rdma: Add SRQ support to backend layer
Date: Sun, 7 Apr 2019 12:13:14 +0300 [thread overview]
Message-ID: <20190407091313.GA6737@lap1> (raw)
In-Reply-To: <7acea8a2-36a1-c9aa-1467-2be7a2f5f3f0@gmail.com>
On Sun, Apr 07, 2019 at 11:13:15AM +0300, Kamal Heib wrote:
>
>
> On 4/3/19 9:05 PM, Yuval Shaia wrote:
> > On Wed, Apr 03, 2019 at 02:33:40PM +0300, Kamal Heib wrote:
> >> Add the required functions and definitions to support shared receive
> >> queues (SRQs) in the backend layer.
> >>
> >> Signed-off-by: Kamal Heib <kamalheib1@gmail.com>
> >> ---
> >> hw/rdma/rdma_backend.c | 116 +++++++++++++++++++++++++++++++++++-
> >> hw/rdma/rdma_backend.h | 12 ++++
> >> hw/rdma/rdma_backend_defs.h | 5 ++
> >> hw/rdma/rdma_rm.c | 2 +
> >> hw/rdma/rdma_rm_defs.h | 1 +
> >> 5 files changed, 134 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c
> >> index d1660b6474fa..04dfd63a573b 100644
> >> --- a/hw/rdma/rdma_backend.c
> >> +++ b/hw/rdma/rdma_backend.c
> >> @@ -40,6 +40,7 @@ typedef struct BackendCtx {
> >> void *up_ctx;
> >> struct ibv_sge sge; /* Used to save MAD recv buffer */
> >> RdmaBackendQP *backend_qp; /* To maintain recv buffers */
> >> + RdmaBackendSRQ *backend_srq;
> >> } BackendCtx;
> >>
> >> struct backend_umad {
> >> @@ -99,6 +100,7 @@ static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
> >> int i, ne, total_ne = 0;
> >> BackendCtx *bctx;
> >> struct ibv_wc wc[2];
> >> + RdmaProtectedGSList *cqe_ctx_list;
> >>
> >> qemu_mutex_lock(&rdma_dev_res->lock);
> >> do {
> >> @@ -116,8 +118,13 @@ static int rdma_poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq)
> >>
> >> comp_handler(bctx->up_ctx, &wc[i]);
> >>
> >> - rdma_protected_gslist_remove_int32(&bctx->backend_qp->cqe_ctx_list,
> >> - wc[i].wr_id);
> >> + if (bctx->backend_qp) {
> >> + cqe_ctx_list = &bctx->backend_qp->cqe_ctx_list;
> >> + } else {
> >> + cqe_ctx_list = &bctx->backend_srq->cqe_ctx_list;
> >> + }
> >> +
> >> + rdma_protected_gslist_remove_int32(cqe_ctx_list, wc[i].wr_id);
> >> rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id);
> >> g_free(bctx);
> >> }
> >> @@ -662,6 +669,60 @@ err_free_bctx:
> >> g_free(bctx);
> >> }
> >>
> >> +void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
> >> + RdmaBackendSRQ *srq, struct ibv_sge *sge,
> >> + uint32_t num_sge, void *ctx)
> >> +{
> >> + BackendCtx *bctx;
> >> + struct ibv_sge new_sge[MAX_SGE];
> >> + uint32_t bctx_id;
> >> + int rc;
> >> + struct ibv_recv_wr wr = {}, *bad_wr;
> >> +
> >> + bctx = g_malloc0(sizeof(*bctx));
> >> + bctx->up_ctx = ctx;
> >> + bctx->backend_srq = srq;
> >> +
> >> + rc = rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx);
> >> + if (unlikely(rc)) {
> >> + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx);
> >> + goto err_free_bctx;
> >> + }
> >> +
> >> + rdma_protected_gslist_append_int32(&srq->cqe_ctx_list, bctx_id);
> >> +
> >> + rc = build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, num_sge,
> >> + &backend_dev->rdma_dev_res->stats.rx_bufs_len);
> >> + if (rc) {
> >> + complete_work(IBV_WC_GENERAL_ERR, rc, ctx);
> >> + goto err_dealloc_cqe_ctx;
> >> + }
> >> +
> >> + wr.num_sge = num_sge;
> >> + wr.sg_list = new_sge;
> >> + wr.wr_id = bctx_id;
> >> + rc = ibv_post_srq_recv(srq->ibsrq, &wr, &bad_wr);
> >> + if (rc) {
> >> + rdma_error_report("ibv_post_srq_recv fail, srqn=0x%x, rc=%d, errno=%d",
> >> + srq->ibsrq->handle, rc, errno);
> >> + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx);
> >> + goto err_dealloc_cqe_ctx;
> >> + }
> >> +
> >> + atomic_inc(&backend_dev->rdma_dev_res->stats.missing_cqe);
> >> + backend_dev->rdma_dev_res->stats.rx_bufs++;
> >> + backend_dev->rdma_dev_res->stats.rx_srq++;
> >
> > You should update function rdma_dump_device_counters with this new
> > counter.
> >
> >> +
> >> + return;
> >> +
> >> +err_dealloc_cqe_ctx:
> >> + backend_dev->rdma_dev_res->stats.rx_bufs_err++;
> >> + rdma_rm_dealloc_cqe_ctx(backend_dev->rdma_dev_res, bctx_id);
> >> +
> >> +err_free_bctx:
> >> + g_free(bctx);
> >> +}
> >> +
> >> int rdma_backend_create_pd(RdmaBackendDev *backend_dev, RdmaBackendPD *pd)
> >> {
> >> pd->ibpd = ibv_alloc_pd(backend_dev->context);
> >> @@ -938,6 +999,55 @@ void rdma_backend_destroy_qp(RdmaBackendQP *qp, RdmaDeviceResources *dev_res)
> >> rdma_protected_gslist_destroy(&qp->cqe_ctx_list);
> >> }
> >>
> >> +int rdma_backend_create_srq(RdmaBackendSRQ *srq, RdmaBackendPD *pd,
> >> + uint32_t max_wr, uint32_t max_sge,
> >> + uint32_t srq_limit)
> >> +{
> >> + struct ibv_srq_init_attr srq_init_attr = {};
> >> +
> >> + srq_init_attr.attr.max_wr = max_wr;
> >> + srq_init_attr.attr.max_sge = max_sge;
> >> + srq_init_attr.attr.srq_limit = srq_limit;
> >> +
> >> + srq->ibsrq = ibv_create_srq(pd->ibpd, &srq_init_attr);
> >> + if (!srq->ibsrq) {
> >> + rdma_error_report("ibv_create_srq failed, errno=%d", errno);
> >> + return -EIO;
> >> + }
> >> +
> >> + rdma_protected_gslist_init(&srq->cqe_ctx_list);
> >> +
> >> + return 0;
> >> +}
> >> +
> >> +int rdma_backend_query_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr)
> >> +{
> >> + if (!srq->ibsrq) {
> >> + return -EINVAL;
> >> + }
> >> +
> >> + return ibv_query_srq(srq->ibsrq, srq_attr);
> >> +}
> >> +
> >> +int rdma_backend_modify_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr,
> >> + int srq_attr_mask)
> >> +{
> >> + if (!srq->ibsrq) {
> >> + return -EINVAL;
> >> + }
> >> +
> >> + return ibv_modify_srq(srq->ibsrq, srq_attr, srq_attr_mask);
> >> +}
> >> +
> >> +void rdma_backend_destroy_srq(RdmaBackendSRQ *srq, RdmaDeviceResources *dev_res)
> >> +{
> >> + if (srq->ibsrq) {
> >> + ibv_destroy_srq(srq->ibsrq);
> >> + }
> >> + g_slist_foreach(srq->cqe_ctx_list.list, free_cqe_ctx, dev_res);
> >> + rdma_protected_gslist_destroy(&srq->cqe_ctx_list);
> >> +}
> >> +
> >> #define CHK_ATTR(req, dev, member, fmt) ({ \
> >> trace_rdma_check_dev_attr(#member, dev.member, req->member); \
> >> if (req->member > dev.member) { \
> >> @@ -960,6 +1070,7 @@ static int init_device_caps(RdmaBackendDev *backend_dev,
> >> }
> >>
> >> dev_attr->max_sge = MAX_SGE;
> >> + dev_attr->max_srq_sge = MAX_SGE;
> >>
> >> CHK_ATTR(dev_attr, bk_dev_attr, max_mr_size, "%" PRId64);
> >> CHK_ATTR(dev_attr, bk_dev_attr, max_qp, "%d");
> >> @@ -970,6 +1081,7 @@ static int init_device_caps(RdmaBackendDev *backend_dev,
> >> CHK_ATTR(dev_attr, bk_dev_attr, max_qp_rd_atom, "%d");
> >> CHK_ATTR(dev_attr, bk_dev_attr, max_qp_init_rd_atom, "%d");
> >> CHK_ATTR(dev_attr, bk_dev_attr, max_ah, "%d");
> >> + CHK_ATTR(dev_attr, bk_dev_attr, max_srq, "%d");
> >>
> >> return 0;
> >> }
> >> diff --git a/hw/rdma/rdma_backend.h b/hw/rdma/rdma_backend.h
> >> index 38056d97c7fc..cad7956d98e8 100644
> >> --- a/hw/rdma/rdma_backend.h
> >> +++ b/hw/rdma/rdma_backend.h
> >> @@ -114,4 +114,16 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev,
> >> RdmaBackendQP *qp, uint8_t qp_type,
> >> struct ibv_sge *sge, uint32_t num_sge, void *ctx);
> >>
> >> +int rdma_backend_create_srq(RdmaBackendSRQ *srq, RdmaBackendPD *pd,
> >> + uint32_t max_wr, uint32_t max_sge,
> >> + uint32_t srq_limit);
> >> +int rdma_backend_query_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr);
> >> +int rdma_backend_modify_srq(RdmaBackendSRQ *srq, struct ibv_srq_attr *srq_attr,
> >> + int srq_attr_mask);
> >> +void rdma_backend_destroy_srq(RdmaBackendSRQ *srq,
> >> + RdmaDeviceResources *dev_res);
> >> +void rdma_backend_post_srq_recv(RdmaBackendDev *backend_dev,
> >> + RdmaBackendSRQ *srq, struct ibv_sge *sge,
> >> + uint32_t num_sge, void *ctx);
> >> +
> >> #endif
> >> diff --git a/hw/rdma/rdma_backend_defs.h b/hw/rdma/rdma_backend_defs.h
> >> index 817153dc8cf4..0b55be35038d 100644
> >> --- a/hw/rdma/rdma_backend_defs.h
> >> +++ b/hw/rdma/rdma_backend_defs.h
> >> @@ -68,4 +68,9 @@ typedef struct RdmaBackendQP {
> >> RdmaProtectedGSList cqe_ctx_list;
> >> } RdmaBackendQP;
> >>
> >> +typedef struct RdmaBackendSRQ {
> >> + struct ibv_srq *ibsrq;
> >> + RdmaProtectedGSList cqe_ctx_list;
> >> +} RdmaBackendSRQ;
> >> +
> >> #endif
> >> diff --git a/hw/rdma/rdma_rm.c b/hw/rdma/rdma_rm.c
> >> index bac3b2f4a6c3..b683506b8616 100644
> >> --- a/hw/rdma/rdma_rm.c
> >> +++ b/hw/rdma/rdma_rm.c
> >> @@ -37,6 +37,8 @@ void rdma_dump_device_counters(Monitor *mon, RdmaDeviceResources *dev_res)
> >> dev_res->stats.tx_err);
> >> monitor_printf(mon, "\trx_bufs : %" PRId64 "\n",
> >> dev_res->stats.rx_bufs);
> >> + monitor_printf(mon, "\trx_srq : %" PRId64 "\n",
> >> + dev_res->stats.rx_srq);
[1]
> >> monitor_printf(mon, "\trx_bufs_len : %" PRId64 "\n",
> >> dev_res->stats.rx_bufs_len);
> >> monitor_printf(mon, "\trx_bufs_err : %" PRId64 "\n",
> >> diff --git a/hw/rdma/rdma_rm_defs.h b/hw/rdma/rdma_rm_defs.h
> >> index c200d311de37..e774af528022 100644
> >> --- a/hw/rdma/rdma_rm_defs.h
> >> +++ b/hw/rdma/rdma_rm_defs.h
> >> @@ -106,6 +106,7 @@ typedef struct RdmaRmStats {
> >> uint64_t rx_bufs;
> >> uint64_t rx_bufs_len;
> >> uint64_t rx_bufs_err;
> >> + uint64_t rx_srq;
> >> uint64_t completions;
> >> uint64_t mad_tx;
> >> uint64_t mad_tx_err;
> >
> > Please make a separate patch to update the function
> > rdma_dump_device_counters.
> >
>
> You mean a separate patch for introducing the "rx_srq" counter & update the
> function rdma_dump_device_counters()?
My bad, missed that ([1]).
No need for a separate patch.
>
> > Besides that patch lgtm.
> >
> > Reviewed-by: Yuval Shaia <yuval.shaia@oracle.com>
> >
> >> --
> >> 2.20.1
> >>
> >>
>
next prev parent reply other threads:[~2019-04-07 9:13 UTC|newest]
Thread overview: 13+ messages / expand[flat|nested] mbox.gz Atom feed top
2019-04-03 11:33 [Qemu-devel] [PATCH v3 0/4] pvrdma: Add support for SRQ Kamal Heib
2019-04-03 11:33 ` [Qemu-devel] [PATCH v3 1/4] hw/rdma: Add SRQ support to backend layer Kamal Heib
2019-04-03 18:05 ` Yuval Shaia
2019-04-07 8:13 ` Kamal Heib
2019-04-07 9:13 ` Yuval Shaia [this message]
2019-04-03 11:33 ` [Qemu-devel] [PATCH 2/4] hw/rdma: Add support for managing SRQ resource Kamal Heib
2019-04-03 18:10 ` Yuval Shaia
2019-04-03 11:33 ` [Qemu-devel] [PATCH v3 3/4] hw/rdma: Modify create/destroy QP to support SRQ Kamal Heib
2019-04-03 18:15 ` Yuval Shaia
2019-04-03 11:33 ` [Qemu-devel] [PATCH v3 4/4] hw/pvrdma: Add support for SRQ Kamal Heib
2019-04-03 18:16 ` Yuval Shaia
2019-04-03 18:19 ` [Qemu-devel] [PATCH v3 0/4] pvrdma: " Yuval Shaia
2019-04-07 9:15 ` Yuval Shaia
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20190407091313.GA6737@lap1 \
--to=yuval.shaia@oracle.com \
--cc=kamalheib1@gmail.com \
--cc=qemu-devel@nongnu.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.