From: Tom Tucker <tom@opengridcomputing.com>
To: bfields@fieldses.org
Cc: thomas.talpey@netapp.com, linux-nfs@vger.kernel.org
Subject: Re: [PATCH 02/12] svcrdma: Add FRMR get/put services
Date: Fri, 03 Oct 2008 20:05:28 -0500 [thread overview]
Message-ID: <48E6C158.8040606@opengridcomputing.com> (raw)
In-Reply-To: <1223069629-5267-3-git-send-email-tom@opengridcomputing.com>
Bruce:
This patch is missing your simplification. I'll add it and repost.
Tom
Tom Tucker wrote:
> Add services for the allocating, freeing, and unmapping Fast Reg MR. These
> services will be used by the transport connection setup, send and receive
> routines.
>
> Signed-off-by: Tom Tucker <tom@opengridcomputing.com>
>
> ---
> include/linux/sunrpc/svc_rdma.h | 3 +
> net/sunrpc/xprtrdma/svc_rdma_transport.c | 122 ++++++++++++++++++++++++++++-
> 2 files changed, 120 insertions(+), 5 deletions(-)
>
> diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
> index 49e458d..3425268 100644
> --- a/include/linux/sunrpc/svc_rdma.h
> +++ b/include/linux/sunrpc/svc_rdma.h
> @@ -214,6 +214,9 @@ extern struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *);
> extern void svc_rdma_put_context(struct svc_rdma_op_ctxt *, int);
> extern struct svc_rdma_req_map *svc_rdma_get_req_map(void);
> extern void svc_rdma_put_req_map(struct svc_rdma_req_map *);
> +extern struct svc_rdma_fastreg_mr *svc_rdma_get_frmr(struct svcxprt_rdma *);
> +extern void svc_rdma_put_frmr(struct svcxprt_rdma *,
> + struct svc_rdma_fastreg_mr *);
> extern void svc_sq_reap(struct svcxprt_rdma *);
> extern void svc_rq_reap(struct svcxprt_rdma *);
> extern struct svc_xprt_class svc_rdma_class;
> diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c
> index 900cb69..dd170af 100644
> --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c
> +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c
> @@ -100,6 +100,7 @@ struct svc_rdma_op_ctxt *svc_rdma_get_context(struct svcxprt_rdma *xprt)
> ctxt->xprt = xprt;
> INIT_LIST_HEAD(&ctxt->dto_q);
> ctxt->count = 0;
> + ctxt->frmr = NULL;
> atomic_inc(&xprt->sc_ctxt_used);
> return ctxt;
> }
> @@ -109,11 +110,19 @@ static void svc_rdma_unmap_dma(struct svc_rdma_op_ctxt *ctxt)
> struct svcxprt_rdma *xprt = ctxt->xprt;
> int i;
> for (i = 0; i < ctxt->count && ctxt->sge[i].length; i++) {
> - atomic_dec(&xprt->sc_dma_used);
> - ib_dma_unmap_single(xprt->sc_cm_id->device,
> - ctxt->sge[i].addr,
> - ctxt->sge[i].length,
> - ctxt->direction);
> + /*
> + * Unmap the DMA addr in the SGE if the lkey matches
> + * the sc_dma_lkey, otherwise, ignore it since it is
> + * an FRMR lkey and will be unmapped later when the
> + * last WR that uses it completes.
> + */
> + if (ctxt->sge[i].lkey == xprt->sc_dma_lkey) {
> + atomic_dec(&xprt->sc_dma_used);
> + ib_dma_unmap_single(xprt->sc_cm_id->device,
> + ctxt->sge[i].addr,
> + ctxt->sge[i].length,
> + ctxt->direction);
> + }
> }
> }
>
> @@ -150,6 +159,7 @@ struct svc_rdma_req_map *svc_rdma_get_req_map(void)
> schedule_timeout_uninterruptible(msecs_to_jiffies(500));
> }
> map->count = 0;
> + map->frmr = NULL;
> return map;
> }
>
> @@ -425,10 +435,12 @@ static struct svcxprt_rdma *rdma_create_xprt(struct svc_serv *serv,
> INIT_LIST_HEAD(&cma_xprt->sc_dto_q);
> INIT_LIST_HEAD(&cma_xprt->sc_rq_dto_q);
> INIT_LIST_HEAD(&cma_xprt->sc_read_complete_q);
> + INIT_LIST_HEAD(&cma_xprt->sc_frmr_q);
> init_waitqueue_head(&cma_xprt->sc_send_wait);
>
> spin_lock_init(&cma_xprt->sc_lock);
> spin_lock_init(&cma_xprt->sc_rq_dto_lock);
> + spin_lock_init(&cma_xprt->sc_frmr_q_lock);
>
> cma_xprt->sc_ord = svcrdma_ord;
>
> @@ -686,6 +698,103 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
> return ERR_PTR(ret);
> }
>
> +static int rdma_alloc_frmr(struct svcxprt_rdma *xprt)
> +{
> + struct ib_mr *mr;
> + struct ib_fast_reg_page_list *pl;
> + struct svc_rdma_fastreg_mr *frmr;
> +
> + mr = ib_alloc_fast_reg_mr(xprt->sc_pd, RPCSVC_MAXPAGES);
> + if (!mr)
> + goto errout;
> + pl = ib_alloc_fast_reg_page_list(xprt->sc_cm_id->device,
> + RPCSVC_MAXPAGES);
> + if (!pl) {
> + ib_dereg_mr(mr);
> + goto errout;
> + }
> + frmr = kmalloc(sizeof(*frmr), GFP_KERNEL);
> + if (!frmr) {
> + ib_dereg_mr(mr);
> + ib_free_fast_reg_page_list(pl);
> + goto errout;
> + }
> + frmr->mr = mr;
> + frmr->page_list = pl;
> + INIT_LIST_HEAD(&frmr->frmr_list);
> + spin_lock_bh(&xprt->sc_frmr_q_lock);
> + list_add(&frmr->frmr_list, &xprt->sc_frmr_q);
> + spin_unlock_bh(&xprt->sc_frmr_q_lock);
> +
> + return 0;
> +
> + errout:
> + return -ENOMEM;
> +}
> +
> +static void rdma_dealloc_frmr_q(struct svcxprt_rdma *xprt)
> +{
> + struct svc_rdma_fastreg_mr *frmr;
> +
> + while (!list_empty(&xprt->sc_frmr_q)) {
> + frmr = list_entry(xprt->sc_frmr_q.next,
> + struct svc_rdma_fastreg_mr, frmr_list);
> + list_del_init(&frmr->frmr_list);
> + ib_dereg_mr(frmr->mr);
> + ib_free_fast_reg_page_list(frmr->page_list);
> + kfree(frmr);
> + }
> +}
> +
> +struct svc_rdma_fastreg_mr *svc_rdma_get_frmr(struct svcxprt_rdma *rdma)
> +{
> + struct svc_rdma_fastreg_mr *frmr = NULL;
> +
> + while (1) {
> + spin_lock_bh(&rdma->sc_frmr_q_lock);
> + if (!list_empty(&rdma->sc_frmr_q)) {
> + frmr = list_entry(rdma->sc_frmr_q.next,
> + struct svc_rdma_fastreg_mr, frmr_list);
> + list_del_init(&frmr->frmr_list);
> + }
> + spin_unlock_bh(&rdma->sc_frmr_q_lock);
> + if (frmr)
> + break;
> + if (rdma_alloc_frmr(rdma))
> + return ERR_PTR(-ENOMEM);
> + }
> + frmr->map_len = 0;
> + frmr->page_list_len = 0;
> +
> + return frmr;
> +}
> +
> +static void frmr_unmap_dma(struct svcxprt_rdma *xprt,
> + struct svc_rdma_fastreg_mr *frmr)
> +{
> + int page_no;
> + for (page_no = 0; page_no < frmr->page_list_len; page_no++) {
> + dma_addr_t addr = frmr->page_list->page_list[page_no];
> + if (ib_dma_mapping_error(frmr->mr->device, addr))
> + continue;
> + atomic_dec(&xprt->sc_dma_used);
> + ib_dma_unmap_single(frmr->mr->device, addr, PAGE_SIZE,
> + frmr->direction);
> + }
> +}
> +
> +void svc_rdma_put_frmr(struct svcxprt_rdma *rdma,
> + struct svc_rdma_fastreg_mr *frmr)
> +{
> + if (frmr) {
> + frmr_unmap_dma(rdma, frmr);
> + spin_lock_bh(&rdma->sc_frmr_q_lock);
> + BUG_ON(!list_empty(&frmr->frmr_list));
> + list_add(&frmr->frmr_list, &rdma->sc_frmr_q);
> + spin_unlock_bh(&rdma->sc_frmr_q_lock);
> + }
> +}
> +
> /*
> * This is the xpo_recvfrom function for listening endpoints. Its
> * purpose is to accept incoming connections. The CMA callback handler
> @@ -961,6 +1070,9 @@ static void __svc_rdma_free(struct work_struct *work)
> WARN_ON(atomic_read(&rdma->sc_ctxt_used) != 0);
> WARN_ON(atomic_read(&rdma->sc_dma_used) != 0);
>
> + /* De-allocate fastreg mr */
> + rdma_dealloc_frmr_q(rdma);
> +
> /* Destroy the QP if present (not a listener) */
> if (rdma->sc_qp && !IS_ERR(rdma->sc_qp))
> ib_destroy_qp(rdma->sc_qp);
> --
> To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>
next prev parent reply other threads:[~2008-10-04 1:05 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2008-10-03 21:33 [PATCH 00/12] svcrdma: Fast memory registration support Tom Tucker
2008-10-03 21:33 ` [PATCH 01/12] svcrdma: Add Fast Reg MR Data Types Tom Tucker
2008-10-03 21:33 ` [PATCH 02/12] svcrdma: Add FRMR get/put services Tom Tucker
2008-10-03 21:33 ` [PATCH 03/12] svcrdma: Query device for Fast Reg support during connection setup Tom Tucker
2008-10-03 21:33 ` [PATCH 04/12] svcrdma: Add a service to register a Fast Reg MR with the device Tom Tucker
2008-10-03 21:33 ` [PATCH 05/12] svcrdma: Modify post recv path to use local dma key Tom Tucker
2008-10-03 21:33 ` [PATCH 06/12] svcrdma: Add support to svc_rdma_send to handle chained WR Tom Tucker
2008-10-03 21:33 ` [PATCH 07/12] svcrdma: Modify the RPC recv path to use FRMR when available Tom Tucker
2008-10-03 21:33 ` [PATCH 08/12] svcrdma: Modify the RPC reply " Tom Tucker
2008-10-03 21:33 ` [PATCH 09/12] svcrdma: Update svc_rdma_send_error to use DMA LKEY Tom Tucker
2008-10-03 21:33 ` [PATCH 10/12] svcrdma: Fix IRD/ORD polarity Tom Tucker
2008-10-03 21:33 ` [PATCH 11/12] svcrdma: Add a message log string to indicate if FastReg is being used Tom Tucker
2008-10-03 21:33 ` [PATCH 12/12] svcrdma: Documentation update for the FastReg memory model Tom Tucker
2008-10-08 22:48 ` [PATCH 11/12] svcrdma: Add a message log string to indicate if FastReg is being used J. Bruce Fields
2008-10-09 6:37 ` Tom Tucker
2008-10-09 16:26 ` J. Bruce Fields
2008-10-09 18:46 ` Tom Tucker
2008-10-10 21:02 ` J. Bruce Fields
2008-10-13 2:18 ` Tom Tucker
2008-10-13 2:20 ` Tom Tucker
2008-10-22 20:23 ` J. Bruce Fields
2008-10-22 21:37 ` Tom Tucker
2008-10-04 1:05 ` Tom Tucker [this message]
2008-10-06 20:02 ` [PATCH 02/12] svcrdma: Add FRMR get/put services Tom Tucker
2008-10-08 22:26 ` [PATCH 00/12] svcrdma: Fast memory registration support J. Bruce Fields
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=48E6C158.8040606@opengridcomputing.com \
--to=tom@opengridcomputing.com \
--cc=bfields@fieldses.org \
--cc=linux-nfs@vger.kernel.org \
--cc=thomas.talpey@netapp.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox