From: Steve Wise <swise@opengridcomputing.com>
To: Sagi Grimberg <sagig@mellanox.com>,
linux-rdma@vger.kernel.org, linux-nfs@vger.kernel.org,
target-devel@vger.kernel.org
Subject: Re: [PATCH for-4.3 02/15] IB: Modify ib_create_mr API
Date: Thu, 30 Jul 2015 09:50:59 -0500 [thread overview]
Message-ID: <55BA39D3.2030009@opengridcomputing.com> (raw)
In-Reply-To: <1438241568-3685-3-git-send-email-sagig@mellanox.com>
On 7/30/2015 2:32 AM, Sagi Grimberg wrote:
> Use ib_alloc_mr with specific parameters.
> Change the existing callers.
>
> Signed-off-by: Sagi Grimberg <sagig@mellanox.com>
> ---
> drivers/infiniband/core/verbs.c | 31 ++++++++++++++++++++------
> drivers/infiniband/hw/mlx5/main.c | 2 +-
> drivers/infiniband/hw/mlx5/mlx5_ib.h | 5 +++--
> drivers/infiniband/hw/mlx5/mr.c | 17 ++++++++++-----
> drivers/infiniband/ulp/iser/iser_verbs.c | 6 ++----
> drivers/infiniband/ulp/isert/ib_isert.c | 6 +-----
> include/rdma/ib_verbs.h | 37 +++++++++++++-------------------
> 7 files changed, 58 insertions(+), 46 deletions(-)
>
> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
> index 003bb62..2ac599b 100644
> --- a/drivers/infiniband/core/verbs.c
> +++ b/drivers/infiniband/core/verbs.c
> @@ -1272,15 +1272,32 @@ int ib_dereg_mr(struct ib_mr *mr)
> }
> EXPORT_SYMBOL(ib_dereg_mr);
>
> -struct ib_mr *ib_create_mr(struct ib_pd *pd,
> - struct ib_mr_init_attr *mr_init_attr)
> +/**
> + * ib_alloc_mr() - Allocates a memory region
> + * @pd: protection domain associated with the region
> + * @mr_type: memory region type
> + * @max_num_sg: maximum sg entries available for registration.
> + *
> + * Notes:
> + * Memory registeration page/sg lists must not exceed max_num_sg.
> + * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
> + * max_num_sg * used_page_size.
> + *
Nit: the above sounds like used_page_size is a variable. Something
like this might work?
"max_num_sg * the page size used for this sg list."
> + */
> +struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
> + enum ib_mr_type mr_type,
> + u32 max_num_sg)
> {
> struct ib_mr *mr;
>
> - if (!pd->device->create_mr)
> - return ERR_PTR(-ENOSYS);
> -
> - mr = pd->device->create_mr(pd, mr_init_attr);
> + if (pd->device->alloc_mr) {
> + mr = pd->device->alloc_mr(pd, mr_type, max_num_sg);
> + } else {
> + if (mr_type != IB_MR_TYPE_MEM_REG ||
> + !pd->device->alloc_fast_reg_mr)
> + return ERR_PTR(-ENOSYS);
> + mr = pd->device->alloc_fast_reg_mr(pd, max_num_sg);
> + }
>
> if (!IS_ERR(mr)) {
> mr->device = pd->device;
> @@ -1292,7 +1309,7 @@ struct ib_mr *ib_create_mr(struct ib_pd *pd,
>
> return mr;
> }
> -EXPORT_SYMBOL(ib_create_mr);
> +EXPORT_SYMBOL(ib_alloc_mr);
>
> struct ib_mr *ib_alloc_fast_reg_mr(struct ib_pd *pd, int max_page_list_len)
> {
> diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
> index 46d1383..2c2a461 100644
> --- a/drivers/infiniband/hw/mlx5/main.c
> +++ b/drivers/infiniband/hw/mlx5/main.c
> @@ -1489,7 +1489,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
> dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach;
> dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach;
> dev->ib_dev.process_mad = mlx5_ib_process_mad;
> - dev->ib_dev.create_mr = mlx5_ib_create_mr;
> + dev->ib_dev.alloc_mr = mlx5_ib_alloc_mr;
> dev->ib_dev.alloc_fast_reg_mr = mlx5_ib_alloc_fast_reg_mr;
> dev->ib_dev.alloc_fast_reg_page_list = mlx5_ib_alloc_fast_reg_page_list;
> dev->ib_dev.free_fast_reg_page_list = mlx5_ib_free_fast_reg_page_list;
> diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
> index 537f42e..3030abe 100644
> --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
> +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
> @@ -572,8 +572,9 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
> int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index,
> int npages, int zap);
> int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
> -struct ib_mr *mlx5_ib_create_mr(struct ib_pd *pd,
> - struct ib_mr_init_attr *mr_init_attr);
> +struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
> + enum ib_mr_type mr_type,
> + u32 max_num_sg);
> struct ib_mr *mlx5_ib_alloc_fast_reg_mr(struct ib_pd *pd,
> int max_page_list_len);
> struct ib_fast_reg_page_list *mlx5_ib_alloc_fast_reg_page_list(struct ib_device *ibdev,
> diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
> index 03cf74e..b0b68bb 100644
> --- a/drivers/infiniband/hw/mlx5/mr.c
> +++ b/drivers/infiniband/hw/mlx5/mr.c
> @@ -1246,14 +1246,15 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
> return 0;
> }
>
> -struct ib_mr *mlx5_ib_create_mr(struct ib_pd *pd,
> - struct ib_mr_init_attr *mr_init_attr)
> +struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
> + enum ib_mr_type mr_type,
> + u32 max_num_sg)
> {
> struct mlx5_ib_dev *dev = to_mdev(pd->device);
> struct mlx5_create_mkey_mbox_in *in;
> struct mlx5_ib_mr *mr;
> int access_mode, err;
> - int ndescs = roundup(mr_init_attr->max_reg_descriptors, 4);
> + int ndescs = roundup(max_num_sg, 4);
>
> mr = kzalloc(sizeof(*mr), GFP_KERNEL);
> if (!mr)
> @@ -1269,9 +1270,11 @@ struct ib_mr *mlx5_ib_create_mr(struct ib_pd *pd,
> in->seg.xlt_oct_size = cpu_to_be32(ndescs);
> in->seg.qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
> in->seg.flags_pd = cpu_to_be32(to_mpd(pd)->pdn);
> - access_mode = MLX5_ACCESS_MODE_MTT;
>
> - if (mr_init_attr->flags & IB_MR_SIGNATURE_EN) {
> + if (mr_type == IB_MR_TYPE_MEM_REG) {
> + access_mode = MLX5_ACCESS_MODE_MTT;
> + in->seg.log2_page_size = PAGE_SHIFT;
> + } else if (mr_type == IB_MR_TYPE_SIGNATURE) {
> u32 psv_index[2];
>
> in->seg.flags_pd = cpu_to_be32(be32_to_cpu(in->seg.flags_pd) |
> @@ -1297,6 +1300,10 @@ struct ib_mr *mlx5_ib_create_mr(struct ib_pd *pd,
> mr->sig->sig_err_exists = false;
> /* Next UMR, Arm SIGERR */
> ++mr->sig->sigerr_count;
> + } else {
> + mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
> + err = -EINVAL;
> + goto err_free_in;
> }
>
> in->seg.flags = MLX5_PERM_UMR_EN | access_mode;
> diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
> index 255321c..9bef5a7 100644
> --- a/drivers/infiniband/ulp/iser/iser_verbs.c
> +++ b/drivers/infiniband/ulp/iser/iser_verbs.c
> @@ -284,9 +284,7 @@ iser_alloc_pi_ctx(struct ib_device *ib_device, struct ib_pd *pd,
> struct fast_reg_descriptor *desc)
> {
> struct iser_pi_context *pi_ctx = NULL;
> - struct ib_mr_init_attr mr_init_attr = {.max_reg_descriptors = 2,
> - .flags = IB_MR_SIGNATURE_EN};
> - int ret = 0;
> + int ret;
>
> desc->pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL);
> if (!desc->pi_ctx)
> @@ -309,7 +307,7 @@ iser_alloc_pi_ctx(struct ib_device *ib_device, struct ib_pd *pd,
> }
> desc->reg_indicators |= ISER_PROT_KEY_VALID;
>
> - pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr);
> + pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2);
> if (IS_ERR(pi_ctx->sig_mr)) {
> ret = PTR_ERR(pi_ctx->sig_mr);
> goto sig_mr_failure;
> diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
> index 559e6be..23a793a 100644
> --- a/drivers/infiniband/ulp/isert/ib_isert.c
> +++ b/drivers/infiniband/ulp/isert/ib_isert.c
> @@ -497,7 +497,6 @@ isert_create_pi_ctx(struct fast_reg_descriptor *desc,
> struct ib_device *device,
> struct ib_pd *pd)
> {
> - struct ib_mr_init_attr mr_init_attr;
> struct pi_context *pi_ctx;
> int ret;
>
> @@ -525,10 +524,7 @@ isert_create_pi_ctx(struct fast_reg_descriptor *desc,
> }
> desc->ind |= ISERT_PROT_KEY_VALID;
>
> - memset(&mr_init_attr, 0, sizeof(mr_init_attr));
> - mr_init_attr.max_reg_descriptors = 2;
> - mr_init_attr.flags |= IB_MR_SIGNATURE_EN;
> - pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr);
> + pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2);
> if (IS_ERR(pi_ctx->sig_mr)) {
> isert_err("Failed to allocate signature enabled mr err=%ld\n",
> PTR_ERR(pi_ctx->sig_mr));
> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
> index a534ee5..110044d 100644
> --- a/include/rdma/ib_verbs.h
> +++ b/include/rdma/ib_verbs.h
> @@ -556,20 +556,18 @@ __attribute_const__ int ib_rate_to_mult(enum ib_rate rate);
> */
> __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate);
>
> -enum ib_mr_create_flags {
> - IB_MR_SIGNATURE_EN = 1,
> -};
>
> /**
> - * ib_mr_init_attr - Memory region init attributes passed to routine
> - * ib_create_mr.
> - * @max_reg_descriptors: max number of registration descriptors that
> - * may be used with registration work requests.
> - * @flags: MR creation flags bit mask.
> + * enum ib_mr_type - memory region type
> + * @IB_MR_TYPE_MEM_REG: memory region that is used for
> + * normal registration
> + * @IB_MR_TYPE_SIGNATURE: memory region that is used for
> + * signature operations (data-integrity
> + * capable regions)
> */
> -struct ib_mr_init_attr {
> - int max_reg_descriptors;
> - u32 flags;
> +enum ib_mr_type {
> + IB_MR_TYPE_MEM_REG,
> + IB_MR_TYPE_SIGNATURE,
> };
>
> /**
> @@ -1670,8 +1668,9 @@ struct ib_device {
> int (*query_mr)(struct ib_mr *mr,
> struct ib_mr_attr *mr_attr);
> int (*dereg_mr)(struct ib_mr *mr);
> - struct ib_mr * (*create_mr)(struct ib_pd *pd,
> - struct ib_mr_init_attr *mr_init_attr);
> + struct ib_mr * (*alloc_mr)(struct ib_pd *pd,
> + enum ib_mr_type mr_type,
> + u32 max_num_sg);
> struct ib_mr * (*alloc_fast_reg_mr)(struct ib_pd *pd,
> int max_page_list_len);
> struct ib_fast_reg_page_list * (*alloc_fast_reg_page_list)(struct ib_device *device,
> @@ -2815,15 +2814,9 @@ int ib_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr);
> */
> int ib_dereg_mr(struct ib_mr *mr);
>
> -
> -/**
> - * ib_create_mr - Allocates a memory region that may be used for
> - * signature handover operations.
> - * @pd: The protection domain associated with the region.
> - * @mr_init_attr: memory region init attributes.
> - */
> -struct ib_mr *ib_create_mr(struct ib_pd *pd,
> - struct ib_mr_init_attr *mr_init_attr);
> +struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
> + enum ib_mr_type mr_type,
> + u32 max_num_sg);
>
> /**
> * ib_alloc_fast_reg_mr - Allocates memory region usable with the
next prev parent reply other threads:[~2015-07-30 14:50 UTC|newest]
Thread overview: 33+ messages / expand[flat|nested] mbox.gz Atom feed top
2015-07-30 7:32 [PATCH for-4.3 00/15] Modify MR allocation API Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 01/15] IB/core: Get rid of redundant verb ib_destroy_mr Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 02/15] IB: Modify ib_create_mr API Sagi Grimberg
2015-07-30 14:50 ` Steve Wise [this message]
2015-07-30 7:32 ` [PATCH for-4.3 03/15] IB/iser: Convert to ib_alloc_mr Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 04/15] iser-target: " Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 05/15] IB/srp: " Sagi Grimberg
2015-07-31 22:41 ` Bart Van Assche
2015-07-30 7:32 ` [PATCH for-4.3 06/15] xprtrdma, svcrdma: " Sagi Grimberg
2015-08-06 14:22 ` Anna Schumaker
2015-07-30 7:32 ` [PATCH for-4.3 07/15] RDS: " Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 08/15] mlx5: Drop mlx5_ib_alloc_fast_reg_mr Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 09/15] mlx4: Support ib_alloc_mr verb Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 10/15] ocrdma: " Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 11/15] iw_cxgb4: " Sagi Grimberg
2015-08-07 15:06 ` Steve Wise
2015-08-07 15:13 ` Christoph Hellwig
2015-08-07 16:19 ` Steve Wise
2015-08-07 16:26 ` Steve Wise
2015-08-07 16:26 ` 'Christoph Hellwig'
2015-08-07 16:29 ` Steve Wise
2015-08-07 16:31 ` 'Christoph Hellwig'
2015-07-30 7:32 ` [PATCH for-4.3 12/15] cxgb3: " Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 13/15] nes: " Sagi Grimberg
2015-07-30 7:32 ` [PATCH for-4.3 14/15] qib: " Sagi Grimberg
2015-07-31 11:41 ` Marciniszyn, Mike
2015-07-31 12:52 ` Doug Ledford
2015-08-07 14:47 ` Marciniszyn, Mike
2015-07-30 7:32 ` [PATCH for-4.3 15/15] IB/core: Drop ib_alloc_fast_reg_mr Sagi Grimberg
2015-07-30 13:53 ` [PATCH for-4.3 00/15] Modify MR allocation API Christoph Hellwig
2015-07-30 14:56 ` Steve Wise
2015-07-30 16:42 ` Doug Ledford
2015-07-30 17:22 ` Jason Gunthorpe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=55BA39D3.2030009@opengridcomputing.com \
--to=swise@opengridcomputing.com \
--cc=linux-nfs@vger.kernel.org \
--cc=linux-rdma@vger.kernel.org \
--cc=sagig@mellanox.com \
--cc=target-devel@vger.kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).