Re: [PATCH rdma-core 1/5] verbs: Support dma-buf based memory region

[Yishai Hadas <yishaih@nvidia.com>]

On 11/23/2020 7:53 PM, Jianxin Xiong wrote:
> Add new API function and new provider method for registering dma-buf
> based memory region. Update the man page and bump the API version.
>
> Signed-off-by: Jianxin Xiong <jianxin.xiong@intel.com>
> ---
>   kernel-headers/rdma/ib_user_ioctl_cmds.h | 14 ++++++++++++
>   libibverbs/cmd_mr.c                      | 38 ++++++++++++++++++++++++++++++++
>   libibverbs/driver.h                      |  7 ++++++
>   libibverbs/dummy_ops.c                   | 11 +++++++++
>   libibverbs/libibverbs.map.in             |  6 +++++
>   libibverbs/man/ibv_reg_mr.3              | 21 ++++++++++++++++--
>   libibverbs/verbs.c                       | 19 ++++++++++++++++
>   libibverbs/verbs.h                       | 10 +++++++++
>   8 files changed, 124 insertions(+), 2 deletions(-)
>
> diff --git a/kernel-headers/rdma/ib_user_ioctl_cmds.h b/kernel-headers/rdma/ib_user_ioctl_cmds.h
> index 7968a18..dafc7eb 100644
> --- a/kernel-headers/rdma/ib_user_ioctl_cmds.h
> +++ b/kernel-headers/rdma/ib_user_ioctl_cmds.h
> @@ -1,5 +1,6 @@
>   /*
>    * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
> + * Copyright (c) 2020, Intel Corporation. All rights reserved.
>    *
>    * This software is available to you under a choice of one of two
>    * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -251,6 +252,7 @@ enum uverbs_methods_mr {
>   	UVERBS_METHOD_MR_DESTROY,
>   	UVERBS_METHOD_ADVISE_MR,
>   	UVERBS_METHOD_QUERY_MR,
> +	UVERBS_METHOD_REG_DMABUF_MR,
>   };
>   
>   enum uverbs_attrs_mr_destroy_ids {
> @@ -272,6 +274,18 @@ enum uverbs_attrs_query_mr_cmd_attr_ids {
>   	UVERBS_ATTR_QUERY_MR_RESP_IOVA,
>   };
>   
> +enum uverbs_attrs_reg_dmabuf_mr_cmd_attr_ids {
> +	UVERBS_ATTR_REG_DMABUF_MR_HANDLE,
> +	UVERBS_ATTR_REG_DMABUF_MR_PD_HANDLE,
> +	UVERBS_ATTR_REG_DMABUF_MR_OFFSET,
> +	UVERBS_ATTR_REG_DMABUF_MR_LENGTH,
> +	UVERBS_ATTR_REG_DMABUF_MR_IOVA,
> +	UVERBS_ATTR_REG_DMABUF_MR_FD,
> +	UVERBS_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,
> +	UVERBS_ATTR_REG_DMABUF_MR_RESP_LKEY,
> +	UVERBS_ATTR_REG_DMABUF_MR_RESP_RKEY,
> +};
> +
>   enum uverbs_attrs_create_counters_cmd_attr_ids {
>   	UVERBS_ATTR_CREATE_COUNTERS_HANDLE,
>   };
> diff --git a/libibverbs/cmd_mr.c b/libibverbs/cmd_mr.c
> index 42dbe42..91ce2ef 100644
> --- a/libibverbs/cmd_mr.c
> +++ b/libibverbs/cmd_mr.c
> @@ -1,5 +1,6 @@
>   /*
>    * Copyright (c) 2018 Mellanox Technologies, Ltd.  All rights reserved.
> + * Copyright (c) 2020 Intel Corporation.  All rights reserved.
>    *
>    * This software is available to you under a choice of one of two
>    * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -116,3 +117,40 @@ int ibv_cmd_query_mr(struct ibv_pd *pd, struct verbs_mr *vmr,
>   	return 0;
>   }
>   
> +int ibv_cmd_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length,
> +			  uint64_t iova, int fd, int access,
> +			  struct verbs_mr *vmr)
> +{
> +	DECLARE_COMMAND_BUFFER(cmdb, UVERBS_OBJECT_MR,
> +			       UVERBS_METHOD_REG_DMABUF_MR,
> +			       9);
> +	struct ib_uverbs_attr *handle;
> +	uint32_t lkey, rkey;
> +	int ret;
> +
> +	handle = fill_attr_out_obj(cmdb, UVERBS_ATTR_REG_DMABUF_MR_HANDLE);
> +	fill_attr_out_ptr(cmdb, UVERBS_ATTR_REG_DMABUF_MR_RESP_LKEY, &lkey);
> +	fill_attr_out_ptr(cmdb, UVERBS_ATTR_REG_DMABUF_MR_RESP_RKEY, &rkey);
> +
> +	fill_attr_in_obj(cmdb, UVERBS_ATTR_REG_DMABUF_MR_PD_HANDLE, pd->handle);
> +	fill_attr_in_uint64(cmdb, UVERBS_ATTR_REG_DMABUF_MR_OFFSET, offset);
> +	fill_attr_in_uint64(cmdb, UVERBS_ATTR_REG_DMABUF_MR_LENGTH, length);
> +	fill_attr_in_uint64(cmdb, UVERBS_ATTR_REG_DMABUF_MR_IOVA, iova);
> +	fill_attr_in_uint32(cmdb, UVERBS_ATTR_REG_DMABUF_MR_FD, fd);
> +	fill_attr_in_uint32(cmdb, UVERBS_ATTR_REG_DMABUF_MR_ACCESS_FLAGS, access);
> +
> +	ret = execute_ioctl(pd->context, cmdb);
> +	if (ret)
> +		return errno;
> +
> +	vmr->ibv_mr.handle =
> +		read_attr_obj(UVERBS_ATTR_REG_DMABUF_MR_HANDLE, handle);
> +	vmr->ibv_mr.context = pd->context;
> +	vmr->ibv_mr.lkey    = lkey;
> +	vmr->ibv_mr.rkey    = rkey;
> +	vmr->ibv_mr.pd	    = pd;
> +	vmr->ibv_mr.addr    = (void *)offset;
> +	vmr->ibv_mr.length  = length;
> +	vmr->mr_type        = IBV_MR_TYPE_MR;
> +	return 0;
> +}
> diff --git a/libibverbs/driver.h b/libibverbs/driver.h
> index ab80f4b..d6a9d0a 100644
> --- a/libibverbs/driver.h
> +++ b/libibverbs/driver.h
> @@ -2,6 +2,7 @@
>    * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
>    * Copyright (c) 2005, 2006 Cisco Systems, Inc.  All rights reserved.
>    * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
> + * Copyright (c) 2020 Intel Corporation. All rights reserved.
>    *
>    * This software is available to you under a choice of one of two
>    * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -373,6 +374,9 @@ struct verbs_context_ops {
>   	struct ibv_mr *(*reg_dm_mr)(struct ibv_pd *pd, struct ibv_dm *dm,
>   				    uint64_t dm_offset, size_t length,
>   				    unsigned int access);
> +	struct ibv_mr *(*reg_dmabuf_mr)(struct ibv_pd *pd, uint64_t offset,
> +					size_t length, uint64_t iova,
> +					int fd, int access);
>   	struct ibv_mr *(*reg_mr)(struct ibv_pd *pd, void *addr, size_t length,
>   				 uint64_t hca_va, int access);
>   	int (*req_notify_cq)(struct ibv_cq *cq, int solicited_only);
> @@ -498,6 +502,9 @@ int ibv_cmd_advise_mr(struct ibv_pd *pd,
>   		      uint32_t flags,
>   		      struct ibv_sge *sg_list,
>   		      uint32_t num_sge);
> +int ibv_cmd_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length,
> +			  uint64_t iova, int fd, int access,
> +			  struct verbs_mr *vmr);
>   int ibv_cmd_alloc_mw(struct ibv_pd *pd, enum ibv_mw_type type,
>   		     struct ibv_mw *mw, struct ibv_alloc_mw *cmd,
>   		     size_t cmd_size,
> diff --git a/libibverbs/dummy_ops.c b/libibverbs/dummy_ops.c
> index e5af9e4..dca96d2 100644
> --- a/libibverbs/dummy_ops.c
> +++ b/libibverbs/dummy_ops.c
> @@ -1,5 +1,6 @@
>   /*
>    * Copyright (c) 2017 Mellanox Technologies, Inc.  All rights reserved.
> + * Copyright (c) 2020 Intel Corporation.  All rights reserved.
>    *
>    * This software is available to you under a choice of one of two
>    * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -452,6 +453,14 @@ static struct ibv_mr *reg_mr(struct ibv_pd *pd, void *addr, size_t length,
>   	return NULL;
>   }
>   
> +static struct ibv_mr *reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset,
> +				    size_t length, uint64_t iova,
> +				    int fd, int access)
> +{
> +	errno = ENOSYS;
> +	return NULL;
> +}
> +
>   static int req_notify_cq(struct ibv_cq *cq, int solicited_only)
>   {
>   	return EOPNOTSUPP;
> @@ -560,6 +569,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
>   	query_srq,
>   	read_counters,
>   	reg_dm_mr,
> +	reg_dmabuf_mr,
>   	reg_mr,
>   	req_notify_cq,
>   	rereg_mr,
> @@ -689,6 +699,7 @@ void verbs_set_ops(struct verbs_context *vctx,
>   	SET_PRIV_OP_IC(vctx, set_ece);
>   	SET_PRIV_OP_IC(vctx, unimport_mr);
>   	SET_PRIV_OP_IC(vctx, unimport_pd);
> +	SET_OP(vctx, reg_dmabuf_mr);
>   
>   #undef SET_OP
>   #undef SET_OP2
> diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in
> index b5ccaca..f67e1ef 100644
> --- a/libibverbs/libibverbs.map.in
> +++ b/libibverbs/libibverbs.map.in
> @@ -148,6 +148,11 @@ IBVERBS_1.11 {
>   		_ibv_query_gid_table;
>   } IBVERBS_1.10;
>   
> +IBVERBS_1.12 {
> +	global:
> +		ibv_reg_dmabuf_mr;
> +} IBVERBS_1.11;
> +
>   /* If any symbols in this stanza change ABI then the entire staza gets a new symbol
>      version. See the top level CMakeLists.txt for this setting. */
>   
> @@ -211,6 +216,7 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ {
>   		ibv_cmd_query_srq;
>   		ibv_cmd_read_counters;
>   		ibv_cmd_reg_dm_mr;
> +		ibv_cmd_reg_dmabuf_mr;
>   		ibv_cmd_reg_mr;
>   		ibv_cmd_req_notify_cq;
>   		ibv_cmd_rereg_mr;
> diff --git a/libibverbs/man/ibv_reg_mr.3 b/libibverbs/man/ibv_reg_mr.3
> index 2bfc955..4975c79 100644
> --- a/libibverbs/man/ibv_reg_mr.3
> +++ b/libibverbs/man/ibv_reg_mr.3
> @@ -3,7 +3,7 @@
>   .\"
>   .TH IBV_REG_MR 3 2006-10-31 libibverbs "Libibverbs Programmer's Manual"
>   .SH "NAME"
> -ibv_reg_mr, ibv_reg_mr_iova, ibv_dereg_mr \- register or deregister a memory region (MR)
> +ibv_reg_mr, ibv_reg_mr_iova, ibv_reg_dmabuf_mr, ibv_dereg_mr \- register or deregister a memory region (MR)
>   .SH "SYNOPSIS"
>   .nf
>   .B #include <infiniband/verbs.h>
> @@ -15,6 +15,9 @@ ibv_reg_mr, ibv_reg_mr_iova, ibv_dereg_mr \- register or deregister a memory reg
>   .BI "                               size_t " "length" ", uint64_t " "hca_va" ,
>   .BI "                               int " "access" );
>   .sp
> +.BI "struct ibv_mr *ibv_reg_dmabuf_mr(struct ibv_pd " "*pd" ", uint64_t " "offset" ,
> +.BI "                                 size_t " "length" ", int " "access" );
> +.sp


This misses the 'fd' parameter.

>   .BI "int ibv_dereg_mr(struct ibv_mr " "*mr" );
>   .fi
>   .SH "DESCRIPTION"
> @@ -71,11 +74,25 @@ a lkey or rkey. The offset in the memory region is computed as 'addr +
>   (iova - hca_va)'. Specifying 0 for hca_va has the same effect as
>   IBV_ACCESS_ZERO_BASED.
>   .PP
> +.B ibv_reg_dmabuf_mr()
> +registers a dma-buf based memory region (MR) associated with the protection domain
> +.I pd\fR.
> +The MR starts at
> +.I offset
> +of the dma-buf and its size is
> +.I length\fR.
> +The dma-buf is identified by the file descriptor
> +.I fd\fR.
> +The argument
> +.I access
> +describes the desired memory protection attributes; it is similar to the ibv_reg_mr case except that only the following flags are supported:
> +.B IBV_ACCESS_LOCAL_WRITE, IBV_ACCESS_REMOTE_WRITE, IBV_ACCESS_REMOTE_READ, IBV_ACCESS_REMOTE_ATOMIC, IBV_ACCESS_RELAXED_ORDERING.
> +.PP
>   .B ibv_dereg_mr()
>   deregisters the MR
>   .I mr\fR.
>   .SH "RETURN VALUE"
> -.B ibv_reg_mr() / ibv_reg_mr_iova()
> +.B ibv_reg_mr() / ibv_reg_mr_iova() / ibv_reg_dmabuf_mr()
>   returns a pointer to the registered MR, or NULL if the request fails.
>   The local key (\fBL_Key\fR) field
>   .B lkey
> diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c
> index 2b0ede8..84ddac7 100644
> --- a/libibverbs/verbs.c
> +++ b/libibverbs/verbs.c
> @@ -1,6 +1,7 @@
>   /*
>    * Copyright (c) 2005 Topspin Communications.  All rights reserved.
>    * Copyright (c) 2006, 2007 Cisco Systems, Inc.  All rights reserved.
> + * Copyright (c) 2020 Intel Corperation.  All rights reserved.
>    *
>    * This software is available to you under a choice of one of two
>    * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -367,6 +368,24 @@ void ibv_unimport_mr(struct ibv_mr *mr)
>   	get_ops(mr->context)->unimport_mr(mr);
>   }
>   
> +LATEST_SYMVER_FUNC(ibv_reg_dmabuf_mr, 1_12, "IBVERBS_1.12",
> +		   struct ibv_mr *,
> +		   struct ibv_pd *pd, uint64_t offset,
> +		   size_t length, int fd, int access)
> +{


  ibv_dereg_mr() -> ibv_dofork_range() doesn't seem to be applicable in 
this case, right ? if offset / addr won't be 0 it may be activated on 
the address range.

Do we rely on applications to *not* turn on fork support in libibverbs ? 
(e.g. call ibv_fork_init()), what if some other registrations may need 
being fork safe ?

> +	struct ibv_mr *mr;
> +
> +	mr = get_ops(pd->context)->reg_dmabuf_mr(pd, offset, length, offset,
> +						 fd, access);
> +	if (mr) {
> +		mr->context = pd->context;
> +		mr->pd      = pd;
> +		mr->addr    = (void *)offset;
> +		mr->length  = length;
> +	}
> +	return mr;
> +}
> +
>   LATEST_SYMVER_FUNC(ibv_rereg_mr, 1_1, "IBVERBS_1.1",
>   		   int,
>   		   struct ibv_mr *mr, int flags,
> diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
> index ee57e05..3961b1e 100644
> --- a/libibverbs/verbs.h
> +++ b/libibverbs/verbs.h
> @@ -3,6 +3,7 @@
>    * Copyright (c) 2004, 2011-2012 Intel Corporation.  All rights reserved.
>    * Copyright (c) 2005, 2006, 2007 Cisco Systems, Inc.  All rights reserved.
>    * Copyright (c) 2005 PathScale, Inc.  All rights reserved.
> + * Copyright (c) 2020 Intel Corporation.  All rights reserved.
>    *
>    * This software is available to you under a choice of one of two
>    * licenses.  You may choose to be licensed under the terms of the GNU
> @@ -2133,6 +2134,9 @@ struct verbs_context {
>   	struct ibv_xrcd *	(*open_xrcd)(struct ibv_context *context,
>   					     struct ibv_xrcd_init_attr *xrcd_init_attr);
>   	int			(*close_xrcd)(struct ibv_xrcd *xrcd);
> +	struct ibv_mr *		(*reg_dmabuf_mr)(struct ibv_pd *pd, uint64_t offset,
> +						 size_t length, uint64_t iova,
> +						 int fd, int access);
>   	uint64_t _ABI_placeholder3;
>   	size_t   sz;			/* Must be immediately before struct ibv_context */
>   	struct ibv_context context;	/* Must be last field in the struct */
> @@ -2535,6 +2539,12 @@ __ibv_reg_mr_iova(struct ibv_pd *pd, void *addr, size_t length, uint64_t iova,
>   			  __builtin_constant_p(                                \
>   				  ((access) & IBV_ACCESS_OPTIONAL_RANGE) == 0))
>   
> +/**
> + * ibv_reg_dmabuf_mr - Register a dambuf-based memory region
> + */
> +struct ibv_mr *ibv_reg_dmabuf_mr(struct ibv_pd *pd, uint64_t offset, size_t length,
> +				 int fd, int access);
> +
>   enum ibv_rereg_mr_err_code {
>   	/* Old MR is valid, invalid input */
>   	IBV_REREG_MR_ERR_INPUT = -1,