[PATCH for-next v3 0/3] RDMA/erdma: Add DMA-BUF memory registration

[PATCH for-next v3 0/3] RDMA/erdma: Add DMA-BUF memory registration

[Boshi Yu]

Hi,

This patch series adds DMA-BUF memory registration support to the erdma
driver:

- #1 renames get/put_mtt_entries to erdma_mem_init/uninit to better reflect
     their purpose of initializing the struct erdma_mem.
- #2 introduces struct erdma_mem_init_attr to pass parameters to
     erdma_mem_init(), improving code maintainability and preparing for
     DMA-BUF support.
- #3 implements erdma_reg_user_mr_dmabuf() to enable DMA-BUF based user
     memory registration using the revocable pinned dmabuf interface.

One known limitation: if the DEREG_MR command fails in the revoke
callback, the driver can only log an error, since erdma does not
support a function-level reset like irdma's request_reset() yet.
We plan to introduce such a mechanism in future work.

Thanks,
Boshi Yu

---

v3:
 - Patch#3: Switch to the revocable pinned dmabuf interface as
   suggested by Jason Gunthorpe.

v2:
 - Patch#2: Add validation for the return value of ib_umem_find_best_pgsz().
  link: https://lore.kernel.org/all/20260518120637.16831-1-boshiyu@linux.alibaba.com/

v1:
  link: https://lore.kernel.org/all/20260507053437.46211-1-boshiyu@linux.alibaba.com/

Boshi Yu (3):
  RDMA/erdma: Rename get/put_mtt_entries to erdma_mem_init/uninit
  RDMA/erdma: Introduce struct erdma_mem_init_attr
  RDMA/erdma: Implement erdma_reg_user_mr_dmabuf

 drivers/infiniband/hw/erdma/erdma_main.c  |   1 +
 drivers/infiniband/hw/erdma/erdma_verbs.c | 262 +++++++++++++++-------
 drivers/infiniband/hw/erdma/erdma_verbs.h |  17 ++
 3 files changed, 201 insertions(+), 79 deletions(-)

-- 
2.46.0

[PATCH for-next v3 1/3] RDMA/erdma: Rename get/put_mtt_entries to erdma_mem_init/uninit

[Boshi Yu]

The get_mtt_entries() interface actually initializes the struct
erdma_mem. Rename the get_mtt_entries/put_mtt_entries to
erdma_mem_init/erdma_mem_uninit, respectively.

Reviewed-by: Cheng Xu <chengyou@linux.alibaba.com>
Signed-off-by: Boshi Yu <boshiyu@linux.alibaba.com>
---
 drivers/infiniband/hw/erdma/erdma_verbs.c | 64 +++++++++++------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 74afe6eb18b0..92cd8cd846b6 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -828,9 +828,9 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
 	}
 }
 
-static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
-			   u64 start, u64 len, int access, u64 virt,
-			   unsigned long req_page_size, bool force_continuous)
+static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
+			  u64 start, u64 len, int access, u64 virt,
+			  unsigned long req_page_size, bool force_continuous)
 {
 	int ret = 0;
 
@@ -867,7 +867,7 @@ static int get_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem,
 	return ret;
 }
 
-static void put_mtt_entries(struct erdma_dev *dev, struct erdma_mem *mem)
+static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem)
 {
 	if (mem->mtt)
 		erdma_destroy_mtt(dev, mem->mtt);
@@ -951,45 +951,45 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
 		   qp->attrs.rq_size * RQE_SIZE))
 		return -EINVAL;
 
-	ret = get_mtt_entries(qp->dev, &qp->user_qp.sq_mem, va,
-			      qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
-			      (SZ_1M - SZ_4K), true);
+	ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, va,
+			     qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
+			     (SZ_1M - SZ_4K), true);
 	if (ret)
 		return ret;
 
 	rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
 	qp->user_qp.rq_offset = rq_offset;
 
-	ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
-			      qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
-			      (SZ_1M - SZ_4K), true);
+	ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
+			     qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
+			     (SZ_1M - SZ_4K), true);
 	if (ret)
-		goto put_sq_mtt;
+		goto uninit_sq_mem;
 
 	ret = erdma_map_user_dbrecords(uctx, dbrec_va,
 				       &qp->user_qp.user_dbr_page,
 				       &dbrec_dma);
 	if (ret)
-		goto put_rq_mtt;
+		goto uninit_rq_mem;
 
 	qp->user_qp.sq_dbrec_dma = dbrec_dma;
 	qp->user_qp.rq_dbrec_dma = dbrec_dma + ERDMA_DB_SIZE;
 
 	return 0;
 
-put_rq_mtt:
-	put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
+uninit_rq_mem:
+	erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
 
-put_sq_mtt:
-	put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
+uninit_sq_mem:
+	erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
 
 	return ret;
 }
 
 static void free_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx)
 {
-	put_mtt_entries(qp->dev, &qp->user_qp.sq_mem);
-	put_mtt_entries(qp->dev, &qp->user_qp.rq_mem);
+	erdma_mem_uninit(qp->dev, &qp->user_qp.sq_mem);
+	erdma_mem_uninit(qp->dev, &qp->user_qp.rq_mem);
 	erdma_unmap_user_dbrecords(uctx, &qp->user_qp.user_dbr_page);
 }
 
@@ -1251,14 +1251,14 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	ret = get_mtt_entries(dev, &mr->mem, start, len, access, virt,
-			      SZ_2G - SZ_4K, false);
+	ret = erdma_mem_init(dev, &mr->mem, start, len, access, virt,
+			     SZ_2G - SZ_4K, false);
 	if (ret)
 		goto err_out_free;
 
 	ret = erdma_create_stag(dev, &stag);
 	if (ret)
-		goto err_out_put_mtt;
+		goto err_uninit_mem;
 
 	mr->ibmr.lkey = mr->ibmr.rkey = stag;
 	mr->ibmr.pd = ibpd;
@@ -1278,8 +1278,8 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
 		       mr->ibmr.lkey >> 8);
 
-err_out_put_mtt:
-	put_mtt_entries(dev, &mr->mem);
+err_uninit_mem:
+	erdma_mem_uninit(dev, &mr->mem);
 
 err_out_free:
 	kfree(mr);
@@ -1309,7 +1309,7 @@ int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 
 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
 
-	put_mtt_entries(dev, &mr->mem);
+	erdma_mem_uninit(dev, &mr->mem);
 
 	kfree(mr);
 	return 0;
@@ -1340,7 +1340,7 @@ int erdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
 			      cq->kern_cq.dbrec_dma);
 	} else {
 		erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 	}
 
 	xa_erase(&dev->cq_xa, cq->cqn);
@@ -1387,8 +1387,8 @@ int erdma_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 	if (rdma_is_kernel_res(&qp->ibqp.res)) {
 		free_kernel_qp(qp);
 	} else {
-		put_mtt_entries(dev, &qp->user_qp.sq_mem);
-		put_mtt_entries(dev, &qp->user_qp.rq_mem);
+		erdma_mem_uninit(dev, &qp->user_qp.sq_mem);
+		erdma_mem_uninit(dev, &qp->user_qp.rq_mem);
 		erdma_unmap_user_dbrecords(ctx, &qp->user_qp.user_dbr_page);
 	}
 
@@ -1910,9 +1910,9 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 	int ret;
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
 
-	ret = get_mtt_entries(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
-			      ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
-			      true);
+	ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
+			     ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
+			     true);
 	if (ret)
 		return ret;
 
@@ -1920,7 +1920,7 @@ static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 				       &cq->user_cq.user_dbr_page,
 				       &cq->user_cq.dbrec_dma);
 	if (ret)
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 
 	return ret;
 }
@@ -2011,7 +2011,7 @@ int erdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 err_free_res:
 	if (!rdma_is_kernel_res(&ibcq->res)) {
 		erdma_unmap_user_dbrecords(ctx, &cq->user_cq.user_dbr_page);
-		put_mtt_entries(dev, &cq->user_cq.qbuf_mem);
+		erdma_mem_uninit(dev, &cq->user_cq.qbuf_mem);
 	} else {
 		dma_free_coherent(&dev->pdev->dev, depth << CQE_SHIFT,
 				  cq->kern_cq.qbuf, cq->kern_cq.qbuf_dma_addr);
-- 
2.46.0

[PATCH for-next v3 2/3] RDMA/erdma: Introduce struct erdma_mem_init_attr

[Boshi Yu]

Pass struct erdma_mem_init_attr to the erdma_mem_init() interface.
Additionally, validate the return value of ib_umem_find_best_pgsz().

Reviewed-by: Cheng Xu <chengyou@linux.alibaba.com>
Signed-off-by: Boshi Yu <boshiyu@linux.alibaba.com>
---
 drivers/infiniband/hw/erdma/erdma_verbs.c | 65 +++++++++++++++--------
 drivers/infiniband/hw/erdma/erdma_verbs.h | 13 +++++
 2 files changed, 55 insertions(+), 23 deletions(-)

diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 92cd8cd846b6..98278432ba80 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -829,26 +829,33 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
 }
 
 static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
-			  u64 start, u64 len, int access, u64 virt,
-			  unsigned long req_page_size, bool force_continuous)
+			  struct erdma_mem_init_attr *attr)
 {
 	int ret = 0;
 
-	mem->umem = ib_umem_get_va(&dev->ibdev, start, len, access);
+	mem->umem =
+		ib_umem_get_va(&dev->ibdev, attr->start, attr->len, attr->access);
 	if (IS_ERR(mem->umem)) {
 		ret = PTR_ERR(mem->umem);
 		mem->umem = NULL;
 		return ret;
 	}
 
-	mem->va = virt;
-	mem->len = len;
-	mem->page_size = ib_umem_find_best_pgsz(mem->umem, req_page_size, virt);
-	mem->page_offset = start & (mem->page_size - 1);
+	mem->va = attr->virt;
+	mem->len = attr->len;
+
+	mem->page_size = ib_umem_find_best_pgsz(mem->umem, attr->req_page_size,
+						attr->virt);
+	if (!mem->page_size) {
+		ret = -EINVAL;
+		goto error_ret;
+	}
+
+	mem->page_offset = attr->start & (mem->page_size - 1);
 	mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
 	mem->page_cnt = mem->mtt_nents;
 	mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
-				    force_continuous);
+				    attr->flags & ERDMA_MEM_FLAG_MTT_PHYS_CONT);
 	if (IS_ERR(mem->mtt)) {
 		ret = PTR_ERR(mem->mtt);
 		goto error_ret;
@@ -943,6 +950,7 @@ erdma_unmap_user_dbrecords(struct erdma_ucontext *ctx,
 static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
 			u64 va, u32 len, u64 dbrec_va)
 {
+	struct erdma_mem_init_attr attr = {};
 	dma_addr_t dbrec_dma;
 	u32 rq_offset;
 	int ret;
@@ -951,18 +959,22 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
 		   qp->attrs.rq_size * RQE_SIZE))
 		return -EINVAL;
 
-	ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, va,
-			     qp->attrs.sq_size << SQEBB_SHIFT, 0, va,
-			     (SZ_1M - SZ_4K), true);
+	attr.virt = va;
+	attr.start = va;
+	attr.req_page_size = SZ_1M - SZ_4K;
+	attr.len = qp->attrs.sq_size << SQEBB_SHIFT;
+	attr.flags = ERDMA_MEM_FLAG_MTT_PHYS_CONT;
+	ret = erdma_mem_init(qp->dev, &qp->user_qp.sq_mem, &attr);
 	if (ret)
 		return ret;
 
 	rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
 	qp->user_qp.rq_offset = rq_offset;
 
-	ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, va + rq_offset,
-			     qp->attrs.rq_size << RQE_SHIFT, 0, va + rq_offset,
-			     (SZ_1M - SZ_4K), true);
+	attr.virt = va + rq_offset;
+	attr.start = va + rq_offset;
+	attr.len = qp->attrs.rq_size << RQE_SHIFT;
+	ret = erdma_mem_init(qp->dev, &qp->user_qp.rq_mem, &attr);
 	if (ret)
 		goto uninit_sq_mem;
 
@@ -1236,8 +1248,9 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 				u64 virt, int access, struct ib_dmah *dmah,
 				struct ib_udata *udata)
 {
-	struct erdma_mr *mr = NULL;
 	struct erdma_dev *dev = to_edev(ibpd->device);
+	struct erdma_mem_init_attr attr = {};
+	struct erdma_mr *mr = NULL;
 	u32 stag;
 	int ret;
 
@@ -1251,8 +1264,12 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
 
-	ret = erdma_mem_init(dev, &mr->mem, start, len, access, virt,
-			     SZ_2G - SZ_4K, false);
+	attr.len = len;
+	attr.virt = virt;
+	attr.start = start;
+	attr.access = access;
+	attr.req_page_size = SZ_2G - SZ_4K;
+	ret = erdma_mem_init(dev, &mr->mem, &attr);
 	if (ret)
 		goto err_out_free;
 
@@ -1262,8 +1279,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 
 	mr->ibmr.lkey = mr->ibmr.rkey = stag;
 	mr->ibmr.pd = ibpd;
-	mr->mem.va = virt;
-	mr->mem.len = len;
 	mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
 	mr->valid = 1;
 	mr->type = ERDMA_MR_TYPE_NORMAL;
@@ -1907,12 +1922,16 @@ int erdma_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 static int erdma_init_user_cq(struct erdma_ucontext *ctx, struct erdma_cq *cq,
 			      struct erdma_ureq_create_cq *ureq)
 {
-	int ret;
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
+	struct erdma_mem_init_attr attr = {};
+	int ret;
 
-	ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, ureq->qbuf_va,
-			     ureq->qbuf_len, 0, ureq->qbuf_va, SZ_64M - SZ_4K,
-			     true);
+	attr.len = ureq->qbuf_len;
+	attr.virt = ureq->qbuf_va;
+	attr.start = ureq->qbuf_va;
+	attr.req_page_size = SZ_64M - SZ_4K;
+	attr.flags = ERDMA_MEM_FLAG_MTT_PHYS_CONT;
+	ret = erdma_mem_init(dev, &cq->user_cq.qbuf_mem, &attr);
 	if (ret)
 		return ret;
 
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index 7d8d3fe501d5..79b3a90b03e7 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -108,6 +108,19 @@ struct erdma_mtt {
 	struct erdma_mtt *low_level;
 };
 
+enum erdma_mem_flags {
+	ERDMA_MEM_FLAG_MTT_PHYS_CONT = (1 << 0),
+};
+
+struct erdma_mem_init_attr {
+	u64 start;
+	u64 virt;
+	u64 len;
+	unsigned long req_page_size;
+	int access;
+	u32 flags;
+};
+
 struct erdma_mem {
 	struct ib_umem *umem;
 	struct erdma_mtt *mtt;
-- 
2.46.0

[PATCH for-next v3 3/3] RDMA/erdma: Implement erdma_reg_user_mr_dmabuf

[Boshi Yu]

Support DMA-BUF based user memory registration using the pinned
revocable dmabuf interface, so that erdma can import dmabufs from
exporters that may require revocation, such as VFIO.

On revocation, the revoke callback destroys the MPT entry in hardware
to stop the device from accessing the revoked memory, while the stag
index is reserved until the MR is fully deregistered, ensuring the key
cannot be obtained by a new region in the meantime.

Refactor the existing erdma_reg_user_mr() into a common
_erdma_reg_user_mr() function that handles both regular and DMA-BUF
memory registration.

Reviewed-by: Cheng Xu <chengyou@linux.alibaba.com>
Signed-off-by: Boshi Yu <boshiyu@linux.alibaba.com>
---
 drivers/infiniband/hw/erdma/erdma_main.c  |   1 +
 drivers/infiniband/hw/erdma/erdma_verbs.c | 189 ++++++++++++++++------
 drivers/infiniband/hw/erdma/erdma_verbs.h |   4 +
 3 files changed, 142 insertions(+), 52 deletions(-)

diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c
index 7e87a815e853..6e4860428e5b 100644
--- a/drivers/infiniband/hw/erdma/erdma_main.c
+++ b/drivers/infiniband/hw/erdma/erdma_main.c
@@ -526,6 +526,7 @@ static const struct ib_device_ops erdma_device_ops = {
 	.query_qp = erdma_query_qp,
 	.req_notify_cq = erdma_req_notify_cq,
 	.reg_user_mr = erdma_reg_user_mr,
+	.reg_user_mr_dmabuf = erdma_reg_user_mr_dmabuf,
 	.modify_qp = erdma_modify_qp,
 
 	INIT_RDMA_OBJ_SIZE(ib_cq, erdma_cq, ibcq),
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 98278432ba80..5c6d84d83fc9 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -189,6 +189,20 @@ static int regmr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
 				   true);
 }
 
+static int dereg_mr_cmd(struct erdma_dev *dev, struct erdma_mr *mr)
+{
+	struct erdma_cmdq_dereg_mr_req req = {};
+
+	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
+				CMDQ_OPCODE_DEREG_MR);
+
+	req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, mr->ibmr.lkey >> 8) |
+		  FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, mr->ibmr.lkey & 0xFF);
+
+	return erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
+				   true);
+}
+
 static int create_cq_cmd(struct erdma_ucontext *uctx, struct erdma_cq *cq)
 {
 	struct erdma_dev *dev = to_edev(cq->ibcq.device);
@@ -828,50 +842,50 @@ static void erdma_destroy_mtt(struct erdma_dev *dev, struct erdma_mtt *mtt)
 	}
 }
 
-static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
-			  struct erdma_mem_init_attr *attr)
+static int erdma_mem_init_by_umem(struct erdma_dev *dev, struct erdma_mem *mem,
+				  struct erdma_mem_init_attr *attr,
+				  struct ib_umem *umem)
 {
-	int ret = 0;
-
-	mem->umem =
-		ib_umem_get_va(&dev->ibdev, attr->start, attr->len, attr->access);
-	if (IS_ERR(mem->umem)) {
-		ret = PTR_ERR(mem->umem);
-		mem->umem = NULL;
-		return ret;
-	}
-
+	mem->umem = umem;
 	mem->va = attr->virt;
 	mem->len = attr->len;
 
 	mem->page_size = ib_umem_find_best_pgsz(mem->umem, attr->req_page_size,
 						attr->virt);
-	if (!mem->page_size) {
-		ret = -EINVAL;
-		goto error_ret;
-	}
+	if (!mem->page_size)
+		return -EINVAL;
 
 	mem->page_offset = attr->start & (mem->page_size - 1);
 	mem->mtt_nents = ib_umem_num_dma_blocks(mem->umem, mem->page_size);
 	mem->page_cnt = mem->mtt_nents;
 	mem->mtt = erdma_create_mtt(dev, MTT_SIZE(mem->page_cnt),
 				    attr->flags & ERDMA_MEM_FLAG_MTT_PHYS_CONT);
-	if (IS_ERR(mem->mtt)) {
-		ret = PTR_ERR(mem->mtt);
-		goto error_ret;
-	}
+	if (IS_ERR(mem->mtt))
+		return PTR_ERR(mem->mtt);
 
 	erdma_fill_bottom_mtt(dev, mem);
 
 	return 0;
+}
 
-error_ret:
-	if (mem->umem) {
-		ib_umem_release(mem->umem);
-		mem->umem = NULL;
+static int erdma_mem_init(struct erdma_dev *dev, struct erdma_mem *mem,
+			  struct erdma_mem_init_attr *attr)
+{
+	struct ib_umem *umem;
+	int ret;
+
+	umem = ib_umem_get_va(&dev->ibdev, attr->start, attr->len,
+			      attr->access);
+	if (IS_ERR(umem))
+		return PTR_ERR(umem);
+
+	ret = erdma_mem_init_by_umem(dev, mem, attr, umem);
+	if (ret) {
+		ib_umem_release(umem);
+		return ret;
 	}
 
-	return ret;
+	return 0;
 }
 
 static void erdma_mem_uninit(struct erdma_dev *dev, struct erdma_mem *mem)
@@ -1244,9 +1258,20 @@ int erdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
 	return num;
 }
 
-struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
-				u64 virt, int access, struct ib_dmah *dmah,
-				struct ib_udata *udata)
+static void erdma_umem_dmabuf_revoke(void *priv)
+{
+	struct erdma_mr *mr = priv;
+	struct erdma_dev *dev = to_edev(mr->ibmr.device);
+	int ret;
+
+	ret = dereg_mr_cmd(dev, mr);
+	if (ret)
+		ibdev_err(&dev->ibdev, "dmabuf mr revoke failed %d", ret);
+}
+
+static struct ib_mr *
+_erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len, u64 virt, int access,
+		   struct ib_umem *umem, struct ib_dmah *dmah)
 {
 	struct erdma_dev *dev = to_edev(ibpd->device);
 	struct erdma_mem_init_attr attr = {};
@@ -1254,12 +1279,6 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	u32 stag;
 	int ret;
 
-	if (dmah)
-		return ERR_PTR(-EOPNOTSUPP);
-
-	if (!len || len > dev->attrs.max_mr_size)
-		return ERR_PTR(-EINVAL);
-
 	mr = kzalloc_obj(*mr);
 	if (!mr)
 		return ERR_PTR(-ENOMEM);
@@ -1269,16 +1288,17 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	attr.start = start;
 	attr.access = access;
 	attr.req_page_size = SZ_2G - SZ_4K;
-	ret = erdma_mem_init(dev, &mr->mem, &attr);
+	ret = erdma_mem_init_by_umem(dev, &mr->mem, &attr, umem);
 	if (ret)
 		goto err_out_free;
 
 	ret = erdma_create_stag(dev, &stag);
 	if (ret)
-		goto err_uninit_mem;
+		goto err_destroy_mtt;
 
 	mr->ibmr.lkey = mr->ibmr.rkey = stag;
 	mr->ibmr.pd = ibpd;
+	mr->ibmr.device = ibpd->device;
 	mr->access = ERDMA_MR_ACC_LR | to_erdma_access_flags(access);
 	mr->valid = 1;
 	mr->type = ERDMA_MR_TYPE_NORMAL;
@@ -1293,8 +1313,8 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX],
 		       mr->ibmr.lkey >> 8);
 
-err_uninit_mem:
-	erdma_mem_uninit(dev, &mr->mem);
+err_destroy_mtt:
+	erdma_destroy_mtt(dev, mr->mem.mtt);
 
 err_out_free:
 	kfree(mr);
@@ -1302,30 +1322,95 @@ struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 	return ERR_PTR(ret);
 }
 
+struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
+				u64 virt, int access, struct ib_dmah *dmah,
+				struct ib_udata *udata)
+{
+	struct erdma_dev *dev = to_edev(ibpd->device);
+	struct ib_umem *umem;
+	struct ib_mr *ibmr;
+
+	if (dmah)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	if (!len || len > dev->attrs.max_mr_size)
+		return ERR_PTR(-EINVAL);
+
+	umem = ib_umem_get_va(&dev->ibdev, start, len, access);
+	if (IS_ERR(umem))
+		return ERR_CAST(umem);
+
+	ibmr = _erdma_reg_user_mr(ibpd, start, len, virt, access, umem, dmah);
+	if (IS_ERR(ibmr)) {
+		ib_umem_release(umem);
+		return ibmr;
+	}
+
+	return ibmr;
+}
+
+struct ib_mr *erdma_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 len,
+				       u64 virt, int fd, int access,
+				       struct ib_dmah *dmah,
+				       struct uverbs_attr_bundle *attrs)
+{
+	struct erdma_dev *dev = to_edev(ibpd->device);
+	struct ib_umem_dmabuf *umem_dmabuf;
+	struct ib_mr *ibmr;
+
+	if (dmah)
+		return ERR_PTR(-EOPNOTSUPP);
+
+	if (!len || len > dev->attrs.max_mr_size)
+		return ERR_PTR(-EINVAL);
+
+	umem_dmabuf =
+		ib_umem_dmabuf_get_pinned_revocable_and_lock(&dev->ibdev, start,
+							     len, fd, access);
+	if (IS_ERR(umem_dmabuf))
+		return ERR_CAST(umem_dmabuf);
+
+	ibmr = _erdma_reg_user_mr(ibpd, start, len, virt, access,
+				  &umem_dmabuf->umem, dmah);
+	if (IS_ERR(ibmr)) {
+		ib_umem_dmabuf_revoke_unlock(umem_dmabuf);
+		ib_umem_release(&umem_dmabuf->umem);
+		return ibmr;
+	}
+
+	ib_umem_dmabuf_set_revoke_locked(umem_dmabuf, erdma_umem_dmabuf_revoke,
+					 to_emr(ibmr));
+	ib_umem_dmabuf_revoke_unlock(umem_dmabuf);
+
+	return ibmr;
+}
+
 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
 {
-	struct erdma_mr *mr;
 	struct erdma_dev *dev = to_edev(ibmr->device);
-	struct erdma_cmdq_dereg_mr_req req;
+	struct erdma_mr *mr = to_emr(ibmr);
+	bool dmabuf;
 	int ret;
 
-	mr = to_emr(ibmr);
-
-	erdma_cmdq_build_reqhdr(&req.hdr, CMDQ_SUBMOD_RDMA,
-				CMDQ_OPCODE_DEREG_MR);
+	dmabuf = mr->mem.umem && mr->mem.umem->is_dmabuf;
+	if (!dmabuf) {
+		ret = dereg_mr_cmd(dev, mr);
+		if (ret)
+			return ret;
+	}
 
-	req.cfg = FIELD_PREP(ERDMA_CMD_MR_MPT_IDX_MASK, ibmr->lkey >> 8) |
-		  FIELD_PREP(ERDMA_CMD_MR_KEY_MASK, ibmr->lkey & 0xFF);
+	if (mr->mem.umem) {
+		ib_umem_release(mr->mem.umem);
+		mr->mem.umem = NULL;
+	}
 
-	ret = erdma_post_cmd_wait(&dev->cmdq, &req, sizeof(req), NULL, NULL,
-				  true);
-	if (ret)
-		return ret;
+	if (mr->mem.mtt) {
+		erdma_destroy_mtt(dev, mr->mem.mtt);
+		mr->mem.mtt = NULL;
+	}
 
 	erdma_free_idx(&dev->res_cb[ERDMA_RES_TYPE_STAG_IDX], ibmr->lkey >> 8);
 
-	erdma_mem_uninit(dev, &mr->mem);
-
 	kfree(mr);
 	return 0;
 }
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index 79b3a90b03e7..f576f062aff3 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -467,6 +467,10 @@ int erdma_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
 struct ib_mr *erdma_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
 				u64 virt, int access, struct ib_dmah *dmah,
 				struct ib_udata *udata);
+struct ib_mr *erdma_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start, u64 len,
+				       u64 virt, int fd, int access,
+				       struct ib_dmah *dmah,
+				       struct uverbs_attr_bundle *attrs);
 struct ib_mr *erdma_get_dma_mr(struct ib_pd *ibpd, int rights);
 int erdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *data);
 int erdma_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma);
-- 
2.46.0

Re: [PATCH for-next v3 3/3] RDMA/erdma: Implement erdma_reg_user_mr_dmabuf

[Jacob Moroni]

Reviewed-by: Jacob Moroni <jmoroni@google.com>

Re: [PATCH for-next v3 0/3] RDMA/erdma: Add DMA-BUF memory registration

[Leon Romanovsky]

On Thu, Jun 18, 2026 at 12:51:02PM +0800, Boshi Yu wrote:
> Hi,
> 
> This patch series adds DMA-BUF memory registration support to the erdma
> driver:
> 
> - #1 renames get/put_mtt_entries to erdma_mem_init/uninit to better reflect
>      their purpose of initializing the struct erdma_mem.
> - #2 introduces struct erdma_mem_init_attr to pass parameters to
>      erdma_mem_init(), improving code maintainability and preparing for
>      DMA-BUF support.
> - #3 implements erdma_reg_user_mr_dmabuf() to enable DMA-BUF based user
>      memory registration using the revocable pinned dmabuf interface.
> 
> One known limitation: if the DEREG_MR command fails in the revoke
> callback, the driver can only log an error, since erdma does not
> support a function-level reset like irdma's request_reset() yet.

I hoped someone would comment on this. If revocation fails, the
hardware may continue accessing memory that the system considers
released.

From comment:
 * When a revocation occurs, the revoke callback will be called. The driver must
 * ensure that the region is no longer accessed when the callback returns. Any
 * subsequent access attempts should also probably cause an AE for MRs.

> We plan to introduce such a mechanism in future work.

That mechanism needs to be included in this series.

Thanks

> 
> Thanks,
> Boshi Yu
> 
> ---
> 
> v3:
>  - Patch#3: Switch to the revocable pinned dmabuf interface as
>    suggested by Jason Gunthorpe.
> 
> v2:
>  - Patch#2: Add validation for the return value of ib_umem_find_best_pgsz().
>   link: https://lore.kernel.org/all/20260518120637.16831-1-boshiyu@linux.alibaba.com/
> 
> v1:
>   link: https://lore.kernel.org/all/20260507053437.46211-1-boshiyu@linux.alibaba.com/
> 
> Boshi Yu (3):
>   RDMA/erdma: Rename get/put_mtt_entries to erdma_mem_init/uninit
>   RDMA/erdma: Introduce struct erdma_mem_init_attr
>   RDMA/erdma: Implement erdma_reg_user_mr_dmabuf
> 
>  drivers/infiniband/hw/erdma/erdma_main.c  |   1 +
>  drivers/infiniband/hw/erdma/erdma_verbs.c | 262 +++++++++++++++-------
>  drivers/infiniband/hw/erdma/erdma_verbs.h |  17 ++
>  3 files changed, 201 insertions(+), 79 deletions(-)
> 
> -- 
> 2.46.0
> 
> 

Re: [PATCH for-next v3 0/3] RDMA/erdma: Add DMA-BUF memory registration

[Boshi Yu]

On 7/5/26 6:08 PM, Leon Romanovsky wrote:
> On Thu, Jun 18, 2026 at 12:51:02PM +0800, Boshi Yu wrote:
>> Hi,
>>
>> This patch series adds DMA-BUF memory registration support to the erdma
>> driver:
>>
>> - #1 renames get/put_mtt_entries to erdma_mem_init/uninit to better reflect
>>       their purpose of initializing the struct erdma_mem.
>> - #2 introduces struct erdma_mem_init_attr to pass parameters to
>>       erdma_mem_init(), improving code maintainability and preparing for
>>       DMA-BUF support.
>> - #3 implements erdma_reg_user_mr_dmabuf() to enable DMA-BUF based user
>>       memory registration using the revocable pinned dmabuf interface.
>>
>> One known limitation: if the DEREG_MR command fails in the revoke
>> callback, the driver can only log an error, since erdma does not
>> support a function-level reset like irdma's request_reset() yet.
> 
> I hoped someone would comment on this. If revocation fails, the
> hardware may continue accessing memory that the system considers
> released.
> 
>  From comment:
>   * When a revocation occurs, the revoke callback will be called. The driver must
>   * ensure that the region is no longer accessed when the callback returns. Any
>   * subsequent access attempts should also probably cause an AE for MRs.
> 
>> We plan to introduce such a mechanism in future work.
> 
> That mechanism needs to be included in this series.
> 
> Thanks
> 

Hi, Leon

Thanks for your suggestion. I will implement the reset mechanism in the 
next version of this series.

Thanks,
Boshi Yu

>>
>> Thanks,
>> Boshi Yu
>>
>> ---
>>
>> v3:
>>   - Patch#3: Switch to the revocable pinned dmabuf interface as
>>     suggested by Jason Gunthorpe.
>>
>> v2:
>>   - Patch#2: Add validation for the return value of ib_umem_find_best_pgsz().
>>    link: https://lore.kernel.org/all/20260518120637.16831-1-boshiyu@linux.alibaba.com/
>>
>> v1:
>>    link: https://lore.kernel.org/all/20260507053437.46211-1-boshiyu@linux.alibaba.com/
>>
>> Boshi Yu (3):
>>    RDMA/erdma: Rename get/put_mtt_entries to erdma_mem_init/uninit
>>    RDMA/erdma: Introduce struct erdma_mem_init_attr
>>    RDMA/erdma: Implement erdma_reg_user_mr_dmabuf
>>
>>   drivers/infiniband/hw/erdma/erdma_main.c  |   1 +
>>   drivers/infiniband/hw/erdma/erdma_verbs.c | 262 +++++++++++++++-------
>>   drivers/infiniband/hw/erdma/erdma_verbs.h |  17 ++
>>   3 files changed, 201 insertions(+), 79 deletions(-)
>>
>> -- 
>> 2.46.0
>>
>>