[PATCH v4 3/5] RDMA/core: add MR support for bvec-based RDMA operations

[Chuck Lever <cel@kernel.org>]

From: Chuck Lever <chuck.lever@oracle.com>

The bvec-based RDMA API currently returns -EOPNOTSUPP when Memory
Region registration is required. This prevents iWARP devices from
using the bvec path, since iWARP requires MR registration for RDMA
READ operations. The force_mr debug parameter is also unusable with
bvec input.

Add rdma_rw_init_mr_wrs_bvec() to handle MR registration for bvec
arrays. The approach creates a synthetic scatterlist populated with
DMA addresses from the bvecs, then reuses the existing ib_map_mr_sg()
infrastructure. This avoids driver changes while keeping the
implementation small.

The synthetic scatterlist is stored in the rdma_rw_ctx for cleanup.
On destroy, the MRs are returned to the pool and the bvec DMA
mappings are released using the stored addresses.

Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
---
 drivers/infiniband/core/rw.c            | 262 +++++++++++++++++-------
 drivers/infiniband/ulp/isert/ib_isert.c |   4 +-
 drivers/nvme/target/rdma.c              |   4 +-
 include/rdma/rw.h                       |  17 +-
 4 files changed, 202 insertions(+), 85 deletions(-)

diff --git a/drivers/infiniband/core/rw.c b/drivers/infiniband/core/rw.c
index c2fc8cba972e..f6d3c0b84df1 100644
--- a/drivers/infiniband/core/rw.c
+++ b/drivers/infiniband/core/rw.c
@@ -122,6 +122,36 @@ static int rdma_rw_init_one_mr(struct ib_qp *qp, u32 port_num,
 	return count;
 }
 
+static int rdma_rw_init_reg_wr(struct rdma_rw_reg_ctx *reg,
+		struct rdma_rw_reg_ctx *prev, struct ib_qp *qp, u32 port_num,
+		u64 remote_addr, u32 rkey, enum dma_data_direction dir)
+{
+	if (prev) {
+		if (reg->mr->need_inval)
+			prev->wr.wr.next = &reg->inv_wr;
+		else
+			prev->wr.wr.next = &reg->reg_wr.wr;
+	}
+
+	reg->reg_wr.wr.next = &reg->wr.wr;
+
+	reg->wr.wr.sg_list = &reg->sge;
+	reg->wr.wr.num_sge = 1;
+	reg->wr.remote_addr = remote_addr;
+	reg->wr.rkey = rkey;
+
+	if (dir == DMA_TO_DEVICE) {
+		reg->wr.wr.opcode = IB_WR_RDMA_WRITE;
+	} else if (!rdma_cap_read_inv(qp->device, port_num)) {
+		reg->wr.wr.opcode = IB_WR_RDMA_READ;
+	} else {
+		reg->wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV;
+		reg->wr.wr.ex.invalidate_rkey = reg->mr->lkey;
+	}
+
+	return 1;
+}
+
 static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 		u32 port_num, struct scatterlist *sg, u32 sg_cnt, u32 offset,
 		u64 remote_addr, u32 rkey, enum dma_data_direction dir)
@@ -132,14 +162,14 @@ static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 	int i, j, ret = 0, count = 0;
 
 	ctx->nr_ops = DIV_ROUND_UP(sg_cnt, pages_per_mr);
-	ctx->reg = kcalloc(ctx->nr_ops, sizeof(*ctx->reg), GFP_KERNEL);
-	if (!ctx->reg) {
+	ctx->reg.ctx = kcalloc(ctx->nr_ops, sizeof(*ctx->reg.ctx), GFP_KERNEL);
+	if (!ctx->reg.ctx) {
 		ret = -ENOMEM;
 		goto out;
 	}
 
 	for (i = 0; i < ctx->nr_ops; i++) {
-		struct rdma_rw_reg_ctx *reg = &ctx->reg[i];
+		struct rdma_rw_reg_ctx *reg = &ctx->reg.ctx[i];
 		u32 nents = min(sg_cnt, pages_per_mr);
 
 		ret = rdma_rw_init_one_mr(qp, port_num, reg, sg, sg_cnt,
@@ -147,30 +177,8 @@ static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 		if (ret < 0)
 			goto out_free;
 		count += ret;
-
-		if (prev) {
-			if (reg->mr->need_inval)
-				prev->wr.wr.next = &reg->inv_wr;
-			else
-				prev->wr.wr.next = &reg->reg_wr.wr;
-		}
-
-		reg->reg_wr.wr.next = &reg->wr.wr;
-
-		reg->wr.wr.sg_list = &reg->sge;
-		reg->wr.wr.num_sge = 1;
-		reg->wr.remote_addr = remote_addr;
-		reg->wr.rkey = rkey;
-		if (dir == DMA_TO_DEVICE) {
-			reg->wr.wr.opcode = IB_WR_RDMA_WRITE;
-		} else if (!rdma_cap_read_inv(qp->device, port_num)) {
-			reg->wr.wr.opcode = IB_WR_RDMA_READ;
-		} else {
-			reg->wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV;
-			reg->wr.wr.ex.invalidate_rkey = reg->mr->lkey;
-		}
-		count++;
-
+		count += rdma_rw_init_reg_wr(reg, prev, qp, port_num,
+				remote_addr, rkey, dir);
 		remote_addr += reg->sge.length;
 		sg_cnt -= nents;
 		for (j = 0; j < nents; j++)
@@ -187,12 +195,95 @@ static int rdma_rw_init_mr_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 
 out_free:
 	while (--i >= 0)
-		ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr);
-	kfree(ctx->reg);
+		ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg.ctx[i].mr);
+	kfree(ctx->reg.ctx);
 out:
 	return ret;
 }
 
+static int rdma_rw_init_mr_wrs_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
+		u32 port_num, const struct bio_vec *bvecs, u32 nr_bvec,
+		struct bvec_iter *iter, u64 remote_addr, u32 rkey,
+		enum dma_data_direction dir)
+{
+	struct ib_device *dev = qp->pd->device;
+	struct rdma_rw_reg_ctx *prev = NULL;
+	u32 pages_per_mr = rdma_rw_fr_page_list_len(dev, qp->integrity_en);
+	struct scatterlist *sg;
+	int i, ret, count = 0;
+	u32 nents = 0;
+
+	/*
+	 * Build scatterlist from bvecs using the iterator. This follows
+	 * the pattern from __blk_rq_map_sg.
+	 */
+	ctx->reg.sgt.sgl = kmalloc_array(nr_bvec, sizeof(*ctx->reg.sgt.sgl),
+					 GFP_KERNEL);
+	if (!ctx->reg.sgt.sgl)
+		return -ENOMEM;
+	sg_init_table(ctx->reg.sgt.sgl, nr_bvec);
+
+	for (sg = ctx->reg.sgt.sgl; iter->bi_size; sg = sg_next(sg)) {
+		struct bio_vec bv = mp_bvec_iter_bvec(bvecs, *iter);
+
+		if (nents >= nr_bvec) {
+			ret = -EINVAL;
+			goto out_free_sgl;
+		}
+		sg_set_page(sg, bv.bv_page, bv.bv_len, bv.bv_offset);
+		bvec_iter_advance(bvecs, iter, bv.bv_len);
+		nents++;
+	}
+	sg_mark_end(sg_last(ctx->reg.sgt.sgl, nents));
+	ctx->reg.sgt.orig_nents = nents;
+
+	/* DMA map the scatterlist */
+	ret = ib_dma_map_sgtable_attrs(dev, &ctx->reg.sgt, dir, 0);
+	if (ret)
+		goto out_free_sgl;
+
+	ctx->nr_ops = DIV_ROUND_UP(ctx->reg.sgt.nents, pages_per_mr);
+	ctx->reg.ctx = kcalloc(ctx->nr_ops, sizeof(*ctx->reg.ctx), GFP_KERNEL);
+	if (!ctx->reg.ctx) {
+		ret = -ENOMEM;
+		goto out_unmap_sgt;
+	}
+
+	sg = ctx->reg.sgt.sgl;
+	nents = ctx->reg.sgt.nents;
+	for (i = 0; i < ctx->nr_ops; i++) {
+		struct rdma_rw_reg_ctx *reg = &ctx->reg.ctx[i];
+		u32 sge_cnt = min(nents, pages_per_mr);
+
+		ret = rdma_rw_init_one_mr(qp, port_num, reg, sg, sge_cnt, 0);
+		if (ret < 0)
+			goto out_free_mrs;
+		count += ret;
+		count += rdma_rw_init_reg_wr(reg, prev, qp, port_num,
+				remote_addr, rkey, dir);
+		remote_addr += reg->sge.length;
+		nents -= sge_cnt;
+		sg += sge_cnt;
+		prev = reg;
+	}
+
+	if (prev)
+		prev->wr.wr.next = NULL;
+
+	ctx->type = RDMA_RW_MR;
+	return count;
+
+out_free_mrs:
+	while (--i >= 0)
+		ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg.ctx[i].mr);
+	kfree(ctx->reg.ctx);
+out_unmap_sgt:
+	ib_dma_unmap_sgtable_attrs(dev, &ctx->reg.sgt, dir, 0);
+out_free_sgl:
+	kfree(ctx->reg.sgt.sgl);
+	return ret;
+}
+
 static int rdma_rw_init_map_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 		struct scatterlist *sg, u32 sg_cnt, u32 offset,
 		u64 remote_addr, u32 rkey, enum dma_data_direction dir)
@@ -547,19 +638,13 @@ EXPORT_SYMBOL(rdma_rw_ctx_init);
  * @rkey:	remote key to operate on
  * @dir:	%DMA_TO_DEVICE for RDMA WRITE, %DMA_FROM_DEVICE for RDMA READ
  *
- * Accepts bio_vec arrays directly, avoiding scatterlist conversion for
- * callers that already have data in bio_vec form. Prefer this over
- * rdma_rw_ctx_init() when the source data is a bio_vec array.
- *
- * This function does not support devices requiring memory registration.
- * iWARP devices and configurations with force_mr=1 should use
- * rdma_rw_ctx_init() with a scatterlist instead.
+ * Maps the bio_vec array directly, avoiding intermediate scatterlist
+ * conversion. Supports MR registration for iWARP devices and force_mr mode.
  *
  * Returns the number of WQEs that will be needed on the workqueue if
  * successful, or a negative error code:
  *
  *   * -EINVAL  - @nr_bvec is zero or @iter.bi_size is zero
- *   * -EOPNOTSUPP - device requires MR path (iWARP or force_mr=1)
  *   * -ENOMEM - DMA mapping or memory allocation failed
  */
 int rdma_rw_ctx_init_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
@@ -567,14 +652,24 @@ int rdma_rw_ctx_init_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 		struct bvec_iter iter, u64 remote_addr, u32 rkey,
 		enum dma_data_direction dir)
 {
+	struct ib_device *dev = qp->pd->device;
 	int ret;
 
 	if (nr_bvec == 0 || iter.bi_size == 0)
 		return -EINVAL;
 
-	/* MR path not supported for bvec - reject iWARP and force_mr */
-	if (rdma_rw_io_needs_mr(qp->device, port_num, dir, nr_bvec))
-		return -EOPNOTSUPP;
+	/*
+	 * iWARP requires MR registration for all RDMA READs. The force_mr
+	 * debug option also mandates MR usage.
+	 */
+	if (dir == DMA_FROM_DEVICE && rdma_protocol_iwarp(dev, port_num))
+		return rdma_rw_init_mr_wrs_bvec(ctx, qp, port_num, bvecs,
+						nr_bvec, &iter, remote_addr,
+						rkey, dir);
+	if (unlikely(rdma_rw_force_mr))
+		return rdma_rw_init_mr_wrs_bvec(ctx, qp, port_num, bvecs,
+						nr_bvec, &iter, remote_addr,
+						rkey, dir);
 
 	if (nr_bvec == 1)
 		return rdma_rw_init_single_wr_bvec(ctx, qp, bvecs, &iter,
@@ -582,13 +677,23 @@ int rdma_rw_ctx_init_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 
 	/*
 	 * Try IOVA-based mapping first for multi-bvec transfers.
-	 * This reduces IOTLB sync overhead by batching all mappings.
+	 * IOVA coalesces bvecs into a single DMA-contiguous region,
+	 * reducing the number of WRs needed and avoiding MR overhead.
 	 */
 	ret = rdma_rw_init_iova_wrs_bvec(ctx, qp, bvecs, &iter, remote_addr,
 			rkey, dir);
 	if (ret != -EOPNOTSUPP)
 		return ret;
 
+	/*
+	 * IOVA mapping not available. Check if MR registration provides
+	 * better performance than multiple SGE entries.
+	 */
+	if (rdma_rw_io_needs_mr(dev, port_num, dir, nr_bvec))
+		return rdma_rw_init_mr_wrs_bvec(ctx, qp, port_num, bvecs,
+						nr_bvec, &iter, remote_addr,
+						rkey, dir);
+
 	return rdma_rw_init_map_wrs_bvec(ctx, qp, bvecs, nr_bvec, &iter,
 			remote_addr, rkey, dir);
 }
@@ -649,23 +754,23 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 
 	ctx->type = RDMA_RW_SIG_MR;
 	ctx->nr_ops = 1;
-	ctx->reg = kzalloc(sizeof(*ctx->reg), GFP_KERNEL);
-	if (!ctx->reg) {
+	ctx->reg.ctx = kzalloc(sizeof(*ctx->reg.ctx), GFP_KERNEL);
+	if (!ctx->reg.ctx) {
 		ret = -ENOMEM;
 		goto out_unmap_prot_sg;
 	}
 
-	ctx->reg->mr = ib_mr_pool_get(qp, &qp->sig_mrs);
-	if (!ctx->reg->mr) {
+	ctx->reg.ctx->mr = ib_mr_pool_get(qp, &qp->sig_mrs);
+	if (!ctx->reg.ctx->mr) {
 		ret = -EAGAIN;
 		goto out_free_ctx;
 	}
 
-	count += rdma_rw_inv_key(ctx->reg);
+	count += rdma_rw_inv_key(ctx->reg.ctx);
 
-	memcpy(ctx->reg->mr->sig_attrs, sig_attrs, sizeof(struct ib_sig_attrs));
+	memcpy(ctx->reg.ctx->mr->sig_attrs, sig_attrs, sizeof(struct ib_sig_attrs));
 
-	ret = ib_map_mr_sg_pi(ctx->reg->mr, sg, sgt.nents, NULL, prot_sg,
+	ret = ib_map_mr_sg_pi(ctx->reg.ctx->mr, sg, sgt.nents, NULL, prot_sg,
 			      prot_sgt.nents, NULL, SZ_4K);
 	if (unlikely(ret)) {
 		pr_err("failed to map PI sg (%u)\n",
@@ -673,24 +778,24 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 		goto out_destroy_sig_mr;
 	}
 
-	ctx->reg->reg_wr.wr.opcode = IB_WR_REG_MR_INTEGRITY;
-	ctx->reg->reg_wr.wr.wr_cqe = NULL;
-	ctx->reg->reg_wr.wr.num_sge = 0;
-	ctx->reg->reg_wr.wr.send_flags = 0;
-	ctx->reg->reg_wr.access = IB_ACCESS_LOCAL_WRITE;
+	ctx->reg.ctx->reg_wr.wr.opcode = IB_WR_REG_MR_INTEGRITY;
+	ctx->reg.ctx->reg_wr.wr.wr_cqe = NULL;
+	ctx->reg.ctx->reg_wr.wr.num_sge = 0;
+	ctx->reg.ctx->reg_wr.wr.send_flags = 0;
+	ctx->reg.ctx->reg_wr.access = IB_ACCESS_LOCAL_WRITE;
 	if (rdma_protocol_iwarp(qp->device, port_num))
-		ctx->reg->reg_wr.access |= IB_ACCESS_REMOTE_WRITE;
-	ctx->reg->reg_wr.mr = ctx->reg->mr;
-	ctx->reg->reg_wr.key = ctx->reg->mr->lkey;
+		ctx->reg.ctx->reg_wr.access |= IB_ACCESS_REMOTE_WRITE;
+	ctx->reg.ctx->reg_wr.mr = ctx->reg.ctx->mr;
+	ctx->reg.ctx->reg_wr.key = ctx->reg.ctx->mr->lkey;
 	count++;
 
-	ctx->reg->sge.addr = ctx->reg->mr->iova;
-	ctx->reg->sge.length = ctx->reg->mr->length;
+	ctx->reg.ctx->sge.addr = ctx->reg.ctx->mr->iova;
+	ctx->reg.ctx->sge.length = ctx->reg.ctx->mr->length;
 	if (sig_attrs->wire.sig_type == IB_SIG_TYPE_NONE)
-		ctx->reg->sge.length -= ctx->reg->mr->sig_attrs->meta_length;
+		ctx->reg.ctx->sge.length -= ctx->reg.ctx->mr->sig_attrs->meta_length;
 
-	rdma_wr = &ctx->reg->wr;
-	rdma_wr->wr.sg_list = &ctx->reg->sge;
+	rdma_wr = &ctx->reg.ctx->wr;
+	rdma_wr->wr.sg_list = &ctx->reg.ctx->sge;
 	rdma_wr->wr.num_sge = 1;
 	rdma_wr->remote_addr = remote_addr;
 	rdma_wr->rkey = rkey;
@@ -698,15 +803,15 @@ int rdma_rw_ctx_signature_init(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 		rdma_wr->wr.opcode = IB_WR_RDMA_WRITE;
 	else
 		rdma_wr->wr.opcode = IB_WR_RDMA_READ;
-	ctx->reg->reg_wr.wr.next = &rdma_wr->wr;
+	ctx->reg.ctx->reg_wr.wr.next = &rdma_wr->wr;
 	count++;
 
 	return count;
 
 out_destroy_sig_mr:
-	ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg->mr);
+	ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg.ctx->mr);
 out_free_ctx:
-	kfree(ctx->reg);
+	kfree(ctx->reg.ctx);
 out_unmap_prot_sg:
 	if (prot_sgt.nents)
 		ib_dma_unmap_sgtable_attrs(dev, &prot_sgt, dir, 0);
@@ -754,16 +859,16 @@ struct ib_send_wr *rdma_rw_ctx_wrs(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 	case RDMA_RW_SIG_MR:
 	case RDMA_RW_MR:
 		for (i = 0; i < ctx->nr_ops; i++) {
-			rdma_rw_update_lkey(&ctx->reg[i],
-				ctx->reg[i].wr.wr.opcode !=
+			rdma_rw_update_lkey(&ctx->reg.ctx[i],
+				ctx->reg.ctx[i].wr.wr.opcode !=
 					IB_WR_RDMA_READ_WITH_INV);
 		}
 
-		if (ctx->reg[0].inv_wr.next)
-			first_wr = &ctx->reg[0].inv_wr;
+		if (ctx->reg.ctx[0].inv_wr.next)
+			first_wr = &ctx->reg.ctx[0].inv_wr;
 		else
-			first_wr = &ctx->reg[0].reg_wr.wr;
-		last_wr = &ctx->reg[ctx->nr_ops - 1].wr.wr;
+			first_wr = &ctx->reg.ctx[0].reg_wr.wr;
+		last_wr = &ctx->reg.ctx[ctx->nr_ops - 1].wr.wr;
 		break;
 	case RDMA_RW_IOVA:
 		first_wr = &ctx->iova.wr.wr;
@@ -833,9 +938,11 @@ void rdma_rw_ctx_destroy(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 
 	switch (ctx->type) {
 	case RDMA_RW_MR:
+		/* Bvec MR contexts must use rdma_rw_ctx_destroy_bvec() */
+		WARN_ON_ONCE(ctx->reg.sgt.sgl);
 		for (i = 0; i < ctx->nr_ops; i++)
-			ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg[i].mr);
-		kfree(ctx->reg);
+			ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg.ctx[i].mr);
+		kfree(ctx->reg.ctx);
 		break;
 	case RDMA_RW_MULTI_WR:
 		kfree(ctx->map.wrs);
@@ -880,6 +987,13 @@ void rdma_rw_ctx_destroy_bvec(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 	u32 i;
 
 	switch (ctx->type) {
+	case RDMA_RW_MR:
+		for (i = 0; i < ctx->nr_ops; i++)
+			ib_mr_pool_put(qp, &qp->rdma_mrs, ctx->reg.ctx[i].mr);
+		kfree(ctx->reg.ctx);
+		ib_dma_unmap_sgtable_attrs(dev, &ctx->reg.sgt, dir, 0);
+		kfree(ctx->reg.sgt.sgl);
+		break;
 	case RDMA_RW_IOVA:
 		dma_iova_destroy(dev->dma_device, &ctx->iova.state,
 				 ctx->iova.mapped_len, dir, 0);
@@ -921,8 +1035,8 @@ void rdma_rw_ctx_destroy_signature(struct rdma_rw_ctx *ctx, struct ib_qp *qp,
 	if (WARN_ON_ONCE(ctx->type != RDMA_RW_SIG_MR))
 		return;
 
-	ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg->mr);
-	kfree(ctx->reg);
+	ib_mr_pool_put(qp, &qp->sig_mrs, ctx->reg.ctx->mr);
+	kfree(ctx->reg.ctx);
 
 	if (prot_sg_cnt)
 		ib_dma_unmap_sg(qp->pd->device, prot_sg, prot_sg_cnt, dir);
diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c
index af811d060cc8..0c6152b7660e 100644
--- a/drivers/infiniband/ulp/isert/ib_isert.c
+++ b/drivers/infiniband/ulp/isert/ib_isert.c
@@ -1589,7 +1589,7 @@ isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
 
 	isert_dbg("Cmd %p\n", isert_cmd);
 
-	ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr);
+	ret = isert_check_pi_status(cmd, isert_cmd->rw.reg.ctx->mr);
 	isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
 
 	if (ret) {
@@ -1635,7 +1635,7 @@ isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
 	iscsit_stop_dataout_timer(cmd);
 
 	if (isert_prot_cmd(isert_conn, se_cmd))
-		ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr);
+		ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg.ctx->mr);
 	isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
 	cmd->write_data_done = 0;
 
diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c
index 9c12b2361a6d..a4aa6719a86e 100644
--- a/drivers/nvme/target/rdma.c
+++ b/drivers/nvme/target/rdma.c
@@ -767,7 +767,7 @@ static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc)
 	}
 
 	if (rsp->req.metadata_len)
-		status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr);
+		status = nvmet_rdma_check_pi_status(rsp->rw.reg.ctx->mr);
 	nvmet_rdma_rw_ctx_destroy(rsp);
 
 	if (unlikely(status))
@@ -808,7 +808,7 @@ static void nvmet_rdma_write_data_done(struct ib_cq *cq, struct ib_wc *wc)
 	 * - if succeeded send good NVMe response
 	 * - if failed send bad NVMe response with appropriate error
 	 */
-	status = nvmet_rdma_check_pi_status(rsp->rw.reg->mr);
+	status = nvmet_rdma_check_pi_status(rsp->rw.reg.ctx->mr);
 	if (unlikely(status))
 		rsp->req.cqe->status = cpu_to_le16(status << 1);
 	nvmet_rdma_rw_ctx_destroy(rsp);
diff --git a/include/rdma/rw.h b/include/rdma/rw.h
index 205e16ed6cd8..53ed0f05fa25 100644
--- a/include/rdma/rw.h
+++ b/include/rdma/rw.h
@@ -41,13 +41,16 @@ struct rdma_rw_ctx {
 		} iova;
 
 		/* for registering multiple WRs: */
-		struct rdma_rw_reg_ctx {
-			struct ib_sge		sge;
-			struct ib_rdma_wr	wr;
-			struct ib_reg_wr	reg_wr;
-			struct ib_send_wr	inv_wr;
-			struct ib_mr		*mr;
-		} *reg;
+		struct {
+			struct rdma_rw_reg_ctx {
+				struct ib_sge		sge;
+				struct ib_rdma_wr	wr;
+				struct ib_reg_wr	reg_wr;
+				struct ib_send_wr	inv_wr;
+				struct ib_mr		*mr;
+			}			*ctx;
+			struct sg_table		sgt;
+		} reg;
 	};
 };
 
-- 
2.52.0